1use std::sync::Arc;
9use std::time::Duration;
10
11use axum::http::HeaderMap;
12use bytes::{Bytes, BytesMut};
13use cpa_common::gostr::GoStr;
14use cpa_core::config::Config;
15use cpa_core::exec::{Caller, ExecError, ExecRequest, ExecStream, Operation, ResponseBody};
16use cpa_core::format::Format;
17use futures_util::StreamExt;
18
19use crate::classify;
20use crate::gojson;
21use crate::registry::{self, AliasResult, Registry};
22use crate::respond;
23use crate::runtime::{AcquireError, Completing, Lease, Outcome, Runtime, Selection};
24use crate::scheduler::{Policy, canonical_model};
25
26#[derive(Clone)]
28pub struct Call {
29 pub entry: Format,
31 pub response: Format,
33 pub operation: Operation,
34 pub model: String,
36 pub body: Bytes,
37 pub stream: bool,
38 pub alt: Option<String>,
39 pub headers: HeaderMap,
40 pub caller: Caller,
41 pub forced_provider: Option<String>,
43 pub selection_model: Option<String>,
45 pub execution_session: Option<String>,
47 pub request_path: String,
49 pub peer: Option<std::net::SocketAddr>,
51 pub turn: Option<Arc<SessionTurn>>,
54 pub media: Option<Arc<Media>>,
57}
58
59pub struct Media {
64 pub kind: MediaKind,
65 pub pinned: Option<String>,
67 pub on_selected: Option<OnSelected>,
69 pub sse: bool,
75 pub disallow_free: bool,
78}
79
80#[derive(Clone)]
84pub struct SseError(pub Bytes);
85
86#[derive(Clone, Copy, Debug, PartialEq, Eq)]
87pub enum MediaKind {
88 Images,
91 Videos,
93 Speech,
96}
97
98impl Call {
99 pub(crate) fn pinned(&self) -> Option<String> {
101 self.turn
102 .as_ref()
103 .and_then(|t| t.pinned.clone())
104 .or_else(|| self.media.as_ref().and_then(|m| m.pinned.clone()))
105 .or_else(|| {
106 crate::plugins::model_callbacks::context()
107 .map(|c| c.auth_id)
108 .filter(|id| !id.is_empty())
109 })
110 }
111
112 fn on_selected(&self) -> Option<&OnSelected> {
114 self.turn
115 .as_ref()
116 .and_then(|t| t.on_selected.as_ref())
117 .or_else(|| self.media.as_ref().and_then(|m| m.on_selected.as_ref()))
118 }
119}
120
121pub struct SessionTurn {
125 pub session: cpa_core::exec::ExecSession,
127 pub pinned: Option<String>,
130 pub on_selected: Option<OnSelected>,
133 pub home: Option<Arc<crate::home_session::SessionHome>>,
136}
137
138pub type OnSelected = Box<dyn Fn(&cpa_core::credential::Credential) + Send + Sync>;
140
141pub type Peer = Option<axum::Extension<axum::extract::ConnectInfo<std::net::SocketAddr>>>;
143
144pub fn peer(peer: Peer) -> Option<std::net::SocketAddr> {
145 peer.map(|axum::Extension(axum::extract::ConnectInfo(addr))| addr)
146}
147
148pub fn route_path(matched: Option<&axum::extract::MatchedPath>, uri: &axum::http::Uri) -> String {
151 let Some(matched) = matched else {
152 return uri.path().to_owned();
153 };
154 matched
155 .as_str()
156 .split('/')
157 .map(
158 |segment| match segment.strip_prefix('{').and_then(|s| s.strip_suffix('}')) {
159 Some(name) if name.starts_with('*') => name.to_owned(),
160 Some(name) => format!(":{name}"),
161 None => segment.to_owned(),
162 },
163 )
164 .collect::<Vec<_>>()
165 .join("/")
166}
167
168pub enum Done {
169 Buffered {
170 headers: HeaderMap,
171 body: Bytes,
172 },
173 Stream {
175 headers: HeaderMap,
176 first: Option<Bytes>,
177 rest: ExecStream,
178 },
179}
180
181#[derive(Debug, Clone)]
182pub enum Failure {
183 Exec(ExecError),
185 Remote { error: ExecError, retry_after: Option<u64> },
188 UnknownModel(String),
190 ImageOnly(String),
192 SpeechOnly(String),
194 Unavailable {
196 code: &'static str,
197 providers: Vec<String>,
198 model: String,
199 cause: Option<String>,
200 retry_after: Option<Duration>,
201 },
202 Cooldown {
204 model: String,
205 provider: String,
206 wait: Duration,
207 cause: Option<String>,
208 },
209}
210
211impl Failure {
212 pub fn status(&self) -> u16 {
213 match self {
214 Failure::Exec(e) | Failure::Remote { error: e, .. } => classify::response_status(e),
215 Failure::UnknownModel(_) | Failure::SpeechOnly(_) => 400,
216 Failure::ImageOnly(_) | Failure::Unavailable { .. } => 503,
217 Failure::Cooldown { .. } => 429,
218 }
219 }
220
221 pub fn text(&self) -> String {
223 match self {
224 Failure::Exec(e) | Failure::Remote { error: e, .. } => classify::error_text(e),
225 Failure::UnknownModel(model) => {
226 let message = gojson::sjson_string(&format!("unknown provider for model {model}"));
227 format!(
228 r#"{{"error":{{"message":{message},"type":"invalid_request_error","code":"model_not_found","param":"model"}}}}"#
229 )
230 }
231 Failure::ImageOnly(model) => format!(
232 "model {} is only supported on /v1/images/generations and /v1/images/edits",
233 route_base_name(model)
234 ),
235 Failure::SpeechOnly(model) => format!(
236 "model {} is only supported on /v1/audio/speech and /v1/tts",
237 route_base_name(model)
238 ),
239 Failure::Unavailable {
240 code,
241 providers,
242 model,
243 cause,
244 ..
245 } => unavailable_text(code, "no auth available", providers, model, cause.as_deref()),
246 Failure::Cooldown {
247 model,
248 provider,
249 wait,
250 cause,
251 } => {
252 let shown = if model.is_empty() { "requested model" } else { model };
253 let mut message = format!("All credentials for model {shown} are cooling down");
254 if !provider.is_empty() {
255 message.push_str(&format!(" via provider {provider}"));
256 }
257 let display = if !wait.is_zero() && *wait < Duration::from_secs(1) {
258 Duration::from_secs(1)
259 } else {
260 Duration::from_secs((wait.as_secs_f64()).round() as u64)
261 };
262 let mut body = serde_json::Map::new();
263 body.insert("code".into(), "model_cooldown".into());
264 body.insert("model".into(), model.clone().into());
265 body.insert("reset_time".into(), gojson::duration(display).into());
266 body.insert("reset_seconds".into(), ceil_seconds(*wait).into());
267 if !provider.is_empty() {
268 body.insert("provider".into(), provider.clone().into());
269 }
270 if let Some(summary) = cause.as_deref().map(upstream_summary).filter(|s| !s.is_empty()) {
271 message.push_str(&format!(" (last error: {summary})"));
272 body.insert("last_upstream_error".into(), summary.into());
273 }
274 body.insert("message".into(), message.into());
275 gojson::sorted(&serde_json::json!({ "error": body }))
276 }
277 }
278 }
279
280 pub fn retry_after(&self) -> Option<u64> {
282 match self {
283 Failure::Cooldown { wait, .. } => Some(ceil_seconds(*wait)),
284 Failure::Remote { retry_after, .. } => *retry_after,
285 Failure::Unavailable {
286 retry_after: Some(wait),
287 ..
288 } if !wait.is_zero() => Some(ceil_seconds(*wait).max(1)),
289 _ => None,
290 }
291 }
292
293 pub fn direct(&self) -> Option<&ExecError> {
295 match self {
297 Failure::Exec(e) | Failure::Remote { error: e, .. } if e.direct => Some(e),
298 _ => None,
299 }
300 }
301}
302
303fn unavailable_text(code: &str, base: &str, providers: &[String], model: &str, cause: Option<&str>) -> String {
306 let providers = if providers.is_empty() {
307 "unknown".into()
308 } else {
309 providers.join(",")
310 };
311 let model = if gojson::trim(model).is_empty() {
312 "unknown"
313 } else {
314 gojson::trim(model)
315 };
316 let mut detail = match cause
317 .map(upstream_summary)
318 .filter(|s| !s.is_empty() && !base.contains(s.as_str()))
319 {
320 Some(summary) => format!("{base} (providers={providers}, model={model}; last upstream error: {summary})"),
321 None => format!("{base} (providers={providers}, model={model})"),
322 };
323 if format!(",{providers},").contains(",claude,") {
324 detail.push_str("; check Claude auth/key session and cooldown state via /v0/management/auth-files");
325 }
326 format!("{code}: {detail}")
327}
328
329fn ceil_seconds(d: Duration) -> u64 {
330 d.as_secs() + u64::from(d.subsec_nanos() > 0)
331}
332
333pub fn upstream_summary(raw: &str) -> String {
335 let raw = raw.trim();
336 if raw.is_empty() {
337 return String::new();
338 }
339 let json_part = match raw.find(": {") {
340 Some(i) if i < 50 => raw[i + 2..].trim(),
341 _ => raw,
342 };
343 if let Ok(v) = serde_json::from_str::<serde_json::Value>(json_part) {
344 let s = |v: Option<&serde_json::Value>| gojson::trim(&gojson::gjson_string(v)).to_owned();
345 let (mut code, mut message) = (String::new(), String::new());
346 match v.get("error") {
347 Some(e @ serde_json::Value::Object(_)) => {
348 code = s(e.get("code"));
349 if code.is_empty() {
350 code = s(e.get("type"));
351 }
352 message = s(e.get("message"));
353 }
354 Some(serde_json::Value::String(m)) => message = m.trim().to_owned(),
355 _ => {}
356 }
357 if code.is_empty() && message.is_empty() {
358 code = s(v.get("code"));
359 if code.is_empty() {
360 code = s(v.get("type"));
361 }
362 message = s(v.get("message"));
363 }
364 let summary = match (code.is_empty(), message.is_empty()) {
365 (false, false) if code.go_eq_fold(&message) || message.go_lower().contains(&code.go_lower()) => message,
366 (false, false) => format!("{code}: {message}"),
367 (true, false) => message,
368 (false, true) => code,
369 (true, true) => String::new(),
370 };
371 if !summary.is_empty() {
372 return crate::sanitize::summary(&summary);
373 }
374 }
375 crate::sanitize::summary(raw)
376}
377
378fn route_base_name(model: &str) -> &str {
380 let model = model.trim();
381 model
382 .rsplit('/')
383 .next()
384 .filter(|s| !s.is_empty())
385 .unwrap_or(model)
386 .trim()
387}
388
389fn check_route_only(base: &str, call: &Call) -> Result<(), RunError> {
393 let name = route_base_name(base).go_lower();
394 let kind = call.media.as_ref().map(|m| m.kind);
395 if IMAGE_ONLY.contains(&name.as_str()) && kind != Some(MediaKind::Images) {
396 return Err(Failure::ImageOnly(base.to_owned()).into());
397 }
398 if SPEECH_ONLY.contains(&name.as_str()) && kind != Some(MediaKind::Speech) {
399 return Err(Failure::SpeechOnly(base.to_owned()).into());
400 }
401 Ok(())
402}
403
404const SPEECH_ONLY: [&str; 2] = ["grok-tts", "grok-voice-tts-1.0"];
406
407const IMAGE_ONLY: [&str; 8] = [
408 "gpt-image-1.5",
409 "gpt-image-2",
410 "gpt-image-2.5-flare",
411 "gpt-image-2.5-sunburst",
412 "gpt-image-2.5",
413 "grok-imagine-image",
414 "grok-imagine-image-quality",
415 "grok-imagine-image-2.0",
416];
417
418fn route(
420 rt: &Runtime,
421 registry: &Registry,
422 call: &Call,
423 routed: Option<&(String, Option<String>)>,
424) -> Result<(Vec<String>, String), RunError> {
425 if let Some(provider) = &call.forced_provider {
426 let model = gojson::trim(&call.model);
427 check_route_only(gojson::trim(canonical_model_raw(model)), call)?;
428 return Ok((vec![provider.clone()], model.to_owned()));
429 }
430 if let Some((provider, model)) = routed {
433 let model = model.clone().unwrap_or_else(|| call.model.clone());
434 check_route_only(gojson::trim(canonical_model_raw(&model)), call)?;
435 let mut providers = vec![provider.clone()];
436 adjust_for_entry(call.entry, &mut providers);
437 return Ok((providers, model));
438 }
439 let model = call.model.as_str();
440 let base = match model.rfind('(') {
441 Some(i) if model.ends_with(')') => &model[..i],
442 _ => model,
443 };
444 let remote = rt.remote_dispatch().is_some();
446 let resolved = if base == "auto" && !remote {
447 let first = registry
448 .resolve_auto(|client, model| rt.suspension(client, model))
449 .unwrap_or_else(|| "auto".into());
450 format!("{first}{}", &model[base.len()..])
451 } else {
452 model.to_owned()
453 };
454 let base = gojson::trim(canonical_model_raw(&resolved)).to_owned();
455 check_route_only(&base, call)?;
456 if remote {
457 return Ok((vec!["home".to_owned()], resolved));
458 }
459 let mut providers = registry.providers(&base);
460 if providers.is_empty() && base != resolved {
461 providers = registry.providers(&resolved);
462 }
463 if providers.is_empty() {
464 return Err(Failure::UnknownModel(call.model.clone()).into());
465 }
466 adjust_for_entry(call.entry, &mut providers);
467 Ok((providers, resolved))
468}
469
470fn adjust_for_entry(entry: Format, providers: &mut Vec<String>) {
472 match entry {
473 Format::Interactions => {
474 if let Some(i) = providers.iter().position(|p| p == "gemini-interactions") {
475 let p = providers.remove(i);
476 providers.insert(0, p);
477 }
478 }
479 Format::OpenAI | Format::OpenAIResponse | Format::Claude | Format::Gemini => {}
480 _ => providers.retain(|p| p != "gemini-interactions"),
481 }
482}
483
484fn canonical_model_raw(model: &str) -> &str {
486 match model.rfind('(') {
487 Some(i) if model.ends_with(')') => &model[..i],
488 _ => model,
489 }
490}
491
492pub async fn serve<F, Fut>(rt: &Arc<Runtime>, call: Call, render: F) -> axum::response::Response
502where
503 F: FnOnce(Result<Done, Failure>) -> Fut + Send + 'static,
504 Fut: std::future::Future<Output = axum::response::Response> + Send + 'static,
505{
506 let sse = call.media.as_ref().is_some_and(|m| m.sse);
507 let speech = call.media.as_ref().is_some_and(|m| m.kind == MediaKind::Speech);
508 let interval = if sse {
509 crate::respond::keepalive(&rt.config())
510 } else {
511 nonstream_keepalive(&call, &rt.config())
512 };
513 let trace = Arc::new(Trace::with_request_id(crate::observability::current_request_id()));
514 let mut run = Box::pin(crate::remote::keep_query_credential({
516 let (rt, trace) = (rt.clone(), trace.clone());
517 async move {
518 let result = run_with_bootstrap_retries(&rt, call, &trace).await;
519 let cfg = rt.config();
520 let upstream = upstream_headers(&cfg, &result);
521 let result = match result {
522 Ok(Done::Buffered { headers, body }) if speech => {
526 let headers = if respond::passthrough_headers(&cfg) {
527 respond::filter_upstream_headers(&headers)
528 } else {
529 respond::filter_upstream_headers(&trace.intercepted_headers_snapshot())
531 };
532 Ok(Done::Buffered { headers, body })
533 }
534 result => result,
535 };
536 (result, upstream)
537 }
538 }));
539 let traced = {
542 let trace = trace.clone();
543 move |mut response: axum::response::Response, upstream: Option<Upstream>| {
544 let passthrough = matches!(upstream, Some(Upstream::Success(_)));
546 match upstream {
547 Some(Upstream::Success(headers)) => respond::write_upstream_headers(response.headers_mut(), &headers),
548 Some(Upstream::Error(headers)) => respond::write_error_headers(response.headers_mut(), &headers),
549 None => {}
550 }
551 let intercepted = trace.intercepted_headers();
554 if !passthrough && !intercepted.is_empty() {
555 respond::write_upstream_headers(
556 response.headers_mut(),
557 &respond::filter_upstream_headers(&intercepted),
558 );
559 }
560 if let Some(value) = trace.header() {
561 response.headers_mut().insert("x-cpa-trace-id", value);
562 }
563 response
564 }
565 };
566 let Some(interval) = interval else {
567 let (result, upstream) = run.await;
568 return traced(render(result).await, upstream);
569 };
570 tokio::select! {
571 biased;
572 (result, upstream) = &mut run => return traced(render(result).await, upstream),
573 () = tokio::time::sleep(interval) => {}
574 }
575 let mut ticks = tokio::time::interval_at(tokio::time::Instant::now() + interval, interval);
578 ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
579 enum Event {
580 Tick,
581 Done(Result<Done, Failure>),
582 }
583 let events = futures_util::stream::unfold(Some((run, ticks)), |state| async move {
584 let (mut run, mut ticks) = state?;
585 tokio::select! {
586 biased;
587 (result, _) = &mut run => Some((Event::Done(result), None)),
588 _ = ticks.tick() => Some((Event::Tick, Some((run, ticks)))),
589 }
590 });
591 let beat = Bytes::from_static(if sse { b": keep-alive\n\n" } else { b"\n" });
592 let once = |bytes: Bytes| futures_util::stream::iter([Ok::<_, axum::Error>(bytes)]);
593 let first = once(beat.clone());
594 let mut render = Some(render);
595 let body = first.chain(events.flat_map(move |event| match event {
596 Event::Tick => once(beat.clone()).left_stream(),
597 Event::Done(result) => {
598 let render = render.take().expect("the run finishes once");
599 let rendered = futures_util::stream::once(render(result));
600 let body = rendered.flat_map(move |response| match response.extensions().get::<SseError>() {
601 Some(SseError(event)) if sse => once(event.clone()).left_stream(),
602 _ => response.into_body().into_data_stream().right_stream(),
603 });
604 body.right_stream()
605 }
606 }));
607 let body = axum::body::Body::from_stream(body);
608 let mut response = if sse {
609 crate::respond::sse(body)
610 } else {
611 let mut response = axum::response::Response::new(body);
612 response.headers_mut().insert(
613 axum::http::header::CONTENT_TYPE,
614 axum::http::HeaderValue::from_static("application/json"),
615 );
616 response
617 };
618 let headers = response.headers_mut();
619 if let Some(value) = trace.header() {
621 headers.insert("x-cpa-trace-id", value);
622 }
623 response
624}
625
626enum Upstream {
628 Success(HeaderMap),
630 Error(HeaderMap),
632}
633
634fn upstream_headers(cfg: &Config, result: &Result<Done, Failure>) -> Option<Upstream> {
637 if !respond::passthrough_headers(cfg) {
638 return None;
639 }
640 match result {
641 Ok(Done::Buffered { headers, .. } | Done::Stream { headers, .. }) => {
642 Some(Upstream::Success(respond::filter_upstream_headers(headers)))
643 }
644 Err(Failure::Exec(error)) if !error.direct => error.addon.as_deref().cloned().map(Upstream::Error),
645 Err(_) => None,
646 }
647}
648
649fn nonstream_keepalive(call: &Call, cfg: &Config) -> Option<Duration> {
654 let speech = call.media.as_ref().is_some_and(|m| m.kind == MediaKind::Speech);
655 if call.stream || call.operation != Operation::Generate || speech {
656 return None;
657 }
658 let seconds = cfg
659 .document
660 .get("requests")
661 .and_then(|r| r.get("nonstream-keepalive-interval"))
662 .and_then(serde_yaml_ng::Value::as_i64)
663 .unwrap_or(0);
664 (seconds > 0).then(|| Duration::from_secs(seconds as u64))
665}
666
667fn bootstrap_retries(cfg: &Config) -> usize {
669 cfg.document
670 .get("requests")
671 .and_then(|r| r.get("streaming"))
672 .and_then(|s| s.get("bootstrap-retries"))
673 .and_then(serde_yaml_ng::Value::as_i64)
674 .unwrap_or(0)
675 .max(0) as usize
676}
677
678pub async fn run_with_bootstrap_retries(rt: &Arc<Runtime>, call: Call, trace: &Trace) -> Result<Done, Failure> {
681 use crate::plugins::execution::{self as plugin_execution, Route};
684 let query = plugin_execution::query();
685 let forced = call.forced_provider.as_deref().map(|p| p.trim().go_lower());
686 let routed = match (plugin_execution::route(rt, &call, &query, trace).await, forced) {
687 (None, _) => None,
688 (Some(Route::Provider { provider, .. }), Some(forced)) if provider == forced => None,
692 (Some(_), Some(_)) => {
693 return Err(Failure::Exec(ExecError::local(
694 400,
695 cpa_core::exec::FailureScope::Request,
696 "agent is only supported for native interactions execution",
697 )));
698 }
699 (Some(Route::Executor(plugin)), None) => {
700 return plugin_execution::execute(rt, &call, &plugin, &query, trace).await;
701 }
702 (Some(Route::Provider { provider, model }), None) => Some((provider, model)),
703 };
704 let resolved = route(rt, &rt.registry(), &call, routed.as_ref()).map_err(|e| e.failure)?;
706 if call.operation == Operation::Generate && rt.usage_queue().accepts() {
708 trace.prompt_prefix(call.entry, &call.body);
709 }
710 let mut call = call;
711 let hooks = crate::plugins::interceptors::start(rt, trace, &mut call, &resolved.1).await?;
712 let result = retry_bootstrap(rt, call, trace, &resolved, hooks.as_ref()).await;
713 match hooks {
714 Some(hooks) => hooks.finish(result, trace).await,
715 None => result,
716 }
717}
718
719async fn retry_bootstrap(
721 rt: &Arc<Runtime>,
722 call: Call,
723 trace: &Trace,
724 resolved: &(Vec<String>, String),
725 hooks: Option<&Arc<crate::plugins::interceptors::Hooks>>,
726) -> Result<Done, Failure> {
727 let max = if call.stream && rt.remote_dispatch().is_none() {
729 bootstrap_retries(&rt.config())
730 } else {
731 0
732 };
733 let mut result = run_primed(rt, call.clone(), trace, resolved.clone(), hooks).await;
734 for _ in 0..max {
735 let original = match &result {
736 Err(RunError {
737 failure: Failure::Exec(e),
738 bootstrap: true,
739 }) if bootstrap_eligible(classify::go_status(e)) => e.clone(),
740 _ => break,
741 };
742 result = match run_primed(rt, call.clone(), trace, resolved.clone(), hooks).await {
743 Err(RunError {
744 failure,
745 bootstrap: false,
746 }) if matches!(failure, Failure::Unavailable { .. }) && classify::response_status(&original) >= 500 => {
747 Err(RunError {
748 failure: Failure::Exec(original),
749 bootstrap: false,
750 })
751 }
752 Err(RunError {
753 failure,
754 bootstrap: false,
755 }) => {
756 return Err(failure);
757 }
758 other => other,
759 };
760 }
761 result.map_err(|e| e.failure)
762}
763
764async fn run_primed(
768 rt: &Arc<Runtime>,
769 call: Call,
770 trace: &Trace,
771 resolved: (Vec<String>, String),
772 hooks: Option<&Arc<crate::plugins::interceptors::Hooks>>,
773) -> Result<Done, RunError> {
774 let result = run_routed(rt, call, trace, resolved, hooks.map(|h| &**h)).await;
775 match (hooks, result) {
776 (Some(hooks), Ok(done @ Done::Stream { .. })) => {
777 hooks.prime(Ok(done), trace).await.map_err(|failure| RunError {
778 failure,
779 bootstrap: true,
780 })
781 }
782 (_, result) => result,
783 }
784}
785
786fn bootstrap_eligible(status: u16) -> bool {
787 matches!(status, 0 | 401 | 402 | 403 | 408 | 429) || status >= 500
788}
789
790#[derive(Default)]
792pub struct Trace(
793 std::sync::Mutex<Option<String>>,
794 std::sync::OnceLock<String>,
795 cpa_core::exec::CaptureSink,
796 std::sync::Mutex<HeaderMap>,
798 Arc<crate::observability::Clock>,
800 std::sync::OnceLock<String>,
803);
804
805impl Trace {
806 pub fn with_request_id(id: Option<String>) -> Self {
809 let trace = Self(
810 Default::default(),
811 Default::default(),
812 crate::request_logging::current()
813 .map(|log| log.capture_sink())
814 .unwrap_or_default(),
815 Default::default(),
816 crate::observability::current_clock(),
817 Default::default(),
818 );
819 if let Some(id) = id.filter(|id| !id.is_empty()) {
820 let _ = trace.1.set(id);
821 }
822 trace
823 }
824
825 pub fn request_id(&self) -> String {
827 self.1.get_or_init(request_id).clone()
828 }
829
830 pub fn with_capture(mut self, capture: cpa_core::exec::CaptureSink) -> Self {
833 self.2 = capture;
834 self
835 }
836
837 pub(crate) fn capture(&self) -> cpa_core::exec::CaptureSink {
839 self.2.clone()
840 }
841
842 pub(crate) fn set_intercepted_headers(&self, headers: HeaderMap) {
845 *self.3.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = headers;
846 }
847
848 fn intercepted_headers_snapshot(&self) -> HeaderMap {
849 self.3.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone()
850 }
851
852 fn intercepted_headers(&self) -> HeaderMap {
853 std::mem::take(&mut *self.3.lock().unwrap_or_else(std::sync::PoisonError::into_inner))
854 }
855
856 fn header(&self) -> Option<axum::http::HeaderValue> {
858 let id = self
859 .0
860 .lock()
861 .unwrap_or_else(std::sync::PoisonError::into_inner)
862 .clone()?;
863 axum::http::HeaderValue::from_str(&id).ok()
864 }
865
866 pub(crate) fn selected(&self, credential: &cpa_core::credential::Credential) {
867 let index = cpa_core::config::credentials::auth_index(credential);
868 if index.is_empty() {
869 return;
870 }
871 let request = self.1.get_or_init(request_id);
872 let stamp = chrono::Local::now().format("%Y%m%d%H%M%S");
873 *self.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(format!("{stamp}-{index}-{request}"));
874 }
875
876 pub(crate) fn id(&self) -> Option<String> {
878 self.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone()
879 }
880
881 pub(crate) fn clock(&self) -> Arc<crate::observability::Clock> {
882 self.4.clone()
883 }
884
885 pub(crate) fn prompt_prefix(&self, format: Format, body: &[u8]) -> String {
888 self.5
889 .get_or_init(|| crate::usage_record::prompt_prefix(format, body))
890 .clone()
891 }
892}
893
894pub fn request_id() -> String {
896 use std::hash::{BuildHasher, Hasher};
897 let random = || {
898 let mut h = std::collections::hash_map::RandomState::new().build_hasher();
899 h.write_u128(
900 std::time::SystemTime::now()
901 .duration_since(std::time::UNIX_EPOCH)
902 .unwrap_or_default()
903 .as_nanos(),
904 );
905 h.finish()
906 };
907 let millis = std::time::SystemTime::now()
908 .duration_since(std::time::UNIX_EPOCH)
909 .unwrap_or_default()
910 .as_millis() as u64;
911 let (a, b) = (random(), random());
912 let hi = (millis << 16) | 0x7000 | (a & 0x0fff);
913 let lo = (b & 0x3fff_ffff_ffff_ffff) | 0x8000_0000_0000_0000;
914 format!(
915 "{:08x}-{:04x}-{:04x}-{:04x}-{:012x}",
916 hi >> 32,
917 (hi >> 16) & 0xffff,
918 hi & 0xffff,
919 lo >> 48,
920 lo & 0xffff_ffff_ffff
921 )
922}
923
924pub struct RunError {
927 pub failure: Failure,
928 pub bootstrap: bool,
929}
930
931impl From<Failure> for RunError {
932 fn from(failure: Failure) -> Self {
933 Self {
934 failure,
935 bootstrap: false,
936 }
937 }
938}
939
940#[derive(Clone)]
942struct Fault {
943 error: ExecError,
944 bootstrap: Option<Box<HeaderMap>>,
947}
948
949impl Fault {
950 fn into_run_error(self) -> RunError {
951 RunError {
952 failure: Failure::Exec(self.error),
953 bootstrap: self.bootstrap.is_some(),
954 }
955 }
956}
957
958pub async fn run(rt: &Arc<Runtime>, call: Call, trace: &Trace) -> Result<Done, RunError> {
961 let resolved = route(rt, &rt.registry(), &call, None)?;
962 run_routed(rt, call, trace, resolved, None).await
963}
964
965async fn run_routed(
967 rt: &Arc<Runtime>,
968 call: Call,
969 trace: &Trace,
970 (providers, model): (Vec<String>, String),
971 hooks: Option<&crate::plugins::interceptors::Hooks>,
972) -> Result<Done, RunError> {
973 let (cfg, policy) = rt.request_snapshot();
974 let registry = rt.registry();
975 let aliases = registry::global_aliases(&cfg);
976 let session = crate::session::resolve(
977 call.entry,
978 &call.headers,
979 &call.body,
980 call.execution_session.as_deref(),
981 &call.caller.principal,
982 );
983 let home_count = call.operation == Operation::CountTokens && rt.remote_dispatch().is_some();
986 let facts = ((call.operation == Operation::Generate
987 && (rt.usage_queue().accepts() || rt.plugins().has_usage_plugins()))
988 || (home_count && rt.usage_queue().accepts()))
989 .then(|| {
990 Arc::new(crate::usage_record::Facts::new(
991 usage_client(&cfg, &call, trace, &session),
992 call.entry,
993 call.response,
994 &call.model,
995 &call.body,
996 call.stream,
997 trace,
998 ))
999 });
1000 let (usage, unauthorized) = if home_count { (None, facts) } else { (facts, None) };
1001 let mut pinned_exclusion: Vec<String> = match call.pinned() {
1004 Some(pinned) if !pinned.is_empty() && rt.remote_dispatch().is_none() => rt
1005 .store()
1006 .snapshot()
1007 .iter()
1008 .filter(|c| c.id != pinned)
1009 .map(|c| c.id.clone())
1010 .collect(),
1011 _ => Vec::new(),
1012 };
1013 if call.media.as_ref().is_some_and(|m| m.disallow_free) {
1016 for credential in rt.store().snapshot() {
1017 let free = credential.provider.trim().eq_ignore_ascii_case("codex")
1018 && credential
1019 .attributes
1020 .get("plan_type")
1021 .is_some_and(|plan| plan.trim().eq_ignore_ascii_case("free"));
1022 if free && !pinned_exclusion.contains(&credential.id) {
1023 pinned_exclusion.push(credential.id.clone());
1024 }
1025 }
1026 }
1027 let mut selection = Selection {
1028 providers: providers.clone(),
1029 model: call.selection_model.clone().unwrap_or_else(|| model.clone()),
1030 session: session.id.clone(),
1031 session_parent: session.parent,
1032 session_fork: session.fork,
1033 exclude: pinned_exclusion.clone(),
1034 ..Selection::default()
1035 };
1036 if rt.plugins().has_scheduler() {
1037 let route_model = selection.model.clone();
1038 selection.scheduler = Some(Arc::new(crate::plugins::execution::scheduler_context(
1039 &call,
1040 &route_model,
1041 policy.session_affinity,
1042 session.derived.as_deref(),
1043 )));
1044 }
1045 let request = ExecRequest {
1046 operation: call.operation,
1047 source_format: call.entry,
1048 response_format: call.response,
1049 requested_model: model.clone(),
1050 model: model.clone(),
1051 original_body: call.body.clone(),
1052 body: call.body.clone(),
1053 stream: call.stream,
1054 stream_payloads: hooks.is_some() || crate::plugins::model_callbacks::context().is_some(),
1055 alt: call.alt.clone(),
1056 session: session.id,
1057 execution_session: call.execution_session.clone(),
1058 derived_session: session.derived,
1059 resolved_model: None,
1060 usage: cpa_core::exec::UsageSink::default().with_capture(trace.2.clone()),
1061 request_path: call.request_path.clone(),
1062 headers: call.headers.clone(),
1063 caller: call.caller.clone(),
1064 };
1065 let compact = call.alt.as_deref() == Some("responses/compact");
1066 if let Some(remote) = rt.remote_dispatch() {
1067 let context = RemoteContext {
1068 rt,
1069 cfg: &cfg,
1070 policy: &policy,
1071 call: &call,
1072 request: &request,
1073 usage: usage.as_ref(),
1074 unauthorized: unauthorized.as_ref(),
1075 aliases: &aliases,
1076 compact,
1077 trace,
1078 user_key: Default::default(),
1079 };
1080 return run_remote(context, remote, selection).await;
1081 }
1082 if policy.session_affinity && !session.explicit {
1085 selection.lcp = crate::lcp::Request::new(call.entry.as_str(), &call.body, &call.caller.principal).map(Arc::new);
1086 }
1087 let mut upstream: Option<Fault> = None;
1090 loop {
1091 let mut last: Option<Fault> = None;
1092 let mut attempted: Vec<String> = Vec::new();
1093 let pick_failure = loop {
1094 if policy.max_retry_credentials > 0 && attempted.len() >= policy.max_retry_credentials {
1095 break None;
1096 }
1097 let lease = match rt.acquire(selection.clone(), &cfg, policy.clone(), ®istry).await {
1098 Ok(lease) => lease,
1099 Err(AcquireError::Prepare { id, error }) => {
1100 attempted.push(id.clone());
1101 selection.exclude.push(id);
1102 last = Some(Fault { error, bootstrap: None });
1103 continue;
1104 }
1105 Err(AcquireError::Cooldown { wait, cause }) => {
1106 let provider = if providers.len() == 1 {
1107 providers[0].clone()
1108 } else {
1109 String::new()
1110 };
1111 break Some(Failure::Cooldown {
1112 model: selection.model.clone(),
1113 provider,
1114 wait,
1115 cause,
1116 });
1117 }
1118 Err(AcquireError::Rejected { code, message }) => {
1119 break Some(scheduler_rejection(code, message, &providers, &model));
1120 }
1121 Err(AcquireError::Scheduler(error)) => break Some(Failure::Exec(error)),
1122 Err(AcquireError::Unavailable { retry_after, cause }) => {
1123 break Some(Failure::Unavailable {
1124 code: if retry_after.is_some() {
1125 "auth_unavailable"
1126 } else {
1127 "auth_not_found"
1128 },
1129 providers: providers.clone(),
1130 model: model.clone(),
1131 cause,
1132 retry_after,
1133 });
1134 }
1135 };
1136 selection.exclude.push(lease.credential.id.clone());
1137 trace.selected(&lease.credential);
1138 let (mut models, alias) = registry::execution_models(&aliases, &lease.credential, &selection.model);
1139 let pooled = models.len() > 1;
1140 if pooled {
1141 let key = format!(
1143 "{}|{}|{}",
1144 lease.credential.id.trim().go_lower(),
1145 registry::provider_key(&lease.credential),
1146 canonical_model(registry::strip_prefix(&selection.model, &lease.credential)).go_lower()
1147 );
1148 let offset = rt.next_pool_offset(&key, models.len());
1149 models.rotate_left(offset);
1150 }
1151 let selection_model = registry::selection_model(&aliases, &lease.credential, &selection.model);
1152 let models: Vec<String> = models
1153 .into_iter()
1154 .filter(|m| {
1155 let state = registry::state_model(&selection_model, &selection.model, m, pooled);
1156 !rt.store().blocked(&lease.credential, &state)
1157 })
1158 .collect();
1159 if models.is_empty() {
1160 continue;
1161 }
1162 attempted.push(lease.credential.id.clone());
1163 let target = Target {
1164 models: &models,
1165 selection_model: &selection_model,
1166 pooled,
1167 alias: &alias,
1168 compact,
1169 keep_model: call.selection_model.is_some(),
1170 unauthorized: None,
1171 };
1172 match attempt(rt, &cfg, &policy, &call, &request, usage.as_ref(), lease, target, hooks).await {
1173 Attempt::Done(done) => return Ok(done),
1174 Attempt::Stop(fault) => return Err(fault.into_run_error()),
1175 Attempt::Next(fault) => {
1176 if classify::upstream_attempted(&fault.error) {
1177 upstream = Some(fault.clone());
1178 }
1179 last = Some(fault);
1180 }
1181 }
1182 };
1183 let round_error: RunError = match (last, pick_failure) {
1187 (Some(fault), _) => fault.into_run_error(),
1188 (None, Some(failure)) => failure.into(),
1189 (None, None) => Failure::Unavailable {
1190 code: "auth_not_found",
1191 providers: providers.clone(),
1192 model: model.clone(),
1193 cause: None,
1194 retry_after: None,
1195 }
1196 .into(),
1197 };
1198 let (status, retry_round) = match &round_error.failure {
1200 Failure::Exec(e) => (
1201 classify::go_status(e),
1202 classify::is_retry_round(e) && !classify::is_request_invalid(e),
1203 ),
1204 Failure::Cooldown { .. } => (429, true),
1205 Failure::Unavailable {
1206 retry_after: Some(_), ..
1207 } => (503, true),
1208 _ => (0, false),
1209 };
1210 let terminal = |round_error: RunError| upstream.clone().map(Fault::into_run_error).unwrap_or(round_error);
1211 let pinned = call.pinned();
1212 let credits = Credits {
1213 providers: &providers,
1214 model: &model,
1215 aliases: &aliases,
1216 pinned: pinned.as_deref(),
1217 compact,
1218 };
1219 if !retry_round {
1220 return credits
1221 .fallback(
1222 rt,
1223 &cfg,
1224 &policy,
1225 &call,
1226 &request,
1227 usage.as_ref(),
1228 hooks,
1229 terminal(round_error),
1230 )
1231 .await;
1232 }
1233 let wait = {
1234 let admit = crate::runtime::admission(®istry, &aliases, &selection, &rt.executors, rt.plugin_runtime());
1235 rt.store().retry_wait(&selection, &policy, status, &attempted, &admit)
1236 };
1237 let Some(wait) = wait else {
1238 return credits
1239 .fallback(
1240 rt,
1241 &cfg,
1242 &policy,
1243 &call,
1244 &request,
1245 usage.as_ref(),
1246 hooks,
1247 terminal(round_error),
1248 )
1249 .await;
1250 };
1251 if !wait.is_zero() {
1252 tokio::time::sleep(jitter(wait, policy.max_retry_interval)).await;
1253 }
1254 selection.retry_round += 1;
1255 selection.exclude.clone_from(&pinned_exclusion);
1256 }
1257}
1258
1259struct RemoteContext<'a> {
1261 rt: &'a Arc<Runtime>,
1262 cfg: &'a Config,
1263 policy: &'a Arc<crate::scheduler::Policy>,
1264 call: &'a Call,
1265 request: &'a ExecRequest,
1266 usage: Option<&'a Arc<crate::usage_record::Facts>>,
1267 unauthorized: Option<&'a Arc<crate::usage_record::Facts>>,
1269 aliases: &'a std::collections::HashMap<String, Vec<cpa_core::registry::dynamic::OAuthAlias>>,
1270 compact: bool,
1271 trace: &'a Trace,
1272 user_key: std::sync::Mutex<String>,
1275}
1276
1277#[derive(Debug, Clone, Copy, PartialEq)]
1279struct Exhausted {
1280 retry_after: Option<Duration>,
1281 retry_now: bool,
1282}
1283
1284#[derive(Debug, Clone)]
1286struct RemoteFault {
1287 failure: Failure,
1288 bootstrap: bool,
1289 upstream: bool,
1291 stop: bool,
1293 invalid: bool,
1295 kind: crate::remote::RemoteErrorKind,
1296 exhausted: Option<Exhausted>,
1297}
1298
1299impl RemoteFault {
1300 fn from_pick(error: crate::remote::RemoteError) -> Self {
1301 Self {
1302 failure: Failure::Exec(error.error),
1303 bootstrap: false,
1304 upstream: false,
1305 stop: false,
1306 invalid: false,
1307 kind: error.kind,
1308 exhausted: None,
1309 }
1310 }
1311
1312 fn from_attempt(fault: Fault, stop: bool) -> Self {
1315 Self {
1316 upstream: classify::upstream_attempted(&fault.error),
1317 invalid: classify::is_request_invalid(&fault.error),
1318 failure: Failure::Exec(fault.error),
1319 bootstrap: fault.bootstrap.is_some(),
1320 stop,
1321 kind: crate::remote::RemoteErrorKind::Plain,
1322 exhausted: None,
1323 }
1324 }
1325
1326 fn local(failure: Failure) -> Self {
1327 Self {
1328 failure,
1329 bootstrap: false,
1330 upstream: false,
1331 stop: false,
1332 invalid: false,
1333 kind: crate::remote::RemoteErrorKind::Plain,
1334 exhausted: None,
1335 }
1336 }
1337
1338 fn safe_retry_after(&self) -> Option<u64> {
1340 let seconds = |d: Duration| (!d.is_zero()).then(|| ceil_seconds(d).max(1));
1341 match (&self.kind, self.exhausted) {
1342 (crate::remote::RemoteErrorKind::Busy { header }, _) => *header,
1343 (_, Some(exhausted)) => exhausted.retry_after.and_then(seconds),
1344 (crate::remote::RemoteErrorKind::Cooldown { retry_after, .. }, None) => retry_after.and_then(seconds),
1345 _ => None,
1346 }
1347 }
1348
1349 fn exhausted(mut self, retry_after: Option<Duration>, retry_now: bool) -> Self {
1351 self.exhausted = Some(Exhausted { retry_after, retry_now });
1352 self
1353 }
1354
1355 fn retry_after(&self) -> Option<Duration> {
1357 if let Some(exhausted) = self.exhausted {
1358 return exhausted.retry_after;
1359 }
1360 match (&self.kind, &self.failure) {
1361 (crate::remote::RemoteErrorKind::Cooldown { retry_after, .. }, _) => *retry_after,
1362 (_, Failure::Exec(e)) => e.retry_after,
1363 _ => None,
1364 }
1365 }
1366
1367 fn status(&self) -> u16 {
1369 match &self.failure {
1370 Failure::Exec(e) | Failure::Remote { error: e, .. } => classify::go_status(e),
1371 other => other.status(),
1372 }
1373 }
1374
1375 fn retry_round(&self) -> bool {
1377 match &self.failure {
1378 Failure::Exec(e) | Failure::Remote { error: e, .. } => classify::is_retry_round(e),
1379 other => matches!(other.status(), 403 | 408 | 429 | 500 | 502 | 503 | 504),
1380 }
1381 }
1382
1383 fn into_run_error(self) -> RunError {
1384 let retry_after = self.safe_retry_after();
1385 let failure = match (retry_after, self.failure) {
1386 (Some(_), Failure::Exec(error)) => Failure::Remote { error, retry_after },
1387 (_, failure) => failure,
1388 };
1389 RunError {
1390 failure,
1391 bootstrap: self.bootstrap,
1392 }
1393 }
1394}
1395
1396fn preferred(fallback: RemoteFault, upstream: Option<&RemoteFault>) -> RemoteFault {
1399 let Some(upstream) = upstream else {
1400 return fallback;
1401 };
1402 let mut preferred = upstream.clone();
1403 preferred.exhausted = fallback.exhausted;
1404 preferred.upstream = true;
1405 preferred
1406}
1407
1408#[derive(Default)]
1411struct RoundTiming {
1412 retry_after: Option<Duration>,
1413 immediate: bool,
1414}
1415
1416impl RoundTiming {
1417 fn observe(&mut self, fault: &RemoteFault) {
1418 if self.immediate {
1419 return;
1420 }
1421 match fault.retry_after() {
1422 Some(d) if d.is_zero() => {
1423 self.retry_after = None;
1424 self.immediate = true;
1425 }
1426 Some(d) if self.retry_after.is_none_or(|current| d < current) => self.retry_after = Some(d),
1427 _ => {}
1428 }
1429 }
1430
1431 fn retry_after(&self) -> Option<Duration> {
1432 if self.immediate { None } else { self.retry_after }
1433 }
1434}
1435
1436async fn run_remote(
1439 cx: RemoteContext<'_>,
1440 remote: Arc<dyn crate::remote::RemoteDispatch>,
1441 base: Selection,
1442) -> Result<Done, RunError> {
1443 let pinned = cx.call.pinned().is_some_and(|p| !p.trim().is_empty());
1444 let max_wait = cx.policy.max_retry_interval;
1445 let default_retry = cx.policy.request_retry as i64;
1446 let allowed = |attempt: i64, limit: i64| home_retry_allowed(default_retry, attempt, limit);
1447 let mut limit: i64 = -1;
1448 let mut attempt: i64 = 0;
1449 let (mut pending, mut waited) = (false, false);
1450 let mut preferred_upstream: Option<RemoteFault> = None;
1451 loop {
1452 let error = match remote_round(&cx, remote.as_ref(), &base, attempt, &mut limit, pinned).await {
1453 Ok(done) => return Ok(done),
1454 Err(error) => *error,
1455 };
1456 if error.upstream {
1457 preferred_upstream = Some(error.clone());
1458 }
1459 if pending {
1460 if error.exhausted.is_none()
1462 && let crate::remote::RemoteErrorKind::Cooldown {
1463 retry_after,
1464 request_retry,
1465 } = error.kind
1466 {
1467 if !pinned && let Some(remote_limit) = request_retry {
1468 limit = remote_limit;
1469 }
1470 if let Some(wait) = retry_after.filter(|w| !w.is_zero() && !max_wait.is_zero() && *w <= max_wait)
1471 && allowed(attempt - 1, limit)
1472 {
1473 if waited {
1474 return Err(error.into_run_error());
1475 }
1476 tokio::time::sleep(wait).await;
1477 waited = true;
1478 continue;
1479 }
1480 }
1481 }
1482 waited = false;
1484 if error.stop {
1485 return Err(error.into_run_error());
1486 }
1487 match remote_retry_wait(&error, attempt, max_wait, limit, pinned, &allowed) {
1488 Some(wait) => {
1489 if !wait.is_zero() {
1490 tokio::time::sleep(wait).await;
1491 }
1492 attempt += 1;
1493 pending = true;
1494 }
1495 None => {
1496 let error = match (&preferred_upstream, error.exhausted) {
1497 (Some(upstream), Some(_)) => preferred(error, Some(upstream)),
1498 _ => error,
1499 };
1500 return Err(error.into_run_error());
1501 }
1502 }
1503 }
1504}
1505
1506fn home_retry_allowed(default_retry: i64, attempt: i64, limit: i64) -> bool {
1509 let limit = if limit < 0 { default_retry.max(0) } else { limit };
1510 attempt >= 0 && attempt < limit
1511}
1512
1513fn remote_retry_wait(
1515 error: &RemoteFault,
1516 attempt: i64,
1517 max_wait: Duration,
1518 mut limit: i64,
1519 pinned: bool,
1520 allowed: &dyn Fn(i64, i64) -> bool,
1521) -> Option<Duration> {
1522 if matches!(error.kind, crate::remote::RemoteErrorKind::Busy { .. })
1523 || error.status() == 200
1524 || error.invalid
1525 || error.stop
1526 {
1527 return None;
1528 }
1529 if let crate::remote::RemoteErrorKind::Cooldown {
1530 request_retry: Some(remote_limit),
1531 ..
1532 } = error.kind
1533 && !pinned
1534 {
1535 limit = remote_limit;
1536 }
1537 let beyond = |wait: Duration| !wait.is_zero() && (max_wait.is_zero() || wait > max_wait);
1538 if let Some(exhausted) = error.exhausted {
1539 if !error.retry_round() || !allowed(attempt, limit) {
1540 return None;
1541 }
1542 if exhausted.retry_now {
1543 return Some(Duration::ZERO);
1544 }
1545 return match exhausted.retry_after {
1546 Some(wait) if beyond(wait) => None,
1547 Some(wait) => Some(wait),
1548 None => Some(Duration::ZERO),
1550 };
1551 }
1552 if error.status() != 429 || !allowed(attempt, limit) {
1553 return None;
1554 }
1555 match error.retry_after() {
1556 Some(wait) if !wait.is_zero() && !beyond(wait) => Some(wait),
1557 _ => None,
1558 }
1559}
1560
1561async fn remote_round(
1565 cx: &RemoteContext<'_>,
1566 remote: &dyn crate::remote::RemoteDispatch,
1567 base: &Selection,
1568 round: i64,
1569 limit: &mut i64,
1570 pinned: bool,
1571) -> Result<Done, Box<RemoteFault>> {
1572 let round = async {
1574 let releases = crate::remote::PendingReleases::default();
1575 let settle = |releases: crate::remote::PendingReleases| async move {
1576 releases.settle().await.map_err(|error| {
1577 RemoteFault::local(Failure::Exec(ExecError::local(
1578 503,
1579 cpa_core::exec::FailureScope::Credential,
1580 format!("home_unavailable: Home did not acknowledge credential release: {error}"),
1581 )))
1582 })
1583 };
1584 let max_credentials = cx.policy.max_retry_credentials;
1585 let stream = cx.call.stream;
1589 let websocket = cx.call.turn.is_some();
1590 let mut tried: Vec<String> = Vec::new();
1591 let mut excluded: Vec<String> = Vec::new();
1592 let mut same_retries: std::collections::HashMap<String, u32> = Default::default();
1593 let mut last_id = String::new();
1594 let mut same_pending = false;
1595 let mut last: Option<RemoteFault> = None;
1596 let mut upstream: Option<RemoteFault> = None;
1597 let mut timing = RoundTiming::default();
1598 let mut count: i64 = 0;
1601 loop {
1602 if !stream || count == 0 {
1603 count += 1;
1604 }
1605 let allow_same =
1606 stream && same_pending && !last_id.is_empty() && same_retries.get(&last_id).copied().unwrap_or(0) == 0;
1607 let capped = max_credentials > 0 && tried.len() >= max_credentials;
1608 if capped && !allow_same {
1609 return Err(match last {
1610 Some(last) => preferred(last, upstream.as_ref()).exhausted(timing.retry_after(), true),
1611 None => RemoteFault::local(no_auth(base)),
1612 });
1613 }
1614 let mut selection = base.clone();
1615 selection.retry_round = round.max(0) as usize;
1616 selection.exclude = if stream { excluded.clone() } else { tried.clone() };
1617 let request = remote_request(cx.call, &selection, count, cx.trace);
1618 let kept = cx
1621 .call
1622 .turn
1623 .as_ref()
1624 .and_then(|turn| turn.home.as_ref())
1625 .and_then(|home| {
1626 home.reuse(
1627 count <= 1 && round <= 0,
1628 &selection.model,
1629 &selection.exclude,
1630 cx.call.pinned().as_deref(),
1631 &releases,
1632 )
1633 });
1634 let picked = match kept {
1635 Some((lease, user_key, request_retry)) => Ok((lease, request_retry, user_key)),
1636 None => {
1637 let home = cx.call.turn.as_ref().and_then(|turn| turn.home.as_ref());
1638 if home.is_some() {
1640 settle(releases.clone()).await?;
1641 }
1642 let picked = cx
1643 .rt
1644 .acquire_remote(remote, selection.clone(), request, &releases)
1645 .await;
1646 if let (Some(home), Ok((_, request_retry, _))) = (home, &picked) {
1647 home.granted(*request_retry);
1648 }
1649 picked
1650 }
1651 };
1652 let (lease, request_retry, user_key) = match picked {
1653 Ok(picked) => picked,
1654 Err(error) => {
1655 let code = error.code.clone();
1656 let pick = RemoteFault::from_pick(error);
1657 let Some(previous) = last else {
1658 return Err(pick);
1659 };
1660 let fallback = preferred(previous, upstream.as_ref());
1661 if let crate::remote::RemoteErrorKind::Cooldown {
1662 retry_after,
1663 request_retry,
1664 } = pick.kind
1665 {
1666 if !pinned && let Some(remote_limit) = request_retry {
1667 *limit = remote_limit;
1668 }
1669 return Err(fallback.exhausted(retry_after, false));
1670 }
1671 if matches!(
1673 code.go_lower().as_str(),
1674 "auth_not_found" | "auth_unavailable" | "request_retry_exceeded"
1675 ) {
1676 return Err(
1677 fallback.exhausted(timing.retry_after(), code.eq_ignore_ascii_case("auth_unavailable"))
1678 );
1679 }
1680 return Err(pick);
1681 }
1682 };
1683 match request_retry.filter(|_| !pinned) {
1685 Some(home_limit) => *limit = home_limit,
1686 None => {
1687 let local = cx.policy.retry_limit(&lease.credential) as i64;
1688 if *limit < 0 || local > *limit {
1689 *limit = local;
1690 }
1691 }
1692 }
1693 let id = lease.credential.id.clone();
1694 if capped && id != last_id {
1695 drop(lease);
1697 settle(releases.clone()).await?;
1698 return Err(match last {
1699 Some(last) => preferred(last, upstream.as_ref()).exhausted(timing.retry_after(), true),
1700 None => RemoteFault::local(no_auth(base)),
1701 });
1702 }
1703 if stream && !last_id.is_empty() && id != last_id {
1704 same_pending = false;
1705 }
1706 if tried.contains(&id) {
1707 if !stream || excluded.contains(&id) {
1708 drop(lease);
1709 settle(releases.clone()).await?;
1710 return Err(match last {
1711 Some(last) => preferred(last, upstream.as_ref()).exhausted(timing.retry_after(), false),
1712 None => RemoteFault::local(Failure::Exec(ExecError::local(
1713 503,
1714 cpa_core::exec::FailureScope::Credential,
1715 "request_retry_exceeded: home returned a previously tried auth",
1716 ))),
1717 });
1718 }
1719 let retries = same_retries.entry(id.clone()).or_default();
1720 *retries += 1;
1721 if *retries > 1 {
1722 excluded.push(id);
1724 drop(lease);
1725 settle(releases.clone()).await?;
1726 continue;
1727 }
1728 } else {
1729 tried.push(id.clone());
1730 }
1731 cx.trace.selected(&lease.credential);
1732 if !user_key.is_empty() {
1737 *cx.user_key.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = user_key;
1738 }
1739 let user_key = cx
1740 .user_key
1741 .lock()
1742 .unwrap_or_else(std::sync::PoisonError::into_inner)
1743 .clone();
1744 let home_caller = (!user_key.is_empty()).then(|| {
1745 let mut request = cx.request.clone();
1746 request.caller.principal = user_key.clone();
1747 let usage = cx.usage.map(|facts| {
1748 let mut facts = (**facts).clone();
1749 facts.client.api_key = user_key.clone();
1750 Arc::new(facts)
1751 });
1752 let unauthorized = cx.unauthorized.map(|facts| {
1756 let mut facts = (**facts).clone();
1757 facts.client.api_key = user_key.clone();
1758 Arc::new(facts)
1759 });
1760 (request, usage, unauthorized)
1761 });
1762 let (request, usage, unauthorized) = match &home_caller {
1763 Some((request, usage, unauthorized)) => (request, usage.as_ref(), unauthorized.as_ref()),
1764 None => (cx.request, cx.usage, cx.unauthorized),
1765 };
1766 let (mut models, mut alias) = registry::execution_models(cx.aliases, &lease.credential, &selection.model);
1769 if let Some(model) = lease
1770 .credential
1771 .attributes
1772 .get(crate::remote::UPSTREAM_MODEL)
1773 .map(|m| m.trim())
1774 .filter(|m| !m.is_empty())
1775 {
1776 models = vec![model.to_owned()];
1777 }
1778 models.truncate(1);
1779 if lease
1782 .credential
1783 .attributes
1784 .get(crate::remote::FORCE_MAPPING)
1785 .is_some_and(|v| v.trim().eq_ignore_ascii_case("true"))
1786 {
1787 alias.force_mapping = true;
1788 alias.original_alias = selection.model.clone();
1789 }
1790 if models.is_empty() {
1791 drop(lease);
1792 settle(releases.clone()).await?;
1793 let fault = RemoteFault::local(Failure::Exec(ExecError::local(
1794 503,
1795 cpa_core::exec::FailureScope::Credential,
1796 "auth_not_found: no execution models available",
1797 )));
1798 timing.observe(&fault);
1799 last = Some(fault);
1800 continue;
1801 }
1802 let selection_model = registry::selection_model(cx.aliases, &lease.credential, &selection.model);
1803 let target = Target {
1804 models: &models,
1805 selection_model: &selection_model,
1806 pooled: false,
1807 alias: &alias,
1808 compact: cx.compact,
1809 keep_model: cx.call.selection_model.is_some(),
1810 unauthorized,
1811 };
1812 let cancelled = lease
1813 .remote_cancelled()
1814 .unwrap_or_else(|| Box::pin(std::future::pending()));
1815 let run = async {
1820 let mut lease = lease;
1821 match cx.rt.prepare_remote(&mut lease, cx.cfg).await {
1822 Ok(()) => (
1823 attempt(cx.rt, cx.cfg, cx.policy, cx.call, request, usage, lease, target, None).await,
1824 false,
1825 ),
1826 Err(error) => {
1827 lease.complete(Outcome::Failure(error.clone()));
1828 (Attempt::Next(Fault { error, bootstrap: None }), true)
1829 }
1830 }
1831 };
1832 let (outcome, prepare_failed) = tokio::select! {
1833 biased;
1834 outcome = run => outcome,
1835 _ = cancelled => (
1836 Attempt::Next(Fault {
1837 error: crate::remote::cancelled_error(),
1838 bootstrap: None,
1839 }),
1840 false,
1841 ),
1842 };
1843 match outcome {
1844 Attempt::Done(done) => return Ok(done),
1845 Attempt::Stop(fault) => {
1846 if stream {
1849 settle(releases.clone()).await?;
1850 }
1851 return Err(RemoteFault::from_attempt(fault, true));
1852 }
1853 Attempt::Next(fault) => {
1854 if stream {
1855 let error = &fault.error;
1857 let exclude = !(lifecycle(error) || (websocket && error.status == 426))
1858 || same_retries.get(&id).copied().unwrap_or(0) > 0;
1859 if exclude && !excluded.contains(&id) {
1860 excluded.push(id.clone());
1861 }
1862 last_id = id.clone();
1863 same_pending = !exclude;
1864 if !prepare_failed {
1865 count += 1;
1866 }
1867 }
1868 let mut fault = RemoteFault::from_attempt(fault, false);
1869 fault.upstream &= !prepare_failed;
1871 if fault.upstream {
1872 upstream = Some(fault.clone());
1873 }
1874 timing.observe(&fault);
1875 last = Some(fault);
1876 settle(releases.clone()).await?;
1877 }
1878 }
1879 }
1880 };
1881 round.await.map_err(Box::new)
1882}
1883
1884fn lifecycle(e: &ExecError) -> bool {
1886 if classify::go_status(e) != 0 {
1887 return false;
1888 }
1889 let lower = classify::error_text(e).trim().go_lower();
1890 matches!(
1891 lower.as_str(),
1892 "context canceled" | "context deadline exceeded" | "eof" | "unexpected eof"
1893 ) || [
1894 "websocket: close 1000",
1895 "websocket: close 1001",
1896 "websocket: close 1006",
1897 "unexpected eof",
1898 ]
1899 .iter()
1900 .any(|needle| lower.contains(needle))
1901}
1902
1903pub(crate) fn scheduler_rejection(code: String, message: String, providers: &[String], model: &str) -> Failure {
1908 match code.as_str() {
1909 "auth_not_found" | "auth_unavailable" => Failure::Exec(ExecError::local(
1912 503,
1913 cpa_core::exec::FailureScope::Request,
1914 unavailable_text(&code, &message, providers, model, None),
1915 )),
1916 _ => Failure::Exec(ExecError::local(
1917 0,
1918 cpa_core::exec::FailureScope::Model,
1919 format!("{code}: {message}"),
1920 )),
1921 }
1922}
1923
1924fn no_auth(base: &Selection) -> Failure {
1926 Failure::Unavailable {
1927 code: "auth_not_found",
1928 providers: base.providers.clone(),
1929 model: base.model.clone(),
1930 cause: None,
1931 retry_after: None,
1932 }
1933}
1934
1935fn remote_request(call: &Call, selection: &Selection, count: i64, trace: &Trace) -> crate::remote::RemoteRequest {
1939 let headers = crate::remote::home_headers(&call.headers, Some(&call.caller));
1940 crate::remote::RemoteRequest {
1941 model: selection.model.clone(),
1942 session_id: selection.session.clone().unwrap_or_default(),
1943 parent_session_id: selection.session_parent.clone().unwrap_or_default(),
1944 headers,
1945 count,
1946 retry_round: selection.retry_round as i64,
1947 excluded: selection.exclude.clone(),
1948 pinned: call.pinned().unwrap_or_default().to_owned(),
1949 request_id: trace.request_id(),
1950 kind: if call.turn.is_some() {
1951 "websocket"
1952 } else if call.stream {
1953 "stream"
1954 } else {
1955 "http"
1956 },
1957 credential_policy: String::new(),
1958 }
1959}
1960
1961pub fn jitter(wait: Duration, max: Duration) -> Duration {
1964 let mut range = (wait / 4).min(Duration::from_secs(2));
1965 if !max.is_zero() {
1966 range = range.min(max.saturating_sub(wait));
1967 }
1968 if range.is_zero() {
1969 return wait;
1970 }
1971 use std::hash::{BuildHasher, Hasher};
1972 let mut hasher = std::collections::hash_map::RandomState::new().build_hasher();
1973 hasher.write_u128(
1974 std::time::SystemTime::now()
1975 .duration_since(std::time::UNIX_EPOCH)
1976 .unwrap_or_default()
1977 .as_nanos(),
1978 );
1979 wait + Duration::from_nanos(hasher.finish() % range.as_nanos().max(1) as u64)
1980}
1981
1982struct Credits<'a> {
1987 providers: &'a [String],
1988 model: &'a str,
1989 aliases: &'a std::collections::HashMap<String, Vec<cpa_core::registry::dynamic::OAuthAlias>>,
1990 pinned: Option<&'a str>,
1991 compact: bool,
1992}
1993
1994impl Credits<'_> {
1995 fn wanted(&self, cfg: &Config, error: &RunError) -> bool {
1998 self.providers
1999 .iter()
2000 .any(|p| p.trim().eq_ignore_ascii_case("antigravity"))
2001 && cpa_exec::antigravity_credits::enabled(cfg)
2002 && match &error.failure {
2003 Failure::Exec(e) => matches!(classify::go_status(e), 429 | 503),
2004 Failure::Cooldown { .. } => true,
2005 Failure::Unavailable { code, .. } => matches!(*code, "auth_not_found" | "auth_unavailable"),
2006 _ => false,
2007 }
2008 }
2009
2010 #[allow(clippy::too_many_arguments)]
2011 async fn fallback(
2012 &self,
2013 rt: &Arc<Runtime>,
2014 cfg: &Config,
2015 policy: &Policy,
2016 call: &Call,
2017 request: &ExecRequest,
2018 usage: Option<&Arc<crate::usage_record::Facts>>,
2019 hooks: Option<&crate::plugins::interceptors::Hooks>,
2020 error: RunError,
2021 ) -> Result<Done, RunError> {
2022 if request.operation == Operation::CountTokens
2024 || !self.wanted(cfg, &error)
2025 || !self.model.go_lower().contains("claude")
2026 {
2027 return Err(error);
2028 }
2029 let snapshot = rt.store().snapshot();
2030 let ids = snapshot
2031 .iter()
2032 .filter(|c| !c.disabled && c.provider.trim().eq_ignore_ascii_case("antigravity"))
2033 .filter(|c| self.pinned.is_none_or(|p| p.is_empty() || c.id == p))
2034 .map(|c| c.id.as_str());
2035 for id in cpa_exec::antigravity_credits::order_candidates(ids) {
2036 let selection = Selection::new("antigravity", self.model);
2037 let Some(lease) = rt
2038 .lease_credential(&id, selection, cfg, cpa_exec::antigravity_credits::request_on)
2039 .await
2040 else {
2041 continue;
2042 };
2043 let (models, alias) = registry::execution_models(self.aliases, &lease.credential, self.model);
2044 if models.is_empty() {
2045 continue;
2046 }
2047 let selection_model = registry::selection_model(self.aliases, &lease.credential, self.model);
2048 let target = Target {
2049 models: &models,
2050 selection_model: &selection_model,
2051 pooled: models.len() > 1,
2052 alias: &alias,
2053 compact: self.compact,
2054 keep_model: call.selection_model.is_some(),
2055 unauthorized: None,
2056 };
2057 let hooks = hooks.filter(|_| request.stream);
2061 if let Attempt::Done(done) = attempt(rt, cfg, policy, call, request, usage, lease, target, hooks).await {
2064 return Ok(done);
2065 }
2066 }
2067 Err(error)
2068 }
2069}
2070
2071enum Attempt {
2072 Done(Done),
2073 Stop(Fault),
2075 Next(Fault),
2077}
2078
2079struct Target<'a> {
2081 models: &'a [String],
2083 selection_model: &'a str,
2084 pooled: bool,
2085 alias: &'a AliasResult,
2086 compact: bool,
2087 keep_model: bool,
2090 unauthorized: Option<&'a Arc<crate::usage_record::Facts>>,
2093}
2094
2095#[allow(clippy::too_many_arguments)]
2096async fn attempt(
2097 rt: &Arc<Runtime>,
2098 cfg: &Config,
2099 policy: &Policy,
2100 call: &Call,
2101 request: &ExecRequest,
2102 usage: Option<&Arc<crate::usage_record::Facts>>,
2103 mut lease: Lease,
2104 target: Target<'_>,
2105 hooks: Option<&crate::plugins::interceptors::Hooks>,
2106) -> Attempt {
2107 let route_model = lease.selection.model.clone();
2108 let mut last = None;
2109 let mut refreshed = false;
2110 let home = call
2113 .turn
2114 .as_ref()
2115 .and_then(|turn| turn.home.as_ref())
2116 .filter(|_| lease.is_remote());
2117 let pick = home.map(|home| home.pick(&lease.credential.id));
2118 let mut guard = pick.clone().map(crate::home_session::PickGuard::new);
2119 let session = call.turn.as_ref().map(|turn| cpa_core::exec::ExecSession {
2120 lease: pick.as_ref().map(crate::home_session::Pick::lease),
2121 ..turn.session.clone()
2122 });
2123 for (i, upstream) in target.models.iter().enumerate() {
2124 let home_model = lease
2126 .credential
2127 .attributes
2128 .get(crate::remote::UPSTREAM_MODEL)
2129 .map(|m| m.trim())
2130 .filter(|m| !m.is_empty());
2131 let state = match home_model {
2132 Some(home) if upstream.trim().is_empty() => home.to_owned(),
2133 Some(_) => upstream.trim().to_owned(),
2134 None => registry::state_model(target.selection_model, &route_model, upstream, target.pooled),
2135 };
2136 lease.execution_model = state.clone();
2137 let mut req = attempt_request(
2138 request,
2139 cfg,
2140 &lease.credential,
2141 &route_model,
2142 upstream,
2143 target.keep_model,
2144 call.stream,
2145 );
2146 let lcp = lease.lcp.clone();
2149 if let Some(m) = &lcp {
2150 req.session = Some(cpa_common::session::bound_session_identity(&m.session));
2151 }
2152 if let Some(on_selected) = call.on_selected() {
2153 on_selected(&lease.credential);
2154 }
2155 if let Some(hooks) = hooks
2158 && let Err(error) = hooks
2159 .after_auth(
2160 &mut req,
2161 &lease.credential,
2162 &route_model,
2163 call.media.as_ref().map(|m| m.kind),
2164 )
2165 .await
2166 {
2167 lease.complete(Outcome::Neutral(error.clone()));
2168 return Attempt::Stop(Fault { error, bootstrap: None });
2169 }
2170 let start = |credential: &cpa_core::credential::Credential, req: &mut ExecRequest| {
2171 usage.map(|facts| {
2172 let mut tracker = crate::usage_record::Tracker::start(rt, facts, credential, upstream);
2173 if let Some(m) = &lcp {
2174 tracker.lcp_session(m);
2175 }
2176 req.usage = tracker.sink().with_capture(req.capture().clone());
2177 tracker
2178 })
2179 };
2180 let observe = |credential: &cpa_core::credential::Credential, req: &mut ExecRequest| {
2182 if let Some(hooks) = hooks {
2183 let observer = hooks.websocket_observer(req, credential, &route_model);
2184 req.usage = std::mem::take(&mut req.usage).with_websocket(observer);
2185 }
2186 };
2187 let mut tracker = start(&lease.credential, &mut req);
2188 observe(&lease.credential, &mut req);
2189 let session = session.as_ref();
2190 let private = lease.home_private().map(|p| p.attributes.clone()).unwrap_or_default();
2191 let private = &private;
2192 let execute = |credential: Arc<cpa_core::credential::Credential>, req: ExecRequest| async move {
2193 let credential = match crate::plugins::model_callbacks::context().filter(|c| !c.proxy_url.is_empty()) {
2194 Some(context) => {
2195 let mut credential = (*credential).clone();
2196 credential.attributes.insert("proxy_url".into(), context.proxy_url);
2197 Arc::new(credential)
2198 }
2199 None => credential,
2200 };
2201 let credential = rt.for_executor(&credential);
2202 if let Some(adapter) = rt.plugin_runtime().executor(&credential, cfg) {
2203 return crate::plugins::execution::execute_credential(rt, call, &adapter, &credential, private, req)
2204 .await;
2205 }
2206 match (session, call.media.as_ref().map(|m| m.kind)) {
2207 (Some(session), _) => rt.executors.execute_in_session(&credential, req, cfg, session).await,
2208 (None, Some(MediaKind::Images)) => {
2210 let path = req.request_path.clone();
2211 rt.executors.images(&credential, req, &path, cfg).await
2212 }
2213 (None, Some(MediaKind::Videos)) => {
2214 let path = req.request_path.clone();
2215 rt.executors.videos(&credential, req, &path, cfg).await
2216 }
2217 (None, Some(MediaKind::Speech)) => rt.executors.speech(&credential, req, cfg).await,
2218 (None, None) => rt.executors.execute(&credential, req, cfg).await,
2219 }
2220 };
2221 tracing::trace!(target: "cpa_latency", stage = "selected");
2222 lease.redispatch();
2225 let mut executed = execute(lease.credential.clone(), req.clone()).await;
2226 if let Err(error) = &executed
2230 && !refreshed
2231 && !lease.is_remote()
2232 && classify::is_unauthorized(error)
2233 && let Some(current) = rt.refresh_after_unauthorized(&lease.credential, cfg).await
2234 {
2235 refreshed = true;
2236 if let Some(t) = tracker.take() {
2237 t.fail(error);
2238 }
2239 lease.credential = current;
2240 lease.redispatch();
2241 tracker = start(&lease.credential, &mut req);
2242 observe(&lease.credential, &mut req);
2243 executed = execute(lease.credential.clone(), req).await;
2244 }
2245 if let (Ok(response), Some(t)) = (&executed, tracker.as_mut()) {
2246 t.arrived(&response.headers);
2247 }
2248 let fault = match executed {
2249 Ok(response) => match finish(call, response).await {
2250 Ok(done) => {
2251 let done = track_usage(done, tracker.take());
2252 let done = if target.alias.force_mapping && !target.alias.original_alias.is_empty() {
2253 rewrite_model(done, &target.alias.original_alias)
2254 } else {
2255 done
2256 };
2257 let user_key = &request.caller.principal;
2258 let Some(lease) = keep_home(home, guard.take(), lease, user_key) else {
2259 return Attempt::Done(done);
2260 };
2261 return Attempt::Done(match done {
2262 Done::Stream { headers, first, rest } => {
2263 lease.accepted();
2266 Done::Stream {
2267 headers,
2268 first,
2269 rest: Completing::new(rest, lease).boxed(),
2270 }
2271 }
2272 buffered => {
2273 lease.complete(Outcome::Success);
2274 buffered
2275 }
2276 });
2277 }
2278 Err(fault) => {
2279 if let Some(t) = tracker.take() {
2280 t.fail(&fault.error);
2281 }
2282 fault
2283 }
2284 },
2285 Err(error) => {
2286 if let Some(t) = tracker.take() {
2287 t.fail(&error);
2288 }
2289 Fault { error, bootstrap: None }
2290 }
2291 };
2292 let error = &fault.error;
2293 if let Some(facts) = target.unauthorized
2294 && classify::is_unauthorized(error)
2295 {
2296 crate::usage_record::publish_home_unauthorized(
2297 rt,
2298 facts,
2299 &lease.credential,
2300 &crate::remote::selection_provider(&lease.credential),
2301 &state,
2302 &classify::error_text(error),
2303 );
2304 }
2305 let action = policy.error_action(&lease.credential, error);
2306 let neutral = (target.compact && classify::is_compact_neutral(error) && !action.force_cooldown)
2307 || (call.operation == Operation::CountTokens
2308 && classify::is_count_endpoint_missing(error, upstream)
2309 && !action.force_cooldown);
2310 let outcome = if neutral {
2311 Outcome::Neutral(error.clone())
2312 } else {
2313 Outcome::Failure(error.clone())
2314 };
2315 let stop = if action.matched {
2316 action.stop
2317 } else {
2318 (target.compact && classify::is_compact_fault(error)) || classify::is_request_invalid(error)
2319 };
2320 let last_here = stop || i + 1 == target.models.len() || classify::credential_scoped(error);
2322 let mut fault = fault;
2326 if stop {
2327 fault.bootstrap = None;
2328 } else if let Some(headers) = &fault.bootstrap {
2329 fault.error.addon = Some(headers.clone());
2330 }
2331 if last_here {
2332 drop(guard.take());
2334 lease.complete(outcome);
2335 return if stop {
2336 Attempt::Stop(fault)
2337 } else {
2338 Attempt::Next(fault)
2339 };
2340 }
2341 lease.note(&state, &outcome);
2342 last = Some(fault);
2343 }
2344 Attempt::Next(last.expect("at least one model was attempted"))
2345}
2346
2347fn keep_home(
2351 home: Option<&Arc<crate::home_session::SessionHome>>,
2352 guard: Option<crate::home_session::PickGuard>,
2353 lease: Lease,
2354 user_key: &str,
2355) -> Option<Lease> {
2356 let (Some(home), Some(mut guard)) = (home, guard) else {
2357 return Some(lease);
2358 };
2359 let pick = guard.defuse()?;
2360 let (credential, route) = (lease.credential.clone(), lease.selection.model.clone());
2361 let availability = lease.home_private().map(|p| p.availability.clone()).unwrap_or_default();
2362 match pick.keep(lease) {
2363 None => {
2364 home.keep(pick, &credential, availability, &route, user_key.to_owned());
2365 None
2366 }
2367 Some(lease) => {
2368 pick.close();
2369 Some(lease)
2370 }
2371 }
2372}
2373
2374fn usage_client(
2377 cfg: &Config,
2378 call: &Call,
2379 trace: &Trace,
2380 session: &crate::session::Session,
2381) -> crate::usage_record::Client {
2382 let nested = (call.caller.principal.is_empty() && call.caller.provider.is_empty())
2385 .then(crate::plugins::model_callbacks::nested_usage_identity)
2386 .flatten();
2387 let (api_key, access_provider) =
2388 nested.unwrap_or_else(|| (call.caller.principal.clone(), call.caller.provider.clone()));
2389 crate::usage_record::Client {
2390 session_id: session.id.clone().unwrap_or_default(),
2391 parent_session_id: session.parent.clone().unwrap_or_default(),
2392 is_fork: session.fork,
2393 node_kind: String::new(),
2394 is_compaction: false,
2395 session_source: session.source,
2396 request_id: trace.request_id(),
2397 endpoint: format!(
2399 "{} {}",
2400 if call.turn.is_some() { "GET" } else { "POST" },
2401 call.request_path
2402 ),
2403 api_key,
2404 access_provider,
2405 ..request_client(cfg, &call.headers, call.peer)
2406 }
2407}
2408
2409pub(crate) fn request_client(
2413 cfg: &Config,
2414 headers: &axum::http::HeaderMap,
2415 peer: Option<std::net::SocketAddr>,
2416) -> crate::usage_record::Client {
2417 let trusted = cfg.derived(|c| cpa_core::config::TrustedProxies::new(&c.trusted_proxies));
2418 let text = |v: &axum::http::HeaderValue| String::from_utf8_lossy(v.as_bytes()).into_owned();
2419 let forwarded: Vec<String> = headers.get_all("x-forwarded-for").iter().map(text).collect();
2420 crate::usage_record::Client {
2421 client_ip: peer.map(|p| p.ip().to_string()).unwrap_or_default(),
2422 resolved_client_ip: trusted
2423 .client_ip(peer, |name| headers.get(name).map(|v| v.as_bytes()))
2424 .trim()
2425 .to_owned(),
2426 x_forwarded_for: forwarded.join(", ").trim().to_owned(),
2427 user_agent: headers
2428 .get("user-agent")
2429 .map(text)
2430 .unwrap_or_default()
2431 .trim()
2432 .to_owned(),
2433 ..Default::default()
2434 }
2435}
2436
2437fn track_usage(done: Done, tracker: Option<crate::usage_record::Tracker>) -> Done {
2440 let Some(mut tracker) = tracker else { return done };
2441 match done {
2442 Done::Buffered { headers, body } => {
2443 tracker.body(&body);
2444 tracker.succeed();
2445 Done::Buffered { headers, body }
2446 }
2447 Done::Stream { headers, first, rest } => {
2448 if let Some(first) = &first {
2449 tracker.event(first);
2450 }
2451 Done::Stream {
2452 headers,
2453 first,
2454 rest: Tracked {
2455 inner: rest,
2456 tracker: Some(tracker),
2457 }
2458 .boxed(),
2459 }
2460 }
2461 }
2462}
2463
2464struct Tracked {
2468 inner: ExecStream,
2469 tracker: Option<crate::usage_record::Tracker>,
2470}
2471
2472impl futures_util::Stream for Tracked {
2473 type Item = Result<Bytes, ExecError>;
2474
2475 fn poll_next(
2476 mut self: std::pin::Pin<&mut Self>,
2477 cx: &mut std::task::Context<'_>,
2478 ) -> std::task::Poll<Option<Self::Item>> {
2479 let item = std::task::ready!(self.inner.poll_next_unpin(cx));
2480 match &item {
2481 Some(Ok(event)) => {
2482 if let Some(t) = self.tracker.as_mut() {
2483 t.event(event);
2484 }
2485 }
2486 Some(Err(error)) => {
2488 self.inner = futures_util::stream::empty().boxed();
2489 if let Some(t) = self.tracker.take() {
2490 t.fail(error);
2491 }
2492 }
2493 None => {
2494 self.inner = futures_util::stream::empty().boxed();
2495 if let Some(t) = self.tracker.take() {
2496 t.succeed();
2497 }
2498 }
2499 }
2500 std::task::Poll::Ready(item)
2501 }
2502}
2503
2504fn attempt_request(
2510 request: &ExecRequest,
2511 cfg: &Config,
2512 credential: &cpa_core::credential::Credential,
2513 route_model: &str,
2514 upstream: &str,
2515 keep_model: bool,
2516 stream: bool,
2517) -> ExecRequest {
2518 let mut req = request.clone();
2519 if !keep_model {
2520 req.model = upstream.to_owned();
2521 }
2522 req.resolved_model = crate::capabilities::resolve_attempt(
2523 cfg,
2524 credential,
2525 route_model,
2526 upstream,
2527 keep_model.then_some(request.model.as_str()),
2528 );
2529 let local = req.resolved_model.as_ref().filter(|_| !stream);
2530 if !keep_model && let Some(home) = crate::capabilities::bind_home(credential, local) {
2531 req.resolved_model = Some(home);
2532 }
2533 req
2534}
2535
2536async fn finish(call: &Call, response: cpa_core::exec::ExecResponse) -> Result<Done, Fault> {
2538 match response.body {
2539 ResponseBody::Buffered(body) => Ok(Done::Buffered {
2540 headers: response.headers,
2541 body,
2542 }),
2543 ResponseBody::Stream(mut stream) if call.stream => {
2544 let bootstrap = |error| Fault {
2545 error,
2546 bootstrap: Some(Box::new(response.headers.clone())),
2547 };
2548 let first = loop {
2549 match stream.next().await {
2550 Some(Ok(bytes)) if bytes.is_empty() => continue,
2551 Some(Ok(bytes)) => break bytes,
2552 Some(Err(error)) => return Err(bootstrap(error)),
2553 None => return Err(bootstrap(classify::empty_stream())),
2555 }
2556 };
2557 Ok(Done::Stream {
2558 headers: response.headers,
2559 first: Some(first),
2560 rest: stream,
2561 })
2562 }
2563 ResponseBody::Stream(mut stream) => {
2564 let mut body = BytesMut::new();
2565 while let Some(event) = stream.next().await {
2566 body.extend_from_slice(&event.map_err(|error| Fault { error, bootstrap: None })?);
2567 }
2568 Ok(Done::Buffered {
2569 headers: response.headers,
2570 body: body.freeze(),
2571 })
2572 }
2573 }
2574}
2575
2576fn rewrite_model(done: Done, target: &str) -> Done {
2579 match done {
2580 Done::Buffered { headers, body } => Done::Buffered {
2581 headers,
2582 body: rewrite_body(&body, target),
2583 },
2584 Done::Stream { headers, first, rest } => {
2585 let target_owned = target.to_owned();
2586 let rest = rest
2587 .map(move |item| item.map(|event| rewrite_event(&event, &target_owned)))
2588 .boxed();
2589 Done::Stream {
2590 headers,
2591 first: first.map(|event| rewrite_event(&event, target)),
2592 rest,
2593 }
2594 }
2595 }
2596}
2597
2598const MODEL_PATHS: [&str; 5] = [
2599 "model",
2600 "modelVersion",
2601 "response.model",
2602 "response.modelVersion",
2603 "message.model",
2604];
2605
2606fn rewrite_json(data: &[u8], target: &str) -> Vec<u8> {
2609 let mut out = data.to_vec();
2610 for path in MODEL_PATHS {
2611 if cpa_common::json::get(&out, path).exists() {
2612 cpa_common::json::set_str(&mut out, path, target);
2613 }
2614 }
2615 out
2616}
2617
2618fn rewrite_body(body: &Bytes, target: &str) -> Bytes {
2620 if body.trim_ascii_start().starts_with(b"{") {
2621 Bytes::from(rewrite_json(body, target))
2622 } else {
2623 rewrite_lines(body, target)
2624 }
2625}
2626
2627fn rewrite_event(event: &Bytes, target: &str) -> Bytes {
2632 let trimmed = event.trim_ascii();
2633 if trimmed.starts_with(b"{") && cpa_common::json::valid(trimmed) {
2634 return Bytes::from(rewrite_json(trimmed, target));
2635 }
2636 rewrite_lines(event, target)
2637}
2638
2639fn rewrite_lines(payload: &Bytes, target: &str) -> Bytes {
2641 let mut changed = false;
2642 let mut out = Vec::with_capacity(payload.len());
2643 for (i, line) in payload.split(|&b| b == b'\n').enumerate() {
2644 if i > 0 {
2645 out.push(b'\n');
2646 }
2647 let data = line.strip_prefix(b"data: ").or_else(|| line.strip_prefix(b"data:"));
2648 match data {
2649 Some(json) if json.starts_with(b"{") && cpa_common::json::valid(json) => {
2650 out.extend_from_slice(&line[..line.len() - json.len()]);
2651 let rewritten = rewrite_json(json, target);
2652 changed |= rewritten != json;
2653 out.extend_from_slice(&rewritten);
2654 }
2655 _ => out.extend_from_slice(line),
2656 }
2657 }
2658 if changed { Bytes::from(out) } else { payload.clone() }
2659}
2660
2661#[cfg(test)]
2662mod tests {
2663 use super::*;
2664
2665 #[test]
2668 fn direct_errors_stay_direct_under_a_home_round_marker() {
2669 let mut raw = ExecError::local(429, cpa_core::exec::FailureScope::Credential, "raw fast-mode body");
2670 raw.direct = true;
2671 raw.headers
2672 .insert(axum::http::header::CONTENT_TYPE, "text/plain".parse().unwrap());
2673 let failure = Failure::Remote {
2674 error: raw,
2675 retry_after: Some(2),
2676 };
2677 assert!(failure.direct().is_some());
2678 let response = crate::errors::write(&failure, |_, _| unreachable!());
2679 assert_eq!(response.status().as_u16(), 429);
2680 assert_eq!(response.headers()["content-type"], "text/plain");
2681 assert!(response.headers().get("retry-after").is_none());
2682 }
2683
2684 #[tokio::test]
2689 async fn stream_records_publish_after_the_upstream_stream_is_dropped() {
2690 use cpa_core::exec::{UpstreamStage, UsageSink};
2691 use std::task::Poll;
2692 struct UpstreamBody(UsageSink);
2694 impl Drop for UpstreamBody {
2695 fn drop(&mut self) {
2696 self.0.upstream_stage(UpstreamStage::BodyRead { bytes: 42 });
2697 }
2698 }
2699 let cfg = Config::parse("observability:\n usage:\n usage-statistics-enabled: true\n").unwrap();
2700 let executors = cpa_exec::Executors {
2701 claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
2702 codex: Default::default(),
2703 devices: Default::default(),
2704 openai: Default::default(),
2705 google: Default::default(),
2706 };
2707 let rt = Arc::new(crate::testing::runtime(cfg.clone(), Vec::new(), executors));
2708 rt.usage_queue().configure(true, &cfg);
2709 let mut metadata = serde_json::Map::new();
2710 metadata.insert("type".into(), "claude".into());
2711 let credential = cpa_core::credential::Credential::from_file(
2712 std::path::Path::new("/auth"),
2713 std::path::Path::new("/auth/a.json"),
2714 metadata,
2715 )
2716 .unwrap();
2717 let facts = Arc::new(crate::usage_record::Facts::new(
2718 Default::default(),
2719 Format::Claude,
2720 Format::Claude,
2721 "m",
2722 b"{}",
2723 true,
2724 &Trace::default(),
2725 ));
2726 for ending in ["error", "end", "client gone"] {
2727 let tracker = crate::usage_record::Tracker::start(&rt, &facts, &credential, "m");
2728 let body = UpstreamBody(tracker.sink());
2729 let rest = futures_util::stream::poll_fn(move |_| {
2731 let _held = &body;
2732 match ending {
2733 "error" => Poll::Ready(Some(Err(ExecError::local(
2734 502,
2735 cpa_core::exec::FailureScope::Request,
2736 "restore failed",
2737 )))),
2738 "end" => Poll::Ready(None),
2739 _ => Poll::Pending,
2740 }
2741 })
2742 .boxed();
2743 let done = track_usage(
2744 Done::Stream {
2745 headers: HeaderMap::new(),
2746 first: None,
2747 rest,
2748 },
2749 Some(tracker),
2750 );
2751 let Done::Stream { mut rest, .. } = done else {
2752 unreachable!()
2753 };
2754 if ending == "client gone" {
2755 assert!(futures_util::poll!(rest.next()).is_pending());
2756 drop(rest);
2757 } else {
2758 assert!(!matches!(rest.next().await, Some(Ok(_))));
2759 }
2760 let records = rt.usage_queue().pop_oldest(10);
2761 assert_eq!(records.len(), 1, "{ending}");
2762 let record: serde_json::Value = serde_json::from_slice(&records[0]).unwrap();
2763 assert_eq!(record["upstream_response_bytes"], 42, "{ending}: {record}");
2764 }
2765 }
2766
2767 #[test]
2769 fn lifecycle_errors_follow_go() {
2770 let transport = |text: &str| ExecError::local(502, cpa_core::exec::FailureScope::Transport, text);
2771 assert!(lifecycle(&transport("websocket: close 1000 (normal)")));
2772 assert!(lifecycle(&transport(
2773 "websocket: close 1006 (abnormal closure): unexpected EOF"
2774 )));
2775 assert!(lifecycle(&transport("EOF")));
2776 assert!(!lifecycle(&transport("websocket: close 1011 (internal server error)")));
2777 assert!(!lifecycle(&transport("read failed")));
2778 let statused = ExecError::local(500, cpa_core::exec::FailureScope::Credential, "unexpected EOF");
2780 assert!(!lifecycle(&statused));
2781 }
2782
2783 #[test]
2786 fn nonstream_keepalive_only_for_nonstream_generate() {
2787 let call = |operation, stream| Call {
2788 entry: Format::Claude,
2789 response: Format::Claude,
2790 operation,
2791 model: "m".into(),
2792 body: Bytes::new(),
2793 stream,
2794 alt: None,
2795 headers: HeaderMap::new(),
2796 caller: Caller {
2797 principal: String::new(),
2798 source: "",
2799 provider: String::new(),
2800 },
2801 forced_provider: None,
2802 selection_model: None,
2803 execution_session: None,
2804 request_path: String::new(),
2805 peer: None,
2806 turn: None,
2807 media: None,
2808 };
2809 let cfg = |yaml: &str| Config::parse(yaml).unwrap();
2810 let on = cfg("requests:\n nonstream-keepalive-interval: 3\n");
2811 assert_eq!(
2812 nonstream_keepalive(&call(Operation::Generate, false), &on),
2813 Some(Duration::from_secs(3))
2814 );
2815 assert_eq!(nonstream_keepalive(&call(Operation::Generate, true), &on), None);
2816 assert_eq!(nonstream_keepalive(&call(Operation::CountTokens, false), &on), None);
2817 for off in [
2818 "{}\n",
2819 "requests:\n nonstream-keepalive-interval: 0\n",
2820 "requests:\n nonstream-keepalive-interval: -5\n",
2821 ] {
2822 assert_eq!(
2823 nonstream_keepalive(&call(Operation::Generate, false), &cfg(off)),
2824 None,
2825 "{off}"
2826 );
2827 }
2828 let legacy = cfg("nonstream-keepalive-interval: 2\n");
2830 assert_eq!(
2831 nonstream_keepalive(&call(Operation::Generate, false), &legacy),
2832 Some(Duration::from_secs(2))
2833 );
2834 }
2835
2836 #[test]
2840 fn attempts_bind_resolved_model_like_go() {
2841 let cfg = Config::parse(
2842 "config-version: 8\napi-keys:\n codex:\n - base-url: https://codex.example\n models:\n - name: gpt-6-sol\n alias: sol\n is-compat: true\n keys:\n - api-key: xk1\n",
2843 )
2844 .unwrap();
2845 let credential = cpa_core::config::credentials::from_config(&cfg).remove(0);
2846 let request = ExecRequest {
2847 operation: Operation::Generate,
2848 source_format: Format::OpenAI,
2849 response_format: Format::OpenAI,
2850 stream_payloads: false,
2851 requested_model: "sol".into(),
2852 model: "sol".into(),
2853 original_body: Bytes::new(),
2854 body: Bytes::new(),
2855 stream: false,
2856 alt: None,
2857 session: None,
2858 execution_session: None,
2859 derived_session: None,
2860 request_path: String::new(),
2861 headers: HeaderMap::new(),
2862 caller: Caller {
2863 principal: String::new(),
2864 source: "",
2865 provider: String::new(),
2866 },
2867 resolved_model: None,
2868 usage: Default::default(),
2869 };
2870 let req = attempt_request(&request, &cfg, &credential, "sol", "gpt-6-sol", false, false);
2871 assert_eq!(req.model, "gpt-6-sol");
2872 let bound = req.resolved_model.unwrap();
2873 assert_eq!(
2874 (bound.info.id.as_str(), bound.source, bound.is_compat()),
2875 ("gpt-6-sol", cpa_core::exec::ResolvedSource::ApiKey, true)
2876 );
2877 let kept = attempt_request(&request, &cfg, &credential, "selection", "gpt-6-sol", true, false);
2878 assert_eq!(kept.model, "sol");
2879 let bound = kept.resolved_model.unwrap();
2880 assert_eq!((bound.info.id.as_str(), bound.is_compat()), ("sol", false));
2881
2882 let mut home = credential.clone();
2886 home.attributes
2887 .insert(crate::remote::MODEL_INFO.into(), r#"{"id":"gpt-6-sol"}"#.into());
2888 let caps = |req: &ExecRequest| {
2889 let bound = req.resolved_model.as_ref().unwrap();
2890 let caps = cpa_common::thinking::ModelCaps::from(&bound.info);
2891 (bound.source, caps.support_configuration_update, bound.is_compat())
2892 };
2893 let cfg = Config::parse(
2894 "config-version: 8\napi-keys:\n codex:\n - base-url: https://codex.example\n models:\n - name: gpt-6-sol\n alias: sol\n is-compat: true\n support-configuration-update: true\n keys:\n - api-key: xk1\n",
2895 )
2896 .unwrap();
2897 let home_source = cpa_core::exec::ResolvedSource::Home;
2898 let buffered = attempt_request(&request, &cfg, &home, "sol", "gpt-6-sol", false, false);
2899 assert_eq!(
2900 caps(&buffered),
2901 (home_source, true, false),
2902 "local is-compat never carries"
2903 );
2904 let streamed = attempt_request(&request, &cfg, &home, "sol", "gpt-6-sol", false, true);
2905 assert_eq!(caps(&streamed), (home_source, false, false));
2906 let kept = attempt_request(&request, &cfg, &home, "selection", "gpt-6-sol", true, false);
2907 assert_eq!(
2908 kept.resolved_model.unwrap().source,
2909 cpa_core::exec::ResolvedSource::ApiKey
2910 );
2911 }
2912
2913 #[test]
2914 fn failure_texts_match_go_shapes() {
2915 assert_eq!(
2916 Failure::UnknownModel("claude-sonnet-4-6".into()).text(),
2917 r#"{"error":{"message":"unknown provider for model claude-sonnet-4-6","type":"invalid_request_error","code":"model_not_found","param":"model"}}"#
2918 );
2919 let cooldown = Failure::Cooldown {
2920 model: "m".into(),
2921 provider: "claude".into(),
2922 wait: Duration::from_millis(59_400),
2923 cause: Some(r#"{"error":{"type":"rate_limit_error","message":"slow down"}}"#.into()),
2924 };
2925 assert_eq!(
2926 cooldown.text(),
2927 r#"{"error":{"code":"model_cooldown","last_upstream_error":"rate_limit_error: slow down","message":"All credentials for model m are cooling down via provider claude (last error: rate_limit_error: slow down)","model":"m","provider":"claude","reset_seconds":60,"reset_time":"59s"}}"#
2928 );
2929 assert_eq!(cooldown.retry_after(), Some(60));
2930 assert_eq!(
2931 (cooldown.status(), Failure::UnknownModel(String::new()).status()),
2932 (429, 400)
2933 );
2934 let none = Failure::Unavailable {
2935 code: "auth_not_found",
2936 providers: vec!["claude".into()],
2937 model: " m ".into(),
2938 cause: None,
2939 retry_after: None,
2940 };
2941 assert_eq!(
2942 none.text(),
2943 "auth_not_found: no auth available (providers=claude, model=m); check Claude auth/key session and cooldown state via /v0/management/auth-files"
2944 );
2945 assert_eq!((none.status(), none.retry_after()), (503, None));
2946 }
2947
2948 #[test]
2949 fn force_mapping_rewrites_json_and_sse_model_fields() {
2950 let body = Bytes::from_static(br#"{"id":"x","model":"claude-opus-5","message":{"model":"claude-opus-5"}}"#);
2951 assert_eq!(
2952 rewrite_body(&body, "opus"),
2953 r#"{"id":"x","model":"opus","message":{"model":"opus"}}"#
2954 );
2955 let event = Bytes::from_static(b"event: message_start\ndata: {\"message\":{\"model\":\"up\"}}\n\n");
2956 assert_eq!(
2957 rewrite_event(&event, "alias"),
2958 "event: message_start\ndata: {\"message\":{\"model\":\"alias\"}}\n\n"
2959 );
2960 let untouched = Bytes::from_static(b"data: [DONE]\n\n");
2961 assert_eq!(rewrite_event(&untouched, "alias"), untouched);
2962 let crlf = Bytes::from_static(b"data: {\"model\" : \"up\", \"x\":\"\\u00e9\", \"n\":1.50}\r\n\r\n");
2964 assert_eq!(
2965 rewrite_event(&crlf, "alias"),
2966 "data: {\"model\" : \"alias\", \"x\":\"\\u00e9\", \"n\":1.50}\r\n\r\n"
2967 );
2968 let spaced = Bytes::from_static(b"data: {\"model\":\"up\"}\n\n");
2970 assert_eq!(rewrite_event(&spaced, "alias"), spaced);
2971 let bare = Bytes::from_static(b" {\"modelVersion\":\"up\"}\n");
2973 assert_eq!(rewrite_event(&bare, "alias"), r#"{"modelVersion":"alias"}"#);
2974 }
2975
2976 fn home_cooldown(retry_after: Duration, request_retry: Option<i64>) -> RemoteFault {
2978 RemoteFault::from_pick(crate::remote::RemoteError {
2979 error: ExecError::local(
2980 429,
2981 cpa_core::exec::FailureScope::Credential,
2982 "model_cooldown: all Home credentials are cooling down",
2983 ),
2984 code: "model_cooldown".into(),
2985 kind: crate::remote::RemoteErrorKind::Cooldown {
2986 retry_after: Some(retry_after),
2987 request_retry,
2988 },
2989 })
2990 }
2991
2992 fn retry_wait(
2995 error: &RemoteFault,
2996 request_retry: i64,
2997 attempt: i64,
2998 max_wait: Duration,
2999 limit: i64,
3000 pinned: bool,
3001 ) -> Option<Duration> {
3002 let allowed = |attempt: i64, limit: i64| home_retry_allowed(request_retry, attempt, limit);
3003 remote_retry_wait(error, attempt, max_wait, limit, pinned, &allowed)
3004 }
3005
3006 #[test]
3010 fn home_cooldowns_retry_within_request_retry_and_the_maximum_wait() {
3011 let ms = Duration::from_millis;
3012 let cooldown = home_cooldown(ms(10), None);
3013 assert_eq!(
3014 retry_wait(&cooldown, 1, 0, Duration::from_secs(1), -1, false),
3015 Some(ms(10))
3016 );
3017 assert_eq!(
3018 retry_wait(&cooldown, 1, 1, Duration::from_secs(1), -1, false),
3019 None,
3020 "after the configured round"
3021 );
3022 assert_eq!(
3023 retry_wait(&cooldown, 1, 0, Duration::ZERO, -1, false),
3024 None,
3025 "no maximum wait"
3026 );
3027 let unavailable = ExecError::local(502, cpa_core::exec::FailureScope::Credential, "upstream unavailable");
3028 let round = RemoteFault::local(Failure::Exec(unavailable)).exhausted(None, false);
3029 assert_eq!(
3030 retry_wait(&round, 1, 0, Duration::ZERO, -1, false),
3031 Some(Duration::ZERO)
3032 );
3033 }
3038
3039 #[test]
3043 fn a_cooldown_request_retry_overrides_the_configured_one_unless_pinned() {
3044 let ms = Duration::from_millis;
3045 let second = Duration::from_secs(1);
3046 let one = home_cooldown(ms(10), Some(1));
3047 assert_eq!(retry_wait(&one, 0, 0, second, -1, false), Some(ms(10)));
3048 assert_eq!(
3049 retry_wait(&one, 0, 1, second, -1, false),
3050 None,
3051 "one additional round only"
3052 );
3053 assert_eq!(
3054 retry_wait(&one, 0, 0, second, -1, true),
3055 None,
3056 "pinned keeps its own limit"
3057 );
3058 let zero = home_cooldown(ms(10), Some(0));
3059 assert_eq!(retry_wait(&zero, 3, 0, second, -1, false), None, "an explicit 0 wins");
3060 }
3061
3062 #[test]
3066 fn a_credential_limit_starts_the_next_round_immediately() {
3067 let mut limited = ExecError::local(429, cpa_core::exec::FailureScope::Credential, "credential rate limited");
3068 limited.retry_after = Some(Duration::from_secs(5));
3069 let round = RemoteFault::local(Failure::Exec(limited)).exhausted(Some(Duration::from_secs(5)), true);
3070 assert_eq!(
3071 retry_wait(&round, 1, 0, Duration::from_secs(1), -1, false),
3072 Some(Duration::ZERO)
3073 );
3074 assert_eq!(round.safe_retry_after(), Some(5));
3075 }
3076
3077 #[test]
3078 fn jitter_stays_within_go_bounds() {
3079 for _ in 0..50 {
3080 let w = jitter(Duration::from_secs(4), Duration::ZERO);
3081 assert!(w >= Duration::from_secs(4) && w < Duration::from_secs(5));
3082 let capped = jitter(Duration::from_secs(20), Duration::from_secs(21));
3083 assert!(capped >= Duration::from_secs(20) && capped < Duration::from_secs(21));
3084 }
3085 assert_eq!(
3086 jitter(Duration::from_secs(5), Duration::from_secs(5)),
3087 Duration::from_secs(5)
3088 );
3089 assert_eq!(jitter(Duration::ZERO, Duration::ZERO), Duration::ZERO);
3090 }
3091
3092 #[test]
3093 fn summaries_follow_go_extract() {
3094 assert_eq!(
3095 upstream_summary(r#"{"error":{"code":"x","message":"boom"}}"#),
3096 "x: boom"
3097 );
3098 assert_eq!(
3099 upstream_summary(r#"{"error":{"type":"overloaded","message":"overloaded now"}}"#),
3100 "overloaded now"
3101 );
3102 assert_eq!(upstream_summary(r#"{"error":"flat"}"#), "flat");
3103 assert_eq!(upstream_summary("plain text"), "plain text");
3104 assert_eq!(upstream_summary("status 500: {\"message\":\"m\"}"), "m");
3105 }
3106}