use std::sync::Arc;
use std::time::Duration;
use axum::http::HeaderMap;
use bytes::{Bytes, BytesMut};
use cpa_common::gostr::GoStr;
use cpa_core::config::Config;
use cpa_core::exec::{Caller, ExecError, ExecRequest, ExecStream, Operation, ResponseBody};
use cpa_core::format::Format;
use futures_util::StreamExt;
use crate::classify;
use crate::gojson;
use crate::registry::{self, AliasResult, Registry};
use crate::respond;
use crate::runtime::{AcquireError, Completing, Lease, Outcome, Runtime, Selection};
use crate::scheduler::{Policy, canonical_model};
#[derive(Clone)]
pub struct Call {
pub entry: Format,
pub response: Format,
pub operation: Operation,
pub model: String,
pub body: Bytes,
pub stream: bool,
pub alt: Option<String>,
pub headers: HeaderMap,
pub caller: Caller,
pub forced_provider: Option<String>,
pub selection_model: Option<String>,
pub execution_session: Option<String>,
pub request_path: String,
pub peer: Option<std::net::SocketAddr>,
pub turn: Option<Arc<SessionTurn>>,
pub media: Option<Arc<Media>>,
}
pub struct Media {
pub kind: MediaKind,
pub pinned: Option<String>,
pub on_selected: Option<OnSelected>,
pub sse: bool,
pub disallow_free: bool,
}
#[derive(Clone)]
pub struct SseError(pub Bytes);
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MediaKind {
Images,
Videos,
Speech,
}
impl Call {
pub(crate) fn pinned(&self) -> Option<String> {
self.turn
.as_ref()
.and_then(|t| t.pinned.clone())
.or_else(|| self.media.as_ref().and_then(|m| m.pinned.clone()))
.or_else(|| {
crate::plugins::model_callbacks::context()
.map(|c| c.auth_id)
.filter(|id| !id.is_empty())
})
}
fn on_selected(&self) -> Option<&OnSelected> {
self.turn
.as_ref()
.and_then(|t| t.on_selected.as_ref())
.or_else(|| self.media.as_ref().and_then(|m| m.on_selected.as_ref()))
}
}
pub struct SessionTurn {
pub session: cpa_core::exec::ExecSession,
pub pinned: Option<String>,
pub on_selected: Option<OnSelected>,
pub home: Option<Arc<crate::home_session::SessionHome>>,
}
pub type OnSelected = Box<dyn Fn(&cpa_core::credential::Credential) + Send + Sync>;
pub type Peer = Option<axum::Extension<axum::extract::ConnectInfo<std::net::SocketAddr>>>;
pub fn peer(peer: Peer) -> Option<std::net::SocketAddr> {
peer.map(|axum::Extension(axum::extract::ConnectInfo(addr))| addr)
}
pub fn route_path(matched: Option<&axum::extract::MatchedPath>, uri: &axum::http::Uri) -> String {
let Some(matched) = matched else {
return uri.path().to_owned();
};
matched
.as_str()
.split('/')
.map(
|segment| match segment.strip_prefix('{').and_then(|s| s.strip_suffix('}')) {
Some(name) if name.starts_with('*') => name.to_owned(),
Some(name) => format!(":{name}"),
None => segment.to_owned(),
},
)
.collect::<Vec<_>>()
.join("/")
}
pub enum Done {
Buffered {
headers: HeaderMap,
body: Bytes,
},
Stream {
headers: HeaderMap,
first: Option<Bytes>,
rest: ExecStream,
},
}
#[derive(Debug, Clone)]
pub enum Failure {
Exec(ExecError),
Remote { error: ExecError, retry_after: Option<u64> },
UnknownModel(String),
ImageOnly(String),
SpeechOnly(String),
Unavailable {
code: &'static str,
providers: Vec<String>,
model: String,
cause: Option<String>,
retry_after: Option<Duration>,
},
Cooldown {
model: String,
provider: String,
wait: Duration,
cause: Option<String>,
},
}
impl Failure {
pub fn status(&self) -> u16 {
match self {
Failure::Exec(e) | Failure::Remote { error: e, .. } => classify::response_status(e),
Failure::UnknownModel(_) | Failure::SpeechOnly(_) => 400,
Failure::ImageOnly(_) | Failure::Unavailable { .. } => 503,
Failure::Cooldown { .. } => 429,
}
}
pub fn text(&self) -> String {
match self {
Failure::Exec(e) | Failure::Remote { error: e, .. } => classify::error_text(e),
Failure::UnknownModel(model) => {
let message = gojson::sjson_string(&format!("unknown provider for model {model}"));
format!(
r#"{{"error":{{"message":{message},"type":"invalid_request_error","code":"model_not_found","param":"model"}}}}"#
)
}
Failure::ImageOnly(model) => format!(
"model {} is only supported on /v1/images/generations and /v1/images/edits",
route_base_name(model)
),
Failure::SpeechOnly(model) => format!(
"model {} is only supported on /v1/audio/speech and /v1/tts",
route_base_name(model)
),
Failure::Unavailable {
code,
providers,
model,
cause,
..
} => unavailable_text(code, "no auth available", providers, model, cause.as_deref()),
Failure::Cooldown {
model,
provider,
wait,
cause,
} => {
let shown = if model.is_empty() { "requested model" } else { model };
let mut message = format!("All credentials for model {shown} are cooling down");
if !provider.is_empty() {
message.push_str(&format!(" via provider {provider}"));
}
let display = if !wait.is_zero() && *wait < Duration::from_secs(1) {
Duration::from_secs(1)
} else {
Duration::from_secs((wait.as_secs_f64()).round() as u64)
};
let mut body = serde_json::Map::new();
body.insert("code".into(), "model_cooldown".into());
body.insert("model".into(), model.clone().into());
body.insert("reset_time".into(), gojson::duration(display).into());
body.insert("reset_seconds".into(), ceil_seconds(*wait).into());
if !provider.is_empty() {
body.insert("provider".into(), provider.clone().into());
}
if let Some(summary) = cause.as_deref().map(upstream_summary).filter(|s| !s.is_empty()) {
message.push_str(&format!(" (last error: {summary})"));
body.insert("last_upstream_error".into(), summary.into());
}
body.insert("message".into(), message.into());
gojson::sorted(&serde_json::json!({ "error": body }))
}
}
}
pub fn retry_after(&self) -> Option<u64> {
match self {
Failure::Cooldown { wait, .. } => Some(ceil_seconds(*wait)),
Failure::Remote { retry_after, .. } => *retry_after,
Failure::Unavailable {
retry_after: Some(wait),
..
} if !wait.is_zero() => Some(ceil_seconds(*wait).max(1)),
_ => None,
}
}
pub fn direct(&self) -> Option<&ExecError> {
match self {
Failure::Exec(e) | Failure::Remote { error: e, .. } if e.direct => Some(e),
_ => None,
}
}
}
fn unavailable_text(code: &str, base: &str, providers: &[String], model: &str, cause: Option<&str>) -> String {
let providers = if providers.is_empty() {
"unknown".into()
} else {
providers.join(",")
};
let model = if gojson::trim(model).is_empty() {
"unknown"
} else {
gojson::trim(model)
};
let mut detail = match cause
.map(upstream_summary)
.filter(|s| !s.is_empty() && !base.contains(s.as_str()))
{
Some(summary) => format!("{base} (providers={providers}, model={model}; last upstream error: {summary})"),
None => format!("{base} (providers={providers}, model={model})"),
};
if format!(",{providers},").contains(",claude,") {
detail.push_str("; check Claude auth/key session and cooldown state via /v0/management/auth-files");
}
format!("{code}: {detail}")
}
fn ceil_seconds(d: Duration) -> u64 {
d.as_secs() + u64::from(d.subsec_nanos() > 0)
}
pub fn upstream_summary(raw: &str) -> String {
let raw = raw.trim();
if raw.is_empty() {
return String::new();
}
let json_part = match raw.find(": {") {
Some(i) if i < 50 => raw[i + 2..].trim(),
_ => raw,
};
if let Ok(v) = serde_json::from_str::<serde_json::Value>(json_part) {
let s = |v: Option<&serde_json::Value>| gojson::trim(&gojson::gjson_string(v)).to_owned();
let (mut code, mut message) = (String::new(), String::new());
match v.get("error") {
Some(e @ serde_json::Value::Object(_)) => {
code = s(e.get("code"));
if code.is_empty() {
code = s(e.get("type"));
}
message = s(e.get("message"));
}
Some(serde_json::Value::String(m)) => message = m.trim().to_owned(),
_ => {}
}
if code.is_empty() && message.is_empty() {
code = s(v.get("code"));
if code.is_empty() {
code = s(v.get("type"));
}
message = s(v.get("message"));
}
let summary = match (code.is_empty(), message.is_empty()) {
(false, false) if code.go_eq_fold(&message) || message.go_lower().contains(&code.go_lower()) => message,
(false, false) => format!("{code}: {message}"),
(true, false) => message,
(false, true) => code,
(true, true) => String::new(),
};
if !summary.is_empty() {
return crate::sanitize::summary(&summary);
}
}
crate::sanitize::summary(raw)
}
fn route_base_name(model: &str) -> &str {
let model = model.trim();
model
.rsplit('/')
.next()
.filter(|s| !s.is_empty())
.unwrap_or(model)
.trim()
}
fn check_route_only(base: &str, call: &Call) -> Result<(), RunError> {
let name = route_base_name(base).go_lower();
let kind = call.media.as_ref().map(|m| m.kind);
if IMAGE_ONLY.contains(&name.as_str()) && kind != Some(MediaKind::Images) {
return Err(Failure::ImageOnly(base.to_owned()).into());
}
if SPEECH_ONLY.contains(&name.as_str()) && kind != Some(MediaKind::Speech) {
return Err(Failure::SpeechOnly(base.to_owned()).into());
}
Ok(())
}
const SPEECH_ONLY: [&str; 2] = ["grok-tts", "grok-voice-tts-1.0"];
const IMAGE_ONLY: [&str; 8] = [
"gpt-image-1.5",
"gpt-image-2",
"gpt-image-2.5-flare",
"gpt-image-2.5-sunburst",
"gpt-image-2.5",
"grok-imagine-image",
"grok-imagine-image-quality",
"grok-imagine-image-2.0",
];
fn route(
rt: &Runtime,
registry: &Registry,
call: &Call,
routed: Option<&(String, Option<String>)>,
) -> Result<(Vec<String>, String), RunError> {
if let Some(provider) = &call.forced_provider {
let model = gojson::trim(&call.model);
check_route_only(gojson::trim(canonical_model_raw(model)), call)?;
return Ok((vec![provider.clone()], model.to_owned()));
}
if let Some((provider, model)) = routed {
let model = model.clone().unwrap_or_else(|| call.model.clone());
check_route_only(gojson::trim(canonical_model_raw(&model)), call)?;
let mut providers = vec![provider.clone()];
adjust_for_entry(call.entry, &mut providers);
return Ok((providers, model));
}
let model = call.model.as_str();
let base = match model.rfind('(') {
Some(i) if model.ends_with(')') => &model[..i],
_ => model,
};
let remote = rt.remote_dispatch().is_some();
let resolved = if base == "auto" && !remote {
let first = registry
.resolve_auto(|client, model| rt.suspension(client, model))
.unwrap_or_else(|| "auto".into());
format!("{first}{}", &model[base.len()..])
} else {
model.to_owned()
};
let base = gojson::trim(canonical_model_raw(&resolved)).to_owned();
check_route_only(&base, call)?;
if remote {
return Ok((vec!["home".to_owned()], resolved));
}
let mut providers = registry.providers(&base);
if providers.is_empty() && base != resolved {
providers = registry.providers(&resolved);
}
if providers.is_empty() {
return Err(Failure::UnknownModel(call.model.clone()).into());
}
adjust_for_entry(call.entry, &mut providers);
Ok((providers, resolved))
}
fn adjust_for_entry(entry: Format, providers: &mut Vec<String>) {
match entry {
Format::Interactions => {
if let Some(i) = providers.iter().position(|p| p == "gemini-interactions") {
let p = providers.remove(i);
providers.insert(0, p);
}
}
Format::OpenAI | Format::OpenAIResponse | Format::Claude | Format::Gemini => {}
_ => providers.retain(|p| p != "gemini-interactions"),
}
}
fn canonical_model_raw(model: &str) -> &str {
match model.rfind('(') {
Some(i) if model.ends_with(')') => &model[..i],
_ => model,
}
}
pub async fn serve<F, Fut>(rt: &Arc<Runtime>, call: Call, render: F) -> axum::response::Response
where
F: FnOnce(Result<Done, Failure>) -> Fut + Send + 'static,
Fut: std::future::Future<Output = axum::response::Response> + Send + 'static,
{
let sse = call.media.as_ref().is_some_and(|m| m.sse);
let speech = call.media.as_ref().is_some_and(|m| m.kind == MediaKind::Speech);
let interval = if sse {
crate::respond::keepalive(&rt.config())
} else {
nonstream_keepalive(&call, &rt.config())
};
let trace = Arc::new(Trace::with_request_id(crate::observability::current_request_id()));
let mut run = Box::pin(crate::remote::keep_query_credential({
let (rt, trace) = (rt.clone(), trace.clone());
async move {
let result = run_with_bootstrap_retries(&rt, call, &trace).await;
let cfg = rt.config();
let upstream = upstream_headers(&cfg, &result);
let result = match result {
Ok(Done::Buffered { headers, body }) if speech => {
let headers = if respond::passthrough_headers(&cfg) {
respond::filter_upstream_headers(&headers)
} else {
respond::filter_upstream_headers(&trace.intercepted_headers_snapshot())
};
Ok(Done::Buffered { headers, body })
}
result => result,
};
(result, upstream)
}
}));
let traced = {
let trace = trace.clone();
move |mut response: axum::response::Response, upstream: Option<Upstream>| {
let passthrough = matches!(upstream, Some(Upstream::Success(_)));
match upstream {
Some(Upstream::Success(headers)) => respond::write_upstream_headers(response.headers_mut(), &headers),
Some(Upstream::Error(headers)) => respond::write_error_headers(response.headers_mut(), &headers),
None => {}
}
let intercepted = trace.intercepted_headers();
if !passthrough && !intercepted.is_empty() {
respond::write_upstream_headers(
response.headers_mut(),
&respond::filter_upstream_headers(&intercepted),
);
}
if let Some(value) = trace.header() {
response.headers_mut().insert("x-cpa-trace-id", value);
}
response
}
};
let Some(interval) = interval else {
let (result, upstream) = run.await;
return traced(render(result).await, upstream);
};
tokio::select! {
biased;
(result, upstream) = &mut run => return traced(render(result).await, upstream),
() = tokio::time::sleep(interval) => {}
}
let mut ticks = tokio::time::interval_at(tokio::time::Instant::now() + interval, interval);
ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
enum Event {
Tick,
Done(Result<Done, Failure>),
}
let events = futures_util::stream::unfold(Some((run, ticks)), |state| async move {
let (mut run, mut ticks) = state?;
tokio::select! {
biased;
(result, _) = &mut run => Some((Event::Done(result), None)),
_ = ticks.tick() => Some((Event::Tick, Some((run, ticks)))),
}
});
let beat = Bytes::from_static(if sse { b": keep-alive\n\n" } else { b"\n" });
let once = |bytes: Bytes| futures_util::stream::iter([Ok::<_, axum::Error>(bytes)]);
let first = once(beat.clone());
let mut render = Some(render);
let body = first.chain(events.flat_map(move |event| match event {
Event::Tick => once(beat.clone()).left_stream(),
Event::Done(result) => {
let render = render.take().expect("the run finishes once");
let rendered = futures_util::stream::once(render(result));
let body = rendered.flat_map(move |response| match response.extensions().get::<SseError>() {
Some(SseError(event)) if sse => once(event.clone()).left_stream(),
_ => response.into_body().into_data_stream().right_stream(),
});
body.right_stream()
}
}));
let body = axum::body::Body::from_stream(body);
let mut response = if sse {
crate::respond::sse(body)
} else {
let mut response = axum::response::Response::new(body);
response.headers_mut().insert(
axum::http::header::CONTENT_TYPE,
axum::http::HeaderValue::from_static("application/json"),
);
response
};
let headers = response.headers_mut();
if let Some(value) = trace.header() {
headers.insert("x-cpa-trace-id", value);
}
response
}
enum Upstream {
Success(HeaderMap),
Error(HeaderMap),
}
fn upstream_headers(cfg: &Config, result: &Result<Done, Failure>) -> Option<Upstream> {
if !respond::passthrough_headers(cfg) {
return None;
}
match result {
Ok(Done::Buffered { headers, .. } | Done::Stream { headers, .. }) => {
Some(Upstream::Success(respond::filter_upstream_headers(headers)))
}
Err(Failure::Exec(error)) if !error.direct => error.addon.as_deref().cloned().map(Upstream::Error),
Err(_) => None,
}
}
fn nonstream_keepalive(call: &Call, cfg: &Config) -> Option<Duration> {
let speech = call.media.as_ref().is_some_and(|m| m.kind == MediaKind::Speech);
if call.stream || call.operation != Operation::Generate || speech {
return None;
}
let seconds = cfg
.document
.get("requests")
.and_then(|r| r.get("nonstream-keepalive-interval"))
.and_then(serde_yaml_ng::Value::as_i64)
.unwrap_or(0);
(seconds > 0).then(|| Duration::from_secs(seconds as u64))
}
fn bootstrap_retries(cfg: &Config) -> usize {
cfg.document
.get("requests")
.and_then(|r| r.get("streaming"))
.and_then(|s| s.get("bootstrap-retries"))
.and_then(serde_yaml_ng::Value::as_i64)
.unwrap_or(0)
.max(0) as usize
}
pub async fn run_with_bootstrap_retries(rt: &Arc<Runtime>, call: Call, trace: &Trace) -> Result<Done, Failure> {
use crate::plugins::execution::{self as plugin_execution, Route};
let query = plugin_execution::query();
let forced = call.forced_provider.as_deref().map(|p| p.trim().go_lower());
let routed = match (plugin_execution::route(rt, &call, &query, trace).await, forced) {
(None, _) => None,
(Some(Route::Provider { provider, .. }), Some(forced)) if provider == forced => None,
(Some(_), Some(_)) => {
return Err(Failure::Exec(ExecError::local(
400,
cpa_core::exec::FailureScope::Request,
"agent is only supported for native interactions execution",
)));
}
(Some(Route::Executor(plugin)), None) => {
return plugin_execution::execute(rt, &call, &plugin, &query, trace).await;
}
(Some(Route::Provider { provider, model }), None) => Some((provider, model)),
};
let resolved = route(rt, &rt.registry(), &call, routed.as_ref()).map_err(|e| e.failure)?;
if call.operation == Operation::Generate && rt.usage_queue().accepts() {
trace.prompt_prefix(call.entry, &call.body);
}
let mut call = call;
let hooks = crate::plugins::interceptors::start(rt, trace, &mut call, &resolved.1).await?;
let result = retry_bootstrap(rt, call, trace, &resolved, hooks.as_ref()).await;
match hooks {
Some(hooks) => hooks.finish(result, trace).await,
None => result,
}
}
async fn retry_bootstrap(
rt: &Arc<Runtime>,
call: Call,
trace: &Trace,
resolved: &(Vec<String>, String),
hooks: Option<&Arc<crate::plugins::interceptors::Hooks>>,
) -> Result<Done, Failure> {
let max = if call.stream && rt.remote_dispatch().is_none() {
bootstrap_retries(&rt.config())
} else {
0
};
let mut result = run_primed(rt, call.clone(), trace, resolved.clone(), hooks).await;
for _ in 0..max {
let original = match &result {
Err(RunError {
failure: Failure::Exec(e),
bootstrap: true,
}) if bootstrap_eligible(classify::go_status(e)) => e.clone(),
_ => break,
};
result = match run_primed(rt, call.clone(), trace, resolved.clone(), hooks).await {
Err(RunError {
failure,
bootstrap: false,
}) if matches!(failure, Failure::Unavailable { .. }) && classify::response_status(&original) >= 500 => {
Err(RunError {
failure: Failure::Exec(original),
bootstrap: false,
})
}
Err(RunError {
failure,
bootstrap: false,
}) => {
return Err(failure);
}
other => other,
};
}
result.map_err(|e| e.failure)
}
async fn run_primed(
rt: &Arc<Runtime>,
call: Call,
trace: &Trace,
resolved: (Vec<String>, String),
hooks: Option<&Arc<crate::plugins::interceptors::Hooks>>,
) -> Result<Done, RunError> {
let result = run_routed(rt, call, trace, resolved, hooks.map(|h| &**h)).await;
match (hooks, result) {
(Some(hooks), Ok(done @ Done::Stream { .. })) => {
hooks.prime(Ok(done), trace).await.map_err(|failure| RunError {
failure,
bootstrap: true,
})
}
(_, result) => result,
}
}
fn bootstrap_eligible(status: u16) -> bool {
matches!(status, 0 | 401 | 402 | 403 | 408 | 429) || status >= 500
}
#[derive(Default)]
pub struct Trace(
std::sync::Mutex<Option<String>>,
std::sync::OnceLock<String>,
cpa_core::exec::CaptureSink,
).
std::sync::Mutex<HeaderMap>,
Arc<crate::observability::Clock>,
std::sync::OnceLock<String>,
);
impl Trace {
pub fn with_request_id(id: Option<String>) -> Self {
let trace = Self(
Default::default(),
Default::default(),
crate::request_logging::current()
.map(|log| log.capture_sink())
.unwrap_or_default(),
Default::default(),
crate::observability::current_clock(),
Default::default(),
);
if let Some(id) = id.filter(|id| !id.is_empty()) {
let _ = trace.1.set(id);
}
trace
}
pub fn request_id(&self) -> String {
self.1.get_or_init(request_id).clone()
}
pub fn with_capture(mut self, capture: cpa_core::exec::CaptureSink) -> Self {
self.2 = capture;
self
}
pub(crate) fn capture(&self) -> cpa_core::exec::CaptureSink {
self.2.clone()
}
pub(crate) fn set_intercepted_headers(&self, headers: HeaderMap) {
*self.3.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = headers;
}
fn intercepted_headers_snapshot(&self) -> HeaderMap {
self.3.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone()
}
fn intercepted_headers(&self) -> HeaderMap {
std::mem::take(&mut *self.3.lock().unwrap_or_else(std::sync::PoisonError::into_inner))
}
fn header(&self) -> Option<axum::http::HeaderValue> {
let id = self
.0
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()?;
axum::http::HeaderValue::from_str(&id).ok()
}
pub(crate) fn selected(&self, credential: &cpa_core::credential::Credential) {
let index = cpa_core::config::credentials::auth_index(credential);
if index.is_empty() {
return;
}
let request = self.1.get_or_init(request_id);
let stamp = chrono::Local::now().format("%Y%m%d%H%M%S");
*self.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(format!("{stamp}-{index}-{request}"));
}
pub(crate) fn id(&self) -> Option<String> {
self.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone()
}
pub(crate) fn clock(&self) -> Arc<crate::observability::Clock> {
self.4.clone()
}
pub(crate) fn prompt_prefix(&self, format: Format, body: &[u8]) -> String {
self.5
.get_or_init(|| crate::usage_record::prompt_prefix(format, body))
.clone()
}
}
pub fn request_id() -> String {
use std::hash::{BuildHasher, Hasher};
let random = || {
let mut h = std::collections::hash_map::RandomState::new().build_hasher();
h.write_u128(
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_nanos(),
);
h.finish()
};
let millis = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_millis() as u64;
let (a, b) = (random(), random());
let hi = (millis << 16) | 0x7000 | (a & 0x0fff);
let lo = (b & 0x3fff_ffff_ffff_ffff) | 0x8000_0000_0000_0000;
format!(
"{:08x}-{:04x}-{:04x}-{:04x}-{:012x}",
hi >> 32,
(hi >> 16) & 0xffff,
hi & 0xffff,
lo >> 48,
lo & 0xffff_ffff_ffff
)
}
pub struct RunError {
pub failure: Failure,
pub bootstrap: bool,
}
impl From<Failure> for RunError {
fn from(failure: Failure) -> Self {
Self {
failure,
bootstrap: false,
}
}
}
#[derive(Clone)]
struct Fault {
error: ExecError,
bootstrap: Option<Box<HeaderMap>>,
}
impl Fault {
fn into_run_error(self) -> RunError {
RunError {
failure: Failure::Exec(self.error),
bootstrap: self.bootstrap.is_some(),
}
}
}
pub async fn run(rt: &Arc<Runtime>, call: Call, trace: &Trace) -> Result<Done, RunError> {
let resolved = route(rt, &rt.registry(), &call, None)?;
run_routed(rt, call, trace, resolved, None).await
}
async fn run_routed(
rt: &Arc<Runtime>,
call: Call,
trace: &Trace,
(providers, model): (Vec<String>, String),
hooks: Option<&crate::plugins::interceptors::Hooks>,
) -> Result<Done, RunError> {
let (cfg, policy) = rt.request_snapshot();
let registry = rt.registry();
let aliases = registry::global_aliases(&cfg);
let session = crate::session::resolve(
call.entry,
&call.headers,
&call.body,
call.execution_session.as_deref(),
&call.caller.principal,
);
let home_count = call.operation == Operation::CountTokens && rt.remote_dispatch().is_some();
let facts = ((call.operation == Operation::Generate
&& (rt.usage_queue().accepts() || rt.plugins().has_usage_plugins()))
|| (home_count && rt.usage_queue().accepts()))
.then(|| {
Arc::new(crate::usage_record::Facts::new(
usage_client(&cfg, &call, trace, &session),
call.entry,
call.response,
&call.model,
&call.body,
call.stream,
trace,
))
});
let (usage, unauthorized) = if home_count { (None, facts) } else { (facts, None) };
let mut pinned_exclusion: Vec<String> = match call.pinned() {
Some(pinned) if !pinned.is_empty() && rt.remote_dispatch().is_none() => rt
.store()
.snapshot()
.iter()
.filter(|c| c.id != pinned)
.map(|c| c.id.clone())
.collect(),
_ => Vec::new(),
};
if call.media.as_ref().is_some_and(|m| m.disallow_free) {
for credential in rt.store().snapshot() {
let free = credential.provider.trim().eq_ignore_ascii_case("codex")
&& credential
.attributes
.get("plan_type")
.is_some_and(|plan| plan.trim().eq_ignore_ascii_case("free"));
if free && !pinned_exclusion.contains(&credential.id) {
pinned_exclusion.push(credential.id.clone());
}
}
}
let mut selection = Selection {
providers: providers.clone(),
model: call.selection_model.clone().unwrap_or_else(|| model.clone()),
session: session.id.clone(),
session_parent: session.parent,
session_fork: session.fork,
exclude: pinned_exclusion.clone(),
..Selection::default()
};
if rt.plugins().has_scheduler() {
let route_model = selection.model.clone();
selection.scheduler = Some(Arc::new(crate::plugins::execution::scheduler_context(
&call,
&route_model,
policy.session_affinity,
session.derived.as_deref(),
)));
}
let request = ExecRequest {
operation: call.operation,
source_format: call.entry,
response_format: call.response,
requested_model: model.clone(),
model: model.clone(),
original_body: call.body.clone(),
body: call.body.clone(),
stream: call.stream,
stream_payloads: hooks.is_some() || crate::plugins::model_callbacks::context().is_some(),
alt: call.alt.clone(),
session: session.id,
execution_session: call.execution_session.clone(),
derived_session: session.derived,
resolved_model: None,
usage: cpa_core::exec::UsageSink::default().with_capture(trace.2.clone()),
request_path: call.request_path.clone(),
headers: call.headers.clone(),
caller: call.caller.clone(),
};
let compact = call.alt.as_deref() == Some("responses/compact");
if let Some(remote) = rt.remote_dispatch() {
let context = RemoteContext {
rt,
cfg: &cfg,
policy: &policy,
call: &call,
request: &request,
usage: usage.as_ref(),
unauthorized: unauthorized.as_ref(),
aliases: &aliases,
compact,
trace,
user_key: Default::default(),
};
return run_remote(context, remote, selection).await;
}
if policy.session_affinity && !session.explicit {
selection.lcp = crate::lcp::Request::new(call.entry.as_str(), &call.body, &call.caller.principal).map(Arc::new);
}
let mut upstream: Option<Fault> = None;
loop {
let mut last: Option<Fault> = None;
let mut attempted: Vec<String> = Vec::new();
let pick_failure = loop {
if policy.max_retry_credentials > 0 && attempted.len() >= policy.max_retry_credentials {
break None;
}
let lease = match rt.acquire(selection.clone(), &cfg, policy.clone(), ®istry).await {
Ok(lease) => lease,
Err(AcquireError::Prepare { id, error }) => {
attempted.push(id.clone());
selection.exclude.push(id);
last = Some(Fault { error, bootstrap: None });
continue;
}
Err(AcquireError::Cooldown { wait, cause }) => {
let provider = if providers.len() == 1 {
providers[0].clone()
} else {
String::new()
};
break Some(Failure::Cooldown {
model: selection.model.clone(),
provider,
wait,
cause,
});
}
Err(AcquireError::Rejected { code, message }) => {
break Some(scheduler_rejection(code, message, &providers, &model));
}
Err(AcquireError::Scheduler(error)) => break Some(Failure::Exec(error)),
Err(AcquireError::Unavailable { retry_after, cause }) => {
break Some(Failure::Unavailable {
code: if retry_after.is_some() {
"auth_unavailable"
} else {
"auth_not_found"
},
providers: providers.clone(),
model: model.clone(),
cause,
retry_after,
});
}
};
selection.exclude.push(lease.credential.id.clone());
trace.selected(&lease.credential);
let (mut models, alias) = registry::execution_models(&aliases, &lease.credential, &selection.model);
let pooled = models.len() > 1;
if pooled {
let key = format!(
"{}|{}|{}",
lease.credential.id.trim().go_lower(),
registry::provider_key(&lease.credential),
canonical_model(registry::strip_prefix(&selection.model, &lease.credential)).go_lower()
);
let offset = rt.next_pool_offset(&key, models.len());
models.rotate_left(offset);
}
let selection_model = registry::selection_model(&aliases, &lease.credential, &selection.model);
let models: Vec<String> = models
.into_iter()
.filter(|m| {
let state = registry::state_model(&selection_model, &selection.model, m, pooled);
!rt.store().blocked(&lease.credential, &state)
})
.collect();
if models.is_empty() {
continue;
}
attempted.push(lease.credential.id.clone());
let target = Target {
models: &models,
selection_model: &selection_model,
pooled,
alias: &alias,
compact,
keep_model: call.selection_model.is_some(),
unauthorized: None,
};
match attempt(rt, &cfg, &policy, &call, &request, usage.as_ref(), lease, target, hooks).await {
Attempt::Done(done) => return Ok(done),
Attempt::Stop(fault) => return Err(fault.into_run_error()),
Attempt::Next(fault) => {
if classify::upstream_attempted(&fault.error) {
upstream = Some(fault.clone());
}
last = Some(fault);
}
}
};
let round_error: RunError = match (last, pick_failure) {
(Some(fault), _) => fault.into_run_error(),
(None, Some(failure)) => failure.into(),
(None, None) => Failure::Unavailable {
code: "auth_not_found",
providers: providers.clone(),
model: model.clone(),
cause: None,
retry_after: None,
}
.into(),
};
let (status, retry_round) = match &round_error.failure {
Failure::Exec(e) => (
classify::go_status(e),
classify::is_retry_round(e) && !classify::is_request_invalid(e),
),
Failure::Cooldown { .. } => (429, true),
Failure::Unavailable {
retry_after: Some(_), ..
} => (503, true),
_ => (0, false),
};
let terminal = |round_error: RunError| upstream.clone().map(Fault::into_run_error).unwrap_or(round_error);
let pinned = call.pinned();
let credits = Credits {
providers: &providers,
model: &model,
aliases: &aliases,
pinned: pinned.as_deref(),
compact,
};
if !retry_round {
return credits
.fallback(
rt,
&cfg,
&policy,
&call,
&request,
usage.as_ref(),
hooks,
terminal(round_error),
)
.await;
}
let wait = {
let admit = crate::runtime::admission(®istry, &aliases, &selection, &rt.executors, rt.plugin_runtime());
rt.store().retry_wait(&selection, &policy, status, &attempted, &admit)
};
let Some(wait) = wait else {
return credits
.fallback(
rt,
&cfg,
&policy,
&call,
&request,
usage.as_ref(),
hooks,
terminal(round_error),
)
.await;
};
if !wait.is_zero() {
tokio::time::sleep(jitter(wait, policy.max_retry_interval)).await;
}
selection.retry_round += 1;
selection.exclude.clone_from(&pinned_exclusion);
}
}
struct RemoteContext<'a> {
rt: &'a Arc<Runtime>,
cfg: &'a Config,
policy: &'a Arc<crate::scheduler::Policy>,
call: &'a Call,
request: &'a ExecRequest,
usage: Option<&'a Arc<crate::usage_record::Facts>>,
unauthorized: Option<&'a Arc<crate::usage_record::Facts>>,
aliases: &'a std::collections::HashMap<String, Vec<cpa_core::registry::dynamic::OAuthAlias>>,
compact: bool,
trace: &'a Trace,
user_key: std::sync::Mutex<String>,
}
#[derive(Debug, Clone, Copy, PartialEq)]
struct Exhausted {
retry_after: Option<Duration>,
retry_now: bool,
}
#[derive(Debug, Clone)]
struct RemoteFault {
failure: Failure,
bootstrap: bool,
upstream: bool,
stop: bool,
invalid: bool,
kind: crate::remote::RemoteErrorKind,
exhausted: Option<Exhausted>,
}
impl RemoteFault {
fn from_pick(error: crate::remote::RemoteError) -> Self {
Self {
failure: Failure::Exec(error.error),
bootstrap: false,
upstream: false,
stop: false,
invalid: false,
kind: error.kind,
exhausted: None,
}
}
fn from_attempt(fault: Fault, stop: bool) -> Self {
Self {
upstream: classify::upstream_attempted(&fault.error),
invalid: classify::is_request_invalid(&fault.error),
failure: Failure::Exec(fault.error),
bootstrap: fault.bootstrap.is_some(),
stop,
kind: crate::remote::RemoteErrorKind::Plain,
exhausted: None,
}
}
fn local(failure: Failure) -> Self {
Self {
failure,
bootstrap: false,
upstream: false,
stop: false,
invalid: false,
kind: crate::remote::RemoteErrorKind::Plain,
exhausted: None,
}
}
fn safe_retry_after(&self) -> Option<u64> {
let seconds = |d: Duration| (!d.is_zero()).then(|| ceil_seconds(d).max(1));
match (&self.kind, self.exhausted) {
(crate::remote::RemoteErrorKind::Busy { header }, _) => *header,
(_, Some(exhausted)) => exhausted.retry_after.and_then(seconds),
(crate::remote::RemoteErrorKind::Cooldown { retry_after, .. }, None) => retry_after.and_then(seconds),
_ => None,
}
}
fn exhausted(mut self, retry_after: Option<Duration>, retry_now: bool) -> Self {
self.exhausted = Some(Exhausted { retry_after, retry_now });
self
}
fn retry_after(&self) -> Option<Duration> {
if let Some(exhausted) = self.exhausted {
return exhausted.retry_after;
}
match (&self.kind, &self.failure) {
(crate::remote::RemoteErrorKind::Cooldown { retry_after, .. }, _) => *retry_after,
(_, Failure::Exec(e)) => e.retry_after,
_ => None,
}
}
fn status(&self) -> u16 {
match &self.failure {
Failure::Exec(e) | Failure::Remote { error: e, .. } => classify::go_status(e),
other => other.status(),
}
}
fn retry_round(&self) -> bool {
match &self.failure {
Failure::Exec(e) | Failure::Remote { error: e, .. } => classify::is_retry_round(e),
other => matches!(other.status(), 403 | 408 | 429 | 500 | 502 | 503 | 504),
}
}
fn into_run_error(self) -> RunError {
let retry_after = self.safe_retry_after();
let failure = match (retry_after, self.failure) {
(Some(_), Failure::Exec(error)) => Failure::Remote { error, retry_after },
(_, failure) => failure,
};
RunError {
failure,
bootstrap: self.bootstrap,
}
}
}
fn preferred(fallback: RemoteFault, upstream: Option<&RemoteFault>) -> RemoteFault {
let Some(upstream) = upstream else {
return fallback;
};
let mut preferred = upstream.clone();
preferred.exhausted = fallback.exhausted;
preferred.upstream = true;
preferred
}
#[derive(Default)]
struct RoundTiming {
retry_after: Option<Duration>,
immediate: bool,
}
impl RoundTiming {
fn observe(&mut self, fault: &RemoteFault) {
if self.immediate {
return;
}
match fault.retry_after() {
Some(d) if d.is_zero() => {
self.retry_after = None;
self.immediate = true;
}
Some(d) if self.retry_after.is_none_or(|current| d < current) => self.retry_after = Some(d),
_ => {}
}
}
fn retry_after(&self) -> Option<Duration> {
if self.immediate { None } else { self.retry_after }
}
}
async fn run_remote(
cx: RemoteContext<'_>,
remote: Arc<dyn crate::remote::RemoteDispatch>,
base: Selection,
) -> Result<Done, RunError> {
let pinned = cx.call.pinned().is_some_and(|p| !p.trim().is_empty());
let max_wait = cx.policy.max_retry_interval;
let default_retry = cx.policy.request_retry as i64;
let allowed = |attempt: i64, limit: i64| home_retry_allowed(default_retry, attempt, limit);
let mut limit: i64 = -1;
let mut attempt: i64 = 0;
let (mut pending, mut waited) = (false, false);
let mut preferred_upstream: Option<RemoteFault> = None;
loop {
let error = match remote_round(&cx, remote.as_ref(), &base, attempt, &mut limit, pinned).await {
Ok(done) => return Ok(done),
Err(error) => *error,
};
if error.upstream {
preferred_upstream = Some(error.clone());
}
if pending {
if error.exhausted.is_none()
&& let crate::remote::RemoteErrorKind::Cooldown {
retry_after,
request_retry,
} = error.kind
{
if !pinned && let Some(remote_limit) = request_retry {
limit = remote_limit;
}
if let Some(wait) = retry_after.filter(|w| !w.is_zero() && !max_wait.is_zero() && *w <= max_wait)
&& allowed(attempt - 1, limit)
{
if waited {
return Err(error.into_run_error());
}
tokio::time::sleep(wait).await;
waited = true;
continue;
}
}
}
waited = false;
if error.stop {
return Err(error.into_run_error());
}
match remote_retry_wait(&error, attempt, max_wait, limit, pinned, &allowed) {
Some(wait) => {
if !wait.is_zero() {
tokio::time::sleep(wait).await;
}
attempt += 1;
pending = true;
}
None => {
let error = match (&preferred_upstream, error.exhausted) {
(Some(upstream), Some(_)) => preferred(error, Some(upstream)),
_ => error,
};
return Err(error.into_run_error());
}
}
}
}
fn home_retry_allowed(default_retry: i64, attempt: i64, limit: i64) -> bool {
let limit = if limit < 0 { default_retry.max(0) } else { limit };
attempt >= 0 && attempt < limit
}
fn remote_retry_wait(
error: &RemoteFault,
attempt: i64,
max_wait: Duration,
mut limit: i64,
pinned: bool,
allowed: &dyn Fn(i64, i64) -> bool,
) -> Option<Duration> {
if matches!(error.kind, crate::remote::RemoteErrorKind::Busy { .. })
|| error.status() == 200
|| error.invalid
|| error.stop
{
return None;
}
if let crate::remote::RemoteErrorKind::Cooldown {
request_retry: Some(remote_limit),
..
} = error.kind
&& !pinned
{
limit = remote_limit;
}
let beyond = |wait: Duration| !wait.is_zero() && (max_wait.is_zero() || wait > max_wait);
if let Some(exhausted) = error.exhausted {
if !error.retry_round() || !allowed(attempt, limit) {
return None;
}
if exhausted.retry_now {
return Some(Duration::ZERO);
}
return match exhausted.retry_after {
Some(wait) if beyond(wait) => None,
Some(wait) => Some(wait),
None => Some(Duration::ZERO),
};
}
if error.status() != 429 || !allowed(attempt, limit) {
return None;
}
match error.retry_after() {
Some(wait) if !wait.is_zero() && !beyond(wait) => Some(wait),
_ => None,
}
}
async fn remote_round(
cx: &RemoteContext<'_>,
remote: &dyn crate::remote::RemoteDispatch,
base: &Selection,
round: i64,
limit: &mut i64,
pinned: bool,
) -> Result<Done, Box<RemoteFault>> {
let round = async {
let releases = crate::remote::PendingReleases::default();
let settle = |releases: crate::remote::PendingReleases| async move {
releases.settle().await.map_err(|error| {
RemoteFault::local(Failure::Exec(ExecError::local(
503,
cpa_core::exec::FailureScope::Credential,
format!("home_unavailable: Home did not acknowledge credential release: {error}"),
)))
})
};
let max_credentials = cx.policy.max_retry_credentials;
let stream = cx.call.stream;
let websocket = cx.call.turn.is_some();
let mut tried: Vec<String> = Vec::new();
let mut excluded: Vec<String> = Vec::new();
let mut same_retries: std::collections::HashMap<String, u32> = Default::default();
let mut last_id = String::new();
let mut same_pending = false;
let mut last: Option<RemoteFault> = None;
let mut upstream: Option<RemoteFault> = None;
let mut timing = RoundTiming::default();
let mut count: i64 = 0;
loop {
if !stream || count == 0 {
count += 1;
}
let allow_same =
stream && same_pending && !last_id.is_empty() && same_retries.get(&last_id).copied().unwrap_or(0) == 0;
let capped = max_credentials > 0 && tried.len() >= max_credentials;
if capped && !allow_same {
return Err(match last {
Some(last) => preferred(last, upstream.as_ref()).exhausted(timing.retry_after(), true),
None => RemoteFault::local(no_auth(base)),
});
}
let mut selection = base.clone();
selection.retry_round = round.max(0) as usize;
selection.exclude = if stream { excluded.clone() } else { tried.clone() };
let request = remote_request(cx.call, &selection, count, cx.trace);
let kept = cx
.call
.turn
.as_ref()
.and_then(|turn| turn.home.as_ref())
.and_then(|home| {
home.reuse(
count <= 1 && round <= 0,
&selection.model,
&selection.exclude,
cx.call.pinned().as_deref(),
&releases,
)
});
let picked = match kept {
Some((lease, user_key, request_retry)) => Ok((lease, request_retry, user_key)),
None => {
let home = cx.call.turn.as_ref().and_then(|turn| turn.home.as_ref());
if home.is_some() {
settle(releases.clone()).await?;
}
let picked = cx
.rt
.acquire_remote(remote, selection.clone(), request, &releases)
.await;
if let (Some(home), Ok((_, request_retry, _))) = (home, &picked) {
home.granted(*request_retry);
}
picked
}
};
let (lease, request_retry, user_key) = match picked {
Ok(picked) => picked,
Err(error) => {
let code = error.code.clone();
let pick = RemoteFault::from_pick(error);
let Some(previous) = last else {
return Err(pick);
};
let fallback = preferred(previous, upstream.as_ref());
if let crate::remote::RemoteErrorKind::Cooldown {
retry_after,
request_retry,
} = pick.kind
{
if !pinned && let Some(remote_limit) = request_retry {
*limit = remote_limit;
}
return Err(fallback.exhausted(retry_after, false));
}
if matches!(
code.go_lower().as_str(),
"auth_not_found" | "auth_unavailable" | "request_retry_exceeded"
) {
return Err(
fallback.exhausted(timing.retry_after(), code.eq_ignore_ascii_case("auth_unavailable"))
);
}
return Err(pick);
}
};
match request_retry.filter(|_| !pinned) {
Some(home_limit) => *limit = home_limit,
None => {
let local = cx.policy.retry_limit(&lease.credential) as i64;
if *limit < 0 || local > *limit {
*limit = local;
}
}
}
let id = lease.credential.id.clone();
if capped && id != last_id {
drop(lease);
settle(releases.clone()).await?;
return Err(match last {
Some(last) => preferred(last, upstream.as_ref()).exhausted(timing.retry_after(), true),
None => RemoteFault::local(no_auth(base)),
});
}
if stream && !last_id.is_empty() && id != last_id {
same_pending = false;
}
if tried.contains(&id) {
if !stream || excluded.contains(&id) {
drop(lease);
settle(releases.clone()).await?;
return Err(match last {
Some(last) => preferred(last, upstream.as_ref()).exhausted(timing.retry_after(), false),
None => RemoteFault::local(Failure::Exec(ExecError::local(
503,
cpa_core::exec::FailureScope::Credential,
"request_retry_exceeded: home returned a previously tried auth",
))),
});
}
let retries = same_retries.entry(id.clone()).or_default();
*retries += 1;
if *retries > 1 {
excluded.push(id);
drop(lease);
settle(releases.clone()).await?;
continue;
}
} else {
tried.push(id.clone());
}
cx.trace.selected(&lease.credential);
if !user_key.is_empty() {
*cx.user_key.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = user_key;
}
let user_key = cx
.user_key
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
let home_caller = (!user_key.is_empty()).then(|| {
let mut request = cx.request.clone();
request.caller.principal = user_key.clone();
let usage = cx.usage.map(|facts| {
let mut facts = (**facts).clone();
facts.client.api_key = user_key.clone();
Arc::new(facts)
});
let unauthorized = cx.unauthorized.map(|facts| {
let mut facts = (**facts).clone();
facts.client.api_key = user_key.clone();
Arc::new(facts)
});
(request, usage, unauthorized)
});
let (request, usage, unauthorized) = match &home_caller {
Some((request, usage, unauthorized)) => (request, usage.as_ref(), unauthorized.as_ref()),
None => (cx.request, cx.usage, cx.unauthorized),
};
let (mut models, mut alias) = registry::execution_models(cx.aliases, &lease.credential, &selection.model);
if let Some(model) = lease
.credential
.attributes
.get(crate::remote::UPSTREAM_MODEL)
.map(|m| m.trim())
.filter(|m| !m.is_empty())
{
models = vec![model.to_owned()];
}
models.truncate(1);
if lease
.credential
.attributes
.get(crate::remote::FORCE_MAPPING)
.is_some_and(|v| v.trim().eq_ignore_ascii_case("true"))
{
alias.force_mapping = true;
alias.original_alias = selection.model.clone();
}
if models.is_empty() {
drop(lease);
settle(releases.clone()).await?;
let fault = RemoteFault::local(Failure::Exec(ExecError::local(
503,
cpa_core::exec::FailureScope::Credential,
"auth_not_found: no execution models available",
)));
timing.observe(&fault);
last = Some(fault);
continue;
}
let selection_model = registry::selection_model(cx.aliases, &lease.credential, &selection.model);
let target = Target {
models: &models,
selection_model: &selection_model,
pooled: false,
alias: &alias,
compact: cx.compact,
keep_model: cx.call.selection_model.is_some(),
unauthorized,
};
let cancelled = lease
.remote_cancelled()
.unwrap_or_else(|| Box::pin(std::future::pending()));
let run = async {
let mut lease = lease;
match cx.rt.prepare_remote(&mut lease, cx.cfg).await {
Ok(()) => (
attempt(cx.rt, cx.cfg, cx.policy, cx.call, request, usage, lease, target, None).await,
false,
),
Err(error) => {
lease.complete(Outcome::Failure(error.clone()));
(Attempt::Next(Fault { error, bootstrap: None }), true)
}
}
};
let (outcome, prepare_failed) = tokio::select! {
biased;
outcome = run => outcome,
_ = cancelled => (
Attempt::Next(Fault {
error: crate::remote::cancelled_error(),
bootstrap: None,
}),
false,
),
};
match outcome {
Attempt::Done(done) => return Ok(done),
Attempt::Stop(fault) => {
if stream {
settle(releases.clone()).await?;
}
return Err(RemoteFault::from_attempt(fault, true));
}
Attempt::Next(fault) => {
if stream {
let error = &fault.error;
let exclude = !(lifecycle(error) || (websocket && error.status == 426))
|| same_retries.get(&id).copied().unwrap_or(0) > 0;
if exclude && !excluded.contains(&id) {
excluded.push(id.clone());
}
last_id = id.clone();
same_pending = !exclude;
if !prepare_failed {
count += 1;
}
}
let mut fault = RemoteFault::from_attempt(fault, false);
fault.upstream &= !prepare_failed;
if fault.upstream {
upstream = Some(fault.clone());
}
timing.observe(&fault);
last = Some(fault);
settle(releases.clone()).await?;
}
}
}
};
round.await.map_err(Box::new)
}
fn lifecycle(e: &ExecError) -> bool {
if classify::go_status(e) != 0 {
return false;
}
let lower = classify::error_text(e).trim().go_lower();
matches!(
lower.as_str(),
"context canceled" | "context deadline exceeded" | "eof" | "unexpected eof"
) || [
"websocket: close 1000",
"websocket: close 1001",
"websocket: close 1006",
"unexpected eof",
]
.iter()
.any(|needle| lower.contains(needle))
}
pub(crate) fn scheduler_rejection(code: String, message: String, providers: &[String], model: &str) -> Failure {
match code.as_str() {
"auth_not_found" | "auth_unavailable" => Failure::Exec(ExecError::local(
503,
cpa_core::exec::FailureScope::Request,
unavailable_text(&code, &message, providers, model, None),
)),
_ => Failure::Exec(ExecError::local(
0,
cpa_core::exec::FailureScope::Model,
format!("{code}: {message}"),
)),
}
}
fn no_auth(base: &Selection) -> Failure {
Failure::Unavailable {
code: "auth_not_found",
providers: base.providers.clone(),
model: base.model.clone(),
cause: None,
retry_after: None,
}
}
fn remote_request(call: &Call, selection: &Selection, count: i64, trace: &Trace) -> crate::remote::RemoteRequest {
let headers = crate::remote::home_headers(&call.headers, Some(&call.caller));
crate::remote::RemoteRequest {
model: selection.model.clone(),
session_id: selection.session.clone().unwrap_or_default(),
parent_session_id: selection.session_parent.clone().unwrap_or_default(),
headers,
count,
retry_round: selection.retry_round as i64,
excluded: selection.exclude.clone(),
pinned: call.pinned().unwrap_or_default().to_owned(),
request_id: trace.request_id(),
kind: if call.turn.is_some() {
"websocket"
} else if call.stream {
"stream"
} else {
"http"
},
credential_policy: String::new(),
}
}
pub fn jitter(wait: Duration, max: Duration) -> Duration {
let mut range = (wait / 4).min(Duration::from_secs(2));
if !max.is_zero() {
range = range.min(max.saturating_sub(wait));
}
if range.is_zero() {
return wait;
}
use std::hash::{BuildHasher, Hasher};
let mut hasher = std::collections::hash_map::RandomState::new().build_hasher();
hasher.write_u128(
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_nanos(),
);
wait + Duration::from_nanos(hasher.finish() % range.as_nanos().max(1) as u64)
}
struct Credits<'a> {
providers: &'a [String],
model: &'a str,
aliases: &'a std::collections::HashMap<String, Vec<cpa_core::registry::dynamic::OAuthAlias>>,
pinned: Option<&'a str>,
compact: bool,
}
impl Credits<'_> {
fn wanted(&self, cfg: &Config, error: &RunError) -> bool {
self.providers
.iter()
.any(|p| p.trim().eq_ignore_ascii_case("antigravity"))
&& cpa_exec::antigravity_credits::enabled(cfg)
&& match &error.failure {
Failure::Exec(e) => matches!(classify::go_status(e), 429 | 503),
Failure::Cooldown { .. } => true,
Failure::Unavailable { code, .. } => matches!(*code, "auth_not_found" | "auth_unavailable"),
_ => false,
}
}
#[allow(clippy::too_many_arguments)]
async fn fallback(
&self,
rt: &Arc<Runtime>,
cfg: &Config,
policy: &Policy,
call: &Call,
request: &ExecRequest,
usage: Option<&Arc<crate::usage_record::Facts>>,
hooks: Option<&crate::plugins::interceptors::Hooks>,
error: RunError,
) -> Result<Done, RunError> {
if request.operation == Operation::CountTokens
|| !self.wanted(cfg, &error)
|| !self.model.go_lower().contains("claude")
{
return Err(error);
}
let snapshot = rt.store().snapshot();
let ids = snapshot
.iter()
.filter(|c| !c.disabled && c.provider.trim().eq_ignore_ascii_case("antigravity"))
.filter(|c| self.pinned.is_none_or(|p| p.is_empty() || c.id == p))
.map(|c| c.id.as_str());
for id in cpa_exec::antigravity_credits::order_candidates(ids) {
let selection = Selection::new("antigravity", self.model);
let Some(lease) = rt
.lease_credential(&id, selection, cfg, cpa_exec::antigravity_credits::request_on)
.await
else {
continue;
};
let (models, alias) = registry::execution_models(self.aliases, &lease.credential, self.model);
if models.is_empty() {
continue;
}
let selection_model = registry::selection_model(self.aliases, &lease.credential, self.model);
let target = Target {
models: &models,
selection_model: &selection_model,
pooled: models.len() > 1,
alias: &alias,
compact: self.compact,
keep_model: call.selection_model.is_some(),
unauthorized: None,
};
let hooks = hooks.filter(|_| request.stream);
if let Attempt::Done(done) = attempt(rt, cfg, policy, call, request, usage, lease, target, hooks).await {
return Ok(done);
}
}
Err(error)
}
}
enum Attempt {
Done(Done),
Stop(Fault),
Next(Fault),
}
struct Target<'a> {
models: &'a [String],
selection_model: &'a str,
pooled: bool,
alias: &'a AliasResult,
compact: bool,
keep_model: bool,
unauthorized: Option<&'a Arc<crate::usage_record::Facts>>,
}
#[allow(clippy::too_many_arguments)]
async fn attempt(
rt: &Arc<Runtime>,
cfg: &Config,
policy: &Policy,
call: &Call,
request: &ExecRequest,
usage: Option<&Arc<crate::usage_record::Facts>>,
mut lease: Lease,
target: Target<'_>,
hooks: Option<&crate::plugins::interceptors::Hooks>,
) -> Attempt {
let route_model = lease.selection.model.clone();
let mut last = None;
let mut refreshed = false;
let home = call
.turn
.as_ref()
.and_then(|turn| turn.home.as_ref())
.filter(|_| lease.is_remote());
let pick = home.map(|home| home.pick(&lease.credential.id));
let mut guard = pick.clone().map(crate::home_session::PickGuard::new);
let session = call.turn.as_ref().map(|turn| cpa_core::exec::ExecSession {
lease: pick.as_ref().map(crate::home_session::Pick::lease),
..turn.session.clone()
});
for (i, upstream) in target.models.iter().enumerate() {
let home_model = lease
.credential
.attributes
.get(crate::remote::UPSTREAM_MODEL)
.map(|m| m.trim())
.filter(|m| !m.is_empty());
let state = match home_model {
Some(home) if upstream.trim().is_empty() => home.to_owned(),
Some(_) => upstream.trim().to_owned(),
None => registry::state_model(target.selection_model, &route_model, upstream, target.pooled),
};
lease.execution_model = state.clone();
let mut req = attempt_request(
request,
cfg,
&lease.credential,
&route_model,
upstream,
target.keep_model,
call.stream,
);
let lcp = lease.lcp.clone();
if let Some(m) = &lcp {
req.session = Some(cpa_common::session::bound_session_identity(&m.session));
}
if let Some(on_selected) = call.on_selected() {
on_selected(&lease.credential);
}
if let Some(hooks) = hooks
&& let Err(error) = hooks
.after_auth(
&mut req,
&lease.credential,
&route_model,
call.media.as_ref().map(|m| m.kind),
)
.await
{
lease.complete(Outcome::Neutral(error.clone()));
return Attempt::Stop(Fault { error, bootstrap: None });
}
let start = |credential: &cpa_core::credential::Credential, req: &mut ExecRequest| {
usage.map(|facts| {
let mut tracker = crate::usage_record::Tracker::start(rt, facts, credential, upstream);
if let Some(m) = &lcp {
tracker.lcp_session(m);
}
req.usage = tracker.sink().with_capture(req.capture().clone());
tracker
})
};
let observe = |credential: &cpa_core::credential::Credential, req: &mut ExecRequest| {
if let Some(hooks) = hooks {
let observer = hooks.websocket_observer(req, credential, &route_model);
req.usage = std::mem::take(&mut req.usage).with_websocket(observer);
}
};
let mut tracker = start(&lease.credential, &mut req);
observe(&lease.credential, &mut req);
let session = session.as_ref();
let private = lease.home_private().map(|p| p.attributes.clone()).unwrap_or_default();
let private = &private;
let execute = |credential: Arc<cpa_core::credential::Credential>, req: ExecRequest| async move {
let credential = match crate::plugins::model_callbacks::context().filter(|c| !c.proxy_url.is_empty()) {
Some(context) => {
let mut credential = (*credential).clone();
credential.attributes.insert("proxy_url".into(), context.proxy_url);
Arc::new(credential)
}
None => credential,
};
let credential = rt.for_executor(&credential);
if let Some(adapter) = rt.plugin_runtime().executor(&credential, cfg) {
return crate::plugins::execution::execute_credential(rt, call, &adapter, &credential, private, req)
.await;
}
match (session, call.media.as_ref().map(|m| m.kind)) {
(Some(session), _) => rt.executors.execute_in_session(&credential, req, cfg, session).await,
(None, Some(MediaKind::Images)) => {
let path = req.request_path.clone();
rt.executors.images(&credential, req, &path, cfg).await
}
(None, Some(MediaKind::Videos)) => {
let path = req.request_path.clone();
rt.executors.videos(&credential, req, &path, cfg).await
}
(None, Some(MediaKind::Speech)) => rt.executors.speech(&credential, req, cfg).await,
(None, None) => rt.executors.execute(&credential, req, cfg).await,
}
};
tracing::trace!(target: "cpa_latency", stage = "selected");
lease.redispatch();
let mut executed = execute(lease.credential.clone(), req.clone()).await;
if let Err(error) = &executed
&& !refreshed
&& !lease.is_remote()
&& classify::is_unauthorized(error)
&& let Some(current) = rt.refresh_after_unauthorized(&lease.credential, cfg).await
{
refreshed = true;
if let Some(t) = tracker.take() {
t.fail(error);
}
lease.credential = current;
lease.redispatch();
tracker = start(&lease.credential, &mut req);
observe(&lease.credential, &mut req);
executed = execute(lease.credential.clone(), req).await;
}
if let (Ok(response), Some(t)) = (&executed, tracker.as_mut()) {
t.arrived(&response.headers);
}
let fault = match executed {
Ok(response) => match finish(call, response).await {
Ok(done) => {
let done = track_usage(done, tracker.take());
let done = if target.alias.force_mapping && !target.alias.original_alias.is_empty() {
rewrite_model(done, &target.alias.original_alias)
} else {
done
};
let user_key = &request.caller.principal;
let Some(lease) = keep_home(home, guard.take(), lease, user_key) else {
return Attempt::Done(done);
};
return Attempt::Done(match done {
Done::Stream { headers, first, rest } => {
lease.accepted();
Done::Stream {
headers,
first,
rest: Completing::new(rest, lease).boxed(),
}
}
buffered => {
lease.complete(Outcome::Success);
buffered
}
});
}
Err(fault) => {
if let Some(t) = tracker.take() {
t.fail(&fault.error);
}
fault
}
},
Err(error) => {
if let Some(t) = tracker.take() {
t.fail(&error);
}
Fault { error, bootstrap: None }
}
};
let error = &fault.error;
if let Some(facts) = target.unauthorized
&& classify::is_unauthorized(error)
{
crate::usage_record::publish_home_unauthorized(
rt,
facts,
&lease.credential,
&crate::remote::selection_provider(&lease.credential),
&state,
&classify::error_text(error),
);
}
let action = policy.error_action(&lease.credential, error);
let neutral = (target.compact && classify::is_compact_neutral(error) && !action.force_cooldown)
|| (call.operation == Operation::CountTokens
&& classify::is_count_endpoint_missing(error, upstream)
&& !action.force_cooldown);
let outcome = if neutral {
Outcome::Neutral(error.clone())
} else {
Outcome::Failure(error.clone())
};
let stop = if action.matched {
action.stop
} else {
(target.compact && classify::is_compact_fault(error)) || classify::is_request_invalid(error)
};
let last_here = stop || i + 1 == target.models.len() || classify::credential_scoped(error);
let mut fault = fault;
if stop {
fault.bootstrap = None;
} else if let Some(headers) = &fault.bootstrap {
fault.error.addon = Some(headers.clone());
}
if last_here {
drop(guard.take());
lease.complete(outcome);
return if stop {
Attempt::Stop(fault)
} else {
Attempt::Next(fault)
};
}
lease.note(&state, &outcome);
last = Some(fault);
}
Attempt::Next(last.expect("at least one model was attempted"))
}
fn keep_home(
home: Option<&Arc<crate::home_session::SessionHome>>,
guard: Option<crate::home_session::PickGuard>,
lease: Lease,
user_key: &str,
) -> Option<Lease> {
let (Some(home), Some(mut guard)) = (home, guard) else {
return Some(lease);
};
let pick = guard.defuse()?;
let (credential, route) = (lease.credential.clone(), lease.selection.model.clone());
let availability = lease.home_private().map(|p| p.availability.clone()).unwrap_or_default();
match pick.keep(lease) {
None => {
home.keep(pick, &credential, availability, &route, user_key.to_owned());
None
}
Some(lease) => {
pick.close();
Some(lease)
}
}
}
fn usage_client(
cfg: &Config,
call: &Call,
trace: &Trace,
session: &crate::session::Session,
) -> crate::usage_record::Client {
let nested = (call.caller.principal.is_empty() && call.caller.provider.is_empty())
.then(crate::plugins::model_callbacks::nested_usage_identity)
.flatten();
let (api_key, access_provider) =
nested.unwrap_or_else(|| (call.caller.principal.clone(), call.caller.provider.clone()));
crate::usage_record::Client {
session_id: session.id.clone().unwrap_or_default(),
parent_session_id: session.parent.clone().unwrap_or_default(),
is_fork: session.fork,
node_kind: String::new(),
is_compaction: false,
session_source: session.source,
request_id: trace.request_id(),
endpoint: format!(
"{} {}",
if call.turn.is_some() { "GET" } else { "POST" },
call.request_path
),
api_key,
access_provider,
..request_client(cfg, &call.headers, call.peer)
}
}
pub(crate) fn request_client(
cfg: &Config,
headers: &axum::http::HeaderMap,
peer: Option<std::net::SocketAddr>,
) -> crate::usage_record::Client {
let trusted = cfg.derived(|c| cpa_core::config::TrustedProxies::new(&c.trusted_proxies));
let text = |v: &axum::http::HeaderValue| String::from_utf8_lossy(v.as_bytes()).into_owned();
let forwarded: Vec<String> = headers.get_all("x-forwarded-for").iter().map(text).collect();
crate::usage_record::Client {
client_ip: peer.map(|p| p.ip().to_string()).unwrap_or_default(),
resolved_client_ip: trusted
.client_ip(peer, |name| headers.get(name).map(|v| v.as_bytes()))
.trim()
.to_owned(),
x_forwarded_for: forwarded.join(", ").trim().to_owned(),
user_agent: headers
.get("user-agent")
.map(text)
.unwrap_or_default()
.trim()
.to_owned(),
..Default::default()
}
}
fn track_usage(done: Done, tracker: Option<crate::usage_record::Tracker>) -> Done {
let Some(mut tracker) = tracker else { return done };
match done {
Done::Buffered { headers, body } => {
tracker.body(&body);
tracker.succeed();
Done::Buffered { headers, body }
}
Done::Stream { headers, first, rest } => {
if let Some(first) = &first {
tracker.event(first);
}
Done::Stream {
headers,
first,
rest: Tracked {
inner: rest,
tracker: Some(tracker),
}
.boxed(),
}
}
}
}
struct Tracked {
inner: ExecStream,
tracker: Option<crate::usage_record::Tracker>,
}
impl futures_util::Stream for Tracked {
type Item = Result<Bytes, ExecError>;
fn poll_next(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Self::Item>> {
let item = std::task::ready!(self.inner.poll_next_unpin(cx));
match &item {
Some(Ok(event)) => {
if let Some(t) = self.tracker.as_mut() {
t.event(event);
}
}
Some(Err(error)) => {
self.inner = futures_util::stream::empty().boxed();
if let Some(t) = self.tracker.take() {
t.fail(error);
}
}
None => {
self.inner = futures_util::stream::empty().boxed();
if let Some(t) = self.tracker.take() {
t.succeed();
}
}
}
std::task::Poll::Ready(item)
}
}
fn attempt_request(
request: &ExecRequest,
cfg: &Config,
credential: &cpa_core::credential::Credential,
route_model: &str,
upstream: &str,
keep_model: bool,
stream: bool,
) -> ExecRequest {
let mut req = request.clone();
if !keep_model {
req.model = upstream.to_owned();
}
req.resolved_model = crate::capabilities::resolve_attempt(
cfg,
credential,
route_model,
upstream,
keep_model.then_some(request.model.as_str()),
);
let local = req.resolved_model.as_ref().filter(|_| !stream);
if !keep_model && let Some(home) = crate::capabilities::bind_home(credential, local) {
req.resolved_model = Some(home);
}
req
}
async fn finish(call: &Call, response: cpa_core::exec::ExecResponse) -> Result<Done, Fault> {
match response.body {
ResponseBody::Buffered(body) => Ok(Done::Buffered {
headers: response.headers,
body,
}),
ResponseBody::Stream(mut stream) if call.stream => {
let bootstrap = |error| Fault {
error,
bootstrap: Some(Box::new(response.headers.clone())),
};
let first = loop {
match stream.next().await {
Some(Ok(bytes)) if bytes.is_empty() => continue,
Some(Ok(bytes)) => break bytes,
Some(Err(error)) => return Err(bootstrap(error)),
None => return Err(bootstrap(classify::empty_stream())),
}
};
Ok(Done::Stream {
headers: response.headers,
first: Some(first),
rest: stream,
})
}
ResponseBody::Stream(mut stream) => {
let mut body = BytesMut::new();
while let Some(event) = stream.next().await {
body.extend_from_slice(&event.map_err(|error| Fault { error, bootstrap: None })?);
}
Ok(Done::Buffered {
headers: response.headers,
body: body.freeze(),
})
}
}
}
fn rewrite_model(done: Done, target: &str) -> Done {
match done {
Done::Buffered { headers, body } => Done::Buffered {
headers,
body: rewrite_body(&body, target),
},
Done::Stream { headers, first, rest } => {
let target_owned = target.to_owned();
let rest = rest
.map(move |item| item.map(|event| rewrite_event(&event, &target_owned)))
.boxed();
Done::Stream {
headers,
first: first.map(|event| rewrite_event(&event, target)),
rest,
}
}
}
}
const MODEL_PATHS: [&str; 5] = [
"model",
"modelVersion",
"response.model",
"response.modelVersion",
"message.model",
];
fn rewrite_json(data: &[u8], target: &str) -> Vec<u8> {
let mut out = data.to_vec();
for path in MODEL_PATHS {
if cpa_common::json::get(&out, path).exists() {
cpa_common::json::set_str(&mut out, path, target);
}
}
out
}
fn rewrite_body(body: &Bytes, target: &str) -> Bytes {
if body.trim_ascii_start().starts_with(b"{") {
Bytes::from(rewrite_json(body, target))
} else {
rewrite_lines(body, target)
}
}
fn rewrite_event(event: &Bytes, target: &str) -> Bytes {
let trimmed = event.trim_ascii();
if trimmed.starts_with(b"{") && cpa_common::json::valid(trimmed) {
return Bytes::from(rewrite_json(trimmed, target));
}
rewrite_lines(event, target)
}
fn rewrite_lines(payload: &Bytes, target: &str) -> Bytes {
let mut changed = false;
let mut out = Vec::with_capacity(payload.len());
for (i, line) in payload.split(|&b| b == b'\n').enumerate() {
if i > 0 {
out.push(b'\n');
}
let data = line.strip_prefix(b"data: ").or_else(|| line.strip_prefix(b"data:"));
match data {
Some(json) if json.starts_with(b"{") && cpa_common::json::valid(json) => {
out.extend_from_slice(&line[..line.len() - json.len()]);
let rewritten = rewrite_json(json, target);
changed |= rewritten != json;
out.extend_from_slice(&rewritten);
}
_ => out.extend_from_slice(line),
}
}
if changed { Bytes::from(out) } else { payload.clone() }
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn direct_errors_stay_direct_under_a_home_round_marker() {
let mut raw = ExecError::local(429, cpa_core::exec::FailureScope::Credential, "raw fast-mode body");
raw.direct = true;
raw.headers
.insert(axum::http::header::CONTENT_TYPE, "text/plain".parse().unwrap());
let failure = Failure::Remote {
error: raw,
retry_after: Some(2),
};
assert!(failure.direct().is_some());
let response = crate::errors::write(&failure, |_, _| unreachable!());
assert_eq!(response.status().as_u16(), 429);
assert_eq!(response.headers()["content-type"], "text/plain");
assert!(response.headers().get("retry-after").is_none());
}
#[tokio::test]
async fn stream_records_publish_after_the_upstream_stream_is_dropped() {
use cpa_core::exec::{UpstreamStage, UsageSink};
use std::task::Poll;
struct UpstreamBody(UsageSink);
impl Drop for UpstreamBody {
fn drop(&mut self) {
self.0.upstream_stage(UpstreamStage::BodyRead { bytes: 42 });
}
}
let cfg = Config::parse("observability:\n usage:\n usage-statistics-enabled: true\n").unwrap();
let executors = cpa_exec::Executors {
claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
codex: Default::default(),
devices: Default::default(),
openai: Default::default(),
google: Default::default(),
};
let rt = Arc::new(crate::testing::runtime(cfg.clone(), Vec::new(), executors));
rt.usage_queue().configure(true, &cfg);
let mut metadata = serde_json::Map::new();
metadata.insert("type".into(), "claude".into());
let credential = cpa_core::credential::Credential::from_file(
std::path::Path::new("/auth"),
std::path::Path::new("/auth/a.json"),
metadata,
)
.unwrap();
let facts = Arc::new(crate::usage_record::Facts::new(
Default::default(),
Format::Claude,
Format::Claude,
"m",
b"{}",
true,
&Trace::default(),
));
for ending in ["error", "end", "client gone"] {
let tracker = crate::usage_record::Tracker::start(&rt, &facts, &credential, "m");
let body = UpstreamBody(tracker.sink());
let rest = futures_util::stream::poll_fn(move |_| {
let _held = &body;
match ending {
"error" => Poll::Ready(Some(Err(ExecError::local(
502,
cpa_core::exec::FailureScope::Request,
"restore failed",
)))),
"end" => Poll::Ready(None),
_ => Poll::Pending,
}
})
.boxed();
let done = track_usage(
Done::Stream {
headers: HeaderMap::new(),
first: None,
rest,
},
Some(tracker),
);
let Done::Stream { mut rest, .. } = done else {
unreachable!()
};
if ending == "client gone" {
assert!(futures_util::poll!(rest.next()).is_pending());
drop(rest);
} else {
assert!(!matches!(rest.next().await, Some(Ok(_))));
}
let records = rt.usage_queue().pop_oldest(10);
assert_eq!(records.len(), 1, "{ending}");
let record: serde_json::Value = serde_json::from_slice(&records[0]).unwrap();
assert_eq!(record["upstream_response_bytes"], 42, "{ending}: {record}");
}
}
#[test]
fn lifecycle_errors_follow_go() {
let transport = |text: &str| ExecError::local(502, cpa_core::exec::FailureScope::Transport, text);
assert!(lifecycle(&transport("websocket: close 1000 (normal)")));
assert!(lifecycle(&transport(
"websocket: close 1006 (abnormal closure): unexpected EOF"
)));
assert!(lifecycle(&transport("EOF")));
assert!(!lifecycle(&transport("websocket: close 1011 (internal server error)")));
assert!(!lifecycle(&transport("read failed")));
let statused = ExecError::local(500, cpa_core::exec::FailureScope::Credential, "unexpected EOF");
assert!(!lifecycle(&statused));
}
#[test]
fn nonstream_keepalive_only_for_nonstream_generate() {
let call = |operation, stream| Call {
entry: Format::Claude,
response: Format::Claude,
operation,
model: "m".into(),
body: Bytes::new(),
stream,
alt: None,
headers: HeaderMap::new(),
caller: Caller {
principal: String::new(),
source: "",
provider: String::new(),
},
forced_provider: None,
selection_model: None,
execution_session: None,
request_path: String::new(),
peer: None,
turn: None,
media: None,
};
let cfg = |yaml: &str| Config::parse(yaml).unwrap();
let on = cfg("requests:\n nonstream-keepalive-interval: 3\n");
assert_eq!(
nonstream_keepalive(&call(Operation::Generate, false), &on),
Some(Duration::from_secs(3))
);
assert_eq!(nonstream_keepalive(&call(Operation::Generate, true), &on), None);
assert_eq!(nonstream_keepalive(&call(Operation::CountTokens, false), &on), None);
for off in [
"{}\n",
"requests:\n nonstream-keepalive-interval: 0\n",
"requests:\n nonstream-keepalive-interval: -5\n",
] {
assert_eq!(
nonstream_keepalive(&call(Operation::Generate, false), &cfg(off)),
None,
"{off}"
);
}
let legacy = cfg("nonstream-keepalive-interval: 2\n");
assert_eq!(
nonstream_keepalive(&call(Operation::Generate, false), &legacy),
Some(Duration::from_secs(2))
);
}
#[test]
fn attempts_bind_resolved_model_like_go() {
let cfg = Config::parse(
"config-version: 8\napi-keys:\n codex:\n - base-url: https://codex.example\n models:\n - name: gpt-6-sol\n alias: sol\n is-compat: true\n keys:\n - api-key: xk1\n",
)
.unwrap();
let credential = cpa_core::config::credentials::from_config(&cfg).remove(0);
let request = ExecRequest {
operation: Operation::Generate,
source_format: Format::OpenAI,
response_format: Format::OpenAI,
stream_payloads: false,
requested_model: "sol".into(),
model: "sol".into(),
original_body: Bytes::new(),
body: Bytes::new(),
stream: false,
alt: None,
session: None,
execution_session: None,
derived_session: None,
request_path: String::new(),
headers: HeaderMap::new(),
caller: Caller {
principal: String::new(),
source: "",
provider: String::new(),
},
resolved_model: None,
usage: Default::default(),
};
let req = attempt_request(&request, &cfg, &credential, "sol", "gpt-6-sol", false, false);
assert_eq!(req.model, "gpt-6-sol");
let bound = req.resolved_model.unwrap();
assert_eq!(
(bound.info.id.as_str(), bound.source, bound.is_compat()),
("gpt-6-sol", cpa_core::exec::ResolvedSource::ApiKey, true)
);
let kept = attempt_request(&request, &cfg, &credential, "selection", "gpt-6-sol", true, false);
assert_eq!(kept.model, "sol");
let bound = kept.resolved_model.unwrap();
assert_eq!((bound.info.id.as_str(), bound.is_compat()), ("sol", false));
let mut home = credential.clone();
home.attributes
.insert(crate::remote::MODEL_INFO.into(), r#"{"id":"gpt-6-sol"}"#.into());
let caps = |req: &ExecRequest| {
let bound = req.resolved_model.as_ref().unwrap();
let caps = cpa_common::thinking::ModelCaps::from(&bound.info);
(bound.source, caps.support_configuration_update, bound.is_compat())
};
let cfg = Config::parse(
"config-version: 8\napi-keys:\n codex:\n - base-url: https://codex.example\n models:\n - name: gpt-6-sol\n alias: sol\n is-compat: true\n support-configuration-update: true\n keys:\n - api-key: xk1\n",
)
.unwrap();
let home_source = cpa_core::exec::ResolvedSource::Home;
let buffered = attempt_request(&request, &cfg, &home, "sol", "gpt-6-sol", false, false);
assert_eq!(
caps(&buffered),
(home_source, true, false),
"local is-compat never carries"
);
let streamed = attempt_request(&request, &cfg, &home, "sol", "gpt-6-sol", false, true);
assert_eq!(caps(&streamed), (home_source, false, false));
let kept = attempt_request(&request, &cfg, &home, "selection", "gpt-6-sol", true, false);
assert_eq!(
kept.resolved_model.unwrap().source,
cpa_core::exec::ResolvedSource::ApiKey
);
}
#[test]
fn failure_texts_match_go_shapes() {
assert_eq!(
Failure::UnknownModel("claude-sonnet-4-6".into()).text(),
r#"{"error":{"message":"unknown provider for model claude-sonnet-4-6","type":"invalid_request_error","code":"model_not_found","param":"model"}}"#
);
let cooldown = Failure::Cooldown {
model: "m".into(),
provider: "claude".into(),
wait: Duration::from_millis(59_400),
cause: Some(r#"{"error":{"type":"rate_limit_error","message":"slow down"}}"#.into()),
};
assert_eq!(
cooldown.text(),
r#"{"error":{"code":"model_cooldown","last_upstream_error":"rate_limit_error: slow down","message":"All credentials for model m are cooling down via provider claude (last error: rate_limit_error: slow down)","model":"m","provider":"claude","reset_seconds":60,"reset_time":"59s"}}"#
);
assert_eq!(cooldown.retry_after(), Some(60));
assert_eq!(
(cooldown.status(), Failure::UnknownModel(String::new()).status()),
(429, 400)
);
let none = Failure::Unavailable {
code: "auth_not_found",
providers: vec!["claude".into()],
model: " m ".into(),
cause: None,
retry_after: None,
};
assert_eq!(
none.text(),
"auth_not_found: no auth available (providers=claude, model=m); check Claude auth/key session and cooldown state via /v0/management/auth-files"
);
assert_eq!((none.status(), none.retry_after()), (503, None));
}
#[test]
fn force_mapping_rewrites_json_and_sse_model_fields() {
let body = Bytes::from_static(br#"{"id":"x","model":"claude-opus-5","message":{"model":"claude-opus-5"}}"#);
assert_eq!(
rewrite_body(&body, "opus"),
r#"{"id":"x","model":"opus","message":{"model":"opus"}}"#
);
let event = Bytes::from_static(b"event: message_start\ndata: {\"message\":{\"model\":\"up\"}}\n\n");
assert_eq!(
rewrite_event(&event, "alias"),
"event: message_start\ndata: {\"message\":{\"model\":\"alias\"}}\n\n"
);
let untouched = Bytes::from_static(b"data: [DONE]\n\n");
assert_eq!(rewrite_event(&untouched, "alias"), untouched);
let crlf = Bytes::from_static(b"data: {\"model\" : \"up\", \"x\":\"\\u00e9\", \"n\":1.50}\r\n\r\n");
assert_eq!(
rewrite_event(&crlf, "alias"),
"data: {\"model\" : \"alias\", \"x\":\"\\u00e9\", \"n\":1.50}\r\n\r\n"
);
let spaced = Bytes::from_static(b"data: {\"model\":\"up\"}\n\n");
assert_eq!(rewrite_event(&spaced, "alias"), spaced);
let bare = Bytes::from_static(b" {\"modelVersion\":\"up\"}\n");
assert_eq!(rewrite_event(&bare, "alias"), r#"{"modelVersion":"alias"}"#);
}
fn home_cooldown(retry_after: Duration, request_retry: Option<i64>) -> RemoteFault {
RemoteFault::from_pick(crate::remote::RemoteError {
error: ExecError::local(
429,
cpa_core::exec::FailureScope::Credential,
"model_cooldown: all Home credentials are cooling down",
),
code: "model_cooldown".into(),
kind: crate::remote::RemoteErrorKind::Cooldown {
retry_after: Some(retry_after),
request_retry,
},
})
}
fn retry_wait(
error: &RemoteFault,
request_retry: i64,
attempt: i64,
max_wait: Duration,
limit: i64,
pinned: bool,
) -> Option<Duration> {
let allowed = |attempt: i64, limit: i64| home_retry_allowed(request_retry, attempt, limit);
remote_retry_wait(error, attempt, max_wait, limit, pinned, &allowed)
}
#[test]
fn home_cooldowns_retry_within_request_retry_and_the_maximum_wait() {
let ms = Duration::from_millis;
let cooldown = home_cooldown(ms(10), None);
assert_eq!(
retry_wait(&cooldown, 1, 0, Duration::from_secs(1), -1, false),
Some(ms(10))
);
assert_eq!(
retry_wait(&cooldown, 1, 1, Duration::from_secs(1), -1, false),
None,
"after the configured round"
);
assert_eq!(
retry_wait(&cooldown, 1, 0, Duration::ZERO, -1, false),
None,
"no maximum wait"
);
let unavailable = ExecError::local(502, cpa_core::exec::FailureScope::Credential, "upstream unavailable");
let round = RemoteFault::local(Failure::Exec(unavailable)).exhausted(None, false);
assert_eq!(
retry_wait(&round, 1, 0, Duration::ZERO, -1, false),
Some(Duration::ZERO)
);
}
#[test]
fn a_cooldown_request_retry_overrides_the_configured_one_unless_pinned() {
let ms = Duration::from_millis;
let second = Duration::from_secs(1);
let one = home_cooldown(ms(10), Some(1));
assert_eq!(retry_wait(&one, 0, 0, second, -1, false), Some(ms(10)));
assert_eq!(
retry_wait(&one, 0, 1, second, -1, false),
None,
"one additional round only"
);
assert_eq!(
retry_wait(&one, 0, 0, second, -1, true),
None,
"pinned keeps its own limit"
);
let zero = home_cooldown(ms(10), Some(0));
assert_eq!(retry_wait(&zero, 3, 0, second, -1, false), None, "an explicit 0 wins");
}
#[test]
fn a_credential_limit_starts_the_next_round_immediately() {
let mut limited = ExecError::local(429, cpa_core::exec::FailureScope::Credential, "credential rate limited");
limited.retry_after = Some(Duration::from_secs(5));
let round = RemoteFault::local(Failure::Exec(limited)).exhausted(Some(Duration::from_secs(5)), true);
assert_eq!(
retry_wait(&round, 1, 0, Duration::from_secs(1), -1, false),
Some(Duration::ZERO)
);
assert_eq!(round.safe_retry_after(), Some(5));
}
#[test]
fn jitter_stays_within_go_bounds() {
for _ in 0..50 {
let w = jitter(Duration::from_secs(4), Duration::ZERO);
assert!(w >= Duration::from_secs(4) && w < Duration::from_secs(5));
let capped = jitter(Duration::from_secs(20), Duration::from_secs(21));
assert!(capped >= Duration::from_secs(20) && capped < Duration::from_secs(21));
}
assert_eq!(
jitter(Duration::from_secs(5), Duration::from_secs(5)),
Duration::from_secs(5)
);
assert_eq!(jitter(Duration::ZERO, Duration::ZERO), Duration::ZERO);
}
#[test]
fn summaries_follow_go_extract() {
assert_eq!(
upstream_summary(r#"{"error":{"code":"x","message":"boom"}}"#),
"x: boom"
);
assert_eq!(
upstream_summary(r#"{"error":{"type":"overloaded","message":"overloaded now"}}"#),
"overloaded now"
);
assert_eq!(upstream_summary(r#"{"error":"flat"}"#), "flat");
assert_eq!(upstream_summary("plain text"), "plain text");
assert_eq!(upstream_summary("status 500: {\"message\":\"m\"}"), "m");
}
}