use reqwest::Client;
use serde::Deserialize;
use tracing::{debug, instrument};
use crate::key_pool::{KeyLease, KeyPool};
use crate::protocol::ProtocolHandler;
use crate::schema::{ApiProtocol, ModelSchema, ModelSource, ProprietaryAuth};
use crate::tasks::ContentBlock;
use crate::tls_client::Degradation;
use crate::InferenceError;
use std::collections::HashMap;
use std::sync::{Mutex, OnceLock};
pub(crate) const PARSLEE_ACCESS_TOKEN_ENV: &str = "PARSLEE_ACCESS_TOKEN";
const REMOTE_MAX_ATTEMPTS: u32 = 3;
const REMOTE_BACKOFF_BASE_SECS: u64 = 3;
fn is_transient_http_status(code: u16) -> bool {
matches!(code, 429 | 500 | 502 | 503 | 529)
}
const MAX_ERROR_CHAIN_LEN: usize = 512;
const MAX_ERROR_CHAIN_DEPTH: usize = 8;
const SENSITIVE_QUERY_PARAMS: &[&str] = &[
"key",
"api_key",
"apikey",
"access_token",
"token",
"password",
"secret",
"signature",
"sig",
];
fn redact_sensitive(text: &str) -> String {
let lower = text.to_ascii_lowercase();
let bytes = text.as_bytes();
let mut out = String::with_capacity(text.len());
let mut i = 0usize;
while i < text.len() {
let userinfo_end = if i >= 3 && &bytes[i - 3..i] == b"://" {
let authority_end = text[i..]
.find(|c: char| c.is_whitespace() || matches!(c, '/' | '?' | '#' | ')'))
.map_or(text.len(), |offset| i + offset);
text[i..authority_end]
.rfind('@')
.map(|offset| i + offset + 1)
} else {
None
};
if let Some(end) = userinfo_end {
out.push_str("REDACTED@");
i = end;
continue;
}
let value_start = if i == 0 || matches!(bytes[i - 1], b'?' | b'&') {
SENSITIVE_QUERY_PARAMS
.iter()
.find(|name| {
let end = i + name.len();
end < text.len() && lower[i..].starts_with(**name) && bytes[end] == b'='
})
.map(|name| i + name.len() + 1)
} else {
None
};
match value_start {
Some(start) => {
out.push_str(&text[i..start]);
out.push_str("REDACTED");
i = text[start..]
.find(|c: char| c == '&' || c == ')' || c.is_whitespace())
.map_or(text.len(), |off| start + off);
}
None => {
let step = text[i..].chars().next().map_or(1, char::len_utf8);
out.push_str(&text[i..i + step]);
i += step;
}
}
}
out
}
fn error_chain(err: &dyn std::error::Error) -> String {
let mut rendered = err.to_string();
let mut source = err.source();
let mut depth = 0usize;
while let Some(cause) = source {
if depth >= MAX_ERROR_CHAIN_DEPTH {
rendered.push_str(": ...");
break;
}
let text = cause.to_string();
if !rendered.ends_with(&text) {
rendered.push_str(": ");
rendered.push_str(&text);
}
source = cause.source();
depth += 1;
}
let redacted = redact_sensitive(&rendered);
if redacted.len() <= MAX_ERROR_CHAIN_LEN {
return redacted;
}
let mut end = MAX_ERROR_CHAIN_LEN;
while end > 0 && !redacted.is_char_boundary(end) {
end -= 1;
}
format!("{}... (truncated)", &redacted[..end])
}
fn reqwest_error_is_transient(e: &reqwest::Error) -> bool {
if e.is_builder() {
return false;
}
if e.is_redirect() || e.is_status() {
return false;
}
true
}
struct TransportAttemptError {
error: InferenceError,
retryable: bool,
}
impl TransportAttemptError {
#[cfg(test)]
fn from_reqwest(context: &str, e: &reqwest::Error) -> Self {
Self::from_reqwest_error(
InferenceError::InferenceFailed(format!("{context}: {}", error_chain(e))),
e,
)
}
fn from_reqwest_error(error: InferenceError, e: &reqwest::Error) -> Self {
Self {
error,
retryable: reqwest_error_is_transient(e),
}
}
fn timeout(message: &str) -> Self {
Self {
error: InferenceError::InferenceFailed(message.to_string()),
retryable: true,
}
}
}
fn openrouter_model_is_unavailable(status: reqwest::StatusCode, body: &str) -> bool {
if status == reqwest::StatusCode::NOT_FOUND {
return true;
}
if status != reqwest::StatusCode::BAD_REQUEST {
return false;
}
let message = serde_json::from_str::<serde_json::Value>(body)
.ok()
.and_then(|value| {
value
.pointer("/error/message")
.and_then(serde_json::Value::as_str)
.map(str::to_lowercase)
})
.unwrap_or_default();
[
"no endpoints found",
"model not found",
"invalid model",
"does not exist",
"deprecated",
]
.iter()
.any(|needle| message.contains(needle))
}
pub(crate) fn is_account_http_status(status: u16) -> bool {
matches!(status, 401..=403)
}
const EXPIRY_WARNING_HORIZON_SECS: u64 = 15 * 60;
const EXPIRY_PROBE_INTERVAL_SECS: u64 = 5 * 60;
async fn warn_if_token_expires_soon() {
static LAST_PROBE: OnceLock<Mutex<Option<std::time::Instant>>> = OnceLock::new();
static WARNED_FOR: OnceLock<Mutex<Option<u64>>> = OnceLock::new();
{
let probe_gate = LAST_PROBE.get_or_init(|| Mutex::new(None));
let mut last = match probe_gate.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
if last.is_some_and(|at| {
at.elapsed() < std::time::Duration::from_secs(EXPIRY_PROBE_INTERVAL_SECS)
}) {
return;
}
*last = Some(std::time::Instant::now());
}
let Some(remaining) = car_auth::access_token_lifetime_remaining().await else {
return;
};
if remaining > EXPIRY_WARNING_HORIZON_SECS {
return;
}
let expires_at = crate::remote::epoch_seconds().saturating_add(remaining);
{
let warned_gate = WARNED_FOR.get_or_init(|| Mutex::new(None));
let mut warned = match warned_gate.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
if warned.is_some_and(|seen| seen.abs_diff(expires_at) <= EXPIRY_PROBE_INTERVAL_SECS) {
return;
}
*warned = Some(expires_at);
}
tracing::warn!(
target: "car_inference::auth",
remaining_secs = remaining,
expires_at_unix = expires_at,
"Parslee token expires in {}m — an operation longer than that will fail \
partway unless it can resume. Re-authenticate with `car auth login` first \
if you are starting a long run.",
remaining / 60
);
}
fn epoch_seconds() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0)
}
fn content_refusal_tags(detail: &str) -> Option<(Option<String>, Option<String>)> {
let (kind, code) = crate::stream::error_tags(detail);
let refused = |v: &str| {
let v = v.to_ascii_lowercase();
v.contains("content_policy")
|| v.contains("content_filter")
|| v.contains("moderation")
|| v.contains("safety")
};
(code.is_some_and(refused) || kind.is_some_and(refused))
.then(|| (kind.map(str::to_string), code.map(str::to_string)))
}
fn gateway_unconfigured_detail(status: reqwest::StatusCode, body: &str) -> Option<String> {
if !status.is_server_error() && status != reqwest::StatusCode::NOT_IMPLEMENTED {
return None;
}
let json: serde_json::Value = serde_json::from_str(body).ok()?;
let error = json.get("error")?;
let code = error.get("code").and_then(|c| c.as_str())?;
if !code.ends_with("_not_configured") {
return None;
}
Some(
error
.get("message")
.and_then(|m| m.as_str())
.map(str::trim)
.filter(|m| !m.is_empty())
.unwrap_or(code)
.to_string(),
)
}
fn gateway_unconfigured_error(
status: reqwest::StatusCode,
message: String,
model: &str,
) -> InferenceError {
let namespace = if crate::openrouter::is_curated_managed_gateway_alias(model) {
crate::openrouter::note_gateway_unconfigured();
"parslee/openrouter/"
} else {
"parslee/"
};
InferenceError::GatewayUnconfigured {
provider: "parslee".to_string(),
namespace: namespace.to_string(),
status: status.as_u16(),
message,
}
}
fn account_error_message(protocol: ApiProtocol, status: u16) -> String {
match (protocol, status) {
(ApiProtocol::OpenRouter, 402) => {
"OpenRouter account is out of credits — add credits in OpenRouter and retry".to_string()
}
(ApiProtocol::OpenRouter, _) => {
"OpenRouter key rejected — re-add it with `car keys set openrouter`".to_string()
}
(_, 402) => "provider account is out of credits or over quota".to_string(),
_ => "provider rejected the API key — check the configured credential".to_string(),
}
}
fn openrouter_http_error(status: reqwest::StatusCode, body: &str, model: &str) -> InferenceError {
if openrouter_model_is_unavailable(status, body) {
return InferenceError::InferenceFailed(format!(
"OpenRouter model '{model}' is no longer available on OpenRouter"
));
}
InferenceError::InferenceFailed(format!("OpenRouter request failed with HTTP {status}"))
}
fn google_finish_outcome(reason: &str) -> Result<(), &'static str> {
match reason {
"STOP" | "MAX_TOKENS" => Ok(()),
"SAFETY" => Err("Google blocked the response for safety"),
"RECITATION" => Err("Google blocked the response for recitation"),
"BLOCKLIST" => Err("Google blocked the response because of a blocklist match"),
"PROHIBITED_CONTENT" => Err("Google blocked prohibited response content"),
"SPII" => Err("Google blocked the response because it may contain sensitive information"),
"MALFORMED_FUNCTION_CALL" => Err("Google returned a malformed function call"),
_ => Err("Google inference ended with an abnormal finish reason"),
}
}
fn is_auth_rejection(e: &InferenceError) -> bool {
let m = e.to_string();
m.contains("HTTP 401") || m.contains("HTTP 403")
}
fn find_sse_separator(buf: &[u8]) -> Option<(usize, usize)> {
let lf = buf.windows(2).position(|w| w == b"\n\n").map(|p| (p, 2));
let crlf = buf
.windows(4)
.position(|w| w == b"\r\n\r\n")
.map(|p| (p, 4));
match (lf, crlf) {
(Some(a), Some(b)) => Some(if a.0 <= b.0 { a } else { b }),
(Some(a), None) => Some(a),
(None, Some(b)) => Some(b),
(None, None) => None,
}
}
type ParsleeIdentityCache = HashMap<(String, String), (String, String)>;
static PARSLEE_IDENTITY: OnceLock<tokio::sync::Mutex<ParsleeIdentityCache>> = OnceLock::new();
pub struct RemoteBackend {
pub(crate) client: Client,
pub(crate) tls_degradation: Option<Degradation>,
pub key_pool: KeyPool,
}
pub fn estimate_tokens(text: &str) -> usize {
std::cmp::max(1, text.len() / 4)
}
fn parse_parslee_responses_sse(raw: &str) -> Vec<crate::stream::StreamEvent> {
let mut events = Vec::new();
for block in raw.split("\n\n") {
let block = block.trim();
if block.is_empty() {
continue;
}
let mut event_type = "";
let mut data = String::new();
for line in block.lines() {
if let Some(t) = line.strip_prefix("event:") {
event_type = t.trim();
} else if let Some(d) = line.strip_prefix("data:") {
if !data.is_empty() {
data.push('\n');
}
data.push_str(d.trim());
}
}
if event_type.is_empty() || data.is_empty() {
continue;
}
events.extend(crate::stream::parse_openai_responses_sse_line(
event_type, &data,
));
}
events
}
fn parslee_request_body(
user_id: &str,
input: Vec<serde_json::Value>,
max_tokens: usize,
tools: Option<&[serde_json::Value]>,
model: Option<&str>,
) -> serde_json::Value {
let mut body = serde_json::json!({
"userId": user_id,
"input": input,
"max_output_tokens": max_tokens,
"store": false,
"include": ["reasoning.encrypted_content"],
});
if let Some(tools) = tools {
if !tools.is_empty() {
body["tools"] = serde_json::json!(tools);
}
}
if let Some(model) = model.filter(|model| model.starts_with("parslee/openrouter/")) {
body["model"] = serde_json::json!(model);
}
body
}
fn parslee_input_items(
system: Option<&str>,
messages: &[serde_json::Value],
) -> Vec<serde_json::Value> {
let mut input: Vec<serde_json::Value> = Vec::new();
if let Some(sys) = system {
if !sys.trim().is_empty() {
input.push(serde_json::json!({ "role": "system", "content": sys }));
}
}
for m in messages {
if m.get("role").is_none() && m.get("type").is_some() {
input.push(m.clone());
continue;
}
let role = m.get("role").and_then(|v| v.as_str()).unwrap_or("user");
match role {
"assistant" => {
if let Some(tcs) = m.get("tool_calls").and_then(|v| v.as_array()) {
for tc in tcs {
let call_id = tc.get("id").and_then(|v| v.as_str()).unwrap_or("");
let func = tc.get("function");
let name = func
.and_then(|f| f.get("name"))
.and_then(|v| v.as_str())
.unwrap_or("");
let args = func.and_then(|f| f.get("arguments")).map_or_else(
|| "{}".to_string(),
|a| {
a.as_str()
.map(str::to_string)
.unwrap_or_else(|| a.to_string())
},
);
input.push(serde_json::json!({
"type": "function_call",
"call_id": call_id,
"name": name,
"arguments": args,
}));
}
}
let text = m.get("content").and_then(|v| v.as_str()).unwrap_or("");
if !text.is_empty() {
input.push(serde_json::json!({ "role": "assistant", "content": text }));
}
}
"tool" => {
let call_id = m.get("tool_call_id").and_then(|v| v.as_str()).unwrap_or("");
let output = m.get("content").and_then(|v| v.as_str()).unwrap_or("");
input.push(serde_json::json!({
"type": "function_call_output",
"call_id": call_id,
"output": output,
}));
}
_ => {
let content = m.get("content").and_then(|v| v.as_str()).unwrap_or("");
if !content.is_empty() {
input.push(serde_json::json!({ "role": role, "content": content }));
}
}
}
}
if tracing::enabled!(tracing::Level::DEBUG) {
let fc = input
.iter()
.filter(|i| i["type"] == "function_call")
.count();
let fco = input
.iter()
.filter(|i| i["type"] == "function_call_output")
.count();
tracing::debug!(
target = "car::parslee_input",
messages_in = messages.len(),
items_out = input.len(),
function_calls = fc,
function_call_outputs = fco,
"built parslee Responses input"
);
}
input
}
fn truncate_prompt_to_fit(
prompt: &str,
context: Option<&str>,
tools_json: Option<&[serde_json::Value]>,
max_tokens: usize,
media_tokens: usize,
context_window: usize,
) -> String {
let context_tokens = context.map(estimate_tokens).unwrap_or(0);
let tools_tokens = tools_json
.map(|t| estimate_tokens(&serde_json::to_string(t).unwrap_or_default()))
.unwrap_or(0);
let overhead = 100; let reserved = context_tokens + tools_tokens + media_tokens + max_tokens + overhead;
let available = context_window.saturating_sub(reserved);
let prompt_tokens = estimate_tokens(prompt);
if prompt_tokens <= available {
return prompt.to_string();
}
tracing::warn!(
prompt_tokens = prompt_tokens,
available_tokens = available,
context_window = context_window,
"truncating prompt to fit context window (prefer compaction via car-memgine)"
);
let chars_to_keep = available * 4;
if chars_to_keep >= prompt.len() {
return prompt.to_string();
}
let start = prompt.len().saturating_sub(chars_to_keep);
let safe_start = prompt.ceil_char_boundary(start);
let truncated = &prompt[safe_start..];
let break_point = truncated
.find('\n')
.or_else(|| truncated.find(' '))
.unwrap_or(0);
format!(
"[...truncated...]\n{}",
truncated[break_point..].trim_start()
)
}
impl RemoteBackend {
pub fn new() -> Self {
let (client, tls_degradation) = crate::tls_client::build_client_with_degradation(
&crate::tls_client::REMOTE_BACKEND,
|| {
Client::builder()
.timeout(std::time::Duration::from_secs(300))
.connect_timeout(std::time::Duration::from_secs(10))
.read_timeout(std::time::Duration::from_secs(180))
},
);
Self {
client,
tls_degradation,
key_pool: KeyPool::new(),
}
}
pub(crate) fn request_error(&self, context: &str, err: &reqwest::Error) -> InferenceError {
InferenceError::InferenceFailed(format!(
"{context}: {}{}",
error_chain(err),
self.tls_degradation_note(err)
))
}
fn transport_attempt_error(
&self,
context: &str,
err: &reqwest::Error,
) -> TransportAttemptError {
TransportAttemptError::from_reqwest_error(self.request_error(context, err), err)
}
fn tls_degradation_note(&self, err: &reqwest::Error) -> String {
match &self.tls_degradation {
Some(degradation) if is_trust_related(err.is_connect(), err.is_timeout()) => {
format!(" [{degradation}]")
}
_ => String::new(),
}
}
pub async fn register_model_keys(&self, schema: &ModelSchema) {
if let ModelSource::RemoteApi { ref endpoint, .. } = schema.source {
let env_vars = schema.all_api_key_envs();
if !env_vars.is_empty() {
self.key_pool.register_endpoint(endpoint, env_vars).await;
}
}
}
#[instrument(
name = "inference.remote_call",
skip_all,
fields(
model = %schema.name,
provider = %schema.provider,
),
)]
async fn execute_request(
&self,
schema: &ModelSchema,
req: crate::protocol::ApiRequest,
) -> Result<crate::protocol::ApiResponse, InferenceError> {
let (endpoint, protocol) = extract_remote_endpoint(schema)?;
let lease = self.lease_key(schema, &endpoint).await?;
if matches!(schema.source, ModelSource::Proprietary { .. }) {
if req.response_format.is_some() {
return Err(InferenceError::UnsupportedMode {
mode: "structured-output",
backend: "parslee-inference",
reason: "the Parslee inference endpoint does not accept a \
response_format / JSON schema yet — route structured-output \
requests to a model that supports it",
});
}
match self
.parslee_assistant_request(&endpoint, &lease.api_key, &req)
.await
{
Ok(resp) => return Ok(resp),
Err(e) if is_auth_rejection(&e) => {
match car_auth::force_refresh().await {
Some(fresh) => {
return self
.parslee_assistant_request(&endpoint, &fresh, &req)
.await;
}
None => return Err(e),
}
}
Err(e) => return Err(e),
}
}
if matches!(protocol, ApiProtocol::Bedrock) {
return self.bedrock_converse_request(&endpoint, &req).await;
}
let handler = crate::protocol::handler_for(protocol);
let start = std::time::Instant::now();
let api_version = match &schema.source {
ModelSource::RemoteApi { api_version, .. } => api_version.clone(),
_ => None,
};
let url = if matches!(protocol, ApiProtocol::Google) {
crate::protocol::google_url(&endpoint, &req.model, &lease.api_key)
} else if matches!(protocol, ApiProtocol::VertexAi) {
crate::protocol::vertex_url(&endpoint, &req.model)
} else if matches!(protocol, ApiProtocol::AzureOpenAi) {
let version = api_version.as_deref().unwrap_or("2024-10-21");
format!(
"{}/openai/deployments/{}/chat/completions?api-version={}",
endpoint.trim_end_matches('/'),
req.model,
version
)
} else {
format_endpoint(&endpoint, chat_path_for(schema, handler.as_ref()))
};
let headers = handler.auth_headers(&lease.api_key);
let body = handler.build_request_body(&req);
debug!(url = %url, model = %req.model, "protocol handler request");
let mut attempt = 0u32;
let resp_text = loop {
attempt += 1;
let send_outcome: Result<(reqwest::StatusCode, String), TransportAttemptError> =
async {
let mut builder = self.client.post(&url);
for (name, value) in &headers {
builder = builder.header(name.as_str(), value.as_str());
}
let send_fut = builder.json(&body).send();
let resp = tokio::time::timeout(std::time::Duration::from_secs(300), send_fut)
.await
.map_err(|_| {
TransportAttemptError::timeout(
"request timed out after 300s (tokio safety timeout)",
)
})?
.map_err(|e| self.transport_attempt_error("HTTP error", &e))?;
let status = resp.status();
let body_fut = resp.text();
let txt = tokio::time::timeout(std::time::Duration::from_secs(300), body_fut)
.await
.map_err(|_| {
TransportAttemptError::timeout(
"response body read timed out after 300s",
)
})?
.map_err(|e| self.transport_attempt_error("read body", &e))?;
Ok((status, txt))
}
.await;
match send_outcome {
Ok((status, txt)) if status.is_success() => break txt,
Ok((status, txt)) => {
let is_rl = txt.contains("429") || txt.contains("RESOURCE_EXHAUSTED");
self.key_pool
.report_failure(&endpoint, &lease.env_var, is_rl)
.await;
let err_msg = format!("API returned {status}: {txt}");
let transient = is_transient_http_status(status.as_u16());
if attempt < REMOTE_MAX_ATTEMPTS && transient {
tokio::time::sleep(std::time::Duration::from_secs(
REMOTE_BACKOFF_BASE_SECS * attempt as u64,
))
.await;
continue;
}
if is_account_http_status(status.as_u16()) {
return Err(InferenceError::ProviderAccount {
provider: schema.provider.clone(),
status: status.as_u16(),
message: account_error_message(protocol, status.as_u16()),
});
}
if matches!(protocol, ApiProtocol::OpenRouter) {
return Err(if transient {
InferenceError::Transient {
status: Some(status.as_u16()),
message: format!(
"OpenRouter upstream failed after {REMOTE_MAX_ATTEMPTS} attempts"
),
}
} else {
openrouter_http_error(status, &txt, &req.model)
});
}
return Err(if transient {
InferenceError::Transient {
status: Some(status.as_u16()),
message: err_msg,
}
} else {
InferenceError::InferenceFailed(err_msg)
});
}
Err(e) => {
let transient = e.retryable;
if attempt < REMOTE_MAX_ATTEMPTS && transient {
tokio::time::sleep(std::time::Duration::from_secs(
REMOTE_BACKOFF_BASE_SECS * attempt as u64,
))
.await;
continue;
}
return Err(if transient {
InferenceError::Transient {
status: None,
message: e.error.to_string(),
}
} else {
e.error
});
}
}
};
let latency_ms = start.elapsed().as_millis() as u64;
let est_tokens = req
.messages
.iter()
.filter_map(|m| m.get("content").and_then(|c| c.as_str()))
.map(|s| s.len() / 4)
.sum::<usize>() as u64;
self.key_pool
.report_success(&endpoint, &lease.env_var, latency_ms, est_tokens, 0)
.await;
let mut response = handler.parse_response(&resp_text)?;
if let Some(ref mut usage) = response.usage {
usage.context_window = schema.context_length as u64;
}
Ok(response)
}
async fn bedrock_converse_request(
&self,
region: &str,
req: &crate::protocol::ApiRequest,
) -> Result<crate::protocol::ApiResponse, InferenceError> {
let creds = crate::aws_sigv4::AwsCredentials::from_env().ok_or_else(|| {
InferenceError::InferenceFailed(
"Bedrock requires AWS credentials: set AWS_ACCESS_KEY_ID and \
AWS_SECRET_ACCESS_KEY (plus AWS_SESSION_TOKEN for temporary creds)"
.to_string(),
)
})?;
if region.trim().is_empty() {
return Err(InferenceError::InferenceFailed(
"Bedrock model endpoint must be the AWS region (e.g. \"us-east-1\")".to_string(),
));
}
let handler = crate::protocol::BedrockHandler;
let body = handler.build_request_body(req);
let body_bytes = serde_json::to_vec(&body)
.map_err(|e| InferenceError::InferenceFailed(format!("serialize Bedrock body: {e}")))?;
let host = format!("bedrock-runtime.{region}.amazonaws.com");
let encoded_model = crate::aws_sigv4::uri_encode_segment(&req.model);
let canonical_path = format!("/model/{encoded_model}/converse");
let url = format!("https://{host}{canonical_path}");
let now = chrono::Utc::now();
let amz_date = now.format("%Y%m%dT%H%M%SZ").to_string();
let date_stamp = now.format("%Y%m%d").to_string();
let base_headers = vec![
("host".to_string(), host.clone()),
("content-type".to_string(), "application/json".to_string()),
];
let signed = crate::aws_sigv4::signed_headers(
&creds,
region,
"bedrock",
"POST",
&canonical_path,
"",
&base_headers,
&body_bytes,
&amz_date,
&date_stamp,
);
let mut builder = self.client.post(&url);
for (k, v) in &signed {
if k != "host" {
builder = builder.header(k, v);
}
}
let send_fut = builder.body(body_bytes).send();
let resp = tokio::time::timeout(std::time::Duration::from_secs(300), send_fut)
.await
.map_err(|_| {
InferenceError::InferenceFailed("Bedrock request timed out after 300s".to_string())
})?
.map_err(|e| self.request_error("HTTP error", &e))?;
let status = resp.status();
let txt = resp.text().await.map_err(|e| {
InferenceError::InferenceFailed(format!("read Bedrock body: {}", error_chain(&e)))
})?;
if !status.is_success() {
return Err(InferenceError::InferenceFailed(format!(
"Bedrock returned {status}: {txt}"
)));
}
handler.parse_response(&txt)
}
async fn parslee_identity(
&self,
endpoint: &str,
bearer: &str,
) -> Result<(String, String), InferenceError> {
let cache = PARSLEE_IDENTITY.get_or_init(|| tokio::sync::Mutex::new(HashMap::new()));
let base = endpoint.trim_end_matches('/');
let cache_key = (base.to_string(), bearer.to_string());
if let Some(v) = cache.lock().await.get(&cache_key) {
return Ok(v.clone());
}
let org_url = format!("{base}/api/v1/organizations/me");
let org_resp = self
.client
.get(&org_url)
.bearer_auth(bearer)
.send()
.await
.map_err(|e| self.request_error("Parslee org lookup", &e))?;
if !org_resp.status().is_success() {
let s = org_resp.status();
let b = org_resp.text().await.unwrap_or_default();
return Err(InferenceError::InferenceFailed(format!(
"Parslee org lookup failed: HTTP {s}: {b}"
)));
}
let org_json: serde_json::Value = org_resp.json().await.map_err(|e| {
InferenceError::InferenceFailed(format!("parse Parslee org: {}", error_chain(&e)))
})?;
let org_id = org_json
.get("organizationId")
.or_else(|| org_json.get("OrganizationId"))
.and_then(|v| v.as_str())
.ok_or_else(|| {
InferenceError::InferenceFailed(
"Parslee org response has no organizationId — the signed-in \
account has no workspace yet (sign in via CAR Host.app or \
the web to finish onboarding)"
.to_string(),
)
})?
.to_string();
let sess_url = format!("{base}/connect/session");
let user_id = match self.client.get(&sess_url).bearer_auth(bearer).send().await {
Ok(r) if r.status().is_success() => r
.json::<serde_json::Value>()
.await
.ok()
.and_then(|j| {
j.get("account")
.and_then(|a| a.get("email"))
.and_then(|v| v.as_str())
.map(String::from)
})
.unwrap_or_else(|| "car".to_string()),
_ => "car".to_string(),
};
let pair = (org_id, user_id);
cache.lock().await.insert(cache_key, pair.clone());
Ok(pair)
}
async fn parslee_assistant_request(
&self,
endpoint: &str,
bearer: &str,
req: &crate::protocol::ApiRequest,
) -> Result<crate::protocol::ApiResponse, InferenceError> {
let (org_id, user_id) = self.parslee_identity(endpoint, bearer).await?;
let input = parslee_input_items(req.system.as_deref(), &req.messages);
if input.is_empty() {
return Err(InferenceError::InferenceFailed(
"Parslee inference: empty prompt".to_string(),
));
}
let url = format!(
"{}/api/v1/orgs/{}/inference/responses",
endpoint.trim_end_matches('/'),
org_id
);
let _ = req.temperature;
let body = parslee_request_body(
&user_id,
input,
req.max_tokens,
req.tools.as_deref(),
Some(&req.model),
);
let send_fut = self
.client
.post(&url)
.bearer_auth(bearer)
.header("content-type", "application/json")
.header("accept", "text/event-stream")
.json(&body)
.send();
let resp = tokio::time::timeout(std::time::Duration::from_secs(300), send_fut)
.await
.map_err(|_| {
InferenceError::InferenceFailed("Parslee chat request timed out (150s)".to_string())
})?
.map_err(|e| self.request_error("Parslee chat HTTP error", &e))?;
let status = resp.status();
let text_fut = resp.text();
let raw = tokio::time::timeout(std::time::Duration::from_secs(300), text_fut)
.await
.map_err(|_| {
InferenceError::InferenceFailed(
"Parslee chat body read timed out (120s)".to_string(),
)
})?
.map_err(|e| {
InferenceError::InferenceFailed(format!(
"Parslee chat read body: {}",
error_chain(&e)
))
})?;
if !status.is_success() {
if let Some(detail) = gateway_unconfigured_detail(status, &raw) {
return Err(gateway_unconfigured_error(status, detail, &req.model));
}
return Err(InferenceError::InferenceFailed(format!(
"Parslee chat failed: HTTP {status}: {raw}"
)));
}
let mut acc = crate::stream::StreamAccumulator::default();
let mut stream_error = None;
let mut saw_completed = false;
for event in parse_parslee_responses_sse(&raw) {
if let crate::stream::StreamEvent::Error(message) = &event {
stream_error = Some(message.clone());
}
if matches!(event, crate::stream::StreamEvent::Done { .. }) {
saw_completed = true;
}
acc.push(&event);
}
let (text, tool_calls, usage, stop_reason, provider_output_items) =
acc.finish_with_provider_output_items();
if let Some(message) = stream_error {
if let Some((kind, code)) = content_refusal_tags(&message) {
return Err(InferenceError::ContentRefused {
provider: "parslee".to_string(),
kind,
code,
message,
});
}
return Err(InferenceError::InferenceFailed(message));
}
if !saw_completed {
return Err(InferenceError::InferenceFailed(
"managed inference stream ended without response.completed".to_string(),
));
}
if text.is_empty() && tool_calls.is_empty() {
let snippet: String = raw.trim().chars().take(600).collect();
return Err(InferenceError::InferenceFailed(format!(
"Parslee inference returned no content (HTTP {status}); raw response: {snippet}"
)));
}
Ok(crate::protocol::ApiResponse {
text,
tool_calls,
provider_output_items,
thinking: Vec::new(),
usage,
stop_reason,
})
}
async fn parslee_assistant_stream_request(
&self,
endpoint: &str,
bearer: &str,
req: &crate::protocol::ApiRequest,
spend_guard: Option<crate::routing_ext::MidStreamSpendGuard>,
) -> Result<tokio::sync::mpsc::Receiver<crate::stream::StreamEvent>, InferenceError> {
let (org_id, user_id) = self.parslee_identity(endpoint, bearer).await?;
let input = parslee_input_items(req.system.as_deref(), &req.messages);
if input.is_empty() {
return Err(InferenceError::InferenceFailed(
"Parslee inference: empty prompt".to_string(),
));
}
let url = format!(
"{}/api/v1/orgs/{}/inference/responses",
endpoint.trim_end_matches('/'),
org_id
);
let body = parslee_request_body(
&user_id,
input,
req.max_tokens,
req.tools.as_deref(),
Some(&req.model),
);
let resp = self
.client
.post(url)
.bearer_auth(bearer)
.header("content-type", "application/json")
.header("accept", "text/event-stream")
.json(&body)
.send()
.await
.map_err(|error| self.request_error("Parslee stream HTTP error", &error))?;
let status = resp.status();
if !status.is_success() {
let body = resp.text().await.unwrap_or_default();
if let Some(detail) = gateway_unconfigured_detail(status, &body) {
return Err(gateway_unconfigured_error(status, detail, &req.model));
}
return Err(InferenceError::InferenceFailed(format!(
"Parslee stream failed: HTTP {status}: {body}"
)));
}
let (tx, rx) = tokio::sync::mpsc::channel(64);
tokio::spawn(async move {
use futures::StreamExt;
let mut stream = resp.bytes_stream();
let mut buffer = Vec::<u8>::new();
let mut spend_guard = spend_guard;
let mut saw_completed = false;
while let Some(chunk) = stream.next().await {
match chunk {
Ok(bytes) => buffer.extend_from_slice(&bytes),
Err(error) => {
tracing::warn!(%error, "Parslee stream transport error");
let _ = tx
.send(crate::stream::StreamEvent::Error(
"Parslee stream transport error".to_string(),
))
.await;
return;
}
}
while let Some((position, separator_len)) = find_sse_separator(&buffer) {
let block = String::from_utf8_lossy(&buffer[..position]).into_owned();
buffer.drain(..position + separator_len);
for event in parse_parslee_responses_sse(&block) {
if let Some(guard) = spend_guard.as_mut() {
if let crate::stream::StreamEvent::Usage { input_tokens, .. } = &event {
guard.note_prompt_tokens(*input_tokens);
}
let delta = match &event {
crate::stream::StreamEvent::TextDelta(text) => Some(text.as_str()),
crate::stream::StreamEvent::ToolCallDelta {
arguments_delta,
..
} => Some(arguments_delta.as_str()),
_ => None,
};
if let Some(delta) = delta {
let tokens =
crate::routing_ext::MidStreamSpendGuard::estimate_tokens(delta);
if let Some(exceeded) = guard.record_tokens(tokens) {
let _ = tx.send(event).await;
let _ = tx
.send(crate::stream::StreamEvent::StopReason(format!(
"spend_limit: {exceeded}"
)))
.await;
let _ = tx
.send(crate::stream::StreamEvent::Error(
"stream cancelled because the spend limit was reached"
.to_string(),
))
.await;
return;
}
}
}
if matches!(event, crate::stream::StreamEvent::Done { .. }) {
saw_completed = true;
}
let terminal = matches!(event, crate::stream::StreamEvent::Error(_));
if tx.send(event).await.is_err() || terminal {
return;
}
}
}
}
if !buffer.is_empty() || !saw_completed {
let _ = tx
.send(crate::stream::StreamEvent::Error(
"managed inference stream ended without response.completed".to_string(),
))
.await;
}
});
Ok(rx)
}
pub async fn generate(
&self,
schema: &ModelSchema,
prompt: &str,
context: Option<&str>,
temperature: f64,
max_tokens: usize,
images: Option<&[ContentBlock]>,
) -> Result<String, InferenceError> {
let resp = self
.generate_with_tools_multi(
schema,
prompt,
context,
temperature,
max_tokens,
None,
images,
None,
None,
None,
0,
false,
crate::tasks::generate::CacheTtl::default(),
None,
None,
)
.await?;
Ok(resp.0)
}
pub async fn generate_with_tools(
&self,
schema: &ModelSchema,
prompt: &str,
context: Option<&str>,
temperature: f64,
max_tokens: usize,
tools: Option<&[serde_json::Value]>,
images: Option<&[ContentBlock]>,
) -> Result<(String, Vec<crate::tasks::generate::ToolCall>), InferenceError> {
let (text, calls, _thinking, _provider_items, _usage, _stop) = self
.generate_with_tools_multi(
schema,
prompt,
context,
temperature,
max_tokens,
tools,
images,
None,
None,
None,
0,
false,
crate::tasks::generate::CacheTtl::default(),
None,
None,
)
.await?;
Ok((text, calls))
}
pub async fn generate_with_tools_multi(
&self,
schema: &ModelSchema,
prompt: &str,
context: Option<&str>,
temperature: f64,
max_tokens: usize,
tools: Option<&[serde_json::Value]>,
images: Option<&[ContentBlock]>,
messages: Option<&[crate::tasks::generate::Message]>,
tool_choice: Option<&str>,
parallel_tool_calls: Option<bool>,
budget_tokens: usize,
cache_control: bool,
cache_ttl: crate::tasks::generate::CacheTtl,
system_stable_prefix: Option<&str>,
response_format: Option<&crate::tasks::generate::ResponseFormat>,
) -> Result<
(
String,
Vec<crate::tasks::generate::ToolCall>,
Vec<crate::tasks::generate::ThinkingBlock>,
Vec<serde_json::Value>,
Option<crate::TokenUsage>,
Option<String>,
),
InferenceError,
> {
let (_, protocol) = extract_remote_endpoint(schema)?;
let handler = crate::protocol::handler_for(protocol);
if !handler.supports_video() {
let has_video_in_images =
images.is_some_and(|blocks| blocks.iter().any(ContentBlock::is_video));
let has_video_in_messages = messages.is_some_and(|msgs| {
msgs.iter().any(|msg| match msg {
crate::tasks::generate::Message::UserMultimodal { content } => {
content.iter().any(ContentBlock::is_video)
}
_ => false,
})
});
if has_video_in_images || has_video_in_messages {
return Err(InferenceError::UnsupportedMode {
mode: "video-content-block",
backend: handler.protocol_name(),
reason: "this remote protocol has no native video input path; route to \
a provider that implements ProtocolHandler::supports_video() (Gemini)",
});
}
}
if !handler.supports_audio() {
let has_audio_in_images =
images.is_some_and(|blocks| blocks.iter().any(ContentBlock::is_audio));
let has_audio_in_messages = messages.is_some_and(|msgs| {
msgs.iter().any(|msg| match msg {
crate::tasks::generate::Message::UserMultimodal { content } => {
content.iter().any(ContentBlock::is_audio)
}
_ => false,
})
});
if has_audio_in_images || has_audio_in_messages {
return Err(InferenceError::UnsupportedMode {
mode: "audio-content-block",
backend: handler.protocol_name(),
reason: "this remote protocol has no native audio input path; route to \
a provider that implements ProtocolHandler::supports_audio() (Gemini)",
});
}
}
if let Some(err) = response_format_rejection(handler.as_ref(), response_format) {
return Err(err);
}
let prompt = if schema.context_length > 0 {
let media_tokens =
crate::media_tokens::request_media_and_history_tokens(images, messages);
truncate_prompt_to_fit(
prompt,
context,
tools,
max_tokens,
media_tokens,
schema.context_length,
)
} else {
prompt.to_string()
};
let (api_messages, system) = if matches!(schema.source, ModelSource::Proprietary { .. }) {
crate::protocol::OpenAiResponsesHandler.build_messages(
messages.unwrap_or(&[]),
&prompt,
context,
images,
)
} else {
handler.build_messages(messages.unwrap_or(&[]), &prompt, context, images)
};
let api_tools = tools.map(|t| handler.build_tools(t));
let req = crate::protocol::ApiRequest {
model: request_model_name(schema),
messages: api_messages,
system,
system_stable_prefix: system_stable_prefix.map(str::to_string),
temperature,
max_tokens,
tools: api_tools,
tool_choice: tool_choice.map(str::to_string),
parallel_tool_calls,
stream: false,
budget_tokens,
cache_control,
cache_ttl,
response_format: response_format.cloned(),
};
let response = self.execute_request(schema, req).await?;
let text = response.text;
let mut calls = response.tool_calls;
let thinking = response.thinking;
let provider_output_items = response.provider_output_items;
let usage = response.usage;
let stop_reason = response.stop_reason;
if cache_control {
if let Some(u) = usage.as_ref() {
if u.cache_read_input_tokens == 0 && u.cache_creation_input_tokens == 0 {
tracing::debug!(
model = %request_model_name(schema),
prompt_tokens = u.prompt_tokens,
"prompt caching requested but nothing was cached (read=0, write=0) — \
prompt likely below the model's minimum cacheable length, or the \
cached prefix changed between requests",
);
}
}
}
if !text.is_empty() {
for call in &mut calls {
if call.name == "done" {
let result_val = call
.arguments
.get("result")
.and_then(|v| v.as_str())
.unwrap_or("");
if result_val.len() < 50 && text.len() > result_val.len() {
call.arguments.insert(
"result".to_string(),
serde_json::Value::String(text.clone()),
);
}
}
}
}
Ok((
text,
calls,
thinking,
provider_output_items,
usage,
stop_reason,
))
}
pub async fn embed(
&self,
schema: &ModelSchema,
texts: &[String],
) -> Result<Vec<Vec<f32>>, InferenceError> {
let (endpoint, protocol) = extract_remote_endpoint(schema)?;
let lease = self.lease_key(schema, &endpoint).await?;
let start = std::time::Instant::now();
let result = match protocol {
ApiProtocol::OpenAiCompat => {
self.embed_openai(&endpoint, &lease.api_key, &schema.name, texts)
.await
}
_ => Err(InferenceError::InferenceFailed(format!(
"embedding not supported for {:?} protocol",
protocol
))),
};
let latency_ms = start.elapsed().as_millis() as u64;
match &result {
Ok(_) => {
let est_tokens = texts
.iter()
.map(|t| t.split_whitespace().count() as u64)
.sum();
self.key_pool
.report_success(&endpoint, &lease.env_var, latency_ms, est_tokens, 0)
.await;
}
Err(e) => {
let is_rl =
e.to_string().contains("429") || e.to_string().contains("RESOURCE_EXHAUSTED");
self.key_pool
.report_failure(&endpoint, &lease.env_var, is_rl)
.await;
}
}
result
}
async fn lease_key(
&self,
schema: &ModelSchema,
endpoint: &str,
) -> Result<KeyLease, InferenceError> {
let fallback_env = match &schema.source {
ModelSource::RemoteApi {
protocol: ApiProtocol::OpenRouter,
..
} => {
let (api_key, source) = crate::openrouter::resolve_credential().ok_or_else(|| {
InferenceError::InferenceFailed(
"OpenRouter requires a key — run `car keys set openrouter` or connect your OpenRouter account in CarHost"
.to_string(),
)
})?;
return Ok(KeyLease {
api_key,
env_var: format!("{}:{}", crate::openrouter::API_KEY_ENV, source.as_str()),
});
}
ModelSource::RemoteApi { api_key_env, .. } => api_key_env.as_str(),
ModelSource::Ollama { .. } | ModelSource::VllmMlx { .. } => {
return Ok(KeyLease {
api_key: String::new(),
env_var: String::new(),
})
}
ModelSource::Proprietary {
ref auth,
ref provider,
..
} => {
let (token, source_label) = match auth {
ProprietaryAuth::OAuth2Pkce { .. } => {
if !provider.eq_ignore_ascii_case("parslee")
|| !schema.provider.eq_ignore_ascii_case("parslee")
{
return Err(InferenceError::InferenceFailed(format!(
"OAuth2 PKCE credentials are supported only for provider \
'parslee'; refusing model {} from proprietary provider \
'{provider}'",
schema.id
)));
}
warn_if_token_expires_soon().await;
(
car_auth::access_token_refreshing().await,
PARSLEE_ACCESS_TOKEN_ENV,
)
}
ProprietaryAuth::BearerTokenEnv { env_var }
| ProprietaryAuth::ApiKeyEnv { env_var } => (
car_secrets::resolve_env_or_keychain(env_var),
env_var.as_str(),
),
};
let token = match token {
Some(t) => t,
None => {
let (reason, detail) = if provider.eq_ignore_ascii_case("parslee") {
match car_auth::credential_state().await {
car_auth::CredentialState::Expired { expires_at } => (
crate::CredentialFailure::Expired { expires_at },
format!(
"the Parslee token expired at unix {expires_at} and could \
not be refreshed. Re-authenticate with `car auth login`; \
a job longer than the remaining token lifetime will fail \
partway unless it is resumable"
),
),
car_auth::CredentialState::Unreadable(e) => (
crate::CredentialFailure::StoreUnreadable,
format!(
"the credential store could not be read ({e}). This is \
not a sign-out — on macOS it usually means a keychain \
prompt is waiting, or the helper timed out. Unlock the \
login keychain and retry before re-authenticating"
),
),
car_auth::CredentialState::SignedOut => (
crate::CredentialFailure::SignedOut,
"no account is signed in. Run `car auth login`".to_string(),
),
car_auth::CredentialState::Active => (
crate::CredentialFailure::RaceRetryable,
"the credential read failed but the store now reports an \
active session — retry the request"
.to_string(),
),
}
} else {
(
crate::CredentialFailure::EnvVarMissing {
env_var: source_label.to_string(),
},
format!("set ${source_label} for provider '{provider}'"),
)
};
return Err(InferenceError::CredentialUnavailable {
provider: provider.to_string(),
model: schema.id.clone(),
reason,
detail,
});
}
};
return Ok(KeyLease {
api_key: token,
env_var: String::new(),
});
}
_ => {
return Err(InferenceError::InferenceFailed(format!(
"model {} is not remote",
schema.id
)))
}
};
self.register_model_keys(schema).await;
self.key_pool
.lease_or_env(endpoint, fallback_env)
.await
.ok_or_else(|| {
InferenceError::InferenceFailed(format!(
"no API keys available for endpoint {} (checked env vars: {:?})",
endpoint,
schema.all_api_key_envs()
))
})
}
async fn embed_openai(
&self,
endpoint: &str,
api_key: &str,
model: &str,
texts: &[String],
) -> Result<Vec<Vec<f32>>, InferenceError> {
let url = format_endpoint(endpoint, "/v1/embeddings");
let body = serde_json::json!({
"model": model,
"input": texts,
});
let resp = self
.client
.post(&url)
.header("Authorization", format!("Bearer {api_key}"))
.header("Content-Type", "application/json")
.json(&body)
.send()
.await
.map_err(|e| self.request_error("HTTP error", &e))?;
let status = resp.status();
let text = resp.text().await.map_err(|e| {
InferenceError::InferenceFailed(format!("read body: {}", error_chain(&e)))
})?;
if !status.is_success() {
return Err(InferenceError::InferenceFailed(format!(
"API returned {status}: {text}"
)));
}
let parsed: OpenAiEmbedResponse = serde_json::from_str(&text)
.map_err(|e| InferenceError::InferenceFailed(format!("parse response: {e}")))?;
Ok(order_embeddings(parsed.data))
}
pub async fn generate_stream(
&self,
schema: &ModelSchema,
prompt: &str,
messages: Option<&[crate::tasks::generate::Message]>,
context: Option<&str>,
temperature: f64,
max_tokens: usize,
tools: Option<&[serde_json::Value]>,
images: Option<&[ContentBlock]>,
tool_choice: Option<&str>,
parallel_tool_calls: Option<bool>,
response_format: Option<&crate::tasks::generate::ResponseFormat>,
spend_guard: Option<crate::routing_ext::MidStreamSpendGuard>,
) -> Result<tokio::sync::mpsc::Receiver<crate::stream::StreamEvent>, InferenceError> {
let (endpoint, protocol) = extract_remote_endpoint(schema)?;
let handler = crate::protocol::handler_for(protocol);
if !handler.supports_video()
&& (images.is_some_and(|blocks| blocks.iter().any(ContentBlock::is_video))
|| messages.is_some_and(|messages| {
messages.iter().any(|message| match message {
crate::tasks::generate::Message::UserMultimodal { content } => {
content.iter().any(ContentBlock::is_video)
}
_ => false,
})
}))
{
return Err(InferenceError::UnsupportedMode {
mode: "video-content-block",
backend: handler.protocol_name(),
reason: "this remote protocol has no native video input path; route to \
a provider that implements ProtocolHandler::supports_video() (Gemini)",
});
}
if !handler.supports_audio()
&& (images.is_some_and(|blocks| blocks.iter().any(ContentBlock::is_audio))
|| messages.is_some_and(|messages| {
messages.iter().any(|message| match message {
crate::tasks::generate::Message::UserMultimodal { content } => {
content.iter().any(ContentBlock::is_audio)
}
_ => false,
})
}))
{
return Err(InferenceError::UnsupportedMode {
mode: "audio-content-block",
backend: handler.protocol_name(),
reason: "this remote protocol has no native audio input path; route to \
a provider that implements ProtocolHandler::supports_audio() (Gemini)",
});
}
if let Some(err) = response_format_rejection(handler.as_ref(), response_format) {
return Err(err);
}
let lease = self.lease_key(schema, &endpoint).await?;
let api_key = lease.api_key;
let model = request_model_name(schema);
let (messages, system) = if matches!(schema.source, ModelSource::Proprietary { .. }) {
crate::protocol::OpenAiResponsesHandler.build_messages(
messages.unwrap_or(&[]),
prompt,
context,
images,
)
} else {
handler.build_messages(messages.unwrap_or(&[]), prompt, context, images)
};
let built_tools = tools.map(|t| handler.build_tools(t));
let req = crate::protocol::ApiRequest {
model: model.clone(),
messages,
system,
system_stable_prefix: None,
temperature,
max_tokens,
tools: built_tools,
tool_choice: tool_choice.map(str::to_string),
parallel_tool_calls,
stream: true,
budget_tokens: 0,
cache_control: false,
cache_ttl: crate::tasks::generate::CacheTtl::default(),
response_format: response_format.cloned(),
};
if matches!(schema.source, ModelSource::Proprietary { .. }) {
return self
.parslee_assistant_stream_request(&endpoint, &api_key, &req, spend_guard)
.await;
}
let body = handler.build_request_body(&req);
let url = if matches!(protocol, ApiProtocol::Google) {
crate::protocol::google_stream_url(&endpoint, &model, &api_key)
} else if matches!(protocol, ApiProtocol::VertexAi) {
crate::protocol::vertex_stream_url(&endpoint, &model)
} else if matches!(protocol, ApiProtocol::AzureOpenAi) {
let api_version = match &schema.source {
ModelSource::RemoteApi { api_version, .. } => api_version.clone(),
_ => None,
};
let version = api_version.as_deref().unwrap_or("2024-10-21");
format!(
"{}/openai/deployments/{}/chat/completions?api-version={}",
endpoint.trim_end_matches('/'),
model,
version
)
} else {
format_endpoint(&endpoint, chat_path_for(schema, handler.as_ref()))
};
let mut headers = reqwest::header::HeaderMap::new();
for (name, value) in handler.auth_headers(&api_key) {
headers.insert(
reqwest::header::HeaderName::from_bytes(name.as_bytes()).map_err(|e| {
InferenceError::InferenceFailed(format!("auth header name: {e}"))
})?,
value.parse().map_err(|e| {
InferenceError::InferenceFailed(format!("auth header value: {e}"))
})?,
);
}
let send_fut = self.client.post(&url).headers(headers).json(&body).send();
let resp = tokio::time::timeout(std::time::Duration::from_secs(300), send_fut)
.await
.map_err(|_| {
InferenceError::InferenceFailed(
"stream request timed out after 300s (tokio safety timeout)".to_string(),
)
})?
.map_err(|e| self.request_error("HTTP error", &e))?;
let status = resp.status();
if !status.is_success() {
let err_text = resp.text().await.unwrap_or_default();
if is_account_http_status(status.as_u16()) {
return Err(InferenceError::ProviderAccount {
provider: schema.provider.clone(),
status: status.as_u16(),
message: account_error_message(protocol, status.as_u16()),
});
}
if matches!(protocol, ApiProtocol::OpenRouter) {
return Err(openrouter_http_error(status, &err_text, &model));
}
return Err(InferenceError::InferenceFailed(format!(
"API returned {status}: {err_text}"
)));
}
let (tx, rx) = tokio::sync::mpsc::channel::<crate::stream::StreamEvent>(64);
tokio::spawn(async move {
use futures::StreamExt;
let mut byte_stream = resp.bytes_stream();
let mut buffer: Vec<u8> = Vec::new();
let mut spend_guard = spend_guard;
let mut saw_positive_completion = false;
let mut saw_chat_finish_reason = false;
let mut saw_google_finish_reason = false;
while let Some(chunk_result) = byte_stream.next().await {
match chunk_result {
Ok(bytes) => buffer.extend_from_slice(&bytes),
Err(e) => {
tracing::warn!(error = %e, "remote stream transport error mid-response");
let _ = tx
.send(crate::stream::StreamEvent::Error(
"remote stream transport error".to_string(),
))
.await;
return;
}
};
while let Some((pos, sep_len)) = find_sse_separator(&buffer) {
let event_block = String::from_utf8_lossy(&buffer[..pos]).into_owned();
buffer.drain(..pos + sep_len);
let sse_events = crate::stream::parse_sse_lines(&event_block);
for (event_type, data) in sse_events {
if data == "[DONE]" {
if matches!(
protocol,
ApiProtocol::OpenRouter
| ApiProtocol::OpenAiCompat
| ApiProtocol::AzureOpenAi
) && saw_chat_finish_reason
&& !saw_positive_completion
{
saw_positive_completion = true;
if tx
.send(crate::stream::StreamEvent::Done {
text: String::new(),
tool_calls: Vec::new(),
})
.await
.is_err()
{
return;
}
}
continue;
}
let stream_events = handler.parse_stream_event(&event_type, &data);
for evt in stream_events {
if let Some(guard) = spend_guard.as_mut() {
if let crate::stream::StreamEvent::Usage { input_tokens, .. } = &evt
{
guard.note_prompt_tokens(*input_tokens);
}
let delta_len = match &evt {
crate::stream::StreamEvent::TextDelta(t) => Some(t.as_str()),
crate::stream::StreamEvent::ToolCallDelta {
arguments_delta,
..
} => Some(arguments_delta.as_str()),
_ => None,
};
if let Some(delta) = delta_len {
let n =
crate::routing_ext::MidStreamSpendGuard::estimate_tokens(
delta,
);
if let Some(exceeded) = guard.record_tokens(n) {
tracing::warn!(
limit_usd = exceeded.limit_usd,
current_usd = exceeded.current_usd,
"mid-stream spend limit tripped; cancelling stream"
);
let _ = tx.send(evt).await;
let _ = tx
.send(crate::stream::StreamEvent::StopReason(format!(
"spend_limit: {exceeded}"
)))
.await;
let _ = tx
.send(crate::stream::StreamEvent::Error(
"stream cancelled because the spend limit was reached"
.to_string(),
))
.await;
return; }
}
}
if let crate::stream::StreamEvent::StopReason(reason) = &evt {
if matches!(protocol, ApiProtocol::Google | ApiProtocol::VertexAi) {
match google_finish_outcome(reason) {
Ok(()) => saw_google_finish_reason = true,
Err(message) => {
let _ = tx
.send(crate::stream::StreamEvent::Error(
message.to_string(),
))
.await;
return;
}
}
} else if matches!(
protocol,
ApiProtocol::OpenRouter
| ApiProtocol::OpenAiCompat
| ApiProtocol::AzureOpenAi
) {
saw_chat_finish_reason = true;
}
}
if matches!(evt, crate::stream::StreamEvent::Done { .. }) {
saw_positive_completion = true;
}
let terminal_error =
matches!(&evt, crate::stream::StreamEvent::Error(_));
if tx.send(evt).await.is_err() {
return; }
if terminal_error {
return; }
}
if matches!(protocol, ApiProtocol::Google | ApiProtocol::VertexAi)
&& saw_google_finish_reason
&& !saw_positive_completion
{
saw_positive_completion = true;
if tx
.send(crate::stream::StreamEvent::Done {
text: String::new(),
tool_calls: Vec::new(),
})
.await
.is_err()
{
return;
}
}
if protocol == ApiProtocol::Anthropic && event_type == "message_stop" {
saw_positive_completion = true;
if tx
.send(crate::stream::StreamEvent::Done {
text: String::new(),
tool_calls: Vec::new(),
})
.await
.is_err()
{
return;
}
}
}
}
}
if !buffer.is_empty() || !saw_positive_completion {
let _ = tx
.send(crate::stream::StreamEvent::Error(
"remote inference stream ended before provider completion".to_string(),
))
.await;
}
});
Ok(rx)
}
}
impl Default for RemoteBackend {
fn default() -> Self {
Self::new()
}
}
pub(crate) fn is_trust_related(is_connect: bool, is_timeout: bool) -> bool {
is_connect && !is_timeout
}
fn extract_remote_endpoint(schema: &ModelSchema) -> Result<(String, ApiProtocol), InferenceError> {
match &schema.source {
ModelSource::RemoteApi {
endpoint, protocol, ..
} => {
let endpoint = if matches!(protocol, ApiProtocol::OpenRouter)
&& std::env::var("CAR_OPENROUTER_TEST_MODE").as_deref() == Ok("1")
{
std::env::var("OPENROUTER_API_BASE").unwrap_or_else(|_| endpoint.clone())
} else {
endpoint.clone()
};
Ok((endpoint, *protocol))
}
ModelSource::Ollama { host, .. } => Ok((host.clone(), ApiProtocol::OpenAiCompat)),
ModelSource::VllmMlx { endpoint, .. } => Ok((endpoint.clone(), ApiProtocol::OpenAiCompat)),
ModelSource::Proprietary {
provider,
endpoint,
auth,
..
} => Ok((
if provider.eq_ignore_ascii_case("parslee")
&& matches!(auth, ProprietaryAuth::OAuth2Pkce { .. })
{
car_auth::api_base(None)
} else {
endpoint.clone()
},
ApiProtocol::OpenAiCompat,
)),
_ => Err(InferenceError::InferenceFailed(format!(
"model {} is not remote",
schema.id
))),
}
}
fn response_format_rejection(
handler: &dyn crate::protocol::ProtocolHandler,
response_format: Option<&crate::tasks::generate::ResponseFormat>,
) -> Option<InferenceError> {
match response_format {
Some(rf) if !handler.supports_response_format(rf) => {
Some(InferenceError::UnsupportedMode {
mode: match rf {
crate::tasks::generate::ResponseFormat::JsonSchema { .. } => {
"structured-output-json-schema"
}
crate::tasks::generate::ResponseFormat::JsonObject => "structured-output-json",
},
backend: handler.protocol_name(),
reason: "this provider is not wired for a provider-enforced \
response_format under CAR's pinned API version; supply a \
tool whose input schema is your schema plus a forcing \
tool_choice",
})
}
_ => None,
}
}
fn chat_path_for<'a>(
schema: &'a ModelSchema,
handler: &'a dyn crate::protocol::ProtocolHandler,
) -> &'a str {
match &schema.source {
ModelSource::Proprietary { protocol, .. } => protocol.chat_path.as_str(),
_ => handler.endpoint_path(),
}
}
fn request_model_name(schema: &ModelSchema) -> String {
if let Some(canonical_id) = crate::openrouter::canonical_managed_gateway_selector(schema) {
return canonical_id.to_string();
}
match &schema.source {
ModelSource::VllmMlx { model_name, .. } => model_name.clone(),
_ => schema.name.clone(),
}
}
fn format_endpoint(base: &str, path: &str) -> String {
let base = base.trim_end_matches('/');
if base.ends_with(path.trim_start_matches('/')) {
base.to_string()
} else {
format!("{}{}", base, path)
}
}
#[derive(Debug, Deserialize)]
struct OpenAiEmbedResponse {
data: Vec<OpenAiEmbedData>,
}
#[derive(Debug, Deserialize)]
struct OpenAiEmbedData {
embedding: Vec<f32>,
#[serde(default)]
index: usize,
}
fn order_embeddings(mut data: Vec<OpenAiEmbedData>) -> Vec<Vec<f32>> {
data.sort_by_key(|d| d.index);
data.into_iter().map(|d| d.embedding).collect()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::schema::ProprietaryProtocol;
use std::time::Duration;
fn trust_store_independent_client() -> reqwest::Client {
reqwest::Client::builder()
.tls_built_in_root_certs(false)
.build()
.expect("a client with no built-in roots has no certificate work to fail at")
}
fn newline_in_api_key_error() -> reqwest::Error {
trust_store_independent_client()
.post("https://example.invalid/v1/chat/completions")
.header("authorization", "Bearer sk-test-key\n")
.build()
.expect_err("a header value containing a newline must fail to build")
}
#[test]
fn builder_error_surfaces_its_underlying_cause() {
let err = newline_in_api_key_error();
let old_rendering = format!("{err}");
assert!(
!old_rendering.contains("failed to parse header value"),
"reqwest's Display is expected to omit the cause; if this now \
includes it, the whole premise of error_chain has changed. \
got: {old_rendering}"
);
let chain = error_chain(&err);
assert!(
chain.contains("failed to parse header value"),
"the source chain must name the real cause, got: {chain}"
);
}
#[test]
fn builder_error_is_classified_permanent() {
let err = newline_in_api_key_error();
assert!(
err.is_builder(),
"precondition: this is a builder-kind error"
);
assert!(
!reqwest_error_is_transient(&err),
"a request that was never constructed must not be retried"
);
let attempt = TransportAttemptError::from_reqwest("HTTP error", &err);
assert!(!attempt.retryable);
assert!(
attempt
.error
.to_string()
.contains("failed to parse header value"),
"got: {}",
attempt.error
);
}
#[tokio::test]
async fn genuine_transport_error_is_classified_transient() {
let err = trust_store_independent_client()
.get("http://127.0.0.1:1/v1/chat/completions")
.send()
.await
.expect_err("nothing listens on loopback port 1");
assert!(
!err.is_builder(),
"precondition: this reached the transport"
);
assert!(
reqwest_error_is_transient(&err),
"a connect failure is worth the retry budget, got: {err}"
);
assert!(TransportAttemptError::from_reqwest("HTTP error", &err).retryable);
}
#[tokio::test]
async fn transport_error_redacts_url_userinfo() {
let mut err = trust_store_independent_client()
.get("http://127.0.0.1:1/v1/chat/completions")
.send()
.await
.expect_err("nothing listens on loopback port 1");
let url = err
.url_mut()
.expect("transport errors retain the request URL");
url.set_username("leaked-user")
.expect("the test username is valid URL userinfo");
url.set_password(Some("leaked-password"))
.expect("the test password is valid URL userinfo");
assert_eq!(url.username(), "leaked-user");
assert_eq!(url.password(), Some("leaked-password"));
let chain = error_chain(&err);
assert!(
!chain.contains("leaked-user"),
"the username must not survive redaction, got: {chain}"
);
assert!(
!chain.contains("leaked-password"),
"the password must not survive redaction, got: {chain}"
);
}
#[test]
fn timeout_attempt_is_retryable() {
assert!(TransportAttemptError::timeout("request timed out after 300s").retryable);
}
#[test]
fn error_chain_redacts_credentials_in_urls() {
let raw = "error sending request for url \
(https://generativelanguage.googleapis.com/v1beta/models/x:generateContent\
?alt=sse&key=AIzaSyREAL_SECRET_VALUE&pretty=1)";
let scrubbed = redact_sensitive(raw);
assert!(
!scrubbed.contains("AIzaSyREAL_SECRET_VALUE"),
"the API key must not survive redaction, got: {scrubbed}"
);
assert!(scrubbed.contains("key=REDACTED"), "got: {scrubbed}");
assert!(scrubbed.contains("alt=sse"), "got: {scrubbed}");
assert!(scrubbed.contains("pretty=1"), "got: {scrubbed}");
assert_eq!(redact_sensitive("?monkey=banana"), "?monkey=banana");
}
#[derive(Debug)]
struct Link {
message: String,
source: Option<Box<Link>>,
}
impl std::fmt::Display for Link {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.message)
}
}
impl std::error::Error for Link {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
self.source
.as_deref()
.map(|s| s as &(dyn std::error::Error + 'static))
}
}
fn distinct_chain(depth: usize, width: usize) -> Link {
let mut err = Link {
message: format!("cause-{depth}-{}", "z".repeat(width)),
source: None,
};
for i in (0..depth).rev() {
err = Link {
message: format!("cause-{i}-{}", "y".repeat(width)),
source: Some(Box::new(err)),
};
}
err
}
#[test]
fn error_chain_is_length_bounded() {
let chain = error_chain(&distinct_chain(40, 400));
assert!(
chain.len() <= MAX_ERROR_CHAIN_LEN + "... (truncated)".len(),
"chain must stay bounded, got {} bytes",
chain.len()
);
assert!(chain.ends_with("... (truncated)"), "got: {chain}");
}
#[test]
fn error_chain_is_depth_bounded() {
let chain = error_chain(&distinct_chain(40, 0));
assert!(chain.ends_with(": ..."), "got: {chain}");
assert!(
chain.contains(&format!("cause-{}", MAX_ERROR_CHAIN_DEPTH - 1)),
"the last link inside the cap must be rendered, got: {chain}"
);
assert!(
!chain.contains(&format!("cause-{}", MAX_ERROR_CHAIN_DEPTH + 1)),
"links past the cap must be dropped, got: {chain}"
);
}
#[test]
fn error_chain_does_not_repeat_a_restated_cause() {
let err = Link {
message: "inference failed: connection reset".to_string(),
source: Some(Box::new(Link {
message: "connection reset".to_string(),
source: None,
})),
};
assert_eq!(error_chain(&err), "inference failed: connection reset");
}
fn openrouter_schema(endpoint: &str) -> ModelSchema {
let mut schema = crate::openrouter::curated_schemas()
.into_iter()
.find(|schema| schema.id == "openrouter/deepseek/deepseek-v3.2")
.expect("curated personal OpenRouter schema");
let ModelSource::RemoteApi {
endpoint: schema_endpoint,
..
} = &mut schema.source
else {
panic!("personal OpenRouter schema must be remote");
};
*schema_endpoint = endpoint.to_string();
schema
}
fn remote_stream_schema(
endpoint: &str,
protocol: ApiProtocol,
model: &str,
api_key_env: &str,
) -> ModelSchema {
ModelSchema {
id: format!("test/{model}"),
name: model.to_string(),
provider: "test".into(),
family: "test".into(),
version: "1".into(),
capabilities: vec![crate::schema::ModelCapability::Generate],
context_length: 128_000,
max_output_tokens: Some(8_192),
param_count: String::new(),
quantization: None,
performance: Default::default(),
cost: Default::default(),
source: ModelSource::RemoteApi {
endpoint: endpoint.to_string(),
api_key_env: api_key_env.to_string(),
api_key_envs: vec![],
api_version: None,
protocol,
},
tags: vec!["test".into()],
supported_params: vec![],
public_benchmarks: vec![],
trust_tier: crate::schema::TrustTier::Community,
deprecated: false,
available: true,
weights_ready: true,
}
}
#[test]
fn a_content_refusal_is_classified_from_the_tags_the_gateway_sent() {
let events = crate::stream::parse_openai_responses_sse_line(
"error",
r#"{"error":{"message":"content refused","type":"invalid_request_error","code":"content_policy_violation"}}"#,
);
let crate::stream::StreamEvent::Error(detail) = events.first().expect("an error event")
else {
panic!("expected an Error event");
};
let (kind, code) = content_refusal_tags(detail).expect("must classify as a refusal");
assert_eq!(code.as_deref(), Some("content_policy_violation"));
assert_eq!(kind.as_deref(), Some("invalid_request_error"));
}
#[test]
fn ordinary_failures_are_not_mistaken_for_refusals() {
for detail in [
"managed inference failed",
"boom (type=server_error, code=internal_error)",
"slow down (type=rate_limit_error, code=rate_limit_exceeded)",
"the safety content policy doc explains this",
"failed (type=",
] {
assert!(
content_refusal_tags(detail).is_none(),
"must not classify as a refusal: {detail}"
);
}
}
#[test]
fn either_tag_alone_is_enough() {
assert!(content_refusal_tags("nope (code=content_filter)").is_some());
assert!(content_refusal_tags("nope (type=content_policy_violation)").is_some());
assert!(content_refusal_tags("nope (code=moderation_blocked)").is_some());
assert!(content_refusal_tags("nope (code=safety_block)").is_some());
}
#[test]
fn gateway_not_configured_body_is_classified() {
let body = r#"{"error":{"code":"openrouter_not_configured","type":"gateway_error",
"message":"OpenRouter inference is not configured on this Parslee environment."}}"#;
let detail = gateway_unconfigured_detail(reqwest::StatusCode::SERVICE_UNAVAILABLE, body)
.expect("the reported 503 body must classify");
assert!(
detail.contains("not configured on this Parslee environment"),
"the operator-facing message must survive for the caller to surface: {detail}"
);
}
#[test]
fn gateway_not_configured_matches_the_code_not_the_prose_or_status() {
let other_upstream = r#"{"error":{"code":"bedrock_not_configured","type":"gateway_error","message":"nope"}}"#;
assert!(
gateway_unconfigured_detail(reqwest::StatusCode::BAD_GATEWAY, other_upstream).is_some(),
"a different upstream, and a different 5xx, is the same condition"
);
let bare = r#"{"error":{"code":"openrouter_not_configured"}}"#;
assert_eq!(
gateway_unconfigured_detail(reqwest::StatusCode::SERVICE_UNAVAILABLE, bare).as_deref(),
Some("openrouter_not_configured")
);
}
#[test]
fn unrelated_gateway_failures_do_not_classify() {
assert!(gateway_unconfigured_detail(
reqwest::StatusCode::INTERNAL_SERVER_ERROR,
r#"{"error":{"code":"internal_error","type":"gateway_error","message":"boom"}}"#
)
.is_none());
assert!(gateway_unconfigured_detail(
reqwest::StatusCode::FORBIDDEN,
r#"{"error":{"code":"openrouter_not_configured"}}"#
)
.is_none());
assert!(gateway_unconfigured_detail(
reqwest::StatusCode::SERVICE_UNAVAILABLE,
r#"{"error":{"code":"overloaded","message":"openrouter is not configured yet"}}"#
)
.is_none());
assert!(gateway_unconfigured_detail(
reqwest::StatusCode::SERVICE_UNAVAILABLE,
"<html>502</html>"
)
.is_none());
}
#[test]
fn parslee_body_uses_max_output_tokens_not_maxtokens() {
let input = vec![serde_json::json!({"role": "user", "content": "hi"})];
let body = parslee_request_body("u1", input, 42, None, None);
assert_eq!(body["max_output_tokens"], 42);
assert!(
body.get("maxTokens").is_none(),
"maxTokens makes the gateway fail inference and CAR fall back to local"
);
assert_eq!(body["userId"], "u1");
assert!(body.get("temperature").is_none());
assert!(body.get("tools").is_none());
let tools = vec![serde_json::json!({"type": "function", "name": "x"})];
let body2 = parslee_request_body(
"u1",
vec![serde_json::json!({"role": "user", "content": "hi"})],
8,
Some(&tools),
None,
);
assert!(body2["tools"].is_array());
let gateway_model = "parslee/openrouter/open-reasoning";
let gateway_body = parslee_request_body(
"u1",
vec![serde_json::json!({"role": "user", "content": "hi"})],
8,
None,
Some(gateway_model),
);
assert_eq!(gateway_body["model"], gateway_model);
let legacy_body = parslee_request_body(
"u1",
vec![serde_json::json!({"role": "user", "content": "hi"})],
8,
None,
Some("parslee/gpt-5.5"),
);
assert!(legacy_body.get("model").is_none());
for schema in crate::openrouter::builtin_schemas()
.into_iter()
.filter(|schema| schema.id.starts_with("parslee/openrouter/"))
{
assert_eq!(
schema.supported_params,
vec![crate::schema::GenerateParam::MaxTokens],
"{} advertises a request control outside this body builder",
schema.id
);
assert_eq!(
gateway_body["max_output_tokens"], 8,
"the advertised MaxTokens control must reach the managed request"
);
}
}
#[test]
fn parslee_input_preserves_multiturn_tool_history() {
let messages = vec![
serde_json::json!({ "role": "user", "content": "fix the bug" }),
serde_json::json!({
"role": "assistant",
"tool_calls": [{
"id": "call_1",
"type": "function",
"function": { "name": "read_file", "arguments": "{\"path\":\"src.py\"}" }
}]
}),
serde_json::json!({
"role": "tool",
"tool_call_id": "call_1",
"content": "def add(a,b): return a-b"
}),
];
let input = parslee_input_items(Some("you are a coder"), &messages);
assert_eq!(input.len(), 4, "no tool-history item may be dropped");
assert_eq!(input[0]["role"], "system");
assert_eq!(input[1]["role"], "user");
assert_eq!(input[2]["type"], "function_call");
assert_eq!(input[2]["call_id"], "call_1");
assert_eq!(input[2]["name"], "read_file");
assert_eq!(input[2]["arguments"], "{\"path\":\"src.py\"}");
assert_eq!(input[3]["type"], "function_call_output");
assert_eq!(input[3]["call_id"], "call_1");
assert_eq!(input[3]["output"], "def add(a,b): return a-b");
assert!(input[3].get("role").is_none());
}
#[test]
fn managed_incomplete_followed_by_done_remains_terminal_error() {
let events = parse_parslee_responses_sse(
concat!(
"event: response.output_text.delta\n",
"data: {\"delta\":\"partial\"}\n\n",
"event: response.incomplete\n",
"data: {\"response\":{\"status\":\"incomplete\",\"incomplete_details\":{\"reason\":\"max_output_tokens\"},\"usage\":{\"input_tokens\":4,\"output_tokens\":2}}}\n\n",
"data: [DONE]\n\n"
),
);
assert!(events.iter().any(
|event| matches!(event, crate::stream::StreamEvent::StopReason(reason) if reason == "max_output_tokens")
));
assert!(events.iter().any(
|event| matches!(event, crate::stream::StreamEvent::Error(message) if message == "managed inference incomplete: max_output_tokens")
));
assert!(
!events
.iter()
.any(|event| matches!(event, crate::stream::StreamEvent::Done { .. })),
"[DONE] after response.incomplete must not upgrade the turn to success"
);
}
#[tokio::test(flavor = "current_thread")]
async fn managed_buffered_response_rejects_partial_text_followed_by_clean_eof() {
use wiremock::matchers::{header, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let server = MockServer::start().await;
let bearer = "buffered-partial-bearer";
Mock::given(method("GET"))
.and(path("/api/v1/organizations/me"))
.and(header("authorization", format!("Bearer {bearer}")))
.respond_with(
ResponseTemplate::new(200)
.set_body_json(serde_json::json!({"organizationId": "org-partial"})),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/connect/session"))
.respond_with(
ResponseTemplate::new(200).set_body_json(
serde_json::json!({"account": {"email": "partial@example.test"}}),
),
)
.mount(&server)
.await;
Mock::given(method("POST"))
.and(path("/api/v1/orgs/org-partial/inference/responses"))
.respond_with(ResponseTemplate::new(200).set_body_raw(
"event: response.output_text.delta\ndata: {\"delta\":\"must not succeed\"}\n\n",
"text/event-stream",
))
.mount(&server)
.await;
let backend = RemoteBackend::new();
let request = crate::protocol::ApiRequest {
model: "managed-alias".into(),
messages: vec![serde_json::json!({"role": "user", "content": "hello"})],
system: None,
system_stable_prefix: None,
temperature: 0.0,
max_tokens: 64,
tools: None,
tool_choice: None,
parallel_tool_calls: None,
stream: false,
budget_tokens: 0,
cache_control: false,
cache_ttl: crate::tasks::generate::CacheTtl::default(),
response_format: None,
};
let error = backend
.parslee_assistant_request(&server.uri(), bearer, &request)
.await
.expect_err("clean EOF without response.completed must fail closed");
assert!(
error.to_string().contains("response.completed"),
"terminal error must tell callers the completion event was missing: {error}"
);
}
#[tokio::test(flavor = "current_thread")]
async fn managed_stream_response_emits_error_after_partial_text_and_clean_eof() {
use wiremock::matchers::{header, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let server = MockServer::start().await;
let bearer = "stream-partial-bearer";
Mock::given(method("GET"))
.and(path("/api/v1/organizations/me"))
.and(header("authorization", format!("Bearer {bearer}")))
.respond_with(
ResponseTemplate::new(200)
.set_body_json(serde_json::json!({"organizationId": "org-stream-partial"})),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/connect/session"))
.respond_with(
ResponseTemplate::new(200).set_body_json(
serde_json::json!({"account": {"email": "partial@example.test"}}),
),
)
.mount(&server)
.await;
Mock::given(method("POST"))
.and(path("/api/v1/orgs/org-stream-partial/inference/responses"))
.respond_with(ResponseTemplate::new(200).set_body_raw(
"event: response.output_text.delta\ndata: {\"delta\":\"partial\"}\n\n",
"text/event-stream",
))
.mount(&server)
.await;
let backend = RemoteBackend::new();
let request = crate::protocol::ApiRequest {
model: "managed-alias".into(),
messages: vec![serde_json::json!({"role": "user", "content": "hello"})],
system: None,
system_stable_prefix: None,
temperature: 0.0,
max_tokens: 64,
tools: None,
tool_choice: None,
parallel_tool_calls: None,
stream: true,
budget_tokens: 0,
cache_control: false,
cache_ttl: crate::tasks::generate::CacheTtl::default(),
response_format: None,
};
let mut receiver = backend
.parslee_assistant_stream_request(&server.uri(), bearer, &request, None)
.await
.expect("HTTP stream starts");
let mut events = Vec::new();
while let Some(event) = receiver.recv().await {
events.push(event);
}
assert!(
matches!(events.first(), Some(crate::stream::StreamEvent::TextDelta(text)) if text == "partial")
);
assert!(
matches!(events.last(), Some(crate::stream::StreamEvent::Error(message)) if message.contains("response.completed")),
"clean EOF without response.completed must append one terminal error: {events:?}"
);
assert!(
!events
.iter()
.any(|event| matches!(event, crate::stream::StreamEvent::Done { .. })),
"partial managed output must never be upgraded to success"
);
}
#[tokio::test(flavor = "current_thread")]
async fn openrouter_wire_success_stream_errors_concurrency_and_live_key_removal() {
use crate::stream::StreamEvent;
use wiremock::matchers::{body_partial_json, header, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let _credential_scope = crate::openrouter::test_credential_scope();
let _provider_env = crate::openrouter::test_environment_scope_async().await;
crate::openrouter::set_test_credential(None);
let missing_schema = openrouter_schema("http://127.0.0.1:9");
let missing = RemoteBackend::new()
.generate(&missing_schema, "hello", None, 0.0, 32, None)
.await
.expect_err("missing OpenRouter key must fail before HTTP");
assert!(missing.to_string().contains("car keys set openrouter"));
crate::openrouter::set_test_credential(Some("test-openrouter-key"));
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.and(header("authorization", "Bearer test-openrouter-key"))
.and(header("x-openrouter-title", "CAR"))
.and(body_partial_json(serde_json::json!({
"model": "deepseek/deepseek-v3.2"
})))
.respond_with(
ResponseTemplate::new(200)
.set_delay(Duration::from_millis(100))
.set_body_json(serde_json::json!({
"choices": [{
"message": {"role": "assistant", "content": "openrouter-ok"},
"finish_reason": "stop"
}],
"usage": {"prompt_tokens": 2, "completion_tokens": 3}
})),
)
.mount(&server)
.await;
let schema = openrouter_schema(&server.uri());
let backend = RemoteBackend::new();
let (first, second) = tokio::join!(
backend.generate(&schema, "one", None, 0.0, 32, None),
backend.generate(&schema, "two", None, 0.0, 32, None),
);
assert_eq!(first.unwrap(), "openrouter-ok");
assert_eq!(second.unwrap(), "openrouter-ok");
let in_flight = backend.generate(&schema, "in flight", None, 0.0, 32, None);
let remove_key = async {
tokio::time::sleep(Duration::from_millis(25)).await;
crate::openrouter::set_test_credential(None);
};
let (in_flight_result, ()) = tokio::join!(in_flight, remove_key);
assert_eq!(in_flight_result.unwrap(), "openrouter-ok");
let next = backend
.generate(&schema, "next", None, 0.0, 32, None)
.await
.expect_err("the next request must observe key removal");
assert!(next.to_string().contains("car keys set openrouter"));
crate::openrouter::set_test_credential(Some("test-openrouter-key"));
let stream_server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(200).set_body_raw(
concat!(
"data: {\"choices\":[{\"delta\":{\"content\":\"open\"}}]}\n\n",
"data: {\"choices\":[{\"delta\":{\"content\":\"router\"},\"finish_reason\":\"stop\"}]}\n\n",
"data: [DONE]\n\n"
),
"text/event-stream",
))
.mount(&stream_server)
.await;
let stream_schema = openrouter_schema(&stream_server.uri());
let mut receiver = backend
.generate_stream(
&stream_schema,
"stream",
None,
None,
0.0,
32,
None,
None,
None,
None,
None,
None,
)
.await
.unwrap();
let mut streamed = String::new();
let mut clean_stop = false;
let mut completed = false;
while let Some(event) = receiver.recv().await {
match event {
StreamEvent::TextDelta(delta) => streamed.push_str(&delta),
StreamEvent::StopReason(reason) if reason == "stop" => clean_stop = true,
StreamEvent::Done { .. } => completed = true,
_ => {}
}
}
assert_eq!(streamed, "openrouter");
assert!(clean_stop, "stream must expose the provider's clean stop");
assert!(
completed,
"a finish_reason followed by [DONE] is positive completion proof"
);
let stream_error_server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(200).set_body_raw(
concat!(
"data: {\"choices\":[{\"delta\":{\"content\":\"partial\"}}]}\n\n",
"data: {\"error\":{\"code\":402,\"message\":\"private balance details\"}}\n\n",
"data: {\"choices\":[{\"delta\":{\"content\":\"must-not-appear\"}}]}\n\n"
),
"text/event-stream",
))
.mount(&stream_error_server)
.await;
let stream_error_schema = openrouter_schema(&stream_error_server.uri());
let mut receiver = backend
.generate_stream(
&stream_error_schema,
"stream error",
None,
None,
0.0,
32,
None,
None,
None,
None,
None,
None,
)
.await
.unwrap();
let mut errors = Vec::new();
let mut text = String::new();
while let Some(event) = receiver.recv().await {
match event {
StreamEvent::Error(message) => errors.push(message),
StreamEvent::TextDelta(delta) => text.push_str(&delta),
_ => {}
}
}
assert_eq!(errors, ["OpenRouter account is out of credits"]);
assert_eq!(text, "partial");
assert!(!errors[0].contains("private balance"));
for (status, body, expected) in [
(
401,
"sensitive upstream diagnostic must not escape",
"key rejected",
),
(
402,
"sensitive upstream diagnostic must not escape",
"out of credits",
),
(
404,
"sensitive upstream diagnostic must not escape",
"no longer available on OpenRouter",
),
(
400,
r#"{"error":{"message":"No endpoints found for this deprecated model; sensitive upstream diagnostic must not escape"}}"#,
"no longer available on OpenRouter",
),
] {
let error_server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(status).set_body_string(body))
.mount(&error_server)
.await;
let error_schema = openrouter_schema(&error_server.uri());
let error = backend
.generate(&error_schema, "fail", None, 0.0, 32, None)
.await
.expect_err("OpenRouter error should be translated");
let message = error.to_string();
assert!(message.contains(expected), "{status}: {message}");
assert!(!message.contains("sensitive upstream diagnostic"));
if status == 404 || status == 400 {
assert!(message.contains("deepseek/deepseek-v3.2"));
}
}
}
#[tokio::test(flavor = "current_thread")]
async fn personal_openrouter_partial_or_unframed_eof_never_emits_done() {
use crate::stream::StreamEvent;
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let _credential_scope = crate::openrouter::test_credential_scope();
crate::openrouter::set_test_credential(Some("terminal-key"));
for body in [
"data: {\"choices\":[{\"delta\":{\"content\":\"partial\"}}]}\n\n",
"data: {\"choices\":[{\"delta\":{\"content\":\"partial\"},\"finish_reason\":\"stop\"}]}\n\n",
concat!(
"data: {\"choices\":[{\"delta\":{\"content\":\"partial\"}}]}\n\n",
"data: [DONE]\n\n"
),
"data: {\"choices\":[{\"delta\":{\"content\":\"trailing\"}}]}",
] {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(
ResponseTemplate::new(200).set_body_raw(body, "text/event-stream"),
)
.mount(&server)
.await;
let schema = openrouter_schema(&server.uri());
let mut receiver = RemoteBackend::new()
.generate_stream(
&schema, "stream", None, None, 0.0, 32, None, None, None, None, None, None,
)
.await
.unwrap();
let mut events = Vec::new();
while let Some(event) = receiver.recv().await {
events.push(event);
}
assert!(
!events
.iter()
.any(|event| matches!(event, StreamEvent::Done { .. })),
"partial sequence must not complete: {events:?}"
);
assert!(
matches!(events.last(), Some(StreamEvent::Error(message)) if message.contains("before provider completion")),
"partial sequence needs one sanitized terminal error: {events:?}"
);
}
}
#[tokio::test(flavor = "current_thread")]
async fn google_and_vertex_stream_only_deliberate_finish_reasons_complete() {
use crate::stream::StreamEvent;
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let _provider_env = crate::openrouter::test_environment_scope_async().await;
unsafe { std::env::set_var("CAR_GOOGLE_STREAM_MATRIX_KEY", "matrix-key") };
for protocol in [ApiProtocol::Google, ApiProtocol::VertexAi] {
for (reason, should_complete) in [
("STOP", true),
("MAX_TOKENS", true),
("SAFETY", false),
("RECITATION", false),
("BLOCKLIST", false),
("PROHIBITED_CONTENT", false),
("SPII", false),
("MALFORMED_FUNCTION_CALL", false),
("UNKNOWN_PROVIDER_REASON", false),
] {
let server = MockServer::start().await;
let expected_path = match protocol {
ApiProtocol::Google => "/v1beta/models/gemini-test:streamGenerateContent",
ApiProtocol::VertexAi => {
"/publishers/google/models/gemini-test:streamGenerateContent"
}
_ => unreachable!(),
};
Mock::given(method("POST"))
.and(path(expected_path))
.respond_with(ResponseTemplate::new(200).set_body_raw(
format!(
"data: {{\"candidates\":[{{\"content\":{{\"parts\":[{{\"text\":\"matrix\"}}]}},\"finishReason\":\"{reason}\"}}]}}\n\n"
),
"text/event-stream",
))
.mount(&server)
.await;
let schema = remote_stream_schema(
&server.uri(),
protocol,
"gemini-test",
"CAR_GOOGLE_STREAM_MATRIX_KEY",
);
let mut receiver = RemoteBackend::new()
.generate_stream(
&schema, "stream", None, None, 0.0, 32, None, None, None, None, None, None,
)
.await
.expect("Google and Vertex must both reach their streaming transports");
let mut events = Vec::new();
while let Some(event) = receiver.recv().await {
events.push(event);
}
assert_eq!(
events
.iter()
.any(|event| matches!(event, StreamEvent::Done { .. })),
should_complete,
"{protocol:?}/{reason}: {events:?}"
);
assert_eq!(
events
.iter()
.any(|event| matches!(event, StreamEvent::Error(_))),
!should_complete,
"{protocol:?}/{reason}: {events:?}"
);
}
}
unsafe { std::env::remove_var("CAR_GOOGLE_STREAM_MATRIX_KEY") };
}
#[tokio::test(flavor = "current_thread")]
async fn azure_personal_openrouter_and_anthropic_terminal_matrix_stays_green() {
use crate::stream::StreamEvent;
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let _credential_scope = crate::openrouter::test_credential_scope();
let _provider_env = crate::openrouter::test_environment_scope_async().await;
crate::openrouter::set_test_credential(Some("matrix-openrouter-key"));
unsafe {
std::env::set_var("CAR_AZURE_STREAM_MATRIX_KEY", "matrix-azure-key");
std::env::set_var("CAR_ANTHROPIC_STREAM_MATRIX_KEY", "matrix-anthropic-key");
}
for (protocol, model, expected_path, body, key_env) in [
(
ApiProtocol::AzureOpenAi,
"azure-deployment",
"/openai/deployments/azure-deployment/chat/completions",
concat!(
"data: {\"choices\":[{\"delta\":{\"content\":\"azure\"},\"finish_reason\":\"stop\"}]}\n\n",
"data: [DONE]\n\n"
),
"CAR_AZURE_STREAM_MATRIX_KEY",
),
(
ApiProtocol::OpenRouter,
"vendor/personal-model",
"/v1/chat/completions",
concat!(
"data: {\"choices\":[{\"delta\":{\"content\":\"personal\"},\"finish_reason\":\"stop\"}]}\n\n",
"data: [DONE]\n\n"
),
"IGNORED_FOR_OPENROUTER",
),
(
ApiProtocol::Anthropic,
"claude-test",
"/v1/messages",
concat!(
"event: content_block_delta\n",
"data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"anthropic\"}}\n\n",
"event: message_delta\n",
"data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":1}}\n\n",
"event: message_stop\n",
"data: {\"type\":\"message_stop\"}\n\n"
),
"CAR_ANTHROPIC_STREAM_MATRIX_KEY",
),
] {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path(expected_path))
.respond_with(
ResponseTemplate::new(200).set_body_raw(body, "text/event-stream"),
)
.mount(&server)
.await;
let schema = remote_stream_schema(&server.uri(), protocol, model, key_env);
let mut receiver = RemoteBackend::new()
.generate_stream(
&schema, "stream", None, None, 0.0, 32, None, None, None, None, None, None,
)
.await
.unwrap();
let mut events = Vec::new();
while let Some(event) = receiver.recv().await {
events.push(event);
}
assert!(
events
.iter()
.any(|event| matches!(event, StreamEvent::Done { .. })),
"{protocol:?}: {events:?}"
);
assert!(
!events
.iter()
.any(|event| matches!(event, StreamEvent::Error(_))),
"{protocol:?}: {events:?}"
);
}
unsafe {
std::env::remove_var("CAR_AZURE_STREAM_MATRIX_KEY");
std::env::remove_var("CAR_ANTHROPIC_STREAM_MATRIX_KEY");
}
}
#[tokio::test(flavor = "current_thread")]
async fn personal_openrouter_spend_limit_is_error_not_success() {
use crate::stream::StreamEvent;
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let _credential_scope = crate::openrouter::test_credential_scope();
crate::openrouter::set_test_credential(Some("spend-key"));
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(200).set_body_raw(
concat!(
"data: {\"choices\":[{\"delta\":{\"content\":\"cross limit\"}}]}\n\n",
"data: {\"choices\":[{\"delta\":{\"content\":\"must not arrive\"},\"finish_reason\":\"stop\"}]}\n\n",
"data: [DONE]\n\n"
),
"text/event-stream",
))
.mount(&server)
.await;
let schema = openrouter_schema(&server.uri());
let guard = crate::routing_ext::MidStreamSpendGuard::new(Some(0.0), 0.0, 0.0, 0.001);
let mut receiver = RemoteBackend::new()
.generate_stream(
&schema,
"stream",
None,
None,
0.0,
32,
None,
None,
None,
None,
None,
Some(guard),
)
.await
.unwrap();
let mut events = Vec::new();
while let Some(event) = receiver.recv().await {
events.push(event);
}
assert!(events
.iter()
.any(|event| matches!(event, StreamEvent::StopReason(reason) if reason.starts_with("spend_limit:"))));
assert!(
matches!(events.last(), Some(StreamEvent::Error(message)) if message == "stream cancelled because the spend limit was reached"),
"spend cancellation must surface as a sanitized terminal error: {events:?}"
);
assert!(!events
.iter()
.any(|event| matches!(event, StreamEvent::Done { .. })));
}
#[tokio::test(flavor = "current_thread")]
async fn managed_openrouter_spend_limit_is_error_not_success() {
use crate::routing_ext::MidStreamSpendGuard;
use crate::stream::StreamEvent;
use wiremock::matchers::{header, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let _provider_env = crate::openrouter::test_environment_scope_async().await;
let server = MockServer::start().await;
let bearer = "managed-spend-limit-bearer";
unsafe {
std::env::set_var(PARSLEE_ACCESS_TOKEN_ENV, bearer);
std::env::set_var(car_auth::PARSLEE_API_BASE_KEY, server.uri());
}
Mock::given(method("GET"))
.and(path("/api/v1/organizations/me"))
.and(header("authorization", format!("Bearer {bearer}")))
.respond_with(
ResponseTemplate::new(200)
.set_body_json(serde_json::json!({"organizationId": "org-spend"})),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/connect/session"))
.respond_with(
ResponseTemplate::new(200)
.set_body_json(serde_json::json!({"account": {"email": "user@example.test"}})),
)
.mount(&server)
.await;
Mock::given(method("POST"))
.and(path("/api/v1/orgs/org-spend/inference/responses"))
.respond_with(ResponseTemplate::new(200).set_body_raw(
concat!(
"event: response.output_text.delta\n",
"data: {\"delta\":\"costly managed delta\"}\n\n",
"event: response.completed\n",
"data: {\"response\":{\"usage\":{\"input_tokens\":4,\"output_tokens\":4}}}\n\n"
),
"text/event-stream",
))
.mount(&server)
.await;
let schema = crate::openrouter::curated_schemas()
.into_iter()
.find(|schema| schema.id == "parslee/openrouter/frontier-general")
.unwrap();
let guard = MidStreamSpendGuard::new(Some(0.0), 0.0, 0.0, 1.0);
let mut rx = RemoteBackend::new()
.generate_stream(
&schema,
"hello",
None,
None,
0.0,
32,
None,
None,
None,
None,
None,
Some(guard),
)
.await
.unwrap();
let mut events = Vec::new();
while let Some(event) = rx.recv().await {
events.push(event);
}
assert!(events.iter().any(
|event| matches!(event, StreamEvent::StopReason(reason) if reason.starts_with("spend_limit:"))
));
assert!(matches!(
events.last(),
Some(StreamEvent::Error(message))
if message == "stream cancelled because the spend limit was reached"
));
assert!(!events
.iter()
.any(|event| matches!(event, StreamEvent::Done { .. })));
unsafe {
std::env::remove_var(PARSLEE_ACCESS_TOKEN_ENV);
std::env::remove_var(car_auth::PARSLEE_API_BASE_KEY);
}
}
#[test]
fn sse_separator_handles_lf_crlf_and_partials() {
assert_eq!(find_sse_separator(b"data: a\n\nrest"), Some((7, 2)));
assert_eq!(find_sse_separator(b"data: a\r\n\r\nrest"), Some((7, 4)));
assert_eq!(find_sse_separator(b"data: a\n"), None);
let buf = b"a\n\nb\r\n\r\nc";
assert_eq!(find_sse_separator(buf), Some((1, 2)));
}
#[test]
fn response_format_guard_rejects_only_unsupported() {
use crate::tasks::generate::ResponseFormat;
let schema = ResponseFormat::JsonSchema {
schema: serde_json::json!({"type": "object"}),
strict: true,
name: None,
};
let obj = ResponseFormat::JsonObject;
let anthropic = crate::protocol::handler_for(crate::schema::ApiProtocol::Anthropic);
for (format, expected_mode) in [
(&schema, "structured-output-json-schema"),
(&obj, "structured-output-json"),
] {
let rej = response_format_rejection(anthropic.as_ref(), Some(format))
.expect("Anthropic response format must be rejected");
match rej {
InferenceError::UnsupportedMode { mode, backend, .. } => {
assert_eq!(mode, expected_mode);
assert_eq!(backend, "anthropic");
}
other => panic!("expected UnsupportedMode, got {other:?}"),
}
}
assert!(
response_format_rejection(anthropic.as_ref(), None).is_none(),
"no response_format is never rejected"
);
let openai = crate::protocol::handler_for(crate::schema::ApiProtocol::OpenAiCompat);
assert!(response_format_rejection(openai.as_ref(), Some(&schema)).is_none());
assert!(response_format_rejection(openai.as_ref(), Some(&obj)).is_none());
}
#[test]
fn auth_rejection_is_anchored_on_http_status() {
use crate::InferenceError;
assert!(is_auth_rejection(&InferenceError::InferenceFailed(
"Parslee chat failed: HTTP 401: {\"error\":\"expired\"}".into()
)));
assert!(is_auth_rejection(&InferenceError::InferenceFailed(
"Parslee org lookup failed: HTTP 403: forbidden".into()
)));
assert!(!is_auth_rejection(&InferenceError::InferenceFailed(
"API returned 400: your last request 401'd upstream".into()
)));
assert!(!is_auth_rejection(&InferenceError::InferenceFailed(
"HTTP 500: server error".into()
)));
}
#[test]
fn embeddings_resorted_to_input_order() {
let data = vec![
OpenAiEmbedData {
embedding: vec![2.0],
index: 2,
},
OpenAiEmbedData {
embedding: vec![0.0],
index: 0,
},
OpenAiEmbedData {
embedding: vec![1.0],
index: 1,
},
];
assert_eq!(
order_embeddings(data),
vec![vec![0.0], vec![1.0], vec![2.0]]
);
}
#[test]
fn format_endpoint_no_dup() {
assert_eq!(
format_endpoint("https://api.openai.com", "/v1/chat/completions"),
"https://api.openai.com/v1/chat/completions"
);
assert_eq!(
format_endpoint(
"https://api.openai.com/v1/chat/completions",
"/v1/chat/completions"
),
"https://api.openai.com/v1/chat/completions"
);
assert_eq!(
format_endpoint("https://api.openai.com/", "/v1/chat/completions"),
"https://api.openai.com/v1/chat/completions"
);
}
#[test]
fn extract_endpoint_from_remote() {
let schema = ModelSchema {
id: "test/model:v1".into(),
name: "Test".into(),
provider: "test".into(),
family: "test".into(),
version: "1".into(),
capabilities: vec![],
context_length: 4096,
max_output_tokens: None,
param_count: String::new(),
quantization: None,
performance: Default::default(),
cost: Default::default(),
source: ModelSource::RemoteApi {
endpoint: "https://api.test.com".into(),
api_key_env: "NONEXISTENT_TEST_KEY_12345".into(),
api_key_envs: vec![],
api_version: None,
protocol: ApiProtocol::OpenAiCompat,
},
tags: vec![],
supported_params: vec![],
public_benchmarks: vec![],
trust_tier: crate::schema::TrustTier::Curated,
deprecated: false,
available: false,
weights_ready: false,
};
let (endpoint, protocol) = extract_remote_endpoint(&schema).unwrap();
assert_eq!(endpoint, "https://api.test.com");
assert_eq!(protocol, ApiProtocol::OpenAiCompat);
}
#[test]
fn extract_endpoint_non_remote_fails() {
let schema = ModelSchema {
id: "local/model:v1".into(),
name: "Local".into(),
provider: "test".into(),
family: "test".into(),
version: "1".into(),
capabilities: vec![],
context_length: 4096,
max_output_tokens: None,
param_count: String::new(),
quantization: None,
performance: Default::default(),
cost: Default::default(),
source: ModelSource::Local {
hf_repo: "test".into(),
hf_filename: "test".into(),
tokenizer_repo: "test".into(),
},
tags: vec![],
supported_params: vec![],
public_benchmarks: vec![],
trust_tier: crate::schema::TrustTier::Curated,
deprecated: false,
available: false,
weights_ready: false,
};
assert!(extract_remote_endpoint(&schema).is_err());
}
#[test]
fn proprietary_endpoint_is_openai_compat() {
let _environment = crate::openrouter::test_environment_scope();
unsafe {
std::env::set_var(car_auth::PARSLEE_API_BASE_KEY, car_auth::DEFAULT_API_BASE);
}
let schema = ModelSchema {
id: "parslee/advisor".into(),
name: "Parslee Advisor".into(),
provider: "parslee".into(),
family: "parslee".into(),
version: "1".into(),
capabilities: vec![],
context_length: 128_000,
max_output_tokens: None,
param_count: String::new(),
quantization: None,
performance: Default::default(),
cost: Default::default(),
source: ModelSource::Proprietary {
provider: "parslee".into(),
endpoint: "https://api.parslee.ai".into(),
auth: ProprietaryAuth::OAuth2Pkce {
authority: "https://api.parslee.ai".into(),
client_id: "parslee-car".into(),
scopes: vec!["inference:invoke".into()],
},
protocol: ProprietaryProtocol {
chat_path: "/api/v1/orgs/{orgId}/inference/responses".into(),
content_type: "application/json".into(),
streaming: false,
extra_headers: Default::default(),
},
},
tags: vec![],
supported_params: vec![],
public_benchmarks: vec![],
trust_tier: crate::schema::TrustTier::Curated,
deprecated: false,
available: true,
weights_ready: true,
};
let (endpoint, protocol) = extract_remote_endpoint(&schema).unwrap();
assert_eq!(endpoint, "https://api.parslee.ai");
assert_eq!(protocol, ApiProtocol::OpenAiCompat);
unsafe {
std::env::remove_var(car_auth::PARSLEE_API_BASE_KEY);
}
}
#[test]
fn managed_parslee_endpoint_is_resolved_at_request_time() {
let _environment = crate::openrouter::test_environment_scope();
unsafe {
std::env::set_var(
car_auth::PARSLEE_API_BASE_KEY,
"https://staging-api.parslee.ai/",
);
}
let schema = ModelSchema {
id: "parslee/openrouter/open-fast".into(),
name: "Parslee OpenRouter Fast".into(),
provider: "parslee".into(),
family: "openrouter".into(),
version: "1".into(),
capabilities: vec![],
context_length: 128_000,
max_output_tokens: None,
param_count: String::new(),
quantization: None,
performance: Default::default(),
cost: Default::default(),
source: ModelSource::Proprietary {
provider: "parslee".into(),
endpoint: car_auth::DEFAULT_API_BASE.into(),
auth: ProprietaryAuth::OAuth2Pkce {
authority: car_auth::DEFAULT_API_BASE.into(),
client_id: "parslee-car".into(),
scopes: vec!["inference:invoke".into()],
},
protocol: ProprietaryProtocol {
chat_path: "/api/v1/orgs/{orgId}/inference/responses".into(),
content_type: "application/json".into(),
streaming: true,
extra_headers: Default::default(),
},
},
tags: vec![],
supported_params: vec![],
public_benchmarks: vec![],
trust_tier: crate::schema::TrustTier::Curated,
deprecated: false,
available: true,
weights_ready: true,
};
let (endpoint, protocol) = extract_remote_endpoint(&schema).unwrap();
assert_eq!(endpoint, "https://staging-api.parslee.ai");
assert_eq!(protocol, ApiProtocol::OpenAiCompat);
unsafe {
std::env::remove_var(car_auth::PARSLEE_API_BASE_KEY);
}
}
#[test]
fn builtin_parslee_aliases_describe_shipped_responses_route() {
let catalog = crate::registry::builtin_catalog();
let aliases = ["parslee/fast", "parslee/reasoning", "parslee/advisor"];
for id in aliases {
let schema = catalog
.iter()
.find(|s| s.id == id)
.unwrap_or_else(|| panic!("builtin catalog is missing managed alias {id}"));
match &schema.source {
ModelSource::Proprietary { protocol, .. } => {
assert_eq!(
protocol.chat_path, "/api/v1/orgs/{orgId}/inference/responses",
"{id} must point at the shipped inference/responses route, \
not the retired /chat/stream stopgap"
);
}
other => panic!("{id} should be a Proprietary managed provider, got {other:?}"),
}
}
}
#[tokio::test(flavor = "current_thread")]
async fn managed_openrouter_stream_replays_provider_items_and_tool_history_on_second_turn() {
use crate::stream::StreamEvent;
use wiremock::matchers::{body_partial_json, header, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let server = MockServer::start().await;
let _provider_env = crate::openrouter::test_environment_scope_async().await;
let bearer = "managed-stream-bearer";
unsafe {
std::env::set_var(PARSLEE_ACCESS_TOKEN_ENV, bearer);
std::env::set_var(car_auth::PARSLEE_API_BASE_KEY, server.uri());
}
Mock::given(method("GET"))
.and(path("/api/v1/organizations/me"))
.and(header("authorization", format!("Bearer {bearer}")))
.respond_with(
ResponseTemplate::new(200)
.set_body_json(serde_json::json!({"organizationId": "org-test"})),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/connect/session"))
.respond_with(
ResponseTemplate::new(200)
.set_body_json(serde_json::json!({"account": {"email": "user@example.test"}})),
)
.mount(&server)
.await;
Mock::given(method("POST"))
.and(path("/api/v1/orgs/org-test/inference/responses"))
.and(body_partial_json(serde_json::json!({
"model": "parslee/openrouter/frontier-general",
"store": false,
"include": ["reasoning.encrypted_content"]
})))
.respond_with(ResponseTemplate::new(200).set_body_raw(
include_str!("../tests/fixtures/parslee-openrouter-reasoning-roundtrip.sse"),
"text/event-stream",
))
.expect(2)
.mount(&server)
.await;
let schema = crate::openrouter::curated_schemas()
.into_iter()
.find(|schema| schema.id == "parslee/openrouter/frontier-general")
.unwrap();
let mut rx = RemoteBackend::new()
.generate_stream(
&schema, "hello", None, None, 0.0, 32, None, None, None, None, None, None,
)
.await
.expect("managed stream should start");
let mut text = String::new();
let mut usage = None;
let mut provider_items = Vec::new();
while let Some(event) = rx.recv().await {
match event {
StreamEvent::TextDelta(delta) => text.push_str(&delta),
StreamEvent::ProviderOutputItem(item) => provider_items.push(item),
StreamEvent::Usage {
input_tokens,
output_tokens,
..
} => usage = Some((input_tokens, output_tokens)),
_ => {}
}
}
assert_eq!(text, "first answer");
assert_eq!(usage, Some((17, 9)));
let expected_reasoning = serde_json::json!({
"type": "reasoning",
"id": "rs_car_roundtrip",
"status": "completed",
"summary": [{"type": "summary_text", "text": "safe summary"}],
"encrypted_content": "opaque-encrypted-reasoning",
});
assert_eq!(provider_items, vec![expected_reasoning.clone()]);
let history = vec![
crate::tasks::generate::Message::User {
content: "first".into(),
},
crate::tasks::generate::Message::ProviderOutputItems {
protocol: crate::protocol::OPENAI_RESPONSES_PROTOCOL.into(),
items: provider_items,
},
crate::tasks::generate::Message::Assistant {
content: text,
tool_calls: vec![crate::tasks::generate::ToolCall {
id: Some("call-stream-history".into()),
name: "lookup".into(),
arguments: HashMap::from([(
"query".into(),
serde_json::Value::String("answer".into()),
)]),
}],
thinking: Vec::new(),
},
crate::tasks::generate::Message::ToolResult {
tool_use_id: "call-stream-history".into(),
content: "tool output".into(),
provenance: Default::default(),
},
crate::tasks::generate::Message::User {
content: "continue".into(),
},
];
let mut second = RemoteBackend::new()
.generate_stream(
&schema,
"",
Some(&history),
None,
0.0,
32,
None,
None,
None,
None,
None,
None,
)
.await
.expect("managed second streaming turn should start");
while second.recv().await.is_some() {}
let requests = server.received_requests().await.unwrap();
let posts: Vec<serde_json::Value> = requests
.iter()
.filter(|request| {
request.method.as_str() == "POST"
&& request.url.path() == "/api/v1/orgs/org-test/inference/responses"
})
.map(|request| serde_json::from_slice(&request.body).unwrap())
.collect();
assert_eq!(posts.len(), 2);
let second_input = posts[1]["input"].as_array().unwrap();
assert!(second_input.contains(&expected_reasoning));
assert!(second_input.iter().any(|item| {
item["type"] == "function_call"
&& item["call_id"] == "call-stream-history"
&& item["name"] == "lookup"
}));
assert!(second_input.iter().any(|item| {
item["type"] == "function_call_output"
&& item["call_id"] == "call-stream-history"
&& item["output"] == "tool output"
}));
assert!(second_input
.iter()
.any(|item| item["role"] == "user" && item["content"] == "continue"));
unsafe {
std::env::remove_var(PARSLEE_ACCESS_TOKEN_ENV);
std::env::remove_var(car_auth::PARSLEE_API_BASE_KEY);
}
}
#[tokio::test]
async fn lease_key_proprietary_oauth2_resolves_env_override() {
let _provider_env = crate::openrouter::test_environment_scope_async().await;
std::env::set_var(PARSLEE_ACCESS_TOKEN_ENV, "test-bearer-abc123");
let schema = ModelSchema {
id: "parslee/advisor".into(),
name: "Parslee Advisor".into(),
provider: "parslee".into(),
family: "parslee".into(),
version: "1".into(),
capabilities: vec![],
context_length: 128_000,
max_output_tokens: None,
param_count: String::new(),
quantization: None,
performance: Default::default(),
cost: Default::default(),
source: ModelSource::Proprietary {
provider: "parslee".into(),
endpoint: "https://api.parslee.ai".into(),
auth: ProprietaryAuth::OAuth2Pkce {
authority: "https://api.parslee.ai".into(),
client_id: "parslee-car".into(),
scopes: vec!["inference:invoke".into()],
},
protocol: ProprietaryProtocol {
chat_path: "/api/v1/orgs/{orgId}/inference/responses".into(),
content_type: "application/json".into(),
streaming: false,
extra_headers: Default::default(),
},
},
tags: vec![],
supported_params: vec![],
public_benchmarks: vec![],
trust_tier: crate::schema::TrustTier::Curated,
deprecated: false,
available: true,
weights_ready: true,
};
let lease = RemoteBackend::new()
.lease_key(&schema, "https://api.parslee.ai")
.await
.expect("proprietary OAuth2 lease should resolve the env-override token");
assert_eq!(lease.api_key, "test-bearer-abc123");
std::env::remove_var(PARSLEE_ACCESS_TOKEN_ENV);
}
#[tokio::test]
async fn non_parslee_oauth_schema_cannot_lease_or_send_parslee_bearer() {
let _provider_env = crate::openrouter::test_environment_scope_async().await;
let server = wiremock::MockServer::start().await;
unsafe {
std::env::set_var(PARSLEE_ACCESS_TOKEN_ENV, "must-not-leave-process");
}
let schema = ModelSchema {
id: "community/custom-oauth".into(),
name: "Custom OAuth".into(),
provider: "community".into(),
family: "community".into(),
version: "1".into(),
capabilities: vec![crate::schema::ModelCapability::Generate],
context_length: 8_192,
max_output_tokens: None,
param_count: "api".into(),
quantization: None,
performance: Default::default(),
cost: Default::default(),
source: ModelSource::Proprietary {
provider: "community".into(),
endpoint: server.uri(),
auth: ProprietaryAuth::OAuth2Pkce {
authority: "https://untrusted.example/authorize".into(),
client_id: "untrusted-client".into(),
scopes: vec!["inference".into()],
},
protocol: ProprietaryProtocol {
chat_path: "/custom/chat".into(),
content_type: "application/json".into(),
streaming: false,
extra_headers: Default::default(),
},
},
tags: vec![],
supported_params: vec![],
public_benchmarks: vec![],
trust_tier: crate::schema::TrustTier::Community,
deprecated: false,
available: true,
weights_ready: true,
};
let lease_error = RemoteBackend::new()
.lease_key(&schema, &server.uri())
.await
.expect_err("non-Parslee OAuth must be rejected before credential resolution");
assert!(
lease_error
.to_string()
.contains("OAuth2 PKCE credentials are supported only for provider 'parslee'"),
"{lease_error}"
);
let inference_error = RemoteBackend::new()
.generate(&schema, "must not send", None, 0.0, 16, None)
.await
.expect_err("explicit non-Parslee OAuth inference must fail closed");
assert!(
inference_error
.to_string()
.contains("OAuth2 PKCE credentials are supported only for provider 'parslee'"),
"{inference_error}"
);
assert!(
server.received_requests().await.unwrap().is_empty(),
"the rejected schema must not send the Parslee bearer or make any network request"
);
unsafe {
std::env::remove_var(PARSLEE_ACCESS_TOKEN_ENV);
}
}
#[test]
fn parse_openai_embed_response() {
let json = r#"{"data":[{"embedding":[0.1,0.2,0.3]}]}"#;
let resp: OpenAiEmbedResponse = serde_json::from_str(json).unwrap();
assert_eq!(resp.data[0].embedding, vec![0.1, 0.2, 0.3]);
}
#[test]
fn request_model_name_uses_vllm_server_model() {
let schema = ModelSchema {
id: "vllm-mlx/test".into(),
name: "Display Name".into(),
provider: "test".into(),
family: "test".into(),
version: "1".into(),
capabilities: vec![],
context_length: 4096,
max_output_tokens: None,
param_count: String::new(),
quantization: None,
performance: Default::default(),
cost: Default::default(),
source: ModelSource::VllmMlx {
endpoint: "http://localhost:8000".into(),
model_name: "mlx-community/Actual-Model".into(),
},
tags: vec![],
supported_params: vec![],
public_benchmarks: vec![],
trust_tier: crate::schema::TrustTier::Curated,
deprecated: false,
available: true,
weights_ready: true,
};
assert_eq!(request_model_name(&schema), "mlx-community/Actual-Model");
}
#[test]
fn managed_gateway_request_body_uses_canonical_id_when_display_name_drifts() {
let mut schema = crate::openrouter::curated_schemas()
.into_iter()
.find(|schema| schema.id == "parslee/openrouter/frontier-general")
.expect("managed frontier alias");
schema.name = "attacker-controlled-upstream-selector".into();
let selector = request_model_name(&schema);
let body = parslee_request_body(
"u1",
vec![serde_json::json!({"role": "user", "content": "hi"})],
8,
None,
Some(&selector),
);
assert_eq!(selector, "parslee/openrouter/frontier-general");
assert_eq!(body["model"], "parslee/openrouter/frontier-general");
}
#[test]
fn truncate_prompt_fits_returns_unchanged() {
let prompt = "short prompt";
let result = truncate_prompt_to_fit(prompt, None, None, 16, 0, 256);
assert_eq!(result, prompt);
}
#[test]
fn truncate_prompt_cjk_mid_codepoint_does_not_panic() {
let prompt: String = std::iter::repeat_n('\u{4E16}', 200).collect();
let result = truncate_prompt_to_fit(&prompt, None, None, 20, 0, 209);
assert!(result.starts_with("[...truncated...]"));
let kept = result.strip_prefix("[...truncated...]\n").unwrap();
assert!(!kept.is_empty());
}
#[test]
fn truncate_prompt_accounts_for_context_and_tools() {
let prompt = "line one\nline two\nline three\n".repeat(50);
let tools = vec![serde_json::json!({"name": "demo_tool"})];
let result = truncate_prompt_to_fit(&prompt, Some("ctx"), Some(&tools), 20, 0, 240);
assert!(result.starts_with("[...truncated...]"));
}
#[test]
fn truncate_prompt_reserves_media_tokens() {
let prompt = "word ".repeat(80);
let untouched = truncate_prompt_to_fit(&prompt, None, None, 20, 0, 500);
assert_eq!(untouched, prompt);
let truncated = truncate_prompt_to_fit(&prompt, None, None, 20, 300, 500);
assert!(truncated.starts_with("[...truncated...]"));
}
#[test]
fn parslee_responses_sse_concatenates_text_and_usage() {
let raw = "event: response.output_text.delta\ndata: {\"delta\":\"Hello\"}\n\n\
event: response.output_text.delta\ndata: {\"delta\":\", world\"}\n\n\
event: response.completed\ndata: {\"response\":{\"usage\":{\"input_tokens\":10,\"output_tokens\":3}}}\n\n";
let mut acc = crate::stream::StreamAccumulator::default();
for ev in parse_parslee_responses_sse(raw) {
acc.push(&ev);
}
let (text, tool_calls, usage, _stop) = acc.finish_with_usage();
assert_eq!(text, "Hello, world");
assert!(tool_calls.is_empty());
let usage = usage.expect("usage reported");
assert_eq!(usage.prompt_tokens, 10);
assert_eq!(usage.completion_tokens, 3);
}
#[test]
fn parslee_responses_sse_parses_tool_call() {
let raw = "event: response.output_item.added\n\
data: {\"output_index\":0,\"item\":{\"type\":\"function_call\",\"name\":\"get_weather\",\"call_id\":\"call_1\"}}\n\n\
event: response.function_call_arguments.delta\n\
data: {\"output_index\":0,\"delta\":\"{\\\"city\\\":\\\"NYC\\\"}\"}\n\n\
event: response.completed\ndata: {\"response\":{\"usage\":{\"input_tokens\":5,\"output_tokens\":2}}}\n\n";
let mut acc = crate::stream::StreamAccumulator::default();
for ev in parse_parslee_responses_sse(raw) {
acc.push(&ev);
}
let (_text, tool_calls) = acc.finish();
assert_eq!(tool_calls.len(), 1);
assert_eq!(tool_calls[0].name, "get_weather");
assert_eq!(tool_calls[0].id.as_deref(), Some("call_1"));
assert_eq!(
tool_calls[0].arguments.get("city").and_then(|v| v.as_str()),
Some("NYC")
);
}
#[test]
fn parslee_responses_sse_ignores_unparseable_frames() {
let raw = "event: response.output_text.delta\ndata: not-json\n\n\
event: response.output_text.delta\ndata: {\"delta\":\"ok\"}\n\n";
let mut acc = crate::stream::StreamAccumulator::default();
for ev in parse_parslee_responses_sse(raw) {
acc.push(&ev);
}
assert_eq!(acc.finish().0, "ok");
}
#[tokio::test]
async fn parslee_identity_resolves_org_and_user() {
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/api/v1/organizations/me"))
.respond_with(
ResponseTemplate::new(200)
.set_body_json(serde_json::json!({ "organizationId": "org_test" })),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/connect/session"))
.respond_with(
ResponseTemplate::new(200).set_body_json(
serde_json::json!({ "account": { "email": "user@example.com" } }),
),
)
.mount(&server)
.await;
let backend = RemoteBackend::new();
let (org, user) = backend
.parslee_identity(&server.uri(), "ident-ok-bearer")
.await
.expect("identity should resolve");
assert_eq!(org, "org_test");
assert_eq!(user, "user@example.com");
}
#[tokio::test]
async fn parslee_identity_cache_is_isolated_by_api_base() {
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let first = MockServer::start().await;
let second = MockServer::start().await;
for (server, org) in [(&first, "org-first"), (&second, "org-second")] {
Mock::given(method("GET"))
.and(path("/api/v1/organizations/me"))
.respond_with(
ResponseTemplate::new(200)
.set_body_json(serde_json::json!({ "organizationId": org })),
)
.mount(server)
.await;
Mock::given(method("GET"))
.and(path("/connect/session"))
.respond_with(ResponseTemplate::new(200).set_body_json(
serde_json::json!({ "account": { "email": "same@example.test" } }),
))
.mount(server)
.await;
}
let backend = RemoteBackend::new();
let first_identity = backend
.parslee_identity(&first.uri(), "same-bearer-across-environments")
.await
.unwrap();
let second_identity = backend
.parslee_identity(&second.uri(), "same-bearer-across-environments")
.await
.unwrap();
assert_eq!(first_identity.0, "org-first");
assert_eq!(second_identity.0, "org-second");
}
#[tokio::test]
async fn parslee_identity_surfaces_org_lookup_401() {
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/api/v1/organizations/me"))
.respond_with(ResponseTemplate::new(401).set_body_string("Authentication required"))
.mount(&server)
.await;
let backend = RemoteBackend::new();
let err = backend
.parslee_identity(&server.uri(), "ident-401-bearer")
.await
.expect_err("a 401 org lookup must error");
let msg = err.to_string();
assert!(msg.contains("org lookup failed"), "got: {msg}");
assert!(msg.contains("HTTP 401"), "got: {msg}");
assert!(
is_auth_rejection(&err),
"401 must be classified as auth rejection"
);
}
use crate::tls_client::test_seam::{
mixed_cert_pem, TrustStoreScope, UNPARSEABLE_CERT_PEM, VALID_CERT_PEM,
};
use crate::tls_client::{TrustFallback, HUGGINGFACE_PROBE, REMOTE_BACKEND, VLLM_HEALTH_CHECK};
fn closed_loopback_url() -> String {
format!("{}/", closed_loopback_endpoint())
}
fn closed_loopback_endpoint() -> String {
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind ephemeral port");
let port = listener.local_addr().expect("local addr").port();
drop(listener);
format!("https://127.0.0.1:{port}")
}
#[tokio::test]
async fn empty_and_mixed_trust_stores_build_without_degrading() {
let mut scope = TrustStoreScope::acquire_async("").await;
crate::tls_client::reset_sites_for_test();
let backend = RemoteBackend::new();
assert!(
backend.tls_degradation.is_none(),
"an empty certificate store must not degrade anything"
);
scope.repoint(&mixed_cert_pem());
crate::tls_client::reset_sites_for_test();
let backend = RemoteBackend::new();
assert!(
backend.tls_degradation.is_none(),
"one readable certificate alongside an unreadable one must not degrade"
);
assert!(
crate::tls_client::last_degradation(&REMOTE_BACKEND).is_none(),
"nothing degraded, so nothing should have been recorded or logged"
);
}
#[tokio::test]
async fn remote_backend_degrades_to_public_cas_instead_of_panicking() {
let scope = TrustStoreScope::acquire_async(UNPARSEABLE_CERT_PEM).await;
scope.assert_breaks_client_construction();
crate::tls_client::reset_sites_for_test();
let backend = RemoteBackend::new();
let degradation = backend
.tls_degradation
.clone()
.expect("a broken trust store must be recorded, not swallowed");
assert_eq!(degradation.fallback, TrustFallback::PublicCaOnly);
assert!(
!degradation.source.is_empty(),
"the original reqwest builder error must be kept"
);
let err = backend
.client
.get(closed_loopback_url())
.send()
.await
.expect_err("nothing listens on that port");
assert!(
err.is_connect(),
"expected a connect-class error, got: {err}"
);
let surfaced = backend
.request_error("remote call failed", &err)
.to_string();
assert!(
surfaced.contains("the OS certificate store failed to load"),
"got: {surfaced}"
);
assert!(
surfaced.contains("restart the daemon"),
"the operator must be told how to recover; got: {surfaced}"
);
}
#[tokio::test]
async fn degraded_backend_carries_its_reason_into_real_inference_requests() {
const KEY_ENV: &str = "CAR_TLS_DEGRADATION_TEST_KEY";
let scope = TrustStoreScope::acquire_async(UNPARSEABLE_CERT_PEM).await;
scope.assert_breaks_client_construction();
crate::tls_client::reset_sites_for_test();
std::env::set_var(KEY_ENV, "test-key");
let backend = RemoteBackend::new();
assert!(
backend.tls_degradation.is_some(),
"setup: must have degraded"
);
let schema = remote_stream_schema(
&closed_loopback_endpoint(),
ApiProtocol::OpenAiCompat,
"test-model",
KEY_ENV,
);
let streamed = backend
.generate_stream(
&schema, "hello", None, None, 0.0, 32, None, None, None, None, None, None,
)
.await
.expect_err("nothing listens on that port")
.to_string();
assert!(
streamed.contains("the OS certificate store failed to load"),
"the streaming inference path dropped the degradation reason: {streamed}"
);
assert!(
streamed.contains("restart the daemon"),
"the operator must be told how inference recovers: {streamed}"
);
let completed = backend
.generate(&schema, "hello", None, 0.0, 32, None)
.await
.expect_err("nothing listens on that port")
.to_string();
assert!(
completed.contains("the OS certificate store failed to load"),
"the completion inference path dropped the degradation reason: {completed}"
);
let mut parslee_schema = remote_stream_schema(
&closed_loopback_endpoint(),
ApiProtocol::OpenAiCompat,
"parslee/test-model",
KEY_ENV,
);
parslee_schema.provider = "parslee".into();
parslee_schema.source = ModelSource::Proprietary {
provider: "parslee".into(),
endpoint: closed_loopback_endpoint(),
auth: ProprietaryAuth::BearerTokenEnv {
env_var: KEY_ENV.into(),
},
protocol: ProprietaryProtocol::default(),
};
let managed = backend
.generate(&parslee_schema, "hello", None, 0.0, 32, None)
.await
.expect_err("the managed identity lookup cannot connect")
.to_string();
assert!(
managed.contains("the OS certificate store failed to load"),
"the managed inference path dropped the degradation reason: {managed}"
);
std::env::remove_var(KEY_ENV);
}
#[tokio::test]
async fn remote_backend_default_matches_new() {
let scope = TrustStoreScope::acquire_async(UNPARSEABLE_CERT_PEM).await;
scope.assert_breaks_client_construction();
crate::tls_client::reset_sites_for_test();
let explicit = RemoteBackend::new();
let defaulted = RemoteBackend::default();
assert_eq!(
explicit.tls_degradation.map(|d| d.fallback),
defaulted.tls_degradation.map(|d| d.fallback),
"`Default` delegates to `new`, so the two must agree on degradation"
);
}
#[tokio::test]
async fn tls_floor_rung_builds_under_a_broken_trust_store() {
let scope = TrustStoreScope::acquire_async(UNPARSEABLE_CERT_PEM).await;
scope.assert_breaks_client_construction();
assert!(
reqwest::Client::builder()
.tls_built_in_native_certs(false)
.build()
.is_ok(),
"rung 2 (built-in public CAs only) must build"
);
assert!(
reqwest::Client::builder()
.tls_built_in_root_certs(false)
.build()
.is_ok(),
"rung 3 (the floor) must build — it is what keeps construction panic-free"
);
}
#[tokio::test]
async fn plain_transport_failures_are_not_reported_as_certificate_problems() {
assert!(is_trust_related(true, false));
assert!(
!is_trust_related(true, true),
"a connect *timeout* is not a trust problem"
);
assert!(!is_trust_related(false, true));
assert!(!is_trust_related(false, false));
let scope = TrustStoreScope::acquire_async(VALID_CERT_PEM).await;
let healthy = RemoteBackend::new();
assert!(healthy.tls_degradation.is_none());
let err = healthy
.client
.get(closed_loopback_url())
.send()
.await
.expect_err("nothing listens on that port");
let surfaced = healthy
.request_error("remote call failed", &err)
.to_string();
assert!(
!surfaced.contains("certificate store"),
"a healthy backend must not blame certificates; got: {surfaced}"
);
drop(scope);
let scope = TrustStoreScope::acquire_async(UNPARSEABLE_CERT_PEM).await;
scope.assert_breaks_client_construction();
crate::tls_client::reset_sites_for_test();
let degraded = RemoteBackend::new();
assert!(degraded.tls_degradation.is_some());
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("GET"))
.and(wiremock::matchers::path("/ok"))
.respond_with(wiremock::ResponseTemplate::new(200).set_body_string("fine"))
.mount(&server)
.await;
wiremock::Mock::given(wiremock::matchers::method("GET"))
.and(wiremock::matchers::path("/denied"))
.respond_with(wiremock::ResponseTemplate::new(401))
.mount(&server)
.await;
let ok = degraded
.client
.get(format!("{}/ok", server.uri()))
.send()
.await
.expect("a degraded client still works against reachable endpoints");
assert!(ok.status().is_success());
assert_eq!(
ok.text().await.expect("body"),
"fine",
"a successful request carries no degradation text"
);
let denied = degraded
.client
.get(format!("{}/denied", server.uri()))
.send()
.await
.expect("a 401 is a response, not a transport failure");
assert_eq!(denied.status().as_u16(), 401);
}
#[test]
fn trust_store_scope_restores_the_previous_override_even_on_panic() {
const FILE_VAR: &str = "SSL_CERT_FILE";
const DIR_VAR: &str = "SSL_CERT_DIR";
let lock = crate::openrouter::test_environment_scope();
let before_file = std::env::var_os(FILE_VAR);
let before_dir = std::env::var_os(DIR_VAR);
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _scope = TrustStoreScope::with_lock(lock, UNPARSEABLE_CERT_PEM);
assert!(
std::env::var_os(FILE_VAR).is_some(),
"the scope must install the override"
);
assert!(
std::env::var_os(DIR_VAR).is_none(),
"the scope must suppress SSL_CERT_DIR so host roots cannot make \
the failure fixture pass"
);
panic!("simulated mid-test failure");
}));
assert!(outcome.is_err(), "the simulated failure must have unwound");
let _relock = crate::openrouter::test_environment_scope();
assert_eq!(
std::env::var_os(FILE_VAR),
before_file,
"the guard must restore the previous SSL_CERT_FILE value"
);
assert_eq!(
std::env::var_os(DIR_VAR),
before_dir,
"the guard must restore the previous SSL_CERT_DIR value so a \
panicking test cannot poison the rest of the suite"
);
}
struct CapturingSubscriber {
events: std::sync::Arc<std::sync::Mutex<Vec<(tracing::Level, String)>>>,
}
#[derive(Default)]
struct FieldCollector(String);
impl tracing::field::Visit for FieldCollector {
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
use std::fmt::Write as _;
let _ = write!(self.0, " {}={:?}", field.name(), value);
}
}
impl tracing::Subscriber for CapturingSubscriber {
fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool {
true
}
fn new_span(&self, _attrs: &tracing::span::Attributes<'_>) -> tracing::span::Id {
tracing::span::Id::from_u64(1)
}
fn record(&self, _span: &tracing::span::Id, _values: &tracing::span::Record<'_>) {}
fn record_follows_from(&self, _span: &tracing::span::Id, _follows: &tracing::span::Id) {}
fn event(&self, event: &tracing::Event<'_>) {
let mut fields = FieldCollector::default();
event.record(&mut fields);
self.events
.lock()
.unwrap_or_else(|p| p.into_inner())
.push((*event.metadata().level(), fields.0));
}
fn enter(&self, _span: &tracing::span::Id) {}
fn exit(&self, _span: &tracing::span::Id) {}
}
#[test]
fn every_site_warns_once_at_warn_level_and_keeps_its_own_record() {
let scope = TrustStoreScope::acquire_blocking(UNPARSEABLE_CERT_PEM);
scope.assert_breaks_client_construction();
crate::tls_client::reset_sites_for_test();
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("current-thread runtime");
let events = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
tracing::subscriber::with_default(
CapturingSubscriber {
events: events.clone(),
},
|| {
let _ = RemoteBackend::new();
let _ = RemoteBackend::new();
let _ = crate::upgrade::HuggingFaceProbe::new();
let _ = crate::upgrade::HuggingFaceProbe::new();
runtime.block_on(async {
let _ = crate::vllm_mlx::health_check("http://127.0.0.1:1").await;
let _ = crate::vllm_mlx::health_check("http://127.0.0.1:1").await;
});
},
);
let captured = events.lock().unwrap_or_else(|p| p.into_inner()).clone();
let warnings: Vec<&String> = captured
.iter()
.filter(|(level, _)| *level <= tracing::Level::WARN)
.map(|(_, message)| message)
.collect();
assert_eq!(
warnings.len(),
3,
"exactly one warning per site, logged once each; captured: {captured:?}"
);
for site in [&REMOTE_BACKEND, &HUGGINGFACE_PROBE, &VLLM_HEALTH_CHECK] {
assert!(
warnings.iter().any(|m| m.contains(site.name())),
"{} degraded without its own warning — an earlier site's warning \
must not silence it; captured: {warnings:?}",
site.name()
);
let record = crate::tls_client::last_degradation(site)
.unwrap_or_else(|| panic!("{} must keep its own record", site.name()));
assert_eq!(record.fallback, TrustFallback::PublicCaOnly);
assert!(!record.source.is_empty());
}
let warning_for = |site: &'static crate::tls_client::Site| -> &String {
warnings
.iter()
.find(|m| m.contains(site.name()))
.unwrap_or_else(|| panic!("no warning for {}", site.name()))
};
for message in &warnings {
assert!(
message.contains("the OS certificate store failed to load"),
"every warning must name the underlying error; got: {message}"
);
}
assert!(
warning_for(&REMOTE_BACKEND).contains("restart the daemon to recover"),
"inference recovers only on a restart and must say so: {}",
warning_for(&REMOTE_BACKEND)
);
for site in [&HUGGINGFACE_PROBE, &VLLM_HEALTH_CHECK] {
let message = warning_for(site);
assert!(
message.contains("with no restart"),
"{} rebuilds its client per call, so it must not demand a restart: {message}",
site.name()
);
assert!(
!message.contains("restart the daemon to recover"),
"{} self-heals — telling the operator to restart is wrong advice: {message}",
site.name()
);
}
}
}