use axum::body::{Body, Bytes};
use axum::http::{HeaderMap, StatusCode};
use axum::response::Response;
use futures::StreamExt;
use super::sched::ActivityGuard;
use super::{
context_overflow_message, ensure_model_for, running_key, AppError, AppState, ContextOverflow,
LoadOptions, Target, Task,
};
pub(super) async fn resolve_hybrid_side(
state: &AppState,
pair: &crate::hybrid::Pair<'_>,
headers: Option<&HeaderMap>,
) -> Result<String, AppError> {
let pin = request_pin(headers)?;
let request_bytes = headers
.and_then(|h| h.get(reqwest::header::CONTENT_LENGTH))
.and_then(|v| v.to_str().ok())
.and_then(|v| v.trim().parse::<u64>().ok());
let budget = match (pin, request_bytes) {
(None, Some(_)) => local_budget(state, pair.local).await,
_ => None,
};
let decision = crate::hybrid::route(pin, request_bytes, budget);
let why = match decision.reason {
crate::hybrid::Reason::Pinned => format!("pinned by {}", crate::hybrid::ROUTE_HEADER),
crate::hybrid::Reason::Overflow { bytes, budget } => format!(
"{} request exceeds the {} this host serves locally",
crate::fmt::human_size(bytes),
crate::fmt::human_size(budget)
),
crate::hybrid::Reason::LocalFirst => "no reason to leave this machine".to_string(),
};
eprintln!(
"[llmman] hybrid {:?} + {:?} -> {} ({why})",
pair.local,
pair.remote_ref(),
decision.side.as_str()
);
Ok(pair.side_ref(decision.side))
}
async fn local_budget(state: &AppState, local: &str) -> Option<u64> {
if state.0.hybrid_local_bytes_explicit {
return state.0.hybrid_local_bytes;
}
let key = running_key(state, local);
let loaded = state
.0
.manager
.lock()
.await
.running
.get_mut(&key)
.and_then(|m| m.process.is_alive().then_some(m.context_window))
.flatten();
loaded
.and_then(crate::hybrid::budget_bytes_for_ctx)
.or(state.0.hybrid_local_bytes)
}
pub(super) fn request_pin(
headers: Option<&HeaderMap>,
) -> Result<Option<crate::hybrid::Side>, AppError> {
let mut values = headers
.map(|h| h.get_all(crate::hybrid::ROUTE_HEADER).iter())
.into_iter()
.flatten();
let value = values.next();
if values.next().is_some() {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
format!("{} given more than once", crate::hybrid::ROUTE_HEADER),
));
}
crate::hybrid::parse_pin(value.map(|v| v.as_bytes()))
.map_err(|e| AppError(e, StatusCode::BAD_REQUEST))
}
fn hybrid_fallback(
model_ref: &str,
headers: Option<&HeaderMap>,
) -> Result<Option<String>, AppError> {
let Some(pair) = crate::hybrid::split_ref(model_ref) else {
return Ok(None);
};
Ok((request_pin(headers)? != Some(crate::hybrid::Side::Local)).then(|| pair.remote_ref()))
}
pub(super) async fn send_with_hybrid_fallback<F, Fut>(
state: &AppState,
model_ref: &str,
headers: Option<&HeaderMap>,
opts: LoadOptions,
send: F,
) -> Result<Response, AppError>
where
F: Fn(String, Target, ActivityGuard) -> Fut,
Fut: std::future::Future<Output = Result<Response, AppError>>,
{
let resolve =
|m: String| async move { ensure_model_for(state, &m, headers, opts, Task::Generate).await };
with_hybrid_fallback(model_ref, headers, resolve, send).await
}
pub(super) async fn with_hybrid_fallback<R, RFut, F, Fut>(
model_ref: &str,
headers: Option<&HeaderMap>,
resolve: R,
send: F,
) -> Result<Response, AppError>
where
R: Fn(String) -> RFut,
RFut: std::future::Future<Output = Result<(String, Target, ActivityGuard), AppError>>,
F: Fn(String, Target, ActivityGuard) -> Fut,
Fut: std::future::Future<Output = Result<Response, AppError>>,
{
let (model, target, guard) = resolve(model_ref.to_string()).await?;
let fallback = match target {
Target::Local(_) => hybrid_fallback(model_ref, headers)?,
_ => None,
};
let Some(cloud) = fallback else {
return send(model, target, guard).await;
};
let refusal = match send(model, target, guard).await {
Ok(resp) => match local_context_overflow(resp).await {
Ok(resp) => return Ok(resp),
Err(refusal) => refusal,
},
Err(err) => match err.0.downcast_ref::<ContextOverflow>() {
Some(overflow) => overflow.refusal.clone(),
None => return Err(err),
},
};
eprintln!("[llmman] hybrid {model_ref:?} -> cloud ({refusal})");
let (model, target, guard) = resolve(cloud).await?;
send(model, target, guard).await
}
const OVERFLOW_BODY_LIMIT: usize = 64 * 1024;
async fn local_context_overflow(resp: Response) -> Result<Response, String> {
if resp.status() != StatusCode::BAD_REQUEST {
return Ok(resp);
}
let (parts, body) = resp.into_parts();
let mut rest = body.into_data_stream();
let mut head = Vec::new();
while head.len() <= OVERFLOW_BODY_LIMIT {
match rest.next().await {
Some(Ok(chunk)) => head.extend_from_slice(&chunk),
Some(Err(_)) | None => break,
}
}
if head.len() <= OVERFLOW_BODY_LIMIT {
if let Some(refusal) =
context_overflow_message(parts.status, &String::from_utf8_lossy(&head))
{
return Err(refusal);
}
}
let head = futures::stream::once(futures::future::ready(Ok(Bytes::from(head))));
Ok(Response::from_parts(
parts,
Body::from_stream(head.chain(rest)),
))
}
#[cfg(test)]
mod tests {
use super::super::test_support::{headers_with, HOSTED, PAIR};
use super::*;
#[test]
fn only_an_unpinned_pair_has_a_hosted_half_to_fall_back_to() {
assert_eq!(
hybrid_fallback(PAIR, Some(&headers_with(&[]))).unwrap(),
Some(HOSTED.to_string())
);
assert_eq!(
hybrid_fallback(PAIR, None).unwrap(),
Some(HOSTED.to_string())
);
assert_eq!(
hybrid_fallback(PAIR, Some(&headers_with(&[("x-llmman-route", "local")]))).unwrap(),
None
);
assert_eq!(hybrid_fallback("gemma4", None).unwrap(), None);
assert_eq!(hybrid_fallback(HOSTED, None).unwrap(), None);
}
#[tokio::test]
async fn a_relayed_response_is_only_intercepted_when_it_is_the_refusal() {
let llama = r#"{"error":{"code":400,"message":"request (9 tokens) exceeds the available context size (8 tokens), try increasing it","type":"exceed_context_size_error"}}"#;
let resp = |status: StatusCode, body: &'static str| {
Response::builder()
.status(status)
.body(Body::from(body))
.unwrap()
};
let refusal = local_context_overflow(resp(StatusCode::BAD_REQUEST, llama))
.await
.expect_err("the refusal must be intercepted");
assert!(
refusal.contains("exceeds the available context size"),
"{refusal}"
);
let other = r#"{"error":{"code":400,"message":"invalid grammar"}}"#;
let passed = local_context_overflow(resp(StatusCode::BAD_REQUEST, other))
.await
.expect("another 400 passes through");
assert_eq!(passed.status(), StatusCode::BAD_REQUEST);
let body = axum::body::to_bytes(passed.into_body(), usize::MAX)
.await
.unwrap();
assert_eq!(body, other.as_bytes(), "body reattached intact");
let ok = local_context_overflow(resp(StatusCode::OK, "data: {}"))
.await
.expect("a success is never read");
assert_eq!(ok.status(), StatusCode::OK);
let big: &'static str = String::from_utf8(vec![b'x'; OVERFLOW_BODY_LIMIT + 10])
.unwrap()
.leak();
let passed = local_context_overflow(resp(StatusCode::BAD_REQUEST, big))
.await
.expect("an oversized 400 passes through");
let body = axum::body::to_bytes(passed.into_body(), usize::MAX)
.await
.unwrap();
assert_eq!(body.len(), big.len());
}
}