use anyhow::Context;
use axum::body::{Body, Bytes};
use axum::http::{HeaderMap, StatusCode};
use axum::response::{IntoResponse, Response};
use axum::Json;
use futures::StreamExt;
use reqwest::Client;
use super::openai::merged_after_refusal;
use super::sched::ActivityGuard;
use super::stream::bytes_to_lines;
use super::{
messages, responses, AppError, ChatBody, ChatUpstream, Target, CHAT_COMPLETIONS_ROUTE,
};
async fn send_proxied(
client: &Client,
target: &Target,
route: &str,
headers: &HeaderMap,
body: Bytes,
) -> Result<reqwest::Response, AppError> {
let resp = post_proxied(client, target, route, headers, body.clone()).await?;
if route != CHAT_COMPLETIONS_ROUTE
|| !matches!(target, Target::Local(_))
|| resp.status() != StatusCode::BAD_REQUEST
{
return Ok(resp);
}
let (status, reply_headers) = (resp.status(), resp.headers().clone());
let reply = resp
.bytes()
.await
.with_context(|| format!("proxy request to {}", target.describe()))?;
if let Some(merged) = merged_after_refusal(target, status, &reply, &body) {
return post_proxied(client, target, route, headers, merged).await;
}
let mut refusal = axum::http::Response::new(reply);
*refusal.status_mut() = status;
*refusal.headers_mut() = reply_headers;
Ok(refusal.into())
}
async fn post_proxied(
client: &Client,
target: &Target,
route: &str,
headers: &HeaderMap,
body: Bytes,
) -> Result<reqwest::Response, AppError> {
let mut req = target.authorize(client.post(target.url(route)).body(body));
if let Some(ct) = headers.get("content-type") {
req = req.header("content-type", ct);
}
Ok(req
.send()
.await
.with_context(|| format!("proxy request to {}", target.describe()))?)
}
pub(super) async fn proxy(
client: &Client,
target: &Target,
route: &str,
headers: &HeaderMap,
body: Bytes,
activity: ActivityGuard,
) -> Result<Response, AppError> {
let resp = send_proxied(client, target, route, headers, body).await?;
Ok(relay(resp, activity))
}
pub(super) fn relay(resp: reqwest::Response, activity: ActivityGuard) -> Response {
let status = resp.status();
let resp_headers = resp.headers().clone();
let stream = resp.bytes_stream().map(move |item| {
let _activity = &activity;
item.map_err(|e| Box::new(e) as Box<dyn std::error::Error + Send + Sync>)
});
let mut builder = Response::builder().status(status.as_u16());
for (k, v) in &resp_headers {
builder = builder.header(k, v);
}
builder.body(Body::from_stream(stream)).unwrap()
}
fn set_response_model(value: &mut serde_json::Value, canonical_model: &str) {
if value.get("model").is_some() {
value["model"] = serde_json::Value::String(canonical_model.to_string());
}
for key in ["response", "message"] {
if let Some(nested) = value.get_mut(key) {
if nested.get("model").is_some() {
nested["model"] = serde_json::Value::String(canonical_model.to_string());
}
}
}
}
fn rewrite_json_response_model(raw: &Bytes, canonical_model: &str) -> Bytes {
match serde_json::from_slice::<serde_json::Value>(raw) {
Ok(mut value) => {
set_response_model(&mut value, canonical_model);
serde_json::to_vec(&value)
.map(Bytes::from)
.unwrap_or_else(|_| raw.clone())
}
Err(_) => raw.clone(),
}
}
fn rewrite_sse_line_model(line: &str, canonical_model: &str) -> String {
match line.strip_prefix("data: ") {
Some(payload) if payload != "[DONE]" => match serde_json::from_str(payload) {
Ok(mut value) => {
set_response_model(&mut value, canonical_model);
format!(
"data: {}",
serde_json::to_string(&value).unwrap_or_else(|_| payload.to_string())
)
}
Err(_) => line.to_string(),
},
_ => line.to_string(),
}
}
pub(super) async fn proxy_rewriting_model(
client: &Client,
target: &Target,
route: &str,
headers: &HeaderMap,
body: Bytes,
activity: ActivityGuard,
canonical_model: &str,
) -> Result<Response, AppError> {
let resp = send_proxied(client, target, route, headers, body).await?;
relay_rewriting_model(resp, activity, canonical_model).await
}
pub(super) async fn proxy_buffered(
client: &Client,
target: &Target,
route: &str,
headers: &HeaderMap,
body: Bytes,
activity: ActivityGuard,
rewrite: impl FnOnce(&Bytes) -> Result<Bytes, AppError>,
) -> Result<Response, AppError> {
let resp = send_proxied(client, target, route, headers, body).await?;
relay_buffered(resp, activity, rewrite).await
}
pub(super) async fn relay_rewriting_model(
resp: reqwest::Response,
activity: ActivityGuard,
canonical_model: &str,
) -> Result<Response, AppError> {
relay_buffered(resp, activity, |raw| {
Ok(rewrite_json_response_model(raw, canonical_model))
})
.await
}
async fn relay_buffered(
resp: reqwest::Response,
activity: ActivityGuard,
rewrite: impl FnOnce(&Bytes) -> Result<Bytes, AppError>,
) -> Result<Response, AppError> {
let status = resp.status();
let resp_headers = resp.headers().clone();
let raw = resp
.bytes()
.await
.context("read inference backend response")?;
drop(activity);
let rewritten = rewrite(&raw)?;
let mut builder = Response::builder().status(status.as_u16());
for (k, v) in &resp_headers {
if k == reqwest::header::CONTENT_LENGTH {
continue;
}
builder = builder.header(k, v);
}
Ok(builder.body(Body::from(rewritten)).unwrap())
}
pub(super) async fn stream_rewriting_model(
client: &Client,
target: &Target,
route: &str,
headers: &HeaderMap,
body: Bytes,
activity: ActivityGuard,
canonical_model: String,
) -> Result<Response, AppError> {
let resp = send_proxied(client, target, route, headers, body).await?;
Ok(relay_stream_rewriting_model(
resp,
activity,
canonical_model,
))
}
pub(super) fn relay_stream_rewriting_model(
resp: reqwest::Response,
activity: ActivityGuard,
canonical_model: String,
) -> Response {
let status = resp.status();
let mut resp_headers = resp.headers().clone();
resp_headers.remove(reqwest::header::CONTENT_LENGTH);
resp_headers.remove(reqwest::header::TRANSFER_ENCODING);
let failure = std::sync::Arc::new(std::sync::Mutex::new(None));
let record = failure.clone();
let upstream = resp.bytes_stream().map(move |item| {
item.or_else(|e| {
*record.lock().unwrap_or_else(|e| e.into_inner()) = Some(e);
Ok(Bytes::new())
})
});
let lines = bytes_to_lines(upstream).map(move |line| {
let _activity = &activity;
let out = rewrite_sse_line_model(&line, &canonical_model) + "\n";
Ok::<_, reqwest::Error>(Bytes::from(out))
});
let error =
futures::stream::once(
async move { failure.lock().unwrap_or_else(|e| e.into_inner()).take() },
)
.filter_map(|e| async move { e.map(Err) });
let mut builder = Response::builder().status(status.as_u16());
for (k, v) in &resp_headers {
builder = builder.header(k, v);
}
builder.body(Body::from_stream(lines.chain(error))).unwrap()
}
pub(super) fn relay_chat_upstream(upstream: ChatUpstream, activity: ActivityGuard) -> Response {
let stream = upstream.body.map(move |item| {
let _activity = &activity;
item.map_err(|e| Box::new(e) as Box<dyn std::error::Error + Send + Sync>)
});
let mut builder = Response::builder().status(upstream.status.as_u16());
for (k, v) in &upstream.headers {
builder = builder.header(k, v);
}
builder.body(Body::from_stream(stream)).unwrap()
}
pub(super) trait SseConverter: Send + 'static {
fn line(&mut self, line: &str) -> String;
fn finish(&mut self) -> String;
fn failed(&self) -> bool;
fn fold(&mut self, lines: Vec<String>) -> serde_json::Value;
}
macro_rules! sse_converter {
($($t:ty),*) => {$(
impl SseConverter for $t {
fn line(&mut self, line: &str) -> String {
Self::line(self, line)
}
fn finish(&mut self) -> String {
Self::finish(self)
}
fn failed(&self) -> bool {
Self::failed(self)
}
fn fold(&mut self, lines: Vec<String>) -> serde_json::Value {
Self::fold(self, lines)
}
}
)*};
}
sse_converter!(responses::StreamConverter, messages::StreamConverter);
pub(super) async fn convert_upstream(
body: ChatBody,
activity: ActivityGuard,
mut converter: impl SseConverter,
streaming: bool,
) -> Response {
if !streaming {
let lines: Vec<String> = bytes_to_lines(body).collect().await;
drop(activity);
let response = converter.fold(lines);
let status = if converter.failed() {
StatusCode::BAD_GATEWAY
} else {
StatusCode::OK
};
return (status, Json(response)).into_response();
}
let sse_stream = bytes_to_lines(body)
.map(Some)
.chain(futures::stream::once(futures::future::ready(None)))
.map(move |line| {
let _activity = &activity;
let out = match line {
Some(line) => converter.line(&line),
None => converter.finish(),
};
Ok::<_, std::convert::Infallible>(Bytes::from(out))
});
Response::builder()
.header("content-type", "text/event-stream")
.header("cache-control", "no-cache")
.body(Body::from_stream(sse_stream))
.unwrap()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn set_response_model_overwrites_an_existing_model_field_and_leaves_a_missing_one_alone() {
let mut with_model = serde_json::json!({"model": "/abs/path/to/model", "id": "x"});
set_response_model(&mut with_model, "gemma4:latest");
assert_eq!(
with_model,
serde_json::json!({"model": "gemma4:latest", "id": "x"})
);
let mut without_model = serde_json::json!({"id": "x"});
set_response_model(&mut without_model, "gemma4:latest");
assert_eq!(without_model, serde_json::json!({"id": "x"}));
let mut event =
serde_json::json!({"type": "response.created", "response": {"model": "wire"}});
set_response_model(&mut event, "canonical");
assert_eq!(event["response"]["model"], "canonical");
}
#[test]
fn rewrite_json_response_model_rewrites_a_json_body_and_leaves_every_other_field_alone() {
let raw = Bytes::from(
serde_json::to_vec(&serde_json::json!({
"id": "chatcmpl-1",
"model": "/home/user/.local/share/llmman/cache/abcd/model-dir",
"choices": [{"message": {"content": "hi"}}]
}))
.unwrap(),
);
let rewritten = rewrite_json_response_model(&raw, "gemma4:latest");
let value: serde_json::Value = serde_json::from_slice(&rewritten).unwrap();
assert_eq!(value["model"], "gemma4:latest");
assert_eq!(value["id"], "chatcmpl-1");
assert_eq!(value["choices"][0]["message"]["content"], "hi");
}
#[test]
fn rewrite_json_response_model_passes_non_json_bodies_through_unchanged() {
let raw = Bytes::from_static(b"not json at all");
assert_eq!(rewrite_json_response_model(&raw, "gemma4:latest"), raw);
}
#[test]
fn rewrite_sse_line_model_rewrites_only_the_model_field_of_a_data_line() {
let line = r#"data: {"id":"1","model":"/abs/path","choices":[{"delta":{"content":"h"}}]}"#;
let rewritten = rewrite_sse_line_model(line, "gemma4:latest");
let payload = rewritten.strip_prefix("data: ").expect("data: prefix");
let value: serde_json::Value = serde_json::from_str(payload).unwrap();
assert_eq!(value["model"], "gemma4:latest");
assert_eq!(value["id"], "1");
assert_eq!(value["choices"][0]["delta"]["content"], "h");
}
#[test]
fn rewrite_sse_line_model_leaves_the_done_sentinel_and_blank_separators_untouched() {
assert_eq!(
rewrite_sse_line_model("data: [DONE]", "gemma4:latest"),
"data: [DONE]"
);
assert_eq!(rewrite_sse_line_model("", "gemma4:latest"), "");
}
#[test]
fn rewrite_sse_line_model_passes_a_non_json_data_line_through_unchanged() {
assert_eq!(
rewrite_sse_line_model("data: not json", "gemma4:latest"),
"data: not json"
);
}
#[test]
fn set_response_model_reaches_a_messages_stream_event() {
let mut event = serde_json::json!({
"type": "message_start",
"message": { "id": "msg_1", "model": "claude-x" }
});
set_response_model(&mut event, "mine");
assert_eq!(event["message"]["model"], "mine");
}
}