use crate::audit::{Audit, AuditEntry};
use crate::error::{Error, Result};
use crate::model::{Auth, Route, Store, Terminator};
use crate::retry::{RetryPolicy, parse_retry_after};
use crate::store::Paths;
use crate::vkey::{self, RouteTable};
use arc_swap::ArcSwap;
use axum::body::Body;
use axum::extract::{Request, State};
use axum::http::{HeaderMap, HeaderName, HeaderValue, Method, StatusCode, header};
use axum::response::{IntoResponse, Response};
use axum::routing::get;
use axum::{Json, Router};
use base64::Engine;
use bytes::Bytes;
use futures_util::{StreamExt, stream};
use serde_json::json;
use std::collections::{HashMap, HashSet};
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
use std::task::{Context, Poll};
use std::time::{Duration, Instant};
use tokio::io::{AsyncReadExt, ReadBuf};
use tracing::Instrument;
static REQ_SEQ: AtomicU64 = AtomicU64::new(1);
const STREAM_READ_TIMEOUT: Duration = Duration::from_secs(60);
pub const BODY_READ_IDLE: Duration = crate::retry::DEFAULT_HEADER_TIMEOUT;
pub const BODY_TOTAL_BUDGET: Duration = Duration::from_secs(10 * 60);
pub const CLIENT_HEADER_TIMEOUT: Duration = Duration::from_secs(30);
pub const CLIENT_BODY_TIMEOUT: Duration = Duration::from_secs(120);
const HTTP2_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(30);
const HTTP2_KEEPALIVE_TIMEOUT: Duration = Duration::from_secs(15);
pub const PERMIT_QUEUE_TIMEOUT: Duration = Duration::from_secs(30);
pub const DRAIN_TIMEOUT: Duration = Duration::from_secs(10);
const TCP_KEEPALIVE: Duration = Duration::from_secs(60);
const RID_DIGITS: &[u8] = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
fn next_rid() -> String {
let n = REQ_SEQ.fetch_add(1, Ordering::Relaxed);
rid_from(n)
}
pub(crate) fn rid_from(n: u64) -> String {
let mut digits = Vec::with_capacity(4);
let mut m = n;
loop {
digits.push(RID_DIGITS[(m % 36) as usize]);
m /= 36;
if m == 0 {
break;
}
}
while digits.len() < 4 {
digits.push(b'0');
}
digits.reverse();
String::from_utf8(digits).unwrap_or_else(|_| "0000".into())
}
pub struct Snapshot {
pub store: Arc<Store>,
pub table: RouteTable,
pub from_backup: bool,
}
#[derive(Default)]
pub struct ReloadStatus {
pub last_ok_secs: AtomicU64,
pub last_tick_secs: AtomicU64,
pub failures: AtomicU32,
pub attempts: AtomicU64,
}
pub struct AppState {
pub snapshot: ArcSwap<Snapshot>,
pub clients: ProxyClients,
pub retry: RetryPolicy,
pub body_limit: usize,
pub body_read_idle: Duration,
pub inbound_body_timeout: Duration,
pub permit_queue_timeout: Duration,
pub body_total_budget: Duration,
pub reload: Arc<ReloadStatus>,
pub counters: Arc<ServeCounters>,
pub inbound_permits: Arc<tokio::sync::Semaphore>,
pub audit: Audit,
pub started: Instant,
}
pub fn build_upstream_client(
proxy: Option<&str>,
read_timeout: Option<Duration>,
) -> Result<reqwest::Client> {
let _ = rustls::crypto::ring::default_provider().install_default();
let mut builder = reqwest::Client::builder()
.pool_max_idle_per_host(64)
.pool_idle_timeout(Duration::from_secs(30))
.tcp_keepalive(Duration::from_secs(60))
.tcp_nodelay(true)
.connect_timeout(Duration::from_secs(10))
.redirect(reqwest::redirect::Policy::none());
if let Some(rt) = read_timeout {
builder = builder.read_timeout(rt);
}
builder = match proxy {
None => builder,
Some(crate::model::PROXY_DIRECT) => builder.no_proxy(),
Some(url) => builder.proxy(
reqwest::Proxy::all(url)
.map_err(|e| Error::msg(format!("invalid proxy URL `{url}`: {e}")))?,
),
};
builder
.build()
.map_err(|e| Error::msg(format!("failed to build HTTP client: {e}")))
}
#[derive(Clone)]
struct ClientPair {
plain: reqwest::Client,
streaming: reqwest::Client,
}
impl ClientPair {
fn build(proxy: Option<&str>) -> Result<Self> {
Ok(Self {
plain: build_upstream_client(proxy, None)?,
streaming: build_upstream_client(proxy, Some(STREAM_READ_TIMEOUT))?,
})
}
fn pick(&self, is_stream: bool) -> reqwest::Client {
if is_stream {
self.streaming.clone()
} else {
self.plain.clone()
}
}
}
pub struct ProxyClients {
base: ClientPair,
direct: ClientPair,
by_url: ArcSwap<HashMap<String, ClientPair>>,
}
impl ProxyClients {
pub fn new() -> Result<Self> {
Ok(Self {
base: ClientPair::build(None)?,
direct: ClientPair::build(Some(crate::model::PROXY_DIRECT))?,
by_url: ArcSwap::from_pointee(HashMap::new()),
})
}
pub fn for_route(&self, route: &Route, is_stream: bool) -> Result<reqwest::Client> {
match route.proxy.as_deref() {
None => Ok(self.base.pick(is_stream)),
Some(crate::model::PROXY_DIRECT) => Ok(self.direct.pick(is_stream)),
Some(url) => {
if let Some(pair) = self.by_url.load().get(url) {
return Ok(pair.pick(is_stream));
}
let pair = ClientPair::build(Some(url))?;
let client = pair.pick(is_stream);
self.by_url.rcu(|cache| {
let mut next = HashMap::clone(cache);
next.entry(url.to_string()).or_insert_with(|| pair.clone());
next
});
Ok(client)
}
}
}
pub fn retain_urls(&self, live: &HashSet<String>) {
self.by_url.rcu(|cache| {
cache
.iter()
.filter(|(url, _)| live.contains(url.as_str()))
.map(|(url, pair)| (url.clone(), pair.clone()))
.collect::<HashMap<_, _>>()
});
}
}
pub fn build_app(state: Arc<AppState>) -> Router {
Router::new()
.route("/__keysink/health", get(health))
.fallback(handler)
.with_state(state)
}
#[derive(Clone)]
pub struct InboundLimits {
pub header_timeout: Duration,
pub max_connections: usize,
pub drain_timeout: Option<Duration>,
pub loop_iterations: Option<Arc<AtomicU64>>,
pub live_connections: Option<Arc<AtomicU64>>,
pub counters: Option<Arc<ServeCounters>>,
}
#[derive(Default)]
pub struct ServeCounters {
pub shed: AtomicU64,
pub accept_errors: AtomicU64,
}
const H2_PREFACE: &[u8] = b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n";
async fn sniff_protocol(
io: &mut (impl tokio::io::AsyncRead + Unpin),
head_timeout: Duration,
) -> std::io::Result<Vec<u8>> {
let deadline = tokio::time::Instant::now() + head_timeout;
let mut seen = Vec::with_capacity(H2_PREFACE.len());
let mut chunk = [0u8; H2_PREFACE.len() + 8];
while seen.len() < H2_PREFACE.len() {
let read = tokio::time::timeout_at(deadline, io.read(&mut chunk))
.await
.map_err(|_| {
std::io::Error::new(std::io::ErrorKind::TimedOut, "protocol sniff timed out")
})??;
if read == 0 {
break;
}
seen.extend_from_slice(&chunk[..read]);
let decided = &seen[..seen.len().min(H2_PREFACE.len())];
if !H2_PREFACE.starts_with(decided) {
break; }
}
Ok(seen)
}
struct PrefaceIo<T> {
io: T,
preface: Vec<u8>,
pos: usize,
}
impl<T: tokio::io::AsyncRead + Unpin> tokio::io::AsyncRead for PrefaceIo<T> {
fn poll_read(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<std::io::Result<()>> {
if self.pos < self.preface.len() {
let n = (self.preface.len() - self.pos).min(buf.remaining());
buf.put_slice(&self.preface[self.pos..self.pos + n]);
self.pos += n;
return Poll::Ready(Ok(()));
}
Pin::new(&mut self.io).poll_read(cx, buf)
}
}
impl<T: tokio::io::AsyncWrite + Unpin> tokio::io::AsyncWrite for PrefaceIo<T> {
fn poll_write(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &[u8],
) -> Poll<std::io::Result<usize>> {
Pin::new(&mut self.io).poll_write(cx, buf)
}
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Pin::new(&mut self.io).poll_flush(cx)
}
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Pin::new(&mut self.io).poll_shutdown(cx)
}
fn poll_write_vectored(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
bufs: &[std::io::IoSlice<'_>],
) -> Poll<std::io::Result<usize>> {
Pin::new(&mut self.io).poll_write_vectored(cx, bufs)
}
fn is_write_vectored(&self) -> bool {
self.io.is_write_vectored()
}
}
pub async fn serve_inbound(
listener: tokio::net::TcpListener,
app: Router,
limits: InboundLimits,
shutdown: impl std::future::Future<Output = ()>,
) -> std::io::Result<()> {
let mut shutdown = std::pin::pin!(shutdown);
let mut conns = tokio::task::JoinSet::new();
let conn_slots = Arc::new(tokio::sync::Semaphore::new(limits.max_connections));
let (drain_tx, drain_rx) = tokio::sync::watch::channel(false);
let mut shed = 0u64;
let mut accept_errors = 0u64;
loop {
if let Some(counter) = &limits.loop_iterations {
counter.fetch_add(1, Ordering::Relaxed);
}
let accepted = tokio::select! {
_ = &mut shutdown => None,
res = conns.join_next(), if !conns.is_empty() => {
if let Some(Err(e)) = res
&& e.is_panic()
{
tracing::error!("inbound connection task panicked: {e}");
}
if let Some(gauge) = &limits.live_connections {
gauge.store(conns.len() as u64, Ordering::Relaxed);
}
continue;
}
accepted = listener.accept() => Some(accepted),
};
let Some(accepted) = accepted else {
break;
};
let (stream, _peer) = match accepted {
Ok(ok) => ok,
Err(e) => {
if is_resource_pressure_accept_error(&e) {
tracing::warn!(error = %e, "transient accept error (resource pressure); backing off 1s");
tokio::select! {
_ = &mut shutdown => break,
_ = tokio::time::sleep(Duration::from_secs(1)) => {}
}
continue;
}
if is_connection_level_accept_error(&e) {
accept_errors += 1;
if let Some(counters) = &limits.counters {
counters
.accept_errors
.store(accept_errors, Ordering::Relaxed);
}
if accept_errors == 1 {
tracing::warn!(error = %e, "per-connection accept error; still accepting");
} else if accept_errors.is_multiple_of(100) {
tracing::debug!(
error = %e,
count = accept_errors,
"per-connection accept error; still accepting"
);
}
continue;
}
return Err(e);
}
};
let Ok(_conn_slot) = conn_slots.clone().try_acquire_owned() else {
shed += 1;
if let Some(counters) = &limits.counters {
counters.shed.store(shed, Ordering::Relaxed);
}
if shed % 100 == 1 {
tracing::warn!(
shed,
"connection cap reached; dropping incoming connections"
);
}
drop(stream);
continue;
};
let _ = stream.set_nodelay(true);
let socket = socket2::SockRef::from(&stream);
let _ = socket.set_tcp_keepalive(
&socket2::TcpKeepalive::new()
.with_time(TCP_KEEPALIVE)
.with_interval(Duration::from_secs(15)),
);
let app = app.clone();
let mut drain_rx = drain_rx.clone();
conns.spawn(async move {
let _conn_slot = _conn_slot;
let mut stream = stream;
let preface = match sniff_protocol(&mut stream, limits.header_timeout).await {
Ok(preface) => preface,
Err(e) => {
tracing::debug!(error = %e, "inbound connection dropped before speaking");
return;
}
};
let mut builder =
hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new());
builder.http1().timer(hyper_util::rt::TokioTimer::new());
builder
.http1()
.header_read_timeout(Some(limits.header_timeout));
builder.http2().timer(hyper_util::rt::TokioTimer::new());
builder
.http2()
.keep_alive_interval(Some(HTTP2_KEEPALIVE_INTERVAL));
builder.http2().keep_alive_timeout(HTTP2_KEEPALIVE_TIMEOUT);
let conn = builder.serve_connection_with_upgrades(
hyper_util::rt::TokioIo::new(PrefaceIo {
io: stream,
preface,
pos: 0,
}),
hyper_util::service::TowerToHyperService::new(app.clone()),
);
let mut conn = std::pin::pin!(conn);
let served = tokio::select! {
res = &mut conn => res,
_ = drain_rx.changed() => {
conn.as_mut().graceful_shutdown();
conn.await
}
};
if let Err(e) = served {
tracing::debug!(error = %e, "inbound connection ended");
}
});
if let Some(gauge) = &limits.live_connections {
gauge.store(conns.len() as u64, Ordering::Relaxed);
}
}
let _ = drain_tx.send(true);
tracing::info!(
connections = conns.len(),
"shutdown: draining in-flight connections (a second signal exits immediately)"
);
let deadline = limits
.drain_timeout
.map(|limit| tokio::time::Instant::now() + limit);
loop {
let joined = match deadline {
Some(deadline) => match tokio::time::timeout_at(deadline, conns.join_next()).await {
Ok(joined) => joined,
Err(_) => {
tracing::warn!(
remaining = conns.len(),
"drain timeout reached; dropping the remaining connections"
);
conns.shutdown().await;
break;
}
},
None => conns.join_next().await,
};
match joined {
Some(Err(e)) if e.is_panic() => {
tracing::error!("inbound connection task panicked during drain: {e}");
}
Some(_) => {}
None => break,
}
}
Ok(())
}
fn is_resource_pressure_accept_error(e: &std::io::Error) -> bool {
#[cfg(unix)]
{
matches!(
e.raw_os_error(),
Some(libc::EMFILE | libc::ENFILE | libc::ENOBUFS | libc::ENOMEM)
)
}
#[cfg(not(unix))]
{
matches!(e.raw_os_error(), Some(10024 | 10055))
}
}
fn is_connection_level_accept_error(e: &std::io::Error) -> bool {
#[cfg(unix)]
{
let Some(code) = e.raw_os_error() else {
return false;
};
let known = [
libc::ECONNABORTED,
libc::EPROTO,
libc::ENETDOWN,
libc::ENETUNREACH,
libc::EHOSTUNREACH,
libc::ENOPROTOOPT,
libc::EOPNOTSUPP,
];
if known.contains(&code) {
return true;
}
#[cfg(target_os = "linux")]
{
if code == libc::ENONET {
return true;
}
}
false
}
#[cfg(not(unix))]
{
matches!(
e.raw_os_error(),
Some(10053 | 10054 | 10050 | 10051 | 10065)
)
}
}
async fn health(State(state): State<Arc<AppState>>) -> impl IntoResponse {
let snap = state.snapshot.load();
let uptime = state.started.elapsed().as_secs();
let last_ok = state.reload.last_ok_secs.load(Ordering::Relaxed);
Json(json!({
"ok": true,
"degraded": snap.from_backup,
"version": env!("CARGO_PKG_VERSION"),
"uptime_secs": uptime,
"routes": snap.table.len(),
"snapshot_age_secs": uptime.saturating_sub(last_ok),
"reload_failures": state.reload.failures.load(Ordering::Relaxed),
"reload_attempts": state.reload.attempts.load(Ordering::Relaxed),
"audit": state.audit.is_enabled(),
"audit_ok": state.audit.healthy(),
"audit_dropped": state.audit.dropped(),
"shed_connections": state.counters.shed.load(Ordering::Relaxed),
"accept_errors": state.counters.accept_errors.load(Ordering::Relaxed),
}))
}
pub async fn handler(State(state): State<Arc<AppState>>, req: Request<Body>) -> Response {
let rid = next_rid();
let span = tracing::info_span!("req", rid = %rid);
async move { handle_inner(state, req, rid).await }
.instrument(span)
.await
}
async fn handle_inner(state: Arc<AppState>, req: Request<Body>, rid: String) -> Response {
let started = Instant::now();
let (parts, body) = req.into_parts();
let method = &parts.method;
let path = parts.uri.path();
let query = parts.uri.query();
let in_headers = parts.headers;
let deny = |route: &str,
hash: Option<&[u8; 32]>,
status: StatusCode,
kind: &'static str,
msg: &str|
-> Response {
record_reject(
state.as_ref(),
&rid,
route,
hash,
method,
path,
status,
started,
);
error_response(status, kind, msg, &rid)
};
if in_headers.contains_key(header::UPGRADE) {
return deny(
"-",
None,
StatusCode::NOT_IMPLEMENTED,
"unsupported",
"keysink does not support WebSocket / Upgrade",
);
}
if matches!(method.as_str(), "CONNECT" | "TRACE" | "TRACK") {
let mut response = deny(
"-",
None,
StatusCode::METHOD_NOT_ALLOWED,
"unsupported",
"keysink does not proxy CONNECT/TRACE/TRACK",
);
response.headers_mut().insert(
header::ALLOW,
HeaderValue::from_static("GET, HEAD, POST, PUT, PATCH, DELETE, OPTIONS"),
);
return response;
}
if !path.is_empty() && !path.starts_with('/') {
return deny(
"-",
None,
StatusCode::BAD_REQUEST,
"invalid_path",
"request target must be origin-form (a path starting with /)",
);
}
if path_has_dot_segment(path) {
return deny(
"-",
None,
StatusCode::NOT_FOUND,
"invalid_path",
"request path contains dot segments",
);
}
let Some(token) = extract_vkey(&in_headers) else {
return deny(
"-",
None,
StatusCode::UNAUTHORIZED,
"authentication_error",
"missing valid virtual key (Authorization: Bearer ks_... or x-api-key)",
);
};
let snapshot = state.snapshot.load_full();
let hash = vkey::hash(&token);
let Some(resolved) = snapshot.table.lookup_hash(&hash) else {
return deny(
"-",
Some(&hash),
StatusCode::UNAUTHORIZED,
"authentication_error",
"virtual key is invalid or has been rotated (if it was just added or rotated, hot reload may not have reached the gateway yet — retry in a second)",
);
};
let route = &resolved.route;
let route_name = resolved.name.as_str();
if !route.enabled {
return deny(
route_name,
Some(&hash),
StatusCode::FORBIDDEN,
"permission_error",
"route is disabled",
);
}
let Some(key) = route.inline_key.as_deref() else {
tracing::error!(route = route_name, "route is missing route-only key");
return deny(
route_name,
Some(&hash),
StatusCode::INTERNAL_SERVER_ERROR,
"configuration_error",
"incomplete route configuration",
);
};
if let Some(len) = in_headers
.get(header::CONTENT_LENGTH)
.and_then(|v| v.to_str().ok())
.and_then(|s| s.parse::<usize>().ok())
&& len > state.body_limit
{
return deny(
route_name,
Some(&hash),
StatusCode::PAYLOAD_TOO_LARGE,
"request_too_large",
&format!("request body exceeds limit of {} bytes", state.body_limit),
);
}
let permit = tokio::time::timeout(
state.permit_queue_timeout,
state.inbound_permits.clone().acquire_owned(),
)
.await;
let _permit = match permit {
Ok(Ok(permit)) => permit,
Err(_elapsed) => {
record_reject(
state.as_ref(),
&rid,
route_name,
Some(&hash),
method,
path,
StatusCode::SERVICE_UNAVAILABLE,
started,
);
let mut response = error_response(
StatusCode::SERVICE_UNAVAILABLE,
"overloaded",
"gateway is at capacity; retry shortly",
&rid,
);
response
.headers_mut()
.insert(header::RETRY_AFTER, HeaderValue::from_static("1"));
return response;
}
Ok(Err(_closed)) => {
return error_response(
StatusCode::SERVICE_UNAVAILABLE,
"overloaded",
"gateway is not accepting requests",
&rid,
);
}
};
let read = tokio::time::timeout(
state.inbound_body_timeout,
axum::body::to_bytes(body, state.body_limit),
)
.await;
let body = match read {
Ok(Ok(bytes)) => bytes,
Ok(Err(e)) => {
let too_large = is_length_limit_error(&e);
let (status, kind, message) = if too_large {
(
StatusCode::PAYLOAD_TOO_LARGE,
"request_too_large",
"request body exceeds limit",
)
} else {
(
StatusCode::BAD_REQUEST,
"invalid_request",
"failed to read request body",
)
};
return deny(route_name, Some(&hash), status, kind, message);
}
Err(_elapsed) => {
return deny(
route_name,
Some(&hash),
StatusCode::REQUEST_TIMEOUT,
"request_timeout",
"timed out reading request body",
);
}
};
let is_stream = is_stream_request(&body);
let fwd_headers = match forward_headers(&in_headers, route, key, is_stream) {
Ok(h) => h,
Err(e) => {
tracing::error!(error = %e, route = route_name, "failed to build upstream headers");
return deny(
route_name,
Some(&hash),
StatusCode::INTERNAL_SERVER_ERROR,
"configuration_error",
"invalid route header configuration",
);
}
};
let url = match build_upstream_url(&route.base_url, path, query, &route.auth, key) {
Ok(u) => u,
Err(e) => {
tracing::error!(error = %e, route = route_name, "failed to build upstream URL");
return deny(
route_name,
Some(&hash),
StatusCode::INTERNAL_SERVER_ERROR,
"configuration_error",
"invalid route URL configuration",
);
}
};
tracing::debug!(
route = route_name,
%method,
%path,
is_stream,
bytes_in = body.len(),
"forwarding request"
);
let client = match state.clients.for_route(route, is_stream) {
Ok(c) => c,
Err(e) => {
tracing::error!(error = %e, route = route_name, "failed to build route proxy client");
return deny(
route_name,
Some(&hash),
StatusCode::INTERNAL_SERVER_ERROR,
"configuration_error",
"invalid route proxy configuration",
);
}
};
let forward = forward_with_retry(
&state,
&client,
route,
route_name,
method.clone(),
url,
fwd_headers,
body.clone(),
is_stream,
&rid,
)
.await;
let mut response = forward.response;
response.headers_mut().insert(
"x-keysink-request-id",
HeaderValue::from_str(&rid).unwrap_or(HeaderValue::from_static("0000")),
);
if state.audit.is_enabled() {
let mut entry = AuditEntry::new(
rid.clone(),
route_name,
vkey::fingerprint(&hash),
method.to_string(),
path,
body.len(),
is_stream,
);
entry.status = response.status().as_u16();
entry.ms = started.elapsed().as_millis() as u64;
entry.attempts = forward.attempts;
state.audit.record(entry);
}
response
}
#[allow(clippy::too_many_arguments)]
fn record_reject(
state: &AppState,
rid: &str,
route: &str,
hash: Option<&[u8; 32]>,
method: &Method,
path: &str,
status: StatusCode,
started: Instant,
) {
let mut entry = AuditEntry::new(
rid.to_string(),
route.to_string(),
hash.map(vkey::fingerprint).unwrap_or_else(|| "-".into()),
method.to_string(),
path.to_string(),
0,
false,
);
entry.status = status.as_u16();
entry.ms = started.elapsed().as_millis() as u64;
state.audit.record_reject(entry);
}
struct Forwarded {
response: Response,
attempts: u8,
}
fn network_retry_allowed(is_connect: bool, idempotent_method: bool) -> bool {
is_connect || idempotent_method
}
fn is_length_limit_error(e: &axum::Error) -> bool {
let mut cur: Option<&dyn std::error::Error> = Some(e);
while let Some(err) = cur {
if err
.downcast_ref::<http_body_util::LengthLimitError>()
.is_some()
|| err.to_string().contains("length limit")
{
return true;
}
cur = err.source();
}
false
}
const DISCARD_BODY_LIMIT: usize = 64 * 1024;
async fn discard_body_limited(mut resp: reqwest::Response, budget: Duration) {
let drain = async {
let mut remaining = DISCARD_BODY_LIMIT;
while let Ok(Some(chunk)) = resp.chunk().await {
if chunk.len() >= remaining {
return;
}
remaining -= chunk.len();
}
};
if tokio::time::timeout(budget, drain).await.is_err() {
tracing::warn!(
"retryable upstream response body did not drain within the read idle; dropping the connection"
);
}
}
#[allow(clippy::too_many_arguments)]
async fn forward_with_retry(
state: &AppState,
client: &reqwest::Client,
route: &Route,
route_name: &str,
method: Method,
url: String,
headers: HeaderMap,
body: Bytes,
is_stream: bool,
rid: &str,
) -> Forwarded {
let policy = state.retry.for_route(route);
let max_retries = policy.max_retries;
let idempotent = matches!(method, Method::GET | Method::HEAD | Method::OPTIONS);
for attempt in 0..=max_retries {
let req = client
.request(method.clone(), &url)
.headers(headers.clone());
let req = if body.is_empty() {
req
} else {
req.body(body.clone())
};
let send = tokio::time::timeout(policy.header_timeout, req.send()).await;
let resp = match send {
Err(_) => {
tracing::warn!(
attempt,
route = route_name,
"timed out waiting for upstream response headers"
);
if idempotent && attempt < max_retries {
tokio::time::sleep(policy.backoff(attempt as u32, None)).await;
continue;
}
return Forwarded {
response: error_response(
StatusCode::GATEWAY_TIMEOUT,
"upstream_timeout",
"upstream response header timeout",
rid,
),
attempts: attempt + 1,
};
}
Ok(Err(e)) => {
tracing::warn!(
attempt,
route = route_name,
timeout = e.is_timeout(),
connect = e.is_connect(),
"upstream request failed"
);
if network_retry_allowed(e.is_connect(), idempotent) && attempt < max_retries {
tokio::time::sleep(policy.backoff(attempt as u32, None)).await;
continue;
}
return Forwarded {
response: error_response(
StatusCode::BAD_GATEWAY,
"upstream_error",
"upstream unreachable",
rid,
),
attempts: attempt + 1,
};
}
Ok(Ok(resp)) => resp,
};
let status = resp.status().as_u16();
if policy.is_retryable(status) && attempt < max_retries {
let retry_after = resp
.headers()
.get(header::RETRY_AFTER)
.and_then(|v| v.to_str().ok())
.and_then(parse_retry_after);
tracing::warn!(
attempt,
status,
route = route_name,
"upstream returned retryable status"
);
discard_body_limited(resp, state.body_read_idle).await;
tokio::time::sleep(policy.backoff(attempt as u32, retry_after)).await;
continue;
}
let response = if is_stream {
stream_response(
resp,
route.sse_terminator,
rid.to_string(),
state.body_read_idle,
state.body_total_budget,
)
.await
} else {
pass_through_response(resp, state.body_read_idle, Some(state.body_total_budget)).await
};
return Forwarded {
response,
attempts: attempt + 1,
};
}
Forwarded {
response: error_response(
StatusCode::BAD_GATEWAY,
"internal_error",
"retry loop exited unexpectedly",
rid,
),
attempts: max_retries + 1,
}
}
pub(crate) fn extract_vkey(headers: &HeaderMap) -> Option<String> {
for name in [
"authorization",
"x-api-key",
"x-goog-api-key",
"api-key",
"anthropic-api-key",
"openai-api-key",
] {
let Some(value) = headers.get(name) else {
continue;
};
let Ok(text) = value.to_str() else {
continue;
};
let text = text.trim();
let text = if name == "authorization" {
match text.get(..7) {
Some(prefix) if prefix.eq_ignore_ascii_case("bearer ") => text[7..].trim_start(),
_ => text,
}
} else {
text
};
if text.starts_with(vkey::PREFIX) {
return Some(text.to_string());
}
}
None
}
fn is_hop_by_hop(name: &str) -> bool {
const HOP: &[&str] = &[
"host",
"content-length",
"connection",
"keep-alive",
"proxy-authenticate",
"proxy-authorization",
"proxy-connection",
"te",
"trailer",
"transfer-encoding",
"upgrade",
];
HOP.iter().any(|h| h.eq_ignore_ascii_case(name))
}
fn is_auth_header(name: &str) -> bool {
const AUTH: &[&str] = &[
"authorization",
"x-api-key",
"x-goog-api-key",
"api-key",
"anthropic-api-key",
"openai-api-key",
];
AUTH.iter().any(|h| h.eq_ignore_ascii_case(name))
}
pub(crate) fn forward_headers(
src: &HeaderMap,
route: &Route,
upstream_key: &str,
is_stream: bool,
) -> Result<HeaderMap> {
let mut out = HeaderMap::with_capacity(src.len() + 2);
for (name, value) in src.iter() {
let n = name.as_str();
if is_hop_by_hop(n) || is_auth_header(n) {
continue;
}
if is_stream && n.eq_ignore_ascii_case("accept-encoding") {
continue;
}
out.append(name.clone(), value.clone());
}
for (key, value) in &route.headers {
if is_stream && key.eq_ignore_ascii_case("accept-encoding") {
continue;
}
let name = HeaderName::from_bytes(key.as_bytes())
.map_err(|e| Error::msg(format!("invalid header name `{key}`: {e}")))?;
let value = HeaderValue::from_str(value)
.map_err(|e| Error::msg(format!("invalid value for header `{key}`: {e}")))?;
out.entry(name).or_insert(value);
}
match &route.auth {
Auth::Bearer => {
out.insert(
header::AUTHORIZATION,
HeaderValue::from_str(&format!("Bearer {upstream_key}"))
.map_err(|e| Error::msg(format!("invalid Bearer value: {e}")))?,
);
}
Auth::XApiKey => {
out.insert(
HeaderName::from_static("x-api-key"),
HeaderValue::from_str(upstream_key)
.map_err(|e| Error::msg(format!("invalid x-api-key value: {e}")))?,
);
}
Auth::Header { name, prefix } => {
let header_name = HeaderName::from_bytes(name.as_bytes())
.map_err(|e| Error::msg(format!("invalid auth header name `{name}`: {e}")))?;
out.insert(
header_name,
HeaderValue::from_str(&format!("{prefix}{upstream_key}"))
.map_err(|e| Error::msg(format!("invalid auth header value: {e}")))?,
);
}
Auth::Basic { username } => {
let raw = format!("{username}:{upstream_key}");
let encoded = base64::engine::general_purpose::STANDARD.encode(raw.as_bytes());
out.insert(
header::AUTHORIZATION,
HeaderValue::from_str(&format!("Basic {encoded}"))
.map_err(|e| Error::msg(format!("invalid Basic value: {e}")))?,
);
}
Auth::Query { .. } | Auth::None => {}
}
Ok(out)
}
pub(crate) fn path_has_dot_segment(path: &str) -> bool {
path.split(['/', '\\']).any(|seg| {
if !seg.contains('%') {
matches!(seg, "." | "..")
} else {
let mut decoded = Vec::with_capacity(seg.len());
let bytes = seg.as_bytes();
let mut i = 0;
while i < bytes.len() {
if bytes[i] == b'%'
&& let Some(hex) = bytes.get(i + 1..i + 3)
&& let Ok(byte) = u8::from_str_radix(std::str::from_utf8(hex).unwrap_or(""), 16)
{
decoded.push(byte);
i += 3;
} else {
decoded.push(bytes[i]);
i += 1;
}
}
matches!(decoded.as_slice(), b"." | b"..")
}
})
}
pub(crate) fn build_upstream_url(
base_url: &str,
path: &str,
query: Option<&str>,
auth: &Auth,
upstream_key: &str,
) -> Result<String> {
let base = base_url.trim_end_matches('/');
let path = if path.is_empty() {
"/"
} else if path.starts_with('/') {
if path_has_dot_segment(path) {
return Err(Error::msg("inbound path contains dot segments"));
}
path
} else {
return Err(Error::msg("inbound path does not start with /"));
};
let mut url =
String::with_capacity(base.len() + path.len() + 64 + query.map_or(0, |q| q.len()));
url.push_str(base);
url.push_str(path);
let query_auth = match auth {
Auth::Query { param } => Some(param.as_str()),
_ => None,
};
let mut parts: Vec<String> = Vec::new();
if let Some(q) = query {
for pair in q.split('&') {
if pair.is_empty() {
continue;
}
if let Some(param) = query_auth {
let key = pair.split('=').next().unwrap_or("");
if key == param {
continue;
}
}
parts.push(pair.to_string());
}
}
if let Some(param) = query_auth {
parts.push(format!(
"{param}={}",
crate::util::percent_encode(upstream_key)
));
}
if !parts.is_empty() {
url.push('?');
url.push_str(&parts.join("&"));
}
Ok(url)
}
pub(crate) fn is_stream_request(body: &[u8]) -> bool {
if memchr::memmem::find(body, b"\"stream\"").is_none() {
return false;
}
top_level_stream_true(body)
}
fn top_level_stream_true(body: &[u8]) -> bool {
let mut i = skip_ws(body, 0);
if body.get(i) != Some(&b'{') {
return false;
}
i += 1;
loop {
i = skip_ws(body, i);
match body.get(i) {
Some(b'}') => return false,
Some(b'"') => {}
_ => return false,
}
let name_start = i + 1;
let Some(name_end) = skip_string(body, i) else {
return false;
};
let name = &body[name_start..name_end - 1];
i = skip_ws(body, name_end);
if body.get(i) != Some(&b':') {
return false;
}
i = skip_ws(body, i + 1);
if name == b"stream" || unescapes_to_stream(name) {
if body[i..].starts_with(b"true")
&& matches!(
body.get(i + 4),
None | Some(b',' | b'}' | b' ' | b'\t' | b'\n' | b'\r')
)
{
return true;
}
}
let Some(next) = skip_value(body, i) else {
return false;
};
i = skip_ws(body, next);
match body.get(i) {
Some(b',') => i += 1,
_ => return false, }
}
}
fn unescapes_to_stream(name: &[u8]) -> bool {
if !name.contains(&b'\\') {
return false;
}
let mut quoted = Vec::with_capacity(name.len() + 2);
quoted.push(b'"');
quoted.extend_from_slice(name);
quoted.push(b'"');
serde_json::from_slice::<String>("ed).is_ok_and(|s| s == "stream")
}
fn skip_ws(body: &[u8], mut i: usize) -> usize {
while matches!(body.get(i), Some(b' ' | b'\t' | b'\n' | b'\r')) {
i += 1;
}
i
}
fn skip_string(body: &[u8], at: usize) -> Option<usize> {
let mut i = at + 1;
loop {
i += memchr::memchr2(b'"', b'\\', &body[i..])?;
match body.get(i)? {
b'"' => return Some(i + 1),
_ => i += 2, }
}
}
fn skip_value(body: &[u8], i: usize) -> Option<usize> {
match *body.get(i)? {
b'"' => skip_string(body, i),
b'[' | b'{' => {
let (open, close) = if body[i] == b'[' {
(b'[', b']')
} else {
(b'{', b'}')
};
let mut depth = 0usize;
let mut j = i;
loop {
match body.get(j)? {
b'"' => {
j = skip_string(body, j)?;
continue;
}
&c if c == open => depth += 1,
&c if c == close => {
depth -= 1;
if depth == 0 {
return Some(j + 1);
}
}
_ => {}
}
j += 1;
}
}
b't' if body[i..].starts_with(b"true") => Some(i + 4),
b'f' if body[i..].starts_with(b"false") => Some(i + 5),
b'n' if body[i..].starts_with(b"null") => Some(i + 4),
c if c == b'-' || c.is_ascii_digit() => {
let mut j = i + 1;
while matches!(
body.get(j),
Some(b'0'..=b'9' | b'-' | b'+' | b'.' | b'e' | b'E')
) {
j += 1;
}
Some(j)
}
_ => None,
}
}
fn drop_hop_by_hop(headers: &mut HeaderMap) {
let hop: Vec<HeaderName> = headers
.iter()
.filter(|(name, _)| is_hop_by_hop(name.as_str()))
.map(|(name, _)| name.clone())
.collect();
for name in hop {
headers.remove(&name);
}
}
pub(crate) fn is_encoded(headers: &HeaderMap) -> bool {
match headers.get(header::CONTENT_ENCODING) {
None => false,
Some(value) => match value.to_str() {
Ok(v) => {
let v = v.trim();
!v.is_empty() && !v.eq_ignore_ascii_case("identity")
}
Err(_) => true,
},
}
}
async fn pass_through_response(
mut resp: reqwest::Response,
body_idle: Duration,
total_budget: Option<Duration>,
) -> Response {
let status = resp.status();
let mut headers = std::mem::take(resp.headers_mut());
drop_hop_by_hop(&mut headers);
let stream = idle_bounded(resp.bytes_stream(), body_idle, total_budget);
build_response(status, headers, Body::from_stream(stream))
}
fn idle_bounded<S>(
inner: S,
idle: Duration,
total: Option<Duration>,
) -> impl futures_util::Stream<Item = std::result::Result<Bytes, std::io::Error>>
where
S: futures_util::Stream<Item = reqwest::Result<Bytes>>,
{
let inner = Box::pin(inner);
let deadline = total.map(|total| tokio::time::Instant::now() + total);
stream::unfold(
(inner, false, deadline),
move |(mut inner, done, deadline)| {
async move {
if done {
return None;
}
let until = match deadline {
Some(deadline) => (tokio::time::Instant::now() + idle).min(deadline),
None => tokio::time::Instant::now() + idle,
};
match tokio::time::timeout_at(until, inner.next()).await {
Ok(Some(Ok(chunk))) => Some((Ok(chunk), (inner, false, deadline))),
Ok(Some(Err(e))) => {
if e.is_timeout() {
tracing::warn!(
"upstream stopped sending the response body (read idle); aborting the response"
);
} else {
tracing::warn!(
timeout = e.is_timeout(),
connect = e.is_connect(),
status = e.status().map(|s| s.as_u16()),
"failed to read upstream response body"
);
}
Some((
Err(std::io::Error::other("upstream body error")),
(inner, true, deadline),
))
}
Ok(None) => None,
Err(_) => {
let expired = deadline.is_some_and(|d| tokio::time::Instant::now() >= d);
let message = if expired {
tracing::warn!(
"upstream response body exceeded the total budget; aborting the response"
);
"upstream body total budget exceeded"
} else {
tracing::warn!(
"upstream response body stalled past the read idle; aborting the response"
);
"upstream body read idle timeout"
};
Some((
Err(std::io::Error::new(std::io::ErrorKind::TimedOut, message)),
(inner, true, deadline),
))
}
}
}
},
)
}
async fn stream_response(
mut resp: reqwest::Response,
terminator: Terminator,
rid: String,
body_idle: Duration,
non_stream_budget: Duration,
) -> Response {
let is_sse = resp
.headers()
.get(header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.map(|v| v.to_ascii_lowercase().starts_with("text/event-stream"))
.unwrap_or(false);
if !is_sse {
return pass_through_response(resp, body_idle, Some(non_stream_budget)).await;
}
if terminator == Terminator::None {
return pass_through_response(resp, body_idle, None).await;
}
if is_encoded(resp.headers()) {
tracing::debug!(rid = %rid, "response is content-encoded, skipping SSE terminator detection");
return pass_through_response(resp, body_idle, None).await;
}
let status = resp.status();
let mut headers = std::mem::take(resp.headers_mut());
drop_hop_by_hop(&mut headers);
let inner = Box::pin(resp.bytes_stream());
let mapped = stream::unfold(
(inner, SseScan::new(terminator, rid)),
move |(mut inner, mut scan)| async move {
if scan.done {
return None;
}
match inner.next().await {
Some(Ok(chunk)) => {
scan.feed(&chunk);
Some((Ok::<Bytes, std::io::Error>(chunk), (inner, scan)))
}
Some(Err(_)) => {
tracing::warn!(rid = %scan.rid, "upstream SSE stream interrupted");
let event = sse_error_event("upstream stream error", terminator);
scan.finish();
Some((Ok(event), (inner, scan)))
}
None => {
if scan.seen {
return None;
}
tracing::warn!(
rid = %scan.rid,
"SSE stream ended early without terminator marker, treating as truncated"
);
let event = sse_error_event("upstream stream ended prematurely", terminator);
scan.finish();
Some((Ok(event), (inner, scan)))
}
}
},
);
build_response(status, headers, Body::from_stream(mapped))
}
const TERMINATOR_MAX_NEEDLE: usize = 21;
struct SseScan {
terminator: Terminator,
seen: bool,
done: bool,
carry: Vec<u8>,
rid: String,
}
impl SseScan {
fn new(terminator: Terminator, rid: String) -> Self {
Self {
terminator,
seen: false,
done: false,
carry: Vec::with_capacity(TERMINATOR_MAX_NEEDLE - 1),
rid,
}
}
fn finish(&mut self) {
self.seen = true;
self.done = true;
}
fn feed(&mut self, chunk: &[u8]) {
if self.seen || self.terminator == Terminator::None {
return;
}
if contains_terminator(chunk, self.terminator) {
self.seen = true;
return;
}
let overlap = TERMINATOR_MAX_NEEDLE - 1;
let head = chunk.len().min(overlap);
if !self.carry.is_empty() && head > 0 {
let mut window = [0u8; 2 * (TERMINATOR_MAX_NEEDLE - 1)];
window[..self.carry.len()].copy_from_slice(&self.carry);
window[self.carry.len()..self.carry.len() + head].copy_from_slice(&chunk[..head]);
if contains_terminator(&window[..self.carry.len() + head], self.terminator) {
self.seen = true;
return;
}
}
if chunk.len() >= overlap {
self.carry.clear();
self.carry
.extend_from_slice(&chunk[chunk.len() - overlap..]);
} else {
let total = self.carry.len() + chunk.len();
if total > overlap {
self.carry.drain(..total - overlap);
}
self.carry.extend_from_slice(chunk);
}
}
}
fn sse_error_event(message: &str, terminator: Terminator) -> Bytes {
if matches!(terminator, Terminator::Anthropic) {
let payload = json!({
"type": "error",
"error": { "type": "api_error", "message": message }
});
Bytes::from(format!("event: error\ndata: {payload}\n\n"))
} else {
let payload = json!({ "error": { "type": "upstream_stream_error", "message": message } });
Bytes::from(format!("data: {payload}\n\ndata: [DONE]\n\n"))
}
}
const TERMINATOR_NEEDLES: [&[u8]; 3] = [
b"data: [DONE]",
b"event: message_stop",
b"\"type\":\"message_stop\"",
];
pub(crate) fn contains_terminator(chunk: &[u8], terminator: Terminator) -> bool {
let needles = match terminator {
Terminator::Openai => &TERMINATOR_NEEDLES[..1],
Terminator::Anthropic => &TERMINATOR_NEEDLES[1..],
Terminator::Auto => &TERMINATOR_NEEDLES[..],
Terminator::None => return false,
};
needles
.iter()
.any(|needle| find_memmem(chunk, needle) || find_memmem_ci(chunk, needle))
}
fn find_memmem(haystack: &[u8], needle: &[u8]) -> bool {
memchr::memmem::find(haystack, needle).is_some()
}
fn find_memmem_ci(haystack: &[u8], needle: &[u8]) -> bool {
if needle.is_empty() || haystack.len() < needle.len() {
return false;
}
let first = needle[0];
let (cand1, cand2) = if first.is_ascii_alphabetic() {
(first.to_ascii_lowercase(), first.to_ascii_uppercase())
} else {
(first, first)
};
let mut from = 0;
while let Some(at) = memchr::memchr2(cand1, cand2, &haystack[from..]) {
let start = from + at;
if start + needle.len() > haystack.len() {
return false;
}
if haystack[start..start + needle.len()].eq_ignore_ascii_case(needle) {
return true;
}
from = start + 1;
}
false
}
fn build_response(status: reqwest::StatusCode, headers: HeaderMap, body: Body) -> Response {
let mut out = Response::new(body);
*out.status_mut() = StatusCode::from_u16(status.as_u16()).unwrap_or(StatusCode::BAD_GATEWAY);
*out.headers_mut() = headers;
out
}
pub fn error_response(status: StatusCode, kind: &str, message: &str, rid: &str) -> Response {
let body = json!({
"error": {
"type": kind,
"message": message,
"request_id": rid,
}
})
.to_string();
let mut response = (status, [(header::CONTENT_TYPE, "application/json")], body).into_response();
if let Ok(value) = HeaderValue::from_str(rid) {
response
.headers_mut()
.insert(HeaderName::from_static("x-keysink-request-id"), value);
}
response
}
fn adopt_snapshot(incoming_from_backup: bool, current_from_backup: Option<bool>) -> bool {
!incoming_from_backup || current_from_backup.is_none_or(|current| current)
}
const RELOAD_POLL: Duration = Duration::from_secs(1);
const MAX_RELOAD_BACKOFF: Duration = Duration::from_secs(30);
fn next_reload_backoff(after: Duration) -> Duration {
after.saturating_mul(2).min(MAX_RELOAD_BACKOFF)
}
const WATCHDOG_INTERVAL: Duration = Duration::from_secs(5);
const WATCHDOG_STALE: Duration = Duration::from_secs(120);
pub fn spawn_watcher(state: Arc<AppState>, paths: Paths) {
tokio::spawn(async move {
loop {
let watcher = tokio::spawn(watcher_loop(state.clone(), paths.clone()));
tokio::pin!(watcher);
let mut stale_warned = false;
let joined = loop {
tokio::select! {
res = &mut watcher => break Some(res),
_ = tokio::time::sleep(WATCHDOG_INTERVAL) => {
let age = tick_age(&state);
if age > WATCHDOG_STALE.as_secs() && !stale_warned {
tracing::error!(
age_secs = age,
"store watcher has not ticked; the vault may no longer be tracked (hung disk read?)"
);
stale_warned = true;
}
}
}
};
match joined {
Some(Ok(())) => tracing::error!("store watcher exited; restarting"),
Some(Err(e)) => tracing::error!(error = %e, "store watcher failed; restarting"),
None => break,
}
tokio::time::sleep(Duration::from_secs(1)).await;
}
});
}
fn tick_age(state: &AppState) -> u64 {
state
.started
.elapsed()
.as_secs()
.saturating_sub(state.reload.last_tick_secs.load(Ordering::Relaxed))
}
async fn watcher_loop(state: Arc<AppState>, paths: Paths) {
let mut last = (0u64, 0u64, 0u64);
let mut failures = 0u32;
let mut backup_warned = false;
let mut delay = RELOAD_POLL;
loop {
tokio::time::sleep(delay).await;
state
.reload
.last_tick_secs
.store(state.started.elapsed().as_secs(), Ordering::Relaxed);
let now = stamp(&paths);
if now == last {
continue;
}
state.reload.attempts.fetch_add(1, Ordering::Relaxed);
let unlocked = {
let paths = paths.clone();
tokio::task::spawn_blocking(move || crate::store::unlock(&paths))
.await
.unwrap_or_else(|e| Err(Error::msg(format!("hot reload worker failed: {e}"))))
};
match unlocked {
Ok(unlocked) => match RouteTable::build(&unlocked.store) {
Ok(table) => {
let from_backup = unlocked.from_backup;
if !adopt_snapshot(from_backup, Some(state.snapshot.load().from_backup)) {
last = now;
delay = RELOAD_POLL;
failures += 1;
state.reload.failures.store(failures, Ordering::Relaxed);
if !backup_warned {
tracing::error!(
"main store failed to decrypt, refusing to overwrite in-memory snapshot with stale .bak (continuing to serve the latest good data); fix the main store ASAP"
);
backup_warned = true;
}
continue;
}
let routes = table.len();
let live_urls: HashSet<String> = unlocked
.store
.routes
.values()
.filter_map(|r| r.proxy.clone())
.filter(|p| p.as_str() != crate::model::PROXY_DIRECT)
.collect();
state.snapshot.store(Arc::new(Snapshot {
store: Arc::new(unlocked.store),
table,
from_backup,
}));
state.clients.retain_urls(&live_urls);
last = now;
failures = 0;
delay = RELOAD_POLL;
backup_warned = false;
state.reload.failures.store(0, Ordering::Relaxed);
state
.reload
.last_ok_secs
.store(state.started.elapsed().as_secs(), Ordering::Relaxed);
if from_backup {
tracing::error!(
"hot reload fell back to .bak backup (main store failed to decrypt): revoked/deleted vkeys may become effective again; fix ASAP"
);
}
tracing::info!(routes, from_backup, "store hot-reloaded");
}
Err(e) => {
failures += 1;
state.reload.failures.store(failures, Ordering::Relaxed);
delay = next_reload_backoff(delay);
if failures % 10 == 1 {
tracing::warn!(error = %e, "hot reload validation failed, keeping old snapshot");
}
}
},
Err(e) => {
failures += 1;
state.reload.failures.store(failures, Ordering::Relaxed);
delay = next_reload_backoff(delay);
if failures % 10 == 1 {
tracing::warn!(error = %e, "hot reload decryption failed, keeping old snapshot");
}
}
}
}
}
fn stamp(paths: &Paths) -> (u64, u64, u64) {
fn one(path: &std::path::Path) -> u64 {
let Ok(md) = std::fs::metadata(path) else {
return 0;
};
let mtime = md
.modified()
.ok()
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
.map(|d| d.as_nanos() as u64)
.unwrap_or(0);
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt as _;
mtime ^ md.len().rotate_left(17) ^ md.ino().rotate_left(41)
}
#[cfg(not(unix))]
{
mtime ^ md.len().rotate_left(17)
}
}
(
one(&paths.store()),
one(&paths.master_key()),
one(&paths.store_bak()),
)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::model::Auth;
use std::collections::BTreeMap;
fn route(auth: Auth) -> Route {
Route::new(
vkey::generate(),
"https://api.example.com".into(),
"real-key-123".into(),
auth,
BTreeMap::new(),
)
}
#[test]
fn extracts_vkey_case_insensitive_and_from_alt_headers() {
let token = vkey::generate();
let mut h = HeaderMap::new();
h.insert(
header::AUTHORIZATION,
HeaderValue::from_str(&format!("BEARER {token}")).unwrap(),
);
assert_eq!(extract_vkey(&h).as_deref(), Some(token.as_str()));
let mut h = HeaderMap::new();
h.insert(
header::AUTHORIZATION,
HeaderValue::from_str(&format!("Bearer {token}")).unwrap(),
);
assert_eq!(extract_vkey(&h).as_deref(), Some(token.as_str()));
let mut h = HeaderMap::new();
h.insert(
header::AUTHORIZATION,
HeaderValue::from_str(&format!("Bearer{token}")).unwrap(),
);
assert_eq!(extract_vkey(&h).as_deref(), None);
let mut h = HeaderMap::new();
h.insert("x-api-key", HeaderValue::from_str(&token).unwrap());
assert_eq!(extract_vkey(&h).as_deref(), Some(token.as_str()));
for name in ["anthropic-api-key", "openai-api-key"] {
let mut h = HeaderMap::new();
h.insert(name, HeaderValue::from_str(&token).unwrap());
assert_eq!(extract_vkey(&h).as_deref(), Some(token.as_str()), "{name}");
}
let mut h = HeaderMap::new();
h.insert(
header::AUTHORIZATION,
HeaderValue::from_static("Bearer sk-real"),
);
assert!(extract_vkey(&h).is_none());
}
#[test]
fn rejects_forbidden_static_headers() {
let mut r = route(Auth::Bearer);
r.headers.insert("host".into(), "evil".into());
assert!(r.validate("r").is_err());
}
#[test]
fn dot_segment_detection_covers_every_form_the_url_parser_collapses() {
for path in [
"/../x",
"/v1/../x",
"/a/./b",
"/a/b/.",
"/..",
"/.",
".",
"..",
"/x/%2e/y",
"/x/%2E%2e/z",
"/x/.%2e/w",
"/x/%2e./w",
"/a\\..\\b",
"\\..",
"/x\\..\\y",
] {
assert!(path_has_dot_segment(path), "{path:?} must be flagged");
}
for path in [
"",
"/",
"/v1/models",
"/a/b/c",
"//",
"/a/.../b",
"/a/..%2fx",
"/a/%2e%2e%2e/x",
"/a/%2fx",
] {
assert!(!path_has_dot_segment(path), "{path:?} must pass");
}
}
#[test]
fn build_upstream_url_refuses_dot_segments() {
for path in ["/../x", "/v1/%2e%2e/x", "/a\\..\\b"] {
assert!(
build_upstream_url("https://up.test/v1", path, None, &Auth::None, "k").is_err(),
"{path:?} must be refused"
);
}
assert!(build_upstream_url("https://up.test/v1", "/v1/ok", None, &Auth::None, "k").is_ok());
}
#[test]
fn detects_stream_flag() {
assert!(is_stream_request(br#"{"stream":true}"#));
assert!(is_stream_request(br#"{"stream": true, "x": 1}"#));
assert!(!is_stream_request(br#"{"stream":false}"#));
assert!(!is_stream_request(br#"{"model":"x"}"#));
assert!(!is_stream_request(b"not json"));
assert!(!is_stream_request(b""));
}
#[test]
fn terminator_needles_fit_the_carry_window() {
for needle in TERMINATOR_NEEDLES {
assert!(
needle.len() <= TERMINATOR_MAX_NEEDLE,
"needle {needle:?} is longer than TERMINATOR_MAX_NEEDLE; raise the const so the boundary window still covers straddling markers"
);
}
}
#[test]
fn scan_finds_markers_split_at_every_point() {
for marker in TERMINATOR_NEEDLES {
for split in 1..marker.len() {
let mut scan = SseScan::new(Terminator::Auto, "R".into());
scan.feed(&marker[..split]);
assert!(
!scan.seen,
"{marker:?} split {split}: a partial marker must not match"
);
scan.feed(&marker[split..]);
assert!(
scan.seen,
"{marker:?} split {split}: a completed marker must be found"
);
}
let mut scan = SseScan::new(Terminator::Auto, "R".into());
for (i, byte) in marker.iter().enumerate() {
scan.feed(std::slice::from_ref(byte));
assert_eq!(
scan.seen,
i + 1 == marker.len(),
"{marker:?} byte {i}: seen must flip exactly at the last byte"
);
}
let mut scan = SseScan::new(Terminator::Auto, "R".into());
scan.feed(b"event: delta\ndata: {\"i\":0}\n\n");
assert!(!scan.seen);
let mut scan = SseScan::new(Terminator::None, "R".into());
scan.feed(b"event: message_stop\n\n");
assert!(!scan.seen && scan.carry.is_empty());
}
}
#[test]
fn builds_url_with_path_and_query() {
let url = build_upstream_url(
"https://api.example.com/",
"/v1/messages",
Some("a=1&b=2"),
&Auth::Bearer,
"secret",
)
.unwrap();
assert_eq!(url, "https://api.example.com/v1/messages?a=1&b=2");
}
#[test]
fn query_auth_replaces_existing_param() {
let url = build_upstream_url(
"https://generativelanguage.googleapis.com",
"/v1beta/models/x:generateContent",
Some("alt=sse&key=old"),
&Auth::Query {
param: "key".into(),
},
"a b&c",
)
.unwrap();
assert_eq!(
url,
"https://generativelanguage.googleapis.com/v1beta/models/x:generateContent?alt=sse&key=a%20b%26c"
);
}
#[test]
fn injects_bearer_and_strips_inbound_auth() {
let mut src = HeaderMap::new();
src.insert(
header::AUTHORIZATION,
HeaderValue::from_static("Bearer ks_xxx"),
);
src.insert("x-api-key", HeaderValue::from_static("ks_xxx"));
src.insert("x-custom", HeaderValue::from_static("ok"));
let out = forward_headers(&src, &route(Auth::Bearer), "real-secret", false).unwrap();
assert_eq!(
out.get(header::AUTHORIZATION).unwrap(),
"Bearer real-secret"
);
assert!(out.get("x-api-key").is_none());
assert_eq!(out.get("x-custom").unwrap(), "ok");
}
#[test]
fn injects_x_api_key_and_static_headers() {
let mut r = route(Auth::XApiKey);
r.headers
.insert("anthropic-version".into(), "2023-06-01".into());
let out = forward_headers(&HeaderMap::new(), &r, "real", false).unwrap();
assert_eq!(out.get("x-api-key").unwrap(), "real");
assert_eq!(out.get("anthropic-version").unwrap(), "2023-06-01");
}
#[test]
fn strips_accept_encoding_only_for_streaming_requests() {
let mut src = HeaderMap::new();
src.insert(
header::ACCEPT_ENCODING,
HeaderValue::from_static("gzip, deflate, br, zstd"),
);
let r = route(Auth::Bearer);
let streamed = forward_headers(&src, &r, "k", true).unwrap();
assert!(streamed.get(header::ACCEPT_ENCODING).is_none());
let plain = forward_headers(&src, &r, "k", false).unwrap();
assert_eq!(
plain.get(header::ACCEPT_ENCODING).unwrap(),
"gzip, deflate, br, zstd"
);
}
#[test]
fn streaming_route_static_accept_encoding_is_dropped() {
let mut r = route(Auth::Bearer);
r.headers.insert("accept-encoding".into(), "gzip".into());
r.headers.insert("ACCEPT-ENCODING".into(), "br".into());
let out = forward_headers(&HeaderMap::new(), &r, "k", true).unwrap();
assert!(out.get(header::ACCEPT_ENCODING).is_none());
let plain = forward_headers(&HeaderMap::new(), &r, "k", false).unwrap();
assert_eq!(plain.get(header::ACCEPT_ENCODING).unwrap(), "br");
}
#[test]
fn detects_content_encoding_but_not_identity() {
let mut headers = HeaderMap::new();
assert!(!is_encoded(&headers));
headers.insert(
header::CONTENT_ENCODING,
HeaderValue::from_static("identity"),
);
assert!(!is_encoded(&headers));
headers.insert(header::CONTENT_ENCODING, HeaderValue::from_static("zstd"));
assert!(is_encoded(&headers));
headers.insert(
header::CONTENT_ENCODING,
HeaderValue::from_static("gzip, br"),
);
assert!(is_encoded(&headers));
}
#[test]
fn basic_auth_encodes_pair() {
let out = forward_headers(
&HeaderMap::new(),
&route(Auth::Basic {
username: "user".into(),
}),
"pass",
false,
)
.unwrap();
assert_eq!(
out.get(header::AUTHORIZATION).unwrap(),
"Basic dXNlcjpwYXNz"
);
}
#[test]
fn detects_sse_terminators() {
assert!(contains_terminator(b"data: [DONE]\n\n", Terminator::Auto));
assert!(contains_terminator(
b"event: message_stop\ndata: {}\n\n",
Terminator::Auto
));
assert!(!contains_terminator(
b"data: {\"i\":0}\n\n",
Terminator::Auto
));
assert!(contains_terminator(b"DATA: [done]\n\n", Terminator::Openai));
assert!(!contains_terminator(
b"data: [DONE]\n\n",
Terminator::Anthropic
));
assert!(!contains_terminator(b"data: [DONE]\n\n", Terminator::None));
}
#[test]
fn stream_probe_semantics() {
assert!(is_stream_request(br#"{"model":"m","stream":true}"#));
assert!(is_stream_request(br#"{"stream":true,"model":"m"}"#));
assert!(!is_stream_request(br#"{"stream":null}"#));
assert!(!is_stream_request(br#"{"stream":"true"}"#));
assert!(!is_stream_request(br#"{"stream":1}"#));
assert!(is_stream_request(
br#"{"stre\u0061m":true,"note":"stream"}"#
));
assert!(!is_stream_request(br#"{"stre\u0061m":true}"#));
assert!(is_stream_request(br#"{"stream":true} trailing garbage"#));
assert!(is_stream_request(br#"{"stream":true,"stream":false}"#));
assert!(!is_stream_request(b"[true]"));
}
#[test]
fn finds_sse_terminators_in_all_case_variants() {
assert!(find_memmem_ci(b"data: [DONE]", b"data: [DONE]"));
assert!(find_memmem_ci(
b"xxxxxxxxxxxxxdata: [DONE]",
b"data: [DONE]"
));
let dense: Vec<u8> = [vec![b'd'; 30], b"data: [DONE]".to_vec()].concat();
assert!(find_memmem_ci(&dense, b"data: [DONE]"));
assert!(find_memmem_ci(
b"{\"i\":0} \"TYPE\":\"MESSAGE_STOP\"",
b"\"type\":\"message_stop\""
));
assert!(!find_memmem_ci(b"xxdata: [DON", b"data: [DONE]"));
assert!(!find_memmem_ci(b"data: [DON", b"data: [DONE]"));
assert!(!find_memmem_ci(b"no marker here", b"data: [DONE]"));
}
#[test]
fn rid_grows_past_four_digits() {
assert_eq!(rid_from(0), "0000");
assert_eq!(rid_from(1), "0001");
assert_eq!(rid_from(1679615), "ZZZZ");
assert_eq!(rid_from(1679616), "10000");
assert_eq!(rid_from(u64::MAX).len(), 13);
}
#[test]
fn sse_error_event_matches_protocol() {
let anthropic = sse_error_event("boom", Terminator::Anthropic);
let text = std::str::from_utf8(&anthropic).unwrap();
assert!(text.starts_with("event: error\n"));
assert!(text.contains("\"type\":\"error\""));
assert!(!text.contains("[DONE]"));
let openai = sse_error_event("boom", Terminator::Openai);
let text = std::str::from_utf8(&openai).unwrap();
assert!(text.starts_with("data: {"));
assert!(text.ends_with("data: [DONE]\n\n"));
let auto = sse_error_event("boom", Terminator::Auto);
assert_eq!(auto, openai);
}
#[test]
fn proxy_clients_follow_route_config() {
let clients = ProxyClients::new().unwrap();
let mut r = route(Auth::Bearer);
r.proxy = None;
assert!(clients.for_route(&r, false).is_ok());
assert!(clients.for_route(&r, true).is_ok());
r.proxy = Some("direct".into());
assert!(clients.for_route(&r, false).is_ok());
r.proxy = Some("http://127.0.0.1:7890".into());
assert!(clients.for_route(&r, true).is_ok());
assert!(clients.for_route(&r, false).is_ok());
r.proxy = Some("http://".into()); assert!(clients.for_route(&r, false).is_err());
}
#[test]
fn retain_urls_evicts_stale_proxy_clients() {
let clients = ProxyClients::new().unwrap();
let mut r = route(Auth::Bearer);
r.proxy = Some("http://127.0.0.1:7890".into());
assert!(clients.for_route(&r, false).is_ok());
let mut live = HashSet::new();
clients.retain_urls(&live);
assert!(clients.for_route(&r, false).is_ok());
live.insert("http://127.0.0.1:7890".into());
clients.for_route(&r, false).unwrap();
clients.retain_urls(&live);
clients.for_route(&r, false).unwrap();
}
#[tokio::test]
async fn classifies_length_limit_by_type_not_string_only() {
let e = axum::body::to_bytes(axum::body::Body::from("hello"), 2)
.await
.unwrap_err();
assert!(is_length_limit_error(&e));
let other = axum::Error::new(std::io::Error::other("boom"));
assert!(!is_length_limit_error(&other));
}
#[test]
fn watcher_never_downgrades_good_snapshot_to_backup() {
assert!(adopt_snapshot(false, Some(false)));
assert!(!adopt_snapshot(true, Some(false)));
assert!(adopt_snapshot(true, None));
assert!(adopt_snapshot(true, Some(true)));
}
#[test]
fn network_errors_retry_only_when_replay_safe() {
assert!(network_retry_allowed(true, false));
assert!(network_retry_allowed(false, true));
assert!(!network_retry_allowed(false, false));
}
#[test]
fn reload_backoff_doubles_and_caps() {
let mut d = RELOAD_POLL;
for _ in 0..10 {
d = next_reload_backoff(d);
}
assert_eq!(d, MAX_RELOAD_BACKOFF);
assert_eq!(
next_reload_backoff(d),
d,
"once capped, backoff stays capped"
);
}
#[cfg(unix)]
#[test]
fn accept_errors_split_into_pressure_connection_and_fatal() {
for code in [libc::EMFILE, libc::ENFILE, libc::ENOBUFS, libc::ENOMEM] {
let e = std::io::Error::from_raw_os_error(code);
assert!(
is_resource_pressure_accept_error(&e),
"errno {code} is resource pressure"
);
assert!(!is_connection_level_accept_error(&e));
}
for code in [
libc::ECONNABORTED,
libc::EPROTO,
libc::ENETDOWN,
libc::ENETUNREACH,
libc::EHOSTUNREACH,
libc::EOPNOTSUPP,
] {
let e = std::io::Error::from_raw_os_error(code);
assert!(
is_connection_level_accept_error(&e),
"errno {code} is per-connection, not fatal"
);
assert!(!is_resource_pressure_accept_error(&e));
}
assert!(!is_connection_level_accept_error(&std::io::Error::other(
"boom"
)));
assert!(!is_connection_level_accept_error(
&std::io::Error::from_raw_os_error(libc::EINVAL)
));
}
#[tokio::test(start_paused = true)]
async fn idle_bounded_passes_chunks_and_ends_cleanly() {
let inner = stream::iter(vec![
Ok::<Bytes, reqwest::Error>(Bytes::from_static(b"a")),
Ok(Bytes::from_static(b"b")),
]);
let mut s = Box::pin(idle_bounded(
inner,
Duration::from_secs(60),
Some(Duration::from_secs(600)),
));
assert_eq!(s.next().await.unwrap().unwrap(), Bytes::from_static(b"a"));
assert_eq!(s.next().await.unwrap().unwrap(), Bytes::from_static(b"b"));
assert!(s.next().await.is_none());
}
#[tokio::test(start_paused = true)]
async fn idle_bounded_aborts_stalled_body_and_stops() {
let inner = stream::once(async { Ok::<Bytes, reqwest::Error>(Bytes::from_static(b"x")) })
.chain(stream::pending::<reqwest::Result<Bytes>>());
let mut s = Box::pin(idle_bounded(
inner,
Duration::from_secs(30),
Some(Duration::from_secs(600)),
));
assert_eq!(s.next().await.unwrap().unwrap(), Bytes::from_static(b"x"));
let err = s.next().await.unwrap().unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
assert!(s.next().await.is_none());
}
#[tokio::test(start_paused = true)]
async fn idle_bounded_tolerates_slow_but_alive_upstream() {
let inner = stream::iter(0u8..3).then(|i| async move {
tokio::time::sleep(Duration::from_secs(10)).await;
Ok::<Bytes, reqwest::Error>(Bytes::from(vec![i]))
});
let mut s = Box::pin(idle_bounded(
inner,
Duration::from_secs(60),
Some(Duration::from_secs(600)),
));
for i in 0u8..3 {
assert_eq!(s.next().await.unwrap().unwrap(), Bytes::from(vec![i]));
}
assert!(s.next().await.is_none());
}
#[tokio::test(start_paused = true)]
async fn idle_bounded_enforces_the_total_budget() {
let inner = stream::iter(0u8..10).then(|i| async move {
tokio::time::sleep(Duration::from_secs(20)).await;
Ok::<Bytes, reqwest::Error>(Bytes::from(vec![i]))
});
let mut s = Box::pin(idle_bounded(
inner,
Duration::from_secs(60),
Some(Duration::from_secs(90)),
));
for i in 0u8..4 {
assert_eq!(s.next().await.unwrap().unwrap(), Bytes::from(vec![i]));
}
let err = s.next().await.unwrap().unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
assert!(
err.to_string().contains("total budget"),
"the total-budget abort must be distinguishable from the idle one: {err}"
);
assert!(s.next().await.is_none());
}
}
#[cfg(test)]
mod serve_inbound_tests {
use super::*;
use crate::model::Store;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
fn test_state() -> Arc<AppState> {
Arc::new(AppState {
snapshot: ArcSwap::from_pointee(Snapshot {
store: Arc::new(Store::new()),
table: RouteTable::build(&Store::new()).unwrap(),
from_backup: false,
}),
clients: ProxyClients::new().unwrap(),
retry: RetryPolicy::default(),
body_limit: 1 << 20,
body_read_idle: BODY_READ_IDLE,
inbound_body_timeout: CLIENT_BODY_TIMEOUT,
permit_queue_timeout: PERMIT_QUEUE_TIMEOUT,
body_total_budget: BODY_TOTAL_BUDGET,
reload: Arc::new(ReloadStatus::default()),
counters: Arc::new(ServeCounters::default()),
inbound_permits: Arc::new(tokio::sync::Semaphore::new(8)),
audit: Audit::disabled(),
started: Instant::now(),
})
}
#[tokio::test]
async fn serve_inbound_accepts_and_answers_while_reaping() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(serve_inbound(
listener,
build_app(test_state()),
InboundLimits {
header_timeout: CLIENT_HEADER_TIMEOUT,
max_connections: 64,
drain_timeout: Some(Duration::from_secs(5)),
loop_iterations: None,
live_connections: None,
counters: None,
},
std::future::pending(),
));
let buf = tokio::time::timeout(Duration::from_secs(10), async {
let mut s = tokio::net::TcpStream::connect(addr).await?;
s.write_all(b"GET /__keysink/health HTTP/1.1\r\nhost: t\r\nconnection: close\r\n\r\n")
.await?;
let mut buf = Vec::new();
s.read_to_end(&mut buf).await?;
Ok::<Vec<u8>, std::io::Error>(buf)
})
.await
.expect("serve_inbound must accept and answer connections")
.unwrap();
assert!(
buf.starts_with(b"HTTP/1.1 200"),
"health must answer 200, got: {}",
String::from_utf8_lossy(&buf)
);
tokio::time::timeout(Duration::from_secs(10), async {
loop {
tokio::time::sleep(Duration::from_millis(20)).await;
let mut s = tokio::net::TcpStream::connect(addr).await?;
s.write_all(
b"GET /__keysink/health HTTP/1.1\r\nhost: t\r\nconnection: close\r\n\r\n",
)
.await?;
let mut buf = Vec::new();
s.read_to_end(&mut buf).await?;
if buf.starts_with(b"HTTP/1.1 200") {
return Ok::<(), std::io::Error>(());
}
}
})
.await
.expect("second connection must also be served after the first was reaped")
.unwrap();
server.abort();
}
#[tokio::test]
async fn serve_inbound_idle_loop_pends_instead_of_spinning() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let iterations = Arc::new(AtomicU64::new(0));
let server = tokio::spawn(serve_inbound(
listener,
build_app(test_state()),
InboundLimits {
header_timeout: CLIENT_HEADER_TIMEOUT,
max_connections: 8,
drain_timeout: Some(Duration::from_secs(5)),
loop_iterations: Some(iterations.clone()),
live_connections: None,
counters: None,
},
std::future::pending(),
));
tokio::time::sleep(Duration::from_millis(300)).await;
let count = iterations.load(Ordering::Relaxed);
assert!(
count < 1_000,
"idle accept loop must pend on accept, not spin; iterations in 300ms: {count}"
);
let buf = tokio::time::timeout(Duration::from_secs(10), async {
let mut s = tokio::net::TcpStream::connect(addr).await?;
s.write_all(b"GET /__keysink/health HTTP/1.1\r\nhost: t\r\nconnection: close\r\n\r\n")
.await?;
let mut buf = Vec::new();
s.read_to_end(&mut buf).await?;
Ok::<Vec<u8>, std::io::Error>(buf)
})
.await
.expect("idle loop must still accept and answer")
.unwrap();
assert!(buf.starts_with(b"HTTP/1.1 200"));
server.abort();
}
#[tokio::test]
async fn serve_inbound_drops_silent_connection_within_head_timeout() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let head_timeout = Duration::from_millis(300);
let server = tokio::spawn(serve_inbound(
listener,
build_app(test_state()),
InboundLimits {
header_timeout: head_timeout,
max_connections: 8,
drain_timeout: Some(Duration::from_secs(5)),
loop_iterations: None,
live_connections: None,
counters: None,
},
std::future::pending(),
));
let closed = tokio::time::timeout(head_timeout * 10, async {
let mut s = tokio::net::TcpStream::connect(addr).await?;
let mut buf = [0u8; 16];
loop {
match s.read(&mut buf).await {
Ok(0) => return Ok::<(), std::io::Error>(()),
Ok(_) => continue,
Err(e) => return Err(e),
}
}
})
.await
.expect("silent connection must be dropped within the head timeout");
assert!(
closed.is_ok(),
"expected clean EOF, got {:?}",
closed.unwrap_err()
);
server.abort();
}
#[tokio::test(start_paused = true)]
async fn sniff_protocol_decides_or_times_out() {
let (mut client, mut server) = tokio::io::duplex(64);
let mut burst = H2_PREFACE.to_vec();
burst.extend_from_slice(&[0, 0, 0, 4, 0, 0, 0, 0]);
client.write_all(&burst).await.unwrap();
let seen = sniff_protocol(&mut server, Duration::from_secs(30))
.await
.unwrap();
assert_eq!(seen, burst, "preface burst must pass through unmodified");
let (mut client, mut server) = tokio::io::duplex(64);
client.write_all(b"GET /x HTTP/1.1\r\n").await.unwrap();
let seen = sniff_protocol(&mut server, Duration::from_secs(30))
.await
.unwrap();
assert_eq!(seen, b"GET /x HTTP/1.1\r\n".to_vec());
let (_client, mut server) = tokio::io::duplex(64);
let err = sniff_protocol(&mut server, Duration::from_secs(30))
.await
.unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
let (mut client, mut server) = tokio::io::duplex(64);
let dribble = tokio::spawn(async move {
loop {
for byte in H2_PREFACE {
if client.write_all(&[*byte]).await.is_err() {
return;
}
tokio::time::sleep(Duration::from_secs(10)).await;
}
}
});
let err = sniff_protocol(&mut server, Duration::from_secs(30))
.await
.unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
dribble.abort();
}
#[tokio::test]
async fn serve_inbound_reaps_finished_connection_tasks() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let live = Arc::new(AtomicU64::new(0));
let server = tokio::spawn(serve_inbound(
listener,
build_app(test_state()),
InboundLimits {
header_timeout: CLIENT_HEADER_TIMEOUT,
max_connections: 16,
drain_timeout: Some(Duration::from_secs(5)),
loop_iterations: None,
live_connections: Some(live.clone()),
counters: None,
},
std::future::pending(),
));
for _ in 0..50 {
let mut s = tokio::net::TcpStream::connect(addr).await.unwrap();
s.write_all(b"GET /__keysink/health HTTP/1.1\r\nhost: t\r\nconnection: close\r\n\r\n")
.await
.unwrap();
let mut buf = Vec::new();
s.read_to_end(&mut buf).await.unwrap();
assert!(buf.starts_with(b"HTTP/1.1 200"));
}
let reaped = tokio::time::timeout(Duration::from_secs(10), async {
loop {
if live.load(Ordering::Relaxed) == 0 {
return true;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.unwrap_or(false);
assert!(
reaped,
"finished connection tasks must be reaped, JoinSet still holds {}",
live.load(Ordering::Relaxed)
);
server.abort();
}
#[tokio::test]
async fn serve_inbound_shutdown_reaps_idle_keep_alive_connection() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
let server = tokio::spawn(serve_inbound(
listener,
build_app(test_state()),
InboundLimits {
header_timeout: CLIENT_HEADER_TIMEOUT,
max_connections: 8,
drain_timeout: Some(Duration::from_secs(5)),
loop_iterations: None,
live_connections: None,
counters: None,
},
async move {
let _ = shutdown_rx.await;
},
));
let mut s = tokio::net::TcpStream::connect(addr).await.unwrap();
s.write_all(b"GET /__keysink/health HTTP/1.1\r\nhost: t\r\n\r\n")
.await
.unwrap();
let mut buf = [0u8; 1024];
let n = s.read(&mut buf).await.unwrap();
assert!(
String::from_utf8_lossy(&buf[..n]).starts_with("HTTP/1.1 200"),
"health must answer before shutdown"
);
let started = Instant::now();
shutdown_tx.send(()).unwrap();
tokio::time::timeout(Duration::from_secs(3), server)
.await
.expect("shutdown must not wait for the 30s idle keep-alive reap")
.unwrap()
.unwrap();
assert!(
started.elapsed() < Duration::from_secs(3),
"drain took {:?}",
started.elapsed()
);
let closed = tokio::time::timeout(Duration::from_secs(2), s.read(&mut buf)).await;
assert!(
matches!(closed, Ok(Ok(0))),
"the drained connection must be closed, got {closed:?}"
);
}
#[tokio::test]
async fn health_reports_reload_and_audit_state() {
let state = test_state();
state.reload.failures.store(3, Ordering::Relaxed);
state.counters.shed.store(7, Ordering::Relaxed);
state.counters.accept_errors.store(2, Ordering::Relaxed);
let resp = health(State(state)).await.into_response();
let bytes = axum::body::to_bytes(resp.into_body(), 8192).await.unwrap();
let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
assert_eq!(v["ok"], true);
assert_eq!(v["reload_failures"], 3);
assert!(
v["snapshot_age_secs"].as_u64().is_some(),
"snapshot age must be reported: {v}"
);
assert_eq!(v["audit"], false);
assert_eq!(v["audit_ok"], true);
assert_eq!(v["audit_dropped"], 0);
assert_eq!(v["shed_connections"], 7);
assert_eq!(v["accept_errors"], 2);
}
#[tokio::test]
async fn preface_io_replays_then_passes_through() {
let (mut client, server) = tokio::io::duplex(8);
let mut io = PrefaceIo {
io: server,
preface: b"PRI * HTTP/2.0".to_vec(),
pos: 0,
};
client.write_all(b"|live|").await.unwrap();
drop(client);
let mut seen = Vec::new();
io.read_to_end(&mut seen).await.unwrap();
assert_eq!(
seen,
b"PRI * HTTP/2.0|live|".to_vec(),
"replayed preface must precede live bytes"
);
}
}