pub mod actions;
use std::convert::Infallible;
use std::net::SocketAddr;
use std::sync::Arc;
use bytes::Bytes;
use http_body_util::Full;
use hyper::body::Incoming;
use hyper::server::conn::http1;
use hyper::service::service_fn;
use hyper::{Request, Response};
use hyper_util::rt::TokioIo;
use tokio::sync::mpsc;
use tracing::{debug, error, info};
use crate::llm::action_helper::call_llm;
use crate::llm::ollama_client::OllamaClient;
use crate::logging::emit::Log;
use crate::protocol::Event;
use crate::server::connection::ConnectionId;
use crate::server::HttpProtocol;
use crate::state::app_state::AppState;
use actions::HTTP_REQUEST_EVENT;
const FIRST_BYTE_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
const IDLE_BETWEEN_REQUESTS_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(75);
const MAX_CONNECTIONS: usize = 128;
const CONNECTION_CAP_REFUSAL: &[u8] = b"HTTP/1.1 503 Service Unavailable\r\n\
Content-Length: 0\r\nRetry-After: 5\r\nConnection: close\r\n\r\n";
pub struct HttpServer;
impl HttpServer {
pub async fn spawn_with_llm_actions(
listen_addr: SocketAddr,
llm_client: OllamaClient,
app_state: Arc<AppState>,
status_tx: mpsc::UnboundedSender<String>,
server_id: crate::state::ServerId,
tls_config: Option<Arc<rustls::ServerConfig>>,
) -> anyhow::Result<SocketAddr> {
let listener =
crate::server::socket_helpers::create_reusable_tcp_listener(listen_addr).await?;
let local_addr = listener.local_addr()?;
let protocol_name = if tls_config.is_some() {
"HTTPS"
} else {
"HTTP"
};
Log::new(Some(&status_tx)).info(format!(
"{} server listening on {}",
protocol_name, local_addr
));
let protocol = Arc::new(HttpProtocol::new());
let filter = Arc::new(
crate::server::http_common::handler::RequestFilter::from_startup_params(
app_state
.get_server(server_id)
.await
.and_then(|s| s.startup_params)
.as_ref(),
),
);
let filter_log = Log::new(Some(&status_tx));
for warning in filter.warnings() {
filter_log.error(format!("HTTP request_filter: {}", warning));
}
let tls_acceptor = tls_config.map(|config| tokio_rustls::TlsAcceptor::from(config));
let task_registrar = app_state.clone();
let limiter = crate::server::accept_bounded::ConnectionLimiter::new(MAX_CONNECTIONS);
let accept_handle = tokio::spawn(async move {
loop {
match crate::server::accept_bounded::accept_bounded(
&listener,
&limiter,
CONNECTION_CAP_REFUSAL,
"HTTP",
Some(&status_tx),
)
.await
{
Ok((stream, remote_addr, permit)) => {
let connection_id =
ConnectionId::new(app_state.get_next_unified_id().await);
let local_addr_conn = stream.local_addr().unwrap_or(local_addr);
info!(
"Accepted {} connection {} from {}",
protocol_name, connection_id, remote_addr
);
use crate::state::server::{
ConnectionState as ServerConnectionState, ConnectionStatus,
ProtocolConnectionInfo,
};
let now = crate::utils::clock::Instant::now();
let conn_state = ServerConnectionState {
id: connection_id,
remote_addr,
local_addr: local_addr_conn,
bytes_sent: 0,
bytes_received: 0,
packets_sent: 0,
packets_received: 0,
last_activity: now,
status: ConnectionStatus::Active,
status_changed_at: now,
protocol_info: ProtocolConnectionInfo::new(serde_json::json!({
"recent_requests": []
})),
};
app_state
.add_connection_to_server(server_id, conn_state)
.await;
let _ = status_tx.send("__UPDATE_UI__".to_string());
let llm_client_clone = llm_client.clone();
let app_state_clone = app_state.clone();
let status_tx_clone = status_tx.clone();
let protocol_clone = protocol.clone();
let tls_acceptor_clone = tls_acceptor.clone();
let filter_clone = filter.clone();
let task_owner = app_state.clone();
task_owner
.spawn_server_task(server_id, async move {
let spoke = matches!(
tokio::time::timeout(
FIRST_BYTE_READ_TIMEOUT,
stream.peek(&mut [0u8; 1]),
)
.await,
Ok(Ok(n)) if n > 0
);
if !spoke {
Log::new(Some(&status_tx_clone)).debug(format!(
"{} peer {} sent nothing for {}s; closing before any \
request",
protocol_name,
remote_addr,
FIRST_BYTE_READ_TIMEOUT.as_secs()
));
app_state_clone
.close_connection_on_server(server_id, connection_id)
.await;
let _ = status_tx_clone.send("__UPDATE_UI__".to_string());
return;
}
let permit = std::sync::Arc::new(permit);
let activity = std::sync::Arc::new(
crate::server::accept_bounded::ConnectionActivity::new(),
);
if let Some(acceptor) = tls_acceptor_clone {
match acceptor.accept(stream).await {
Ok(tls_stream) => {
Log::new(Some(&status_tx_clone)).debug(format!(
"{} TLS handshake complete with {}",
protocol_name, remote_addr
));
let io = TokioIo::new(tls_stream);
Self::serve_connection(
io,
connection_id,
server_id,
llm_client_clone,
app_state_clone.clone(),
status_tx_clone.clone(),
protocol_clone,
filter_clone,
activity,
permit,
)
.await;
}
Err(e) => {
Log::new(Some(&status_tx_clone)).warn(format!(
"{} TLS handshake failed: {}",
protocol_name, e
));
}
}
} else {
let io = TokioIo::new(stream);
Self::serve_connection(
io,
connection_id,
server_id,
llm_client_clone,
app_state_clone.clone(),
status_tx_clone.clone(),
protocol_clone,
filter_clone,
activity,
permit,
)
.await;
}
app_state_clone
.close_connection_on_server(server_id, connection_id)
.await;
Log::new(Some(&status_tx_clone)).info(format!(
"{} connection {connection_id} closed",
protocol_name
));
let _ = status_tx_clone.send("__UPDATE_UI__".to_string());
})
.await;
}
Err(e) => {
Log::new(Some(&status_tx))
.error(format!("Failed to accept HTTP connection: {}", e));
break;
}
}
}
});
task_registrar
.register_server_task(server_id, accept_handle)
.await;
Ok(local_addr)
}
#[allow(clippy::too_many_arguments)]
async fn serve_connection<T>(
io: TokioIo<T>,
connection_id: ConnectionId,
server_id: crate::state::ServerId,
llm_client: OllamaClient,
app_state: Arc<AppState>,
status_tx: mpsc::UnboundedSender<String>,
protocol: Arc<HttpProtocol>,
filter: Arc<crate::server::http_common::handler::RequestFilter>,
activity: Arc<crate::server::accept_bounded::ConnectionActivity>,
permit: Arc<crate::server::accept_bounded::ConnectionPermit>,
) where
T: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static,
{
let status_for_service = status_tx.clone();
let app_state_for_service = app_state.clone();
let activity_for_service = Arc::clone(&activity);
let service = service_fn(move |req: Request<Incoming>| {
let llm_clone = llm_client.clone();
let state_clone = app_state_for_service.clone();
let status_clone = status_for_service.clone();
let protocol_clone = protocol.clone();
let filter_clone = filter.clone();
let activity = Arc::clone(&activity_for_service);
let permit_clone = Arc::clone(&permit);
async move {
let _busy = activity.busy();
handle_http_request_with_llm_actions(
req,
connection_id,
server_id,
llm_clone,
state_clone,
status_clone,
protocol_clone,
filter_clone,
permit_clone,
)
.await
}
});
let conn = http1::Builder::new()
.serve_connection(io, service)
.with_upgrades();
tokio::pin!(conn);
tokio::select! {
result = &mut conn => {
if let Err(err) = result {
error!("Error serving HTTP connection: {:?}", err);
}
}
_ = crate::server::accept_bounded::watch_idle(
Arc::clone(&activity),
IDLE_BETWEEN_REQUESTS_TIMEOUT,
) => {
debug!(
"HTTP connection {} idle for {}s; closing",
connection_id,
IDLE_BETWEEN_REQUESTS_TIMEOUT.as_secs()
);
}
}
}
}
#[allow(clippy::too_many_arguments)]
async fn handle_http_request_with_llm_actions(
req: Request<Incoming>,
connection_id: ConnectionId,
server_id: crate::state::ServerId,
llm_client: OllamaClient,
app_state: Arc<AppState>,
status_tx: mpsc::UnboundedSender<String>,
protocol: Arc<HttpProtocol>,
filter: Arc<crate::server::http_common::handler::RequestFilter>,
connection_permit: Arc<crate::server::accept_bounded::ConnectionPermit>,
) -> Result<Response<Full<Bytes>>, Infallible> {
app_state
.update_connection_stats(server_id, connection_id, None, None, Some(1), None)
.await;
let response = handle_http_request_inner(
req,
connection_id,
server_id,
llm_client,
app_state.clone(),
status_tx,
protocol,
filter,
connection_permit,
)
.await;
let bytes_sent = response
.as_ref()
.ok()
.and_then(|resp| {
use hyper::body::Body;
resp.body().size_hint().exact()
})
.unwrap_or(0);
app_state
.update_connection_stats(
server_id,
connection_id,
None,
Some(bytes_sent),
None,
Some(1),
)
.await;
response
}
#[allow(clippy::too_many_arguments)]
async fn handle_http_request_inner(
req: Request<Incoming>,
connection_id: ConnectionId,
server_id: crate::state::ServerId,
llm_client: OllamaClient,
app_state: Arc<AppState>,
status_tx: mpsc::UnboundedSender<String>,
protocol: Arc<HttpProtocol>,
filter: Arc<crate::server::http_common::handler::RequestFilter>,
connection_permit: Arc<crate::server::accept_bounded::ConnectionPermit>,
) -> Result<Response<Full<Bytes>>, Infallible> {
{
if let Some(upgrade_header) = req.headers().get(hyper::header::UPGRADE) {
if let Ok(upgrade_value) = upgrade_header.to_str() {
if upgrade_value.contains("h2c") {
Log::new(Some(&status_tx)).info(format!(
"HTTP/2 upgrade (h2c) requested on connection {}",
connection_id
));
if req.headers().get("HTTP2-Settings").is_none() {
Log::new(Some(&status_tx)).warn(format!(
"HTTP h2c upgrade on connection {} decision=protocol_error: no \
HTTP2-Settings header (no LLM call)",
connection_id
));
let response = Response::builder()
.status(400) .body(Full::new(Bytes::from(
"HTTP/2 upgrade requires HTTP2-Settings header",
)))
.unwrap();
return Ok(response);
}
let llm_clone = llm_client.clone();
let app_state_clone = app_state.clone();
let status_tx_clone = status_tx.clone();
let protocol_clone = protocol.clone();
let filter_clone = filter.clone();
let permit_clone = Arc::clone(&connection_permit);
let task_owner = app_state.clone();
task_owner
.spawn_server_task(server_id, async move {
let _permit = permit_clone;
match hyper::upgrade::on(req).await {
Ok(upgraded) => {
Log::new(Some(&status_tx_clone)).info(format!(
"Upgraded connection {} to HTTP/2",
connection_id
));
use hyper_util::rt::TokioIo;
let io = TokioIo::new(upgraded);
if let Err(e) = handle_upgraded_h2c_connection(
io,
connection_id,
server_id,
llm_clone,
app_state_clone,
status_tx_clone,
protocol_clone,
filter_clone,
)
.await
{
error!("Error handling upgraded h2c connection: {}", e);
}
}
Err(e) => {
Log::new(Some(&status_tx_clone))
.warn(format!("HTTP/2 upgrade failed: {}", e));
}
}
})
.await;
let response = Response::builder()
.status(101) .header(hyper::header::UPGRADE, "h2c")
.header(hyper::header::CONNECTION, "Upgrade")
.body(Full::new(Bytes::new()))
.unwrap();
return Ok(response);
}
}
}
}
let request_data =
match crate::server::http_common::handler::extract_request_data(req, "HTTP", &status_tx)
.await
{
Ok(data) => data,
Err(too_large) => {
return Ok(
crate::server::http_common::handler::build_payload_too_large_response(
&too_large, "HTTP", &status_tx,
),
);
}
};
if !request_data.body_bytes.is_empty() {
app_state
.update_connection_stats(
server_id,
connection_id,
Some(request_data.body_bytes.len() as u64),
None,
None,
None,
)
.await;
}
let (path, query_string) = if let Some(pos) = request_data.uri.find('?') {
(
request_data.uri[..pos].to_string(),
Some(request_data.uri[pos + 1..].to_string()),
)
} else {
(request_data.uri.clone(), None)
};
if !filter.is_pass_through() && !filter.allows(&request_data, &path) {
let resp = filter.rejection();
Log::new(Some(&status_tx)).info(format!(
"HTTP {} {} decision=refused_by_filter -> {} (no LLM call)",
request_data.method,
path,
resp.status().as_u16()
));
return Ok(resp);
}
let query = if let Some(ref qs) = query_string {
let mut params = serde_json::Map::new();
for pair in qs.split('&') {
if let Some((key, value)) = pair.split_once('=') {
let decoded_key =
urlencoding::decode(key).unwrap_or(std::borrow::Cow::Borrowed(key));
let decoded_value =
urlencoding::decode(value).unwrap_or(std::borrow::Cow::Borrowed(value));
params.insert(
decoded_key.to_string(),
serde_json::Value::String(decoded_value.to_string()),
);
} else {
let decoded_key =
urlencoding::decode(pair).unwrap_or(std::borrow::Cow::Borrowed(pair));
params.insert(
decoded_key.to_string(),
serde_json::Value::String(String::new()),
);
}
}
serde_json::Value::Object(params)
} else {
serde_json::Value::Object(serde_json::Map::new())
};
let body_is_binary = std::str::from_utf8(&request_data.body_bytes).is_err();
let body_text = String::from_utf8_lossy(&request_data.body_bytes);
let mut event_data = serde_json::json!({
"method": request_data.method,
"path": path,
"query": query,
"headers": request_data.headers,
"body": if body_text.is_empty() { "" } else { body_text.as_ref() },
"body_bytes": request_data.body_bytes.len()
});
if body_is_binary {
event_data["body_is_binary"] = serde_json::Value::Bool(true);
}
if let Some(qs) = query_string {
event_data["query_string"] = serde_json::Value::String(qs);
}
let event = Event::new(&HTTP_REQUEST_EVENT, event_data);
match call_llm(
&llm_client,
&app_state,
server_id,
Some(connection_id),
&event,
protocol.as_ref(),
)
.await
{
Ok(execution_result) => {
debug!("LLM HTTP response received");
for msg in execution_result.messages {
let _ = status_tx.send(msg);
}
let produced_response = crate::server::http_common::handler::produced_http_response(
&execution_result.protocol_results,
);
let failure_summary = execution_result
.failures
.iter()
.map(|f| format!("{}: {}", f.action, f.error))
.collect::<Vec<_>>()
.join("; ");
let has_default_response = filter.default_response_parts().is_some();
let fallback = if has_default_response {
"default_response"
} else {
"blank_200"
};
let subject = format!(
"HTTP {} {} (connection {})",
request_data.method, request_data.uri, connection_id
);
let log = Log::new(Some(&status_tx));
if produced_response {
log.info(format!("{} decision=model_answer", subject));
} else if !failure_summary.is_empty() {
log.error(format!(
"{} decision=model_bad_action fallback={}: the model answered but its \
action(s) could not be executed ({}), and the peer is still given an \
affirmative response",
subject, fallback, failure_summary
));
} else if has_default_response {
log.warn(format!(
"{} decision=model_silent fallback=default_response: the model produced no \
send_http_response, so the server's configured default_response was sent",
subject
));
} else {
log.warn(format!(
"{} decision=model_silent fallback=blank_200: the model produced no \
send_http_response and no default_response is configured, so the peer is \
answered 200 with an empty body (fail-open)",
subject
));
}
crate::server::http_common::handler::build_response(
execution_result.protocol_results,
"HTTP",
&request_data.method,
&request_data.uri,
&status_tx,
filter.default_response_parts(),
)
}
Err(e) => {
crate::server::http_common::handler::build_error_response(
e,
"HTTP",
&request_data.method,
&request_data.uri,
&status_tx,
)
}
}
}
#[allow(clippy::too_many_arguments)]
async fn handle_upgraded_h2c_connection<T>(
io: T,
connection_id: ConnectionId,
server_id: crate::state::ServerId,
llm_client: OllamaClient,
app_state: Arc<AppState>,
status_tx: mpsc::UnboundedSender<String>,
_protocol: Arc<HttpProtocol>,
filter: Arc<crate::server::http_common::handler::RequestFilter>,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
where
T: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static,
{
use crate::server::Http2Protocol;
info!("Starting h2c connection for {}", connection_id);
let mut h2_conn = crate::server::http2::h2_server::bounded_h2_builder()
.handshake(io)
.await?;
let body_budget = crate::server::http2::h2_server::BodyBudget::new();
let protocol = Arc::new(Http2Protocol::new());
let activity = Arc::new(crate::server::accept_bounded::ConnectionActivity::new());
loop {
let accepted = tokio::select! {
accepted = h2_conn.accept() => accepted,
_ = crate::server::accept_bounded::watch_idle(
Arc::clone(&activity),
IDLE_BETWEEN_REQUESTS_TIMEOUT,
) => {
debug!(
"H2C connection {} idle for {}s; closing",
connection_id,
IDLE_BETWEEN_REQUESTS_TIMEOUT.as_secs()
);
break;
}
};
match accepted {
Some(result) => {
let (request, send_response) = result?;
let llm_clone = llm_client.clone();
let app_state_clone = app_state.clone();
let status_tx_clone = status_tx.clone();
let protocol_clone = protocol.clone();
let filter_clone = filter.clone();
let activity_clone = Arc::clone(&activity);
let stream_budget = body_budget.clone();
let task_owner = app_state.clone();
task_owner
.spawn_server_task(server_id, async move {
let _busy = activity_clone.busy();
if let Err(e) = crate::server::http2::h2_server::handle_h2_request(
request,
send_response,
connection_id,
server_id,
llm_clone,
app_state_clone,
status_tx_clone,
protocol_clone,
filter_clone,
stream_budget,
)
.await
{
error!("Error handling h2c request: {}", e);
}
})
.await;
}
None => {
info!("H2C connection {} closed", connection_id);
break;
}
}
}
Ok(())
}