use std::time::Duration;
use axum::body::Bytes;
use axum::extract::State;
use axum::http::StatusCode;
use axum::response::sse::{Event, KeepAlive, Sse};
use axum::response::{IntoResponse, Response};
use axum::Json;
use serde_json::Value;
use crate::tenancy::jwt_lifecycle::JwtLifecycle;
use super::handlers::dispatch;
use super::router::{AuthedMcpState, McpState};
use super::types::{JsonRpcError, JsonRpcRequest, JsonRpcResponse};
pub(crate) async fn post_handler(State(state): State<McpState>, body: Bytes) -> Response {
handle_message(&state, &body, None).await
}
pub(crate) async fn handle_message(
state: &McpState,
body: &[u8],
ctx: Option<super::tools::McpContext>,
) -> Response {
let value: Value = match serde_json::from_slice(body) {
Ok(v) => v,
Err(_) => return json_error(Value::Null, JsonRpcError::parse_error()),
};
let recovered_id = value.get("id").cloned().unwrap_or(Value::Null);
let null_id = value.get("id").is_some_and(Value::is_null);
let request: JsonRpcRequest = match serde_json::from_value(value) {
Ok(r) => r,
Err(e) => return json_error(recovered_id, JsonRpcError::invalid_request(e.to_string())),
};
if request.jsonrpc != super::types::JSONRPC_VERSION {
return json_error(
recovered_id,
JsonRpcError::invalid_request("jsonrpc must be \"2.0\""),
);
}
if null_id {
return json_error(
Value::Null,
JsonRpcError::invalid_request("id must not be null"),
);
}
if request.is_notification() {
if request.method == "notifications/cancelled" {
if let (Some(ctx), Some(rid)) = (
ctx.as_ref(),
request
.params
.as_ref()
.and_then(|p| p.get("requestId"))
.map(jsonrpc_id_string),
) {
super::progress::cancel(&ctx.agent.tenant_binding(), ctx.agent.agent_id, &rid);
}
}
return StatusCode::ACCEPTED.into_response();
}
let id = request.id.clone().unwrap_or(Value::Null);
let request_id = jsonrpc_id_string(&id);
match dispatch(
state,
&request.method,
request.params,
ctx,
Some(request_id.as_str()),
)
.await
{
Ok(result) => Json(JsonRpcResponse::success(id, result)).into_response(),
Err(err) => Json(JsonRpcResponse::failure(id, err)).into_response(),
}
}
pub(crate) fn sse_handler<T: Into<crate::extractors::TenantScope>>(
t: T,
axum::extract::State(state): axum::extract::State<AuthedMcpState>,
axum::extract::OriginalUri(uri): axum::extract::OriginalUri,
headers: axum::http::HeaderMap,
extensions: axum::http::Extensions,
) -> impl std::future::Future<Output = Response> + Send {
let t = t.into();
async move { sse_in(t, state.jwt, uri, headers, extensions).await }
}
async fn sse_in(
t: crate::extractors::TenantScope,
jwt: std::sync::Arc<JwtLifecycle>,
uri: axum::http::Uri,
headers: axum::http::HeaderMap,
extensions: axum::http::Extensions,
) -> Response {
let Some(token) = super::auth::bearer(&headers) else {
return super::auth::unauthorized(&headers, &extensions, &uri);
};
let ip = crate::login_throttle::ClientIp::from_parts(&extensions, &headers);
let Some(tenant) = crate::tenancy::TenantSlug::of(&t.org) else {
return (StatusCode::INTERNAL_SERVER_ERROR, "tenant has no id").into_response();
};
let authed = super::auth::authenticate_bearer_until(&jwt, t.pool(), &tenant, token, &ip);
let (agent, exp) = match authed.await {
Ok(v) => v,
Err(e) => return e.into_response(&headers, &extensions, &uri),
};
open_capped(
Liveness::of(t.pool(), &extensions),
agent,
exp.map(|exp| JwtBound { jwt, exp }),
StreamCaps::from_env(),
crate::shutdown::Shutdown::from_extensions(&extensions),
)
}
#[derive(Clone)]
struct Liveness {
pool: crate::sql::Pool,
registry: Option<crate::sql::Pool>,
}
impl Liveness {
fn of(pool: &crate::sql::Pool, extensions: &axum::http::Extensions) -> Self {
Self {
pool: pool.clone(),
registry: crate::tenancy::middleware::session_keys(extensions).map(|k| k.registry),
}
}
async fn check(&self, agent: &super::McpAgent) -> Result<bool, String> {
if let Some(registry) = &self.registry {
let org = crate::tenancy::active_org_by_id(registry, agent.org_id)
.await
.map_err(|e| e.to_string())?;
if org.is_none() {
return Ok(false);
}
}
crate::tenancy::agent_token_still_valid_pool(
&self.pool,
agent.agent_id,
agent.user_id,
&agent.secret_prefix,
)
.await
.map_err(|e| e.to_string())
}
}
fn open_capped(
live: Liveness,
agent: super::McpAgent,
token: Option<JwtBound>,
caps: StreamCaps,
shutdown: crate::shutdown::Shutdown,
) -> Response {
let scope = agent.tenant_binding().namespace();
let Some(slot) = StreamSlot::acquire(&scope, agent.agent_id, caps) else {
return (
StatusCode::TOO_MANY_REQUESTS,
"too many open notification streams",
)
.into_response();
};
agent_sse(
live,
agent,
token,
KEEP_ALIVE * RECHECK_EVERY_KEEP_ALIVES,
slot,
shutdown,
)
}
#[derive(Clone, Copy, Debug)]
struct StreamCaps {
per_agent: usize,
per_tenant: usize,
}
impl StreamCaps {
const PER_AGENT_VAR: &'static str = "RUSTANGO_MCP_SSE_PER_AGENT";
const PER_TENANT_VAR: &'static str = "RUSTANGO_MCP_SSE_PER_TENANT";
fn from_env() -> Self {
let read = |var: &str, default: usize| {
std::env::var(var)
.ok()
.and_then(|v| v.trim().parse().ok())
.unwrap_or(default)
};
Self {
per_agent: read(Self::PER_AGENT_VAR, 8),
per_tenant: read(Self::PER_TENANT_VAR, 256),
}
}
}
#[derive(Default)]
struct OpenStreams {
by_agent: std::collections::HashMap<(String, i64), usize>,
by_tenant: std::collections::HashMap<String, usize>,
}
fn open_streams() -> std::sync::MutexGuard<'static, OpenStreams> {
static OPEN: std::sync::OnceLock<std::sync::Mutex<OpenStreams>> = std::sync::OnceLock::new();
OPEN.get_or_init(Default::default)
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
struct StreamSlot {
tenant: String,
agent_id: i64,
}
impl StreamSlot {
fn acquire(tenant: &str, agent_id: i64, caps: StreamCaps) -> Option<Self> {
let mut open = open_streams();
let key = (tenant.to_owned(), agent_id);
let agent_n = open.by_agent.get(&key).copied().unwrap_or(0);
let tenant_n = open.by_tenant.get(tenant).copied().unwrap_or(0);
let full = |n: usize, cap: usize| cap != 0 && n >= cap;
if full(agent_n, caps.per_agent) || full(tenant_n, caps.per_tenant) {
return None;
}
open.by_agent.insert(key, agent_n + 1);
open.by_tenant.insert(tenant.to_owned(), tenant_n + 1);
Some(Self {
tenant: tenant.to_owned(),
agent_id,
})
}
}
impl Drop for StreamSlot {
fn drop(&mut self) {
let mut open = open_streams();
let key = (std::mem::take(&mut self.tenant), self.agent_id);
if let Some(n) = open.by_agent.get_mut(&key) {
*n -= 1;
if *n == 0 {
open.by_agent.remove(&key);
}
}
if let Some(n) = open.by_tenant.get_mut(&key.0) {
*n -= 1;
if *n == 0 {
open.by_tenant.remove(&key.0);
}
}
}
}
struct JwtBound {
jwt: std::sync::Arc<JwtLifecycle>,
exp: i64,
}
const KEEP_ALIVE: Duration = Duration::from_secs(15);
const RECHECK_EVERY_KEEP_ALIVES: u32 = 4;
const MAX_RECHECK_ERRORS: u32 = 5;
fn agent_sse(
live: Liveness,
agent: super::McpAgent,
token: Option<JwtBound>,
recheck_every: Duration,
slot: StreamSlot,
shutdown: crate::shutdown::Shutdown,
) -> Response {
use tokio::sync::broadcast::error::RecvError;
let tenant = agent.tenant_binding();
let mut rx = super::notifications::subscribe(&tenant);
let exp = token.as_ref().map(|t| t.exp);
let stream = async_stream::stream! {
let _slot = slot;
let expired = async move {
match exp {
Some(exp) => {
let left = exp.saturating_mul(1000) - chrono::Utc::now().timestamp_millis();
tokio::time::sleep(Duration::from_millis(u64::try_from(left).unwrap_or(0))).await;
}
None => std::future::pending().await,
}
};
tokio::pin!(expired);
let start = tokio::time::Instant::now() + recheck_every;
let mut recheck = tokio::time::interval_at(start, recheck_every);
let mut failed = 0;
loop {
let received = tokio::select! {
() = &mut expired => break,
_ = recheck.tick() => {
if let Some(t) = &token {
if t.jwt.is_blacklisted(&agent.jti).await {
break;
}
}
match live.check(&agent).await {
Ok(true) => failed = 0,
Ok(false) => break,
Err(e) => {
failed += 1;
tracing::warn!(error = %e, failed, "mcp sse liveness re-check failed");
if failed >= MAX_RECHECK_ERRORS {
break;
}
}
}
continue;
}
r = rx.recv() => r,
};
match received {
Ok(frame) => {
if super::notifications::frame_visible(&frame, &tenant, agent.agent_id) {
yield Ok::<Event, std::convert::Infallible>(Event::default().data(frame.body));
}
}
Err(RecvError::Lagged(_)) => continue,
Err(RecvError::Closed) => break,
}
}
};
Sse::new(shutdown.cut(stream))
.keep_alive(KeepAlive::new().interval(KEEP_ALIVE))
.into_response()
}
fn json_error(id: Value, error: JsonRpcError) -> Response {
Json(JsonRpcResponse::failure(id, error)).into_response()
}
fn jsonrpc_id_string(id: &Value) -> String {
match id {
Value::String(s) => s.clone(),
other => other.to_string(),
}
}
#[cfg(all(test, feature = "sqlite", feature = "testkit"))]
mod tests {
use super::*;
use crate::shutdown::Shutdown;
use crate::tenancy::{create_agent_pool, rotate_agent_secret_pool};
type World = (
crate::sql::Pool,
super::super::McpAgent,
Option<JwtBound>,
String,
);
async fn world(jwt: JwtLifecycle, raw_key: bool) -> World {
let jwt = std::sync::Arc::new(jwt);
let pool = crate::sql::Pool::connect("sqlite::memory:").await.unwrap();
crate::testkit::migrate_framework(&pool).await.unwrap();
let bot = create_agent_pool(&pool, "bot").await.unwrap();
let ip = crate::login_throttle::ClientIp::default();
let bearer = if raw_key {
bot.token
} else {
let Ok(minted) = super::super::auth::mint_agent_jwt(
&jwt,
&pool,
&crate::tenancy::TenantSlug::new(1, "acme"),
"bot",
&bot.token,
&ip,
)
.await
else {
panic!("mint");
};
minted.token
};
let Ok((agent, exp)) = super::super::auth::authenticate_bearer_until(
&jwt,
&pool,
&crate::tenancy::TenantSlug::new(1, "acme"),
&bearer,
&ip,
)
.await
else {
panic!("authenticate");
};
(pool, agent, exp.map(|exp| JwtBound { jwt, exp }), bearer)
}
fn jwt() -> JwtLifecycle {
JwtLifecycle::new(b"unit-secret-at-least-32-bytes-long!!".to_vec())
}
const NO_CAP: StreamCaps = StreamCaps {
per_agent: 0,
per_tenant: 0,
};
fn open(
pool: crate::sql::Pool,
agent: super::super::McpAgent,
token: Option<JwtBound>,
recheck: Duration,
) -> Response {
open_with(
Liveness {
pool,
registry: None,
},
agent,
token,
recheck,
)
}
fn open_with(
live: Liveness,
agent: super::super::McpAgent,
token: Option<JwtBound>,
recheck: Duration,
) -> Response {
let scope = agent.tenant_binding().namespace();
let slot = StreamSlot::acquire(&scope, agent.agent_id, NO_CAP).unwrap();
agent_sse(live, agent, token, recheck, slot, Shutdown::default())
}
#[test]
fn open_streams_are_capped_per_agent_and_per_tenant() {
let tenant = format!("cap-{}", std::process::id());
let caps = StreamCaps {
per_agent: 2,
per_tenant: 3,
};
let a1 = StreamSlot::acquire(&tenant, 1, caps).unwrap();
let a2 = StreamSlot::acquire(&tenant, 1, caps).unwrap();
assert!(StreamSlot::acquire(&tenant, 1, caps).is_none(), "per agent");
let b1 = StreamSlot::acquire(&tenant, 2, caps).unwrap();
assert!(
StreamSlot::acquire(&tenant, 3, caps).is_none(),
"per tenant"
);
assert!(
StreamSlot::acquire(&format!("{tenant}-other"), 1, caps).is_some(),
"another tenant has its own count"
);
drop(a1);
let again = StreamSlot::acquire(&tenant, 1, caps);
assert!(again.is_some(), "a dropped stream frees its place");
drop((a2, b1, again));
}
#[tokio::test]
async fn the_handler_refuses_past_the_cap() {
let (pool, mut agent, _, _) = world(jwt(), false).await;
agent.agent_id = i64::MAX - 443;
let caps = StreamCaps {
per_agent: 1,
per_tenant: 0,
};
let scope = agent.tenant_binding().namespace();
let held = StreamSlot::acquire(&scope, agent.agent_id, caps).unwrap();
let live = Liveness {
pool,
registry: None,
};
let resp = open_capped(live, agent, None, caps, Shutdown::default());
assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS);
drop(held);
}
async fn assert_ends_on_revoke(
pool: crate::sql::Pool,
agent: super::super::McpAgent,
token: Option<JwtBound>,
) {
let body = drain(open(pool.clone(), agent, token, Duration::from_millis(50)));
tokio::time::sleep(Duration::from_millis(300)).await;
assert!(!body.is_finished(), "a live agent's stream must stay open");
rotate_agent_secret_pool(&pool, "bot").await.unwrap();
let ended = tokio::time::timeout(Duration::from_secs(5), body).await;
assert!(ended.is_ok(), "the stream outlived the revoke");
}
fn drain(resp: Response) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let _ = axum::body::to_bytes(resp.into_body(), usize::MAX).await;
})
}
#[tokio::test]
async fn the_stream_ends_when_the_jwt_expires() {
let (pool, agent, token, _) = world(jwt().with_access_ttl(2), false).await;
assert!(token.is_some());
let body = drain(open(pool, agent, token, Duration::from_secs(3600)));
let ended = tokio::time::timeout(Duration::from_secs(6), body).await;
assert!(ended.is_ok(), "the stream outlived its token");
}
#[tokio::test]
async fn the_stream_ends_when_the_agent_is_revoked() {
let (pool, agent, token, _) = world(jwt(), false).await;
assert_ends_on_revoke(pool, agent, token).await;
}
#[tokio::test]
async fn the_stream_ends_when_the_jwt_is_revoked() {
let (pool, agent, token, bearer) = world(jwt(), false).await;
let jwt = token.as_ref().map(|t| t.jwt.clone()).unwrap();
let body = drain(open(pool, agent, token, Duration::from_millis(50)));
tokio::time::sleep(Duration::from_millis(300)).await;
assert!(!body.is_finished(), "a live token's stream must stay open");
assert!(jwt.revoke(&bearer).await);
let ended = tokio::time::timeout(Duration::from_secs(5), body).await;
assert!(ended.is_ok(), "the stream outlived the JWT revoke");
}
#[tokio::test]
async fn the_stream_ends_at_shutdown() {
let (pool, agent, token, _) = world(jwt(), false).await;
let (stop, on_stop) = tokio::sync::watch::channel(());
let scope = agent.tenant_binding().namespace();
let slot = StreamSlot::acquire(&scope, agent.agent_id, NO_CAP).unwrap();
let resp = agent_sse(
Liveness {
pool,
registry: None,
},
agent,
token,
Duration::from_secs(3600),
slot,
Shutdown::on(on_stop),
);
let body = drain(resp);
stop.send(()).unwrap();
let ended = tokio::time::timeout(Duration::from_secs(5), body).await;
assert!(ended.is_ok(), "the stream outlived the stop signal");
}
#[tokio::test]
async fn a_raw_key_stream_ends_when_the_agent_is_revoked() {
let _g = super::super::auth::raw_key_cache_test_lock().lock().await;
let (pool, agent, token, _) = world(jwt(), true).await;
assert!(token.is_none());
assert_ends_on_revoke(pool, agent, token).await;
}
#[tokio::test]
async fn a_reused_slugs_frames_do_not_reach_the_purged_tenants_stream() {
use http_body_util::BodyExt as _;
let (pool, agent, token, _) = world(jwt(), false).await;
let own = agent.tenant_binding();
let heir = crate::tenancy::TenantSlug::new(own.org_id + 1, own.slug.clone());
let agent_id = agent.agent_id;
let mut body = open(pool, agent, token, Duration::from_secs(3600)).into_body();
super::super::notifications::notify_tools_list_changed(&heir, Some(agent_id));
super::super::notifications::notify_resources_list_changed(&own, None);
let first = tokio::time::timeout(Duration::from_secs(5), async {
while let Some(Ok(frame)) = body.frame().await {
let data = frame.into_data().unwrap_or_default();
let text = String::from_utf8_lossy(&data).into_owned();
if text.contains("list_changed") {
return text;
}
}
String::new()
})
.await
.unwrap();
assert!(
first.contains("resources/list_changed"),
"got the heir's frame: {first}"
);
}
async fn seed_org(pool: &crate::sql::Pool, id: i64, active: bool) {
let mut org = crate::testkit::org_with(|o| {
o.id = crate::sql::Auto::Set(id);
o.slug = format!("live-{id}");
o.active = active;
});
org.insert(pool).await.unwrap();
}
async fn ends_within_a_second(
pool: crate::sql::Pool,
registry: crate::sql::Pool,
agent: super::super::McpAgent,
token: Option<JwtBound>,
) -> bool {
let live = Liveness {
pool,
registry: Some(registry),
};
let body = drain(open_with(live, agent, token, Duration::from_millis(300)));
tokio::time::timeout(Duration::from_secs(1), body)
.await
.is_ok()
}
#[tokio::test]
async fn the_stream_ends_when_the_org_is_gone() {
let (pool, agent, token, _) = world(jwt(), false).await;
assert!(ends_within_a_second(pool.clone(), pool, agent, token).await);
}
#[tokio::test]
async fn the_stream_ends_when_the_org_is_deactivated() {
let (pool, agent, token, _) = world(jwt(), false).await;
seed_org(&pool, agent.org_id, false).await;
assert!(ends_within_a_second(pool.clone(), pool, agent, token).await);
}
#[tokio::test]
async fn a_live_orgs_stream_stays_open() {
let (pool, agent, token, _) = world(jwt(), false).await;
seed_org(&pool, agent.org_id, true).await;
assert!(!ends_within_a_second(pool.clone(), pool, agent, token).await);
}
#[tokio::test]
async fn the_handler_wires_the_registry_from_the_tenant_context() {
use crate::extractors::DatabaseTenantContext;
use crate::tenancy::{session::SessionSecret, BackendKind, ChainResolver, DatabasePools};
let (pool, _, _, _) = world(jwt(), false).await;
assert!(Liveness::of(&pool, &axum::http::Extensions::new())
.registry
.is_none());
let mut ext = axum::http::Extensions::new();
ext.insert(std::sync::Arc::new(DatabaseTenantContext {
pools: std::sync::Arc::new(DatabasePools::<crate::sql::sqlx::Sqlite>::new(
BackendKind::Sqlite,
)),
resolver: ChainResolver::new(),
session_secret: SessionSecret::from_bytes(vec![1; 32]),
operator_secret: SessionSecret::from_bytes(vec![2; 32]),
registry: pool.clone(),
}));
assert!(Liveness::of(&pool, &ext).registry.is_some());
}
#[tokio::test]
async fn another_tenants_burst_does_not_drop_this_tenants_frames() {
use http_body_util::BodyExt as _;
let (pool, agent, token, _) = world(jwt(), false).await;
let tenant = agent.tenant_binding();
let mut body = open(pool, agent, token, Duration::from_secs(3600)).into_body();
super::super::notifications::notify_resources_list_changed(&tenant, None);
let other = crate::tenancy::TenantSlug::new(99, "other-tenant");
for _ in 0..1000 {
super::super::notifications::notify_tools_list_changed(&other, None);
}
let found = tokio::time::timeout(Duration::from_secs(5), async {
while let Some(Ok(frame)) = body.frame().await {
let data = frame.into_data().unwrap_or_default();
if String::from_utf8_lossy(&data).contains("resources/list_changed") {
return true;
}
}
false
})
.await;
assert_eq!(found, Ok(true), "this tenant's frame was lost");
}
#[tokio::test]
async fn a_failing_recheck_keeps_the_stream_open() {
let (pool, agent, token, _) = world(jwt(), false).await;
let rename = |from: &'static str, to: &'static str| {
let pool = pool.clone();
async move {
crate::sql::raw_execute_pool(
&pool,
&format!("ALTER TABLE {from} RENAME TO {to}"),
vec![],
)
.await
.unwrap();
}
};
rename("rustango_agents", "rustango_agents_off").await;
let body = drain(open(pool.clone(), agent, token, Duration::from_millis(100)));
tokio::time::sleep(Duration::from_millis(250)).await;
assert!(!body.is_finished(), "a DB error must not end the stream");
rename("rustango_agents_off", "rustango_agents").await;
rotate_agent_secret_pool(&pool, "bot").await.unwrap();
let ended = tokio::time::timeout(Duration::from_secs(5), body).await;
assert!(ended.is_ok(), "the stream outlived the revoke");
}
#[tokio::test]
async fn the_stream_ends_after_repeated_recheck_errors() {
let _g = super::super::auth::raw_key_cache_test_lock().lock().await;
let (pool, agent, token, _) = world(jwt(), true).await;
assert!(token.is_none());
crate::sql::raw_execute_pool(
&pool,
"ALTER TABLE rustango_agents RENAME TO rustango_agents_gone",
vec![],
)
.await
.unwrap();
let body = drain(open(pool, agent, token, Duration::from_millis(50)));
let ended = tokio::time::timeout(Duration::from_secs(5), body).await;
assert!(
ended.is_ok(),
"the stream outlived a tenant whose pool always errors"
);
}
}