use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::time::{Duration, Instant};
use axum::http::HeaderMap;
use nemo_relay::api::llm::{
LlmAttributes, LlmCallEndParams, LlmCallParams, LlmHandle, LlmRequest, llm_call, llm_call_end,
};
use nemo_relay::api::runtime::{
ScopeStackHandle, TASK_SCOPE_STACK, create_scope_stack, task_scope_push,
};
use nemo_relay::api::scope::{
EmitMarkEventParams, PopScopeParams, PushScopeParams, ScopeHandle, ScopeType,
event as emit_mark_event, get_handle, pop_scope, push_scope,
};
use nemo_relay::api::tool::{
ToolCallEndParams, ToolCallParams, ToolHandle, tool_call, tool_call_end,
tool_conditional_execution,
};
use serde_json::{Map, Value, json};
use tokio::sync::Mutex;
use crate::alignment::{
self, GatewayManagementPolicy, PendingSubagentStart, SessionAlias, SessionAlignmentState,
insert_optional, json_string_at, json_value_at, merge_metadata,
};
use crate::config::{GatewayConfig, SessionConfig};
use crate::error::CliError;
use crate::model::{
AgentKind, LlmEvent, LlmHintEvent, NormalizedEvent, SessionEvent, SubagentEvent, ToolEvent,
};
const LLM_HINT_TTL: Duration = Duration::from_secs(300);
const TOOL_HINT_TTL: Duration = Duration::from_secs(300);
const LAST_OWNER_TTL: Duration = Duration::from_secs(300);
const AGENT_IDLE_TIMEOUT: Duration = Duration::from_secs(30);
const AGENT_IDLE_SWEEP_INTERVAL: Duration = Duration::from_secs(5);
#[derive(Clone)]
pub(crate) struct SessionManager {
inner: Arc<Mutex<HashMap<String, Session>>>,
alignment: Arc<Mutex<SessionAlignmentState>>,
default_config: GatewayConfig,
}
#[derive(Debug, Clone)]
pub(crate) struct LlmGatewayStart {
pub(crate) session_id: Option<String>,
pub(crate) provider: String,
pub(crate) model_name: Option<String>,
pub(crate) subagent_id: Option<String>,
pub(crate) conversation_id: Option<String>,
pub(crate) generation_id: Option<String>,
pub(crate) request_id: Option<String>,
pub(crate) request: LlmRequest,
pub(crate) streaming: bool,
pub(crate) metadata: Value,
}
#[cfg(test)]
#[derive(Debug, Clone)]
pub(crate) struct ActiveLlm {
stack: ScopeStackHandle,
handle: LlmHandle,
session_id: String,
owner_subagent_id: Option<String>,
}
pub(crate) struct GatewayCallPrep {
pub(crate) scope_stack: ScopeStackHandle,
pub(crate) session_id: String,
pub(crate) provider_name: String,
pub(crate) request: LlmRequest,
pub(crate) parent: Option<ScopeHandle>,
pub(crate) attributes: LlmAttributes,
pub(crate) metadata: Value,
pub(crate) model_name: Option<String>,
pub(crate) owner_subagent_id: Option<String>,
pub(crate) bypass_managed_pipeline: bool,
pub(crate) prune_empty_session_on_finish: bool,
}
struct Session {
agent_kind: AgentKind,
session_id: String,
scope_stack: ScopeStackHandle,
session_started: bool,
session_metadata: Value,
agent_scope: Option<ScopeHandle>,
turn_scope: Option<ScopeHandle>,
gateway_request_turn_open: bool,
turn_index: u64,
last_turn_llm_output: Option<Value>,
subagents: HashMap<String, ScopeHandle>,
subagent_stacks: HashMap<String, ScopeStackHandle>,
subagent_stack: Vec<String>,
completed_subagents: HashSet<String>,
llms: HashMap<String, LlmHandle>,
tools: HashMap<String, ActiveTool>,
pending_llm_hints: Vec<PendingLlmHint>,
pending_tool_hints: Vec<PendingToolHint>,
llm_request_affinity: HashMap<String, Option<String>>,
last_llm_owner: Option<LastLlmOwner>,
last_activity: Instant,
active_gateway_calls: usize,
config: SessionConfig,
}
#[derive(Debug, Clone)]
struct ActiveTool {
handle: ToolHandle,
name: String,
arguments: Value,
owner_subagent_id: Option<String>,
}
impl std::ops::Deref for ActiveTool {
type Target = ToolHandle;
fn deref(&self) -> &Self::Target {
&self.handle
}
}
#[derive(Debug, Clone)]
struct PendingLlmHint {
hint: LlmHintEvent,
inserted_at: Instant,
}
#[derive(Debug, Clone)]
struct PendingToolHint {
hint: ToolHint,
inserted_at: Instant,
}
#[derive(Debug, Clone)]
struct ToolHint {
tool_call_id: Option<String>,
tool_name: Option<String>,
subagent_id: Option<String>,
arguments: Value,
source: String,
}
#[derive(Debug, Clone)]
struct LastLlmOwner {
subagent_id: String,
updated_at: Instant,
source: LastLlmOwnerSource,
}
#[derive(Debug, Clone, Copy)]
enum LastLlmOwnerSource {
Llm,
Tool,
SubagentStart,
}
impl LastLlmOwnerSource {
const fn status(self) -> &'static str {
match self {
Self::Llm => "sticky_last_owner",
Self::Tool => "recent_tool_owner",
Self::SubagentStart => "subagent_start",
}
}
const fn metadata_source(self) -> Option<&'static str> {
match self {
Self::Llm => None,
Self::Tool => Some("tool_owner"),
Self::SubagentStart => Some("subagent_start"),
}
}
}
struct LlmOwnerResolution {
parent: Option<ScopeHandle>,
subagent_id: Option<String>,
status: &'static str,
source: Option<String>,
hint: Option<LlmHintEvent>,
metadata: Value,
}
struct ToolOwnerResolution {
parent: Option<ScopeHandle>,
subagent_id: Option<String>,
status: &'static str,
source: Option<String>,
hint: Option<ToolHint>,
}
impl SessionManager {
pub(crate) fn new(default_config: GatewayConfig) -> Self {
Self {
inner: Arc::new(Mutex::new(HashMap::new())),
alignment: Arc::new(Mutex::new(SessionAlignmentState::default())),
default_config,
}
}
pub(crate) fn start_idle_sweeper(&self) {
let inner = Arc::downgrade(&self.inner);
let alignment = Arc::downgrade(&self.alignment);
tokio::spawn(async move {
let mut interval = tokio::time::interval(AGENT_IDLE_SWEEP_INTERVAL);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
interval.tick().await;
let (Some(inner), Some(alignment)) = (inner.upgrade(), alignment.upgrade()) else {
break;
};
if let Err(error) = close_idle_sessions_from_parts(
&inner,
&alignment,
Instant::now(),
AGENT_IDLE_TIMEOUT,
"idle_timeout",
)
.await
{
eprintln!("nemo-relay CLI gateway: idle session teardown failed: {error}");
}
}
});
}
pub(crate) async fn apply_events(
&self,
headers: &HeaderMap,
events: Vec<NormalizedEvent>,
) -> Result<(), CliError> {
let mut alignment_state = self.alignment.lock().await;
let mut sessions = self.inner.lock().await;
for event in events {
let mut event = event;
let config = self.default_config.session_config_from_headers(headers);
if queue_or_promote_child_start(
&mut event,
&mut sessions,
&mut alignment_state,
config.clone(),
)
.await?
{
continue;
}
let Some((event, session_id, is_agent_started)) =
route_event_for_session(event, &mut sessions, &mut alignment_state)
else {
continue;
};
let event_kind = event_agent_kind(&event);
let should_remove_session = apply_event_to_session(
&mut sessions,
&session_id,
event,
event_kind,
config.clone(),
is_agent_started,
)
.await?;
if is_agent_started {
promote_pending_subagents_for_parent(
&mut sessions,
&mut alignment_state,
&session_id,
config.clone(),
)
.await?;
}
if should_remove_session {
sessions.remove(&session_id);
}
}
Ok(())
}
#[cfg(test)]
pub(crate) async fn start_llm(
&self,
headers: &HeaderMap,
start: LlmGatewayStart,
) -> Result<ActiveLlm, CliError> {
let mut start = start;
let config = self.default_config.session_config_from_headers(headers);
let alias = self.resolve_start_alias(&mut start, config.clone()).await?;
let mut sessions = self.inner.lock().await;
let session_id = start
.session_id
.clone()
.or_else(|| single_active_session_id(&sessions))
.unwrap_or_else(|| format!("{}-gateway", AgentKind::Gateway.as_str()));
let inferred_agent_kind = alignment::agent_kind_for_gateway_provider(&start.provider);
let session = sessions
.entry(session_id.clone())
.or_insert_with(|| Session::new(session_id, inferred_agent_kind, config));
let mut active = session.start_llm(start).await?;
if let Some(alias) = alias {
active.session_id = alias.parent_session_id;
active.owner_subagent_id = active.owner_subagent_id.or(Some(alias.subagent_id));
}
Ok(active)
}
pub(crate) async fn prepare_gateway_call(
&self,
headers: &HeaderMap,
start: LlmGatewayStart,
) -> Result<GatewayCallPrep, CliError> {
let mut start = start;
let config = self.default_config.session_config_from_headers(headers);
self.resolve_start_alias(&mut start, config.clone()).await?;
let mut sessions = self.inner.lock().await;
let session_id = start
.session_id
.clone()
.or_else(|| single_active_session_id(&sessions))
.unwrap_or_else(|| format!("{}-gateway", AgentKind::Gateway.as_str()));
let inferred_agent_kind = alignment::agent_kind_for_gateway_provider(&start.provider);
let created_session = !sessions.contains_key(&session_id);
let session = sessions
.entry(session_id.clone())
.or_insert_with(|| Session::new(session_id.clone(), inferred_agent_kind, config));
let result = session.prepare_gateway_call(start).await;
match result {
Ok(mut prep) => {
prep.prune_empty_session_on_finish = prep.bypass_managed_pipeline
&& sessions
.get(&session_id)
.is_some_and(|session| session.is_empty());
Ok(prep)
}
Err(error) => {
if created_session
&& sessions
.get(&session_id)
.is_some_and(|session| session.is_empty())
{
sessions.remove(&session_id);
}
Err(error)
}
}
}
pub(crate) async fn finish_gateway_call(&self, session_id: &str, prune_empty_session: bool) {
let mut sessions = self.inner.lock().await;
if let Some(session) = sessions.get_mut(session_id) {
session.finish_gateway_call();
}
if prune_empty_session
&& sessions
.get(session_id)
.is_some_and(|session| session.is_empty() && session.active_gateway_calls == 0)
{
sessions.remove(session_id);
}
}
pub(crate) async fn has_open_sessions(&self) -> bool {
self.inner
.lock()
.await
.values()
.any(Session::blocks_plugin_idle_shutdown)
}
#[cfg(test)]
pub(crate) async fn end_llm(
&self,
active: ActiveLlm,
response: Value,
metadata: Value,
) -> Result<(), CliError> {
let response_for_hints = response.clone();
let session_id = active.session_id.clone();
let llm_id = active.handle.uuid.to_string();
let owner_subagent_id = active.owner_subagent_id.clone();
{
let mut sessions = self.inner.lock().await;
let Some(session) = sessions.get_mut(&session_id) else {
return Ok(());
};
if session.llms.remove(&llm_id).is_none() {
return Ok(());
}
}
TASK_SCOPE_STACK
.scope(active.stack, async move {
llm_call_end(
LlmCallEndParams::builder()
.handle(&active.handle)
.response(response)
.metadata(metadata)
.build(),
)
.map_err(CliError::from)
})
.await?;
let mut sessions = self.inner.lock().await;
if let Some(session) = sessions.get_mut(&session_id) {
session.record_completed_llm_response(response_for_hints, owner_subagent_id);
}
Ok(())
}
pub(crate) async fn record_gateway_response_hints(
&self,
session_id: &str,
owner_subagent_id: Option<String>,
response: Value,
) {
let alias = {
let alignment_state = self.alignment.lock().await;
alignment_state.alias_for_session(session_id)
};
let (session_id, owner_subagent_id) = match alias {
Some(alias) => (
alias.parent_session_id,
owner_subagent_id.or(Some(alias.subagent_id)),
),
None => (session_id.to_string(), owner_subagent_id),
};
let mut sessions = self.inner.lock().await;
if let Some(session) = sessions.get_mut(&session_id) {
session.record_completed_llm_response(response, owner_subagent_id);
}
}
pub(crate) async fn close_all(&self, reason: &str) -> Result<(), CliError> {
self.alignment.lock().await.clear();
let mut sessions = {
let mut guard = self.inner.lock().await;
guard
.drain()
.map(|(_, session)| session)
.collect::<Vec<_>>()
};
close_sessions_for_shutdown(&mut sessions, reason).await
}
#[cfg(test)]
pub(crate) async fn close_idle_sessions_at(
&self,
now: Instant,
timeout: Duration,
reason: &str,
) -> Result<usize, CliError> {
close_idle_sessions_from_parts(&self.inner, &self.alignment, now, timeout, reason).await
}
async fn resolve_start_alias(
&self,
start: &mut LlmGatewayStart,
config: SessionConfig,
) -> Result<Option<SessionAlias>, CliError> {
let Some(session_id) = start.session_id.clone() else {
return Ok(None);
};
let mut alignment_state = self.alignment.lock().await;
if let Some(alias) = alignment_state.alias_for_session(&session_id) {
apply_start_alias(start, &alias);
return Ok(Some(alias));
}
let Some(pending) = alignment_state.pending_for_session(&session_id) else {
return Ok(None);
};
let mut sessions = self.inner.lock().await;
let alias = promote_pending_subagent(
&mut sessions,
&mut alignment_state,
session_id,
pending,
config,
)
.await?;
if let Some(alias) = alias.as_ref() {
apply_start_alias(start, alias);
}
Ok(alias)
}
}
fn apply_start_alias(start: &mut LlmGatewayStart, alias: &SessionAlias) {
start.session_id = Some(alias.parent_session_id.clone());
start.subagent_id = Some(alias.subagent_id.clone());
start.metadata = merge_metadata(start.metadata.clone(), alias.metadata());
}
async fn queue_or_promote_child_start(
event: &mut NormalizedEvent,
sessions: &mut HashMap<String, Session>,
alignment_state: &mut SessionAlignmentState,
config: SessionConfig,
) -> Result<bool, CliError> {
let Some((child_session_id, pending)) = alignment::pending_subagent_start(event).await else {
return Ok(false);
};
if sessions
.get(&child_session_id)
.is_some_and(|session| !session.can_reparent_as_subagent_alias())
{
return Ok(false);
}
if sessions.contains_key(pending.parent_session_id()) {
alignment_state.remove_pending(&child_session_id);
promote_pending_subagent(sessions, alignment_state, child_session_id, pending, config)
.await?;
} else {
sessions.remove(&child_session_id);
alignment_state.insert_pending(child_session_id, pending);
}
Ok(true)
}
async fn apply_event_to_session(
sessions: &mut HashMap<String, Session>,
session_id: &str,
event: NormalizedEvent,
event_kind: AgentKind,
config: SessionConfig,
is_agent_started: bool,
) -> Result<bool, CliError> {
let session = sessions
.entry(session_id.to_string())
.or_insert_with(|| Session::new(session_id.to_string(), event_kind, config));
if is_agent_started
&& session.agent_kind == AgentKind::Gateway
&& event_kind != AgentKind::Gateway
{
session.agent_kind = event_kind;
}
session.apply(event).await?;
Ok(session.is_empty())
}
async fn promote_pending_subagents_for_parent(
sessions: &mut HashMap<String, Session>,
alignment_state: &mut SessionAlignmentState,
parent_session_id: &str,
config: SessionConfig,
) -> Result<(), CliError> {
for (child_session_id, pending) in alignment_state.pending_for_parent(parent_session_id) {
promote_pending_subagent(
sessions,
alignment_state,
child_session_id,
pending,
config.clone(),
)
.await?;
}
Ok(())
}
async fn promote_pending_subagent(
sessions: &mut HashMap<String, Session>,
alignment_state: &mut SessionAlignmentState,
child_session_id: String,
pending: PendingSubagentStart,
config: SessionConfig,
) -> Result<Option<SessionAlias>, CliError> {
if sessions
.get(&child_session_id)
.is_some_and(|session| !session.can_reparent_as_subagent_alias())
{
return Ok(None);
}
sessions.remove(&child_session_id);
let parent_session_id = pending.parent_session_id().to_string();
let parent_session = sessions
.entry(parent_session_id.clone())
.or_insert_with(|| {
Session::new(parent_session_id.clone(), pending.event.agent_kind, config)
});
if !parent_session.session_started && parent_session.agent_scope.is_none() {
parent_session
.apply(NormalizedEvent::AgentStarted(SessionEvent {
session_id: parent_session_id,
agent_kind: pending.event.agent_kind,
event_name: "implicit_parent_for_aligned_subagent".into(),
payload: Value::Null,
metadata: Value::Null,
}))
.await?;
}
let subagent_event = pending.subagent_start_event();
parent_session
.apply(NormalizedEvent::SubagentStarted(subagent_event))
.await?;
let alias = pending.alias_for_child_session(child_session_id.clone());
alignment_state.insert_alias(child_session_id, alias.clone());
Ok(Some(alias))
}
async fn close_sessions_for_shutdown(
sessions: &mut [Session],
reason: &str,
) -> Result<(), CliError> {
let mut first_error = None;
for session in sessions {
if let Err(error) = session.close_for_shutdown(reason).await
&& first_error.is_none()
{
first_error = Some(error);
}
}
first_error.map_or(Ok(()), Err)
}
async fn close_idle_sessions_from_parts(
inner: &Arc<Mutex<HashMap<String, Session>>>,
alignment: &Arc<Mutex<SessionAlignmentState>>,
now: Instant,
timeout: Duration,
reason: &str,
) -> Result<usize, CliError> {
let mut idle_sessions = Vec::new();
{
let mut sessions = inner.lock().await;
let ids = sessions
.iter()
.filter_map(|(session_id, session)| {
session
.is_idle_for(now, timeout)
.then_some(session_id.clone())
})
.collect::<Vec<_>>();
for session_id in ids {
if let Some(session) = sessions.remove(&session_id) {
idle_sessions.push((session_id, session));
}
}
}
if idle_sessions.is_empty() {
return Ok(0);
}
let mut closed_turns = 0;
let mut closed_subagents = Vec::new();
let mut retained_sessions = Vec::new();
let mut first_error = None;
for (session_id, mut session) in idle_sessions {
let stack = session.scope_stack.clone();
let result = TASK_SCOPE_STACK
.scope(stack, async { session.close_turn_for_reason(reason).await })
.await;
match result {
Ok(subagent_ids) => {
closed_turns += 1;
for subagent_id in subagent_ids {
closed_subagents.push((session_id.clone(), subagent_id));
}
}
Err(error) if first_error.is_none() => first_error = Some(error),
Err(_) => {}
}
if !session.is_empty() {
retained_sessions.push((session_id, session));
}
}
{
let mut sessions = inner.lock().await;
sessions.extend(retained_sessions);
}
if !closed_subagents.is_empty() {
let mut alignment_state = alignment.lock().await;
for (session_id, subagent_id) in closed_subagents {
alignment_state.clear_for_ended_subagent(&session_id, &subagent_id);
}
}
first_error.map_or(Ok(closed_turns), Err)
}
fn route_event_for_session(
event: NormalizedEvent,
sessions: &mut HashMap<String, Session>,
alignment_state: &mut SessionAlignmentState,
) -> Option<(NormalizedEvent, String, bool)> {
let mut event = alignment_state.route_event(event);
let explicit_subagent_alias = alignment::explicit_subagent_alias(&mut event);
let session_id = event.session_id().to_string();
let is_agent_started = matches!(&event, NormalizedEvent::AgentStarted(_));
if event.is_terminal() && !sessions.contains_key(&session_id) {
return None;
}
if !apply_explicit_subagent_alias(
&mut event,
sessions,
alignment_state,
explicit_subagent_alias,
) {
return None;
}
Some((event, session_id, is_agent_started))
}
fn apply_explicit_subagent_alias(
event: &mut NormalizedEvent,
sessions: &mut HashMap<String, Session>,
alignment_state: &mut SessionAlignmentState,
explicit_subagent_alias: Option<(String, SessionAlias)>,
) -> bool {
let Some((child_session_id, alias)) = explicit_subagent_alias else {
alignment_state.align_explicit_subagent_end(event);
return true;
};
if sessions
.get(&child_session_id)
.is_some_and(|session| !session.can_reparent_as_subagent_alias())
{
return false;
}
sessions.remove(&child_session_id);
alignment_state.insert_alias(child_session_id, alias);
alignment_state.align_explicit_subagent_end(event);
true
}
impl Session {
fn new(session_id: String, agent_kind: AgentKind, config: SessionConfig) -> Self {
Self {
agent_kind,
session_id,
scope_stack: create_scope_stack(),
session_started: false,
session_metadata: Value::Null,
agent_scope: None,
turn_scope: None,
gateway_request_turn_open: false,
turn_index: 0,
last_turn_llm_output: None,
subagents: HashMap::new(),
subagent_stacks: HashMap::new(),
subagent_stack: Vec::new(),
completed_subagents: HashSet::new(),
llms: HashMap::new(),
tools: HashMap::new(),
pending_llm_hints: Vec::new(),
pending_tool_hints: Vec::new(),
llm_request_affinity: HashMap::new(),
last_llm_owner: None,
last_activity: Instant::now(),
active_gateway_calls: 0,
config,
}
}
fn can_reparent_as_subagent_alias(&self) -> bool {
self.is_empty()
}
fn is_empty(&self) -> bool {
!self.session_started
&& self.agent_scope.is_none()
&& self.turn_scope.is_none()
&& self.subagents.is_empty()
&& self.subagent_stacks.is_empty()
&& self.subagent_stack.is_empty()
&& self.llms.is_empty()
&& self.tools.is_empty()
}
fn blocks_plugin_idle_shutdown(&self) -> bool {
self.agent_scope.is_some()
|| self.turn_scope.is_some()
|| !self.subagents.is_empty()
|| !self.subagent_stacks.is_empty()
|| !self.subagent_stack.is_empty()
|| !self.llms.is_empty()
|| !self.tools.is_empty()
|| self.active_gateway_calls > 0
}
fn touch_activity(&mut self) {
self.last_activity = Instant::now();
}
fn begin_gateway_call(&mut self) {
self.touch_activity();
self.active_gateway_calls += 1;
}
fn finish_gateway_call(&mut self) {
self.touch_activity();
self.active_gateway_calls = self.active_gateway_calls.saturating_sub(1);
}
fn is_idle_for(&self, now: Instant, timeout: Duration) -> bool {
self.turn_scope.is_some()
&& self.active_gateway_calls == 0
&& self.llms.is_empty()
&& self.tools.is_empty()
&& now.duration_since(self.last_activity) >= timeout
}
async fn apply(&mut self, event: NormalizedEvent) -> Result<(), CliError> {
self.touch_activity();
let stack = self.scope_stack.clone();
TASK_SCOPE_STACK
.scope(stack, async move {
match event {
NormalizedEvent::AgentStarted(event) => self.start_agent(event),
NormalizedEvent::AgentEnded(event) => self.end_agent(event).await,
NormalizedEvent::TurnEnded(event) => self.end_turn(event).await,
NormalizedEvent::SubagentStarted(event) => self.start_subagent(event).await,
NormalizedEvent::SubagentEnded(event) => self.end_subagent(event).await,
NormalizedEvent::LlmHint(event) => self.add_llm_hint(event),
NormalizedEvent::LlmStarted(event) => self.start_hook_llm(event),
NormalizedEvent::LlmEnded(event) => self.end_hook_llm(event),
NormalizedEvent::ToolStarted(event) => self.start_tool(event),
NormalizedEvent::ToolEnded(event) => self.end_tool(event).await,
NormalizedEvent::PromptSubmitted(event) => self.start_turn(event).await,
NormalizedEvent::Compaction(event) => self.mark("compaction", event),
NormalizedEvent::Notification(event) => self.mark("notification", event),
NormalizedEvent::HookMark(event) => self.mark("hook_mark", event),
}
})
.await
}
#[cfg(test)]
async fn start_llm(&mut self, start: LlmGatewayStart) -> Result<ActiveLlm, CliError> {
self.touch_activity();
let stack = self.scope_stack.clone();
TASK_SCOPE_STACK
.scope(stack.clone(), async move {
self.ensure_turn_started_for_gateway(&start)?;
let mut attributes = LlmAttributes::empty();
if start.streaming {
attributes |= LlmAttributes::STREAMING;
}
let owner = self.resolve_llm_owner(&start);
self.record_llm_request_affinity(
&start.provider,
&start.request,
owner.subagent_id.as_deref(),
owner.status,
);
let metadata = merge_metadata(
llm_correlation_metadata(
start.metadata,
owner.status,
owner.source.as_deref(),
owner.subagent_id.as_deref(),
owner.hint.as_ref(),
),
owner.metadata,
);
let handle = llm_call(
LlmCallParams::builder()
.name(start.provider.as_str())
.request(&start.request)
.parent_opt(owner.parent.as_ref())
.attributes(attributes)
.metadata(metadata)
.model_name_opt(start.model_name)
.build(),
)?;
let active = ActiveLlm {
stack,
handle,
session_id: self.session_id.clone(),
owner_subagent_id: owner.subagent_id,
};
self.llms
.insert(active.handle.uuid.to_string(), active.handle.clone());
Ok(active)
})
.await
}
async fn prepare_gateway_call(
&mut self,
start: LlmGatewayStart,
) -> Result<GatewayCallPrep, CliError> {
self.begin_gateway_call();
let stack = self.scope_stack.clone();
let result = TASK_SCOPE_STACK
.scope(stack.clone(), async {
let policy = self.gateway_management_policy(&start);
if !policy.bypasses_managed_pipeline() {
self.ensure_turn_started_for_gateway(&start)?;
}
let mut attributes = LlmAttributes::empty();
if start.streaming {
attributes |= LlmAttributes::STREAMING;
}
let owner = if policy.bypasses_managed_pipeline() {
self.unmanaged_probe_owner(policy)
} else {
self.resolve_llm_owner(&start)
};
self.record_llm_request_affinity(
&start.provider,
&start.request,
owner.subagent_id.as_deref(),
owner.status,
);
let metadata = merge_metadata(
llm_correlation_metadata(
start.metadata,
owner.status,
owner.source.as_deref(),
owner.subagent_id.as_deref(),
owner.hint.as_ref(),
),
owner.metadata,
);
Ok(GatewayCallPrep {
scope_stack: stack.clone(),
session_id: self.session_id.clone(),
provider_name: start.provider,
request: start.request,
parent: owner.parent,
attributes,
metadata,
model_name: start.model_name,
owner_subagent_id: owner.subagent_id,
bypass_managed_pipeline: policy.bypasses_managed_pipeline(),
prune_empty_session_on_finish: false,
})
})
.await;
if result.is_err() {
self.finish_gateway_call();
}
result
}
fn start_agent(&mut self, event: SessionEvent) -> Result<(), CliError> {
self.agent_kind = event.agent_kind;
self.session_started = true;
self.session_metadata =
merge_metadata(self.session_metadata.clone(), event.metadata.clone());
self.ensure_agent_started(event.metadata)
}
fn ensure_agent_started(&mut self, event_metadata: Value) -> Result<(), CliError> {
if self.agent_scope.is_some()
|| !alignment::should_emit_session_agent_scope(self.agent_kind)
{
return Ok(());
}
let _root = get_handle()?;
let metadata = merge_metadata(
self.scope_metadata(event_metadata),
json!({ "nemo_relay_scope_role": "session" }),
);
let scope = push_scope(
PushScopeParams::builder()
.name(self.agent_kind.as_str())
.scope_type(ScopeType::Agent)
.metadata(metadata)
.build(),
)?;
self.agent_scope = Some(scope);
Ok(())
}
async fn start_turn(&mut self, event: SessionEvent) -> Result<(), CliError> {
if alignment::aliased_turn_subagent_id(&event).is_some() {
self.ensure_turn_started(event.metadata.clone())?;
return self.mark("prompt_submitted", event);
}
if self.turn_scope.is_some() {
if self.gateway_request_turn_open {
self.gateway_request_turn_open = false;
return self.mark("prompt_submitted", event);
}
self.close_turn_for_reason("superseded_by_next_turn")
.await?;
}
self.open_turn(event.metadata, event.payload, "user_prompt")
}
fn ensure_turn_started(&mut self, event_metadata: Value) -> Result<(), CliError> {
if self.turn_scope.is_some() {
return Ok(());
}
self.open_turn(event_metadata, Value::Null, "implicit")
}
fn ensure_turn_started_for_gateway(&mut self, start: &LlmGatewayStart) -> Result<(), CliError> {
if self.turn_scope.is_some() {
return Ok(());
}
if let Some(input) =
alignment::gateway_turn_input(self.agent_kind, &start.provider, &start.request)
{
self.open_turn(start.metadata.clone(), input, "gateway_request")?;
self.gateway_request_turn_open = true;
return Ok(());
}
self.open_turn(Value::Null, Value::Null, "implicit")
}
fn gateway_management_policy(&self, start: &LlmGatewayStart) -> GatewayManagementPolicy {
if self.turn_scope.is_some() {
return GatewayManagementPolicy::Managed;
}
alignment::gateway_management_policy(
self.agent_kind,
&start.provider,
start.model_name.as_deref(),
&start.request,
)
}
fn open_turn(
&mut self,
event_metadata: Value,
input: Value,
turn_source: &str,
) -> Result<(), CliError> {
self.ensure_agent_started(event_metadata.clone())?;
self.turn_index += 1;
let metadata = merge_metadata(
self.scope_metadata(event_metadata),
json!({
"nemo_relay_scope_role": "turn",
"turn_index": self.turn_index,
"turn_source": turn_source,
}),
);
let turn_name = self.turn_scope_name();
let scope = push_scope(
PushScopeParams::builder()
.name(turn_name.as_str())
.scope_type(ScopeType::Custom)
.parent_opt(self.agent_scope.as_ref())
.metadata(metadata)
.input(input)
.build(),
)?;
self.turn_scope = Some(scope);
self.gateway_request_turn_open = false;
self.last_turn_llm_output = None;
Ok(())
}
fn turn_scope_name(&self) -> String {
format!("{}-turn", self.agent_kind.as_str())
}
fn scope_metadata(&self, event_metadata: Value) -> Value {
merge_metadata(
merge_metadata(
merge_metadata(
self.config.metadata.clone().unwrap_or(Value::Null),
self.session_metadata.clone(),
),
event_metadata,
),
json!({
"session_id": self.session_id,
"agent_kind": self.agent_kind.as_str(),
"gateway_config_profile": self.config.profile,
"plugin_config": self.config.plugin_config,
"gateway_mode": self.config.gateway_mode,
}),
)
}
async fn end_turn(&mut self, event: SessionEvent) -> Result<(), CliError> {
if let Some(subagent_id) = alignment::aliased_turn_subagent_id(&event) {
self.close_subagent_scope(&subagent_id, event.payload)
.await?;
return Ok(());
}
self.close_turn(event.payload, "closed_by_turn_end").await?;
Ok(())
}
async fn close_turn_for_reason(&mut self, reason: &str) -> Result<Vec<String>, CliError> {
self.close_turn(json!({ "status": reason }), reason).await
}
async fn close_turn(&mut self, output: Value, reason: &str) -> Result<Vec<String>, CliError> {
if self.turn_scope.is_none() {
return Ok(Vec::new());
}
self.close_active_llms(reason)?;
self.close_active_tools(reason)?;
let closed_subagents = self.close_active_subagents(reason).await?;
let output = self.last_turn_llm_output.take().unwrap_or(output);
self.clear_correlation_state();
self.close_turn_scope(output)?;
Ok(closed_subagents)
}
async fn end_agent(&mut self, event: SessionEvent) -> Result<(), CliError> {
if !self.session_started && self.agent_scope.is_none() && self.turn_scope.is_none() {
return Ok(());
}
self.close_turn_for_reason("closed_by_agent_end").await?;
self.clear_correlation_state();
self.close_agent_scope(event.payload)?;
self.session_started = false;
Ok(())
}
async fn close_for_shutdown(&mut self, reason: &str) -> Result<(), CliError> {
let stack = self.scope_stack.clone();
let payload = json!({ "status": reason });
TASK_SCOPE_STACK
.scope(stack, async move {
if self.agent_scope.is_none() && self.turn_scope.is_none() {
return Ok(());
}
self.close_turn_for_reason(reason).await?;
self.clear_correlation_state();
self.close_agent_scope(payload)?;
self.session_started = false;
Ok(())
})
.await
}
fn close_active_llms(&mut self, reason: &str) -> Result<(), CliError> {
let active_llms: Vec<_> = self.llms.drain().map(|(_, handle)| handle).collect();
for handle in active_llms {
llm_call_end(
LlmCallEndParams::builder()
.handle(&handle)
.response(json!({ "status": reason }))
.metadata(json!({ "status": reason }))
.build(),
)?;
}
Ok(())
}
fn close_active_tools(&mut self, reason: &str) -> Result<(), CliError> {
let active_tools: Vec<_> = self
.tools
.drain()
.map(|(_, active)| active.handle)
.collect();
for handle in active_tools {
tool_call_end(
ToolCallEndParams::builder()
.handle(&handle)
.result(json!({ "status": reason }))
.metadata(json!({ "status": reason }))
.build(),
)?;
}
Ok(())
}
async fn close_active_subagents(&mut self, reason: &str) -> Result<Vec<String>, CliError> {
let mut closed = Vec::new();
while let Some(subagent_id) = self.subagent_stack.pop() {
self.close_subagent_scope(&subagent_id, json!({ "status": reason }))
.await?;
closed.push(subagent_id);
}
self.subagents.clear();
self.subagent_stacks.clear();
Ok(closed)
}
fn clear_correlation_state(&mut self) {
self.pending_llm_hints.clear();
self.pending_tool_hints.clear();
self.llm_request_affinity.clear();
self.last_llm_owner = None;
}
fn close_agent_scope(&mut self, payload: Value) -> Result<(), CliError> {
let Some(scope) = self.agent_scope.take() else {
return Ok(());
};
pop_scope(
PopScopeParams::builder()
.handle_uuid(&scope.uuid)
.output(payload)
.build(),
)?;
Ok(())
}
fn close_turn_scope(&mut self, output: Value) -> Result<(), CliError> {
let Some(scope) = self.turn_scope.take() else {
return Ok(());
};
self.gateway_request_turn_open = false;
pop_scope(
PopScopeParams::builder()
.handle_uuid(&scope.uuid)
.output(output)
.build(),
)?;
Ok(())
}
fn root_work_scope(&self) -> Option<ScopeHandle> {
self.turn_scope.clone().or_else(|| self.agent_scope.clone())
}
async fn start_subagent(&mut self, event: SubagentEvent) -> Result<(), CliError> {
self.ensure_turn_started(event.metadata.clone())?;
if self.subagents.contains_key(&event.subagent_id) {
return Ok(());
}
let has_parallel_sibling = !self.subagents.is_empty();
let parent_scope = self
.turn_scope
.clone()
.expect("ensure_turn_started should initialize the turn scope");
let agent_scope = self.agent_scope.clone();
let subagent_id = event.subagent_id;
let subagent_name = format!("subagent:{subagent_id}");
let metadata = merge_metadata(
event.metadata,
json!({ "nemo_relay_scope_role": "subagent" }),
);
let subagent_stack = create_scope_stack();
let scope = TASK_SCOPE_STACK
.scope(subagent_stack.clone(), async {
if let Some(agent_scope) = agent_scope {
task_scope_push(agent_scope);
}
task_scope_push(parent_scope.clone());
push_scope(
PushScopeParams::builder()
.name(subagent_name.as_str())
.scope_type(ScopeType::Agent)
.parent(&parent_scope)
.metadata(metadata)
.input(event.payload)
.build(),
)
.map_err(CliError::from)
})
.await?;
self.completed_subagents.remove(&subagent_id);
if has_parallel_sibling {
self.set_last_subagent_start_owner(Some(subagent_id.clone()));
}
self.subagent_stack.push(subagent_id.clone());
self.subagent_stacks
.insert(subagent_id.clone(), subagent_stack);
self.subagents.insert(subagent_id, scope);
Ok(())
}
async fn end_subagent(&mut self, event: SubagentEvent) -> Result<(), CliError> {
if self.completed_subagents.contains(&event.subagent_id) {
return Ok(());
}
if !self.subagents.contains_key(&event.subagent_id) {
eprintln!(
"nemo-relay CLI gateway: received {} for subagent {} without a matching start",
event.event_name, event.subagent_id
);
if self.agent_kind == AgentKind::ClaudeCode && self.turn_scope.is_none() {
return Ok(());
}
return self.mark(
"subagent_end_without_start",
SessionEvent {
session_id: event.session_id,
agent_kind: event.agent_kind,
event_name: event.event_name,
payload: event.payload,
metadata: event.metadata,
},
);
};
self.ensure_turn_started(event.metadata.clone())?;
self.close_subagent_scope(&event.subagent_id, event.payload)
.await?;
Ok(())
}
async fn close_subagent_scope(
&mut self,
subagent_id: &str,
output: Value,
) -> Result<bool, CliError> {
let Some(scope) = self.subagents.remove(subagent_id) else {
return Ok(false);
};
let stack = self
.subagent_stacks
.remove(subagent_id)
.unwrap_or_else(|| self.scope_stack.clone());
TASK_SCOPE_STACK
.scope(stack, async {
pop_scope(
PopScopeParams::builder()
.handle_uuid(&scope.uuid)
.output(output)
.build(),
)
.map_err(CliError::from)
})
.await?;
self.subagent_stack.retain(|id| id != subagent_id);
self.completed_subagents.insert(subagent_id.to_string());
self.pending_tool_hints
.retain(|pending| pending.hint.subagent_id.as_deref() != Some(subagent_id));
self.llm_request_affinity
.retain(|_, owner| owner.as_deref() != Some(subagent_id));
if self
.last_llm_owner
.as_ref()
.is_some_and(|owner| owner.subagent_id == subagent_id)
{
self.last_llm_owner = None;
}
Ok(true)
}
fn add_llm_hint(&mut self, event: LlmHintEvent) -> Result<(), CliError> {
self.ensure_turn_started(event.metadata.clone())?;
self.cleanup_correlation_state();
let owner_subagent_id = event.subagent_id.clone().or_else(|| event.agent_id.clone());
self.add_tool_hints_from_llm_response(event.payload.clone(), owner_subagent_id);
self.pending_llm_hints.push(PendingLlmHint {
hint: event,
inserted_at: Instant::now(),
});
Ok(())
}
fn start_hook_llm(&mut self, event: LlmEvent) -> Result<(), CliError> {
self.ensure_turn_started(event.metadata.clone())?;
if self.llms.contains_key(&event.api_call_id) {
return Ok(());
}
let (parent, metadata) = self.hook_llm_owner(event.metadata);
let handle = llm_call(
LlmCallParams::builder()
.name(event.provider.as_str())
.request(&LlmRequest {
headers: Map::new(),
content: event.request,
})
.parent_opt(parent.as_ref())
.attributes(LlmAttributes::empty())
.metadata(metadata)
.model_name_opt(event.model_name)
.build(),
)?;
self.llms.insert(event.api_call_id, handle);
Ok(())
}
fn end_hook_llm(&mut self, event: LlmEvent) -> Result<(), CliError> {
self.ensure_turn_started(event.metadata.clone())?;
let (parent, metadata) = self.hook_llm_owner(event.metadata);
let handle = match self.llms.remove(&event.api_call_id) {
Some(handle) => handle,
None => llm_call(
LlmCallParams::builder()
.name(event.provider.as_str())
.request(&LlmRequest {
headers: Map::new(),
content: event.request,
})
.parent_opt(parent.as_ref())
.attributes(LlmAttributes::empty())
.metadata(metadata.clone())
.model_name_opt(event.model_name.clone())
.build(),
)?,
};
let output = event.response;
let root_owned =
json_string_at(&metadata, &[&["llm_correlation_subagent_id"][..]]).is_none();
if root_owned {
self.record_turn_llm_output(output.clone());
}
llm_call_end(
LlmCallEndParams::builder()
.handle(&handle)
.response(output)
.metadata(metadata)
.build(),
)?;
Ok(())
}
fn hook_llm_owner(&mut self, metadata: Value) -> (Option<ScopeHandle>, Value) {
let Some(subagent_id) = json_string_at(&metadata, &[&["llm_correlation_subagent_id"][..]])
else {
return (self.root_work_scope(), metadata);
};
let Some(scope) = self.subagents.get(&subagent_id).cloned() else {
return (self.root_work_scope(), metadata);
};
self.set_last_llm_owner(Some(subagent_id.clone()));
(
Some(scope),
merge_metadata(metadata, self.subagent_llm_metadata(&subagent_id)),
)
}
fn start_tool(&mut self, event: ToolEvent) -> Result<(), CliError> {
self.ensure_turn_started(event.metadata.clone())?;
if self.tools.contains_key(&event.tool_call_id) {
return Ok(());
}
let owner = self.resolve_tool_owner(&event);
let arguments = if event.arguments.is_null() {
owner
.hint
.as_ref()
.map(|hint| hint.arguments.clone())
.unwrap_or(event.arguments)
} else {
event.arguments
};
let active_tool_arguments = arguments.clone();
let active_tool_name = event.tool_name.clone();
let active_tool_owner_subagent_id = owner.subagent_id.clone();
tool_conditional_execution(event.tool_name.as_str(), &arguments)?;
let metadata = tool_correlation_metadata(
event.metadata,
owner.status,
owner.source.as_deref(),
owner.subagent_id.as_deref(),
owner.hint.as_ref(),
);
self.set_last_tool_owner(owner.subagent_id.clone());
let handle = tool_call(
ToolCallParams::builder()
.name(event.tool_name.as_str())
.args(arguments)
.parent_opt(owner.parent.as_ref())
.metadata(metadata)
.tool_call_id(event.tool_call_id.clone())
.build(),
)?;
self.tools.insert(
event.tool_call_id,
ActiveTool {
handle,
name: active_tool_name,
arguments: active_tool_arguments,
owner_subagent_id: active_tool_owner_subagent_id,
},
);
Ok(())
}
async fn end_tool(&mut self, event: ToolEvent) -> Result<(), CliError> {
self.ensure_turn_started(event.metadata.clone())?;
let completed_agent_subagent_id = alignment::completed_subagent_from_tool(&event);
let explicit_subagent_id = event
.subagent_id
.clone()
.filter(|subagent_id| self.subagents.contains_key(subagent_id));
let handle = match self.remove_tool_handle_for_event(&event) {
Some(handle) => handle,
None => {
let owner = self.resolve_tool_owner(&event);
let arguments = if event.arguments.is_null() {
owner
.hint
.as_ref()
.map(|hint| hint.arguments.clone())
.unwrap_or(event.arguments)
} else {
event.arguments
};
let metadata = tool_correlation_metadata(
event.metadata.clone(),
owner.status,
owner.source.as_deref(),
owner.subagent_id.as_deref(),
owner.hint.as_ref(),
);
self.set_last_tool_owner(owner.subagent_id.clone());
tool_call(
ToolCallParams::builder()
.name(event.tool_name.as_str())
.args(arguments)
.parent_opt(owner.parent.as_ref())
.metadata(metadata)
.tool_call_id(event.tool_call_id.clone())
.build(),
)?
}
};
tool_call_end(
ToolCallEndParams::builder()
.handle(&handle)
.result(event.result.clone())
.metadata(merge_metadata(
event.metadata,
json!({ "status": event.status }),
))
.build(),
)?;
self.set_last_tool_owner(explicit_subagent_id);
if let Some(subagent_id) = completed_agent_subagent_id {
self.close_subagent_scope(&subagent_id, event.result)
.await?;
}
Ok(())
}
fn remove_tool_handle_for_event(&mut self, event: &ToolEvent) -> Option<ToolHandle> {
if let Some(active) = self.tools.remove(&event.tool_call_id) {
return Some(active.handle);
}
let owner_subagent_id = self.tool_event_owner_subagent_id(event);
let key = self.matching_active_tool_key(event, owner_subagent_id.as_deref())?;
self.tools.remove(&key).map(|active| active.handle)
}
fn matching_active_tool_key(
&self,
event: &ToolEvent,
owner_subagent_id: Option<&str>,
) -> Option<String> {
if event.arguments.is_null() {
return None;
}
let matches = self
.tools
.iter()
.filter_map(|(key, active)| {
(owner_subagent_id
.is_none_or(|owner| active.owner_subagent_id.as_deref() == Some(owner))
&& active.name == event.tool_name
&& active.arguments == event.arguments)
.then_some(key.clone())
})
.collect::<Vec<_>>();
(matches.len() == 1).then(|| matches[0].clone())
}
fn tool_event_owner_subagent_id(&self, event: &ToolEvent) -> Option<String> {
if let Some(subagent_id) = &event.subagent_id
&& self.subagents.contains_key(subagent_id)
{
return Some(subagent_id.clone());
}
self.matching_tool_hint_index(event)
.and_then(|index| self.pending_tool_hints[index].hint.subagent_id.clone())
.filter(|subagent_id| self.subagents.contains_key(subagent_id))
}
fn mark(&mut self, name: &str, event_payload: SessionEvent) -> Result<(), CliError> {
self.ensure_turn_started(event_payload.metadata.clone())?;
emit_mark_event(
EmitMarkEventParams::builder()
.name(name)
.data(event_payload.payload)
.metadata(event_payload.metadata)
.build(),
)?;
Ok(())
}
fn cleanup_correlation_state(&mut self) {
let now = Instant::now();
self.pending_llm_hints
.retain(|hint| now.duration_since(hint.inserted_at) <= LLM_HINT_TTL);
self.pending_tool_hints
.retain(|hint| now.duration_since(hint.inserted_at) <= TOOL_HINT_TTL);
if self
.last_llm_owner
.as_ref()
.is_some_and(|owner| now.duration_since(owner.updated_at) > LAST_OWNER_TTL)
{
self.last_llm_owner = None;
}
}
fn resolve_llm_owner(&mut self, start: &LlmGatewayStart) -> LlmOwnerResolution {
self.cleanup_correlation_state();
if let Some(resolution) = self.explicit_llm_owner(start) {
return resolution;
}
if let Some(resolution) = self.single_hint_owner() {
return resolution;
}
if let Some(resolution) = self.matched_hint_owner(start) {
return resolution;
}
if let Some(resolution) = self.request_affinity_owner(start) {
return resolution;
}
if let Some(resolution) = self.sticky_llm_owner() {
return resolution;
}
if let Some(resolution) = self.sole_subagent_owner() {
return resolution;
}
self.fallback_llm_owner()
}
fn explicit_llm_owner(&mut self, start: &LlmGatewayStart) -> Option<LlmOwnerResolution> {
if let Some(subagent_id) = &start.subagent_id
&& let Some(scope) = self.subagents.get(subagent_id).cloned()
{
self.set_last_llm_owner(Some(subagent_id.clone()));
return Some(LlmOwnerResolution {
parent: Some(scope),
subagent_id: Some(subagent_id.clone()),
status: "explicit",
source: Some("gateway_header".to_string()),
hint: None,
metadata: self.subagent_llm_metadata(subagent_id),
});
}
None
}
fn single_hint_owner(&mut self) -> Option<LlmOwnerResolution> {
if self.pending_llm_hints.len() == 1 {
let hint = self.pending_llm_hints.remove(0).hint;
return Some(self.resolution_from_hint(hint, "single_hint"));
}
None
}
fn matched_hint_owner(&mut self, start: &LlmGatewayStart) -> Option<LlmOwnerResolution> {
if let Some(index) = self.matching_hint_index(start) {
let hint = self.pending_llm_hints.remove(index).hint;
return Some(self.resolution_from_hint(hint, "matched_hint"));
}
None
}
fn request_affinity_owner(&mut self, start: &LlmGatewayStart) -> Option<LlmOwnerResolution> {
let key = alignment::request_affinity_key(&start.provider, &start.request)?;
let subagent_id = self.llm_request_affinity.get(&key).cloned().flatten()?;
let parent = match self.subagents.get(&subagent_id).cloned() {
Some(parent) => parent,
None => {
self.llm_request_affinity.remove(&key);
return None;
}
};
self.set_last_llm_owner(Some(subagent_id.clone()));
Some(LlmOwnerResolution {
parent: Some(parent),
subagent_id: Some(subagent_id.clone()),
status: "request_affinity",
source: Some("request_payload".to_string()),
hint: None,
metadata: self.subagent_llm_metadata(&subagent_id),
})
}
fn sticky_llm_owner(&self) -> Option<LlmOwnerResolution> {
if let Some(owner) = self.last_llm_owner.as_ref()
&& let Some(parent) = self.subagents.get(&owner.subagent_id).cloned()
{
return Some(LlmOwnerResolution {
parent: Some(parent),
subagent_id: Some(owner.subagent_id.clone()),
status: owner.source.status(),
source: owner.source.metadata_source().map(ToOwned::to_owned),
hint: None,
metadata: self.subagent_llm_metadata(&owner.subagent_id),
});
}
None
}
fn sole_subagent_owner(&mut self) -> Option<LlmOwnerResolution> {
if self.subagents.len() == 1
&& let Some((subagent_id, scope)) = self.subagents.iter().next()
{
let subagent_id = subagent_id.clone();
let scope = scope.clone();
let metadata = self.subagent_llm_metadata(&subagent_id);
self.set_last_llm_owner(Some(subagent_id.clone()));
return Some(LlmOwnerResolution {
parent: Some(scope),
subagent_id: Some(subagent_id),
status: "active_subagent",
source: None,
hint: None,
metadata,
});
}
None
}
fn fallback_llm_owner(&self) -> LlmOwnerResolution {
LlmOwnerResolution {
parent: self.root_work_scope(),
subagent_id: None,
status: if self.pending_llm_hints.is_empty() {
"agent_fallback"
} else {
"ambiguous_fallback"
},
source: None,
hint: None,
metadata: Value::Null,
}
}
fn unmanaged_probe_owner(&self, policy: GatewayManagementPolicy) -> LlmOwnerResolution {
let (status, source) = policy
.bypass_correlation()
.expect("unmanaged probe owner requires unmanaged gateway policy");
LlmOwnerResolution {
parent: self.root_work_scope(),
subagent_id: None,
status,
source: Some(source.to_string()),
hint: None,
metadata: Value::Null,
}
}
fn resolution_from_hint(
&mut self,
hint: LlmHintEvent,
status: &'static str,
) -> LlmOwnerResolution {
let hinted_subagent_id = hint.subagent_id.clone().or_else(|| hint.agent_id.clone());
let (parent, subagent_id, metadata) = match hinted_subagent_id.as_deref() {
Some(id) => match self.subagents.get(id).cloned() {
Some(scope) => (
Some(scope),
Some(id.to_string()),
self.subagent_llm_metadata(id),
),
None => (self.root_work_scope(), None, Value::Null),
},
None => (self.root_work_scope(), None, Value::Null),
};
if parent.is_some() {
self.set_last_llm_owner(subagent_id.clone());
}
LlmOwnerResolution {
parent,
subagent_id,
status,
source: Some(hint.event_name.clone()),
hint: Some(hint),
metadata,
}
}
fn subagent_llm_metadata(&self, subagent_id: &str) -> Value {
let Some(scope) = self.subagents.get(subagent_id) else {
return Value::Null;
};
alignment::llm_owner_metadata(scope.metadata.as_ref())
}
fn matching_hint_index(&self, start: &LlmGatewayStart) -> Option<usize> {
let matches: Vec<_> = self
.pending_llm_hints
.iter()
.enumerate()
.filter_map(|(index, pending)| {
let score = hint_match_score(&pending.hint, start);
(score > 0).then_some((index, score))
})
.collect();
let best_score = matches.iter().map(|(_, score)| *score).max()?;
let best: Vec<_> = matches
.into_iter()
.filter(|(_, score)| *score == best_score)
.collect();
(best.len() == 1).then_some(best[0].0)
}
fn set_last_llm_owner(&mut self, subagent_id: Option<String>) {
self.last_llm_owner = subagent_id.map(|subagent_id| LastLlmOwner {
subagent_id,
updated_at: Instant::now(),
source: LastLlmOwnerSource::Llm,
});
}
fn set_last_tool_owner(&mut self, subagent_id: Option<String>) {
if let Some(subagent_id) = subagent_id {
self.last_llm_owner = Some(LastLlmOwner {
subagent_id,
updated_at: Instant::now(),
source: LastLlmOwnerSource::Tool,
});
}
}
fn set_last_subagent_start_owner(&mut self, subagent_id: Option<String>) {
if let Some(subagent_id) = subagent_id {
self.last_llm_owner = Some(LastLlmOwner {
subagent_id,
updated_at: Instant::now(),
source: LastLlmOwnerSource::SubagentStart,
});
}
}
fn record_llm_request_affinity(
&mut self,
provider: &str,
request: &LlmRequest,
subagent_id: Option<&str>,
status: &str,
) {
if !owner_status_teaches_request_affinity(status) {
return;
}
let Some(subagent_id) = subagent_id else {
return;
};
let Some(key) = alignment::request_affinity_key(provider, request) else {
return;
};
match self.llm_request_affinity.get_mut(&key) {
Some(Some(existing)) if existing == subagent_id => {}
Some(owner) => *owner = None,
None => {
self.llm_request_affinity
.insert(key, Some(subagent_id.to_string()));
}
}
}
fn add_tool_hints_from_llm_response(
&mut self,
response: Value,
owner_subagent_id: Option<String>,
) {
self.cleanup_correlation_state();
let hints = tool_hints_from_llm_response(&response, owner_subagent_id);
self.pending_tool_hints
.extend(hints.into_iter().map(|hint| PendingToolHint {
hint,
inserted_at: Instant::now(),
}));
}
fn record_completed_llm_response(
&mut self,
response: Value,
owner_subagent_id: Option<String>,
) {
if owner_subagent_id.is_none() {
self.record_turn_llm_output(response.clone());
}
self.add_tool_hints_from_llm_response(response, owner_subagent_id);
}
fn record_turn_llm_output(&mut self, response: Value) {
if self.turn_scope.is_some() {
self.last_turn_llm_output = Some(response);
}
}
fn resolve_tool_owner(&mut self, event: &ToolEvent) -> ToolOwnerResolution {
self.cleanup_correlation_state();
if let Some(subagent_id) = &event.subagent_id
&& let Some(scope) = self.subagents.get(subagent_id).cloned()
{
self.consume_matching_tool_hint(event);
return ToolOwnerResolution {
parent: Some(scope),
subagent_id: Some(subagent_id.clone()),
status: "explicit",
source: Some("hook_payload".to_string()),
hint: None,
};
}
if let Some(index) = self.matching_tool_hint_index(event) {
let status = if self.pending_tool_hints.len() == 1 {
"single_hint"
} else {
"matched_hint"
};
let hint = self.pending_tool_hints.remove(index).hint;
return self.tool_resolution_from_hint(hint, status);
}
ToolOwnerResolution {
parent: self.root_work_scope(),
subagent_id: None,
status: if self.pending_tool_hints.is_empty() {
"agent_fallback"
} else {
"ambiguous_fallback"
},
source: None,
hint: None,
}
}
fn tool_resolution_from_hint(
&mut self,
hint: ToolHint,
status: &'static str,
) -> ToolOwnerResolution {
let (parent, subagent_id) = match hint.subagent_id.as_deref() {
Some(id) => match self.subagents.get(id).cloned() {
Some(scope) => (Some(scope), Some(id.to_string())),
None => (self.root_work_scope(), None),
},
None => (self.root_work_scope(), None),
};
ToolOwnerResolution {
parent,
subagent_id,
status,
source: Some(hint.source.clone()),
hint: Some(hint),
}
}
fn consume_matching_tool_hint(&mut self, event: &ToolEvent) {
if let Some(index) = self.matching_tool_hint_index(event) {
self.pending_tool_hints.remove(index);
}
}
fn matching_tool_hint_index(&self, event: &ToolEvent) -> Option<usize> {
let matches: Vec<_> = self
.pending_tool_hints
.iter()
.enumerate()
.filter_map(|(index, pending)| {
let score = tool_hint_match_score(&pending.hint, event);
(score > 0).then_some((index, score))
})
.collect();
let best_score = matches.iter().map(|(_, score)| *score).max()?;
let best: Vec<_> = matches
.into_iter()
.filter(|(_, score)| *score == best_score)
.collect();
(best.len() == 1).then_some(best[0].0)
}
}
fn hint_match_score(hint: &LlmHintEvent, start: &LlmGatewayStart) -> u8 {
let mut score = 0;
if same_optional(hint.subagent_id.as_deref(), start.subagent_id.as_deref())
|| same_optional(hint.agent_id.as_deref(), start.subagent_id.as_deref())
{
score += 8;
}
if same_optional(
hint.conversation_id.as_deref(),
start.conversation_id.as_deref(),
) {
score += 4;
}
if same_optional(
hint.generation_id.as_deref(),
start.generation_id.as_deref(),
) {
score += 4;
}
if same_optional(hint.request_id.as_deref(), start.request_id.as_deref()) {
score += 4;
}
if same_optional(hint.model.as_deref(), start.model_name.as_deref()) {
score += 1;
}
score
}
fn tool_hints_from_llm_response(
response: &Value,
owner_subagent_id: Option<String>,
) -> Vec<ToolHint> {
let mut hints = Vec::new();
collect_openai_chat_tool_hints(response, owner_subagent_id.as_deref(), &mut hints);
collect_openai_response_tool_hints(response, owner_subagent_id.as_deref(), &mut hints);
collect_anthropic_tool_hints(response, owner_subagent_id.as_deref(), &mut hints);
hints
}
fn collect_openai_chat_tool_hints(
response: &Value,
owner_subagent_id: Option<&str>,
hints: &mut Vec<ToolHint>,
) {
let Some(choices) = response.get("choices").and_then(Value::as_array) else {
return;
};
for choice in choices {
let Some(tool_calls) = choice
.get("message")
.and_then(|message| message.get("tool_calls"))
.and_then(Value::as_array)
else {
continue;
};
for call in tool_calls {
push_tool_hint(
hints,
call,
owner_subagent_id,
"openai_chat_tool_call",
&[&["id"][..], &["call_id"][..]],
&[&["function", "name"][..], &["name"][..]],
&[&["function", "arguments"][..], &["arguments"][..]],
);
}
}
}
fn collect_openai_response_tool_hints(
response: &Value,
owner_subagent_id: Option<&str>,
hints: &mut Vec<ToolHint>,
) {
let Some(output) = response.get("output").and_then(Value::as_array) else {
return;
};
for item in output {
if item.get("type").and_then(Value::as_str) != Some("function_call") {
continue;
}
push_tool_hint(
hints,
item,
owner_subagent_id,
"openai_response_tool_call",
&[&["call_id"][..], &["id"][..]],
&[&["name"][..], &["tool_name"][..]],
&[&["arguments"][..], &["input"][..]],
);
}
}
fn collect_anthropic_tool_hints(
response: &Value,
owner_subagent_id: Option<&str>,
hints: &mut Vec<ToolHint>,
) {
for content in [
response.get("content"),
response
.get("message")
.and_then(|message| message.get("content")),
]
.into_iter()
.flatten()
.filter_map(Value::as_array)
{
for block in content {
if json_string_at(block, &[&["type"][..]]).as_deref() == Some("tool_use") {
push_tool_hint(
hints,
block,
owner_subagent_id,
"anthropic_tool_use",
&[&["id"][..], &["tool_use_id"][..]],
&[&["name"][..], &["tool_name"][..]],
&[&["input"][..], &["arguments"][..]],
);
}
}
}
}
fn push_tool_hint(
hints: &mut Vec<ToolHint>,
object: &Value,
owner_subagent_id: Option<&str>,
source: &str,
id_paths: &[&[&str]],
name_paths: &[&[&str]],
argument_paths: &[&[&str]],
) {
let tool_call_id = json_string_at(object, id_paths);
let tool_name = json_string_at(object, name_paths);
let arguments = json_value_at(object, argument_paths)
.map(normalize_tool_arguments)
.unwrap_or(Value::Null);
if tool_call_id.is_none() && (tool_name.is_none() || arguments.is_null()) {
return;
}
hints.push(ToolHint {
tool_call_id,
tool_name,
subagent_id: owner_subagent_id.map(ToOwned::to_owned),
arguments,
source: source.to_string(),
});
}
fn tool_hint_match_score(hint: &ToolHint, event: &ToolEvent) -> u8 {
let mut score = 0;
let id_matches = same_optional(
hint.tool_call_id.as_deref(),
Some(event.tool_call_id.as_str()),
);
let name_matches = same_optional(hint.tool_name.as_deref(), Some(event.tool_name.as_str()));
let arguments_match = !hint.arguments.is_null()
&& !event.arguments.is_null()
&& hint.arguments == event.arguments;
if id_matches {
score += 12;
}
if id_matches && name_matches {
score += 4;
}
if id_matches && arguments_match {
score += 1;
}
if !id_matches && name_matches && arguments_match {
score += 5;
}
score
}
fn same_optional(left: Option<&str>, right: Option<&str>) -> bool {
matches!((left, right), (Some(left), Some(right)) if left == right)
}
fn owner_status_teaches_request_affinity(status: &str) -> bool {
matches!(
status,
"explicit"
| "single_hint"
| "matched_hint"
| "active_subagent"
| "subagent_start"
| "request_affinity"
)
}
fn normalize_tool_arguments(arguments: Value) -> Value {
match arguments {
Value::String(raw) => serde_json::from_str(&raw).unwrap_or(Value::String(raw)),
value => value,
}
}
fn llm_correlation_metadata(
metadata: Value,
status: &str,
source: Option<&str>,
subagent_id: Option<&str>,
hint: Option<&LlmHintEvent>,
) -> Value {
let mut correlation = Map::new();
correlation.insert("llm_correlation_status".into(), json!(status));
if let Some(source) = source {
correlation.insert("llm_correlation_source".into(), json!(source));
}
if let Some(subagent_id) = subagent_id {
correlation.insert("llm_correlation_subagent_id".into(), json!(subagent_id));
}
if let Some(hint) = hint {
insert_optional(
&mut correlation,
"llm_correlation_conversation_id",
hint.conversation_id.as_deref(),
);
insert_optional(
&mut correlation,
"llm_correlation_generation_id",
hint.generation_id.as_deref(),
);
insert_optional(
&mut correlation,
"llm_correlation_request_id",
hint.request_id.as_deref(),
);
insert_optional(
&mut correlation,
"llm_correlation_agent_type",
hint.agent_type.as_deref(),
);
}
merge_metadata(metadata, Value::Object(correlation))
}
fn tool_correlation_metadata(
metadata: Value,
status: &str,
source: Option<&str>,
subagent_id: Option<&str>,
hint: Option<&ToolHint>,
) -> Value {
let mut correlation = Map::new();
correlation.insert("tool_correlation_status".into(), json!(status));
if let Some(source) = source {
correlation.insert("tool_correlation_source".into(), json!(source));
}
if let Some(subagent_id) = subagent_id {
correlation.insert("tool_correlation_subagent_id".into(), json!(subagent_id));
}
if let Some(hint) = hint {
insert_optional(
&mut correlation,
"tool_correlation_tool_call_id",
hint.tool_call_id.as_deref(),
);
insert_optional(
&mut correlation,
"tool_correlation_tool_name",
hint.tool_name.as_deref(),
);
if !hint.arguments.is_null() {
correlation.insert("tool_correlation_arguments".into(), hint.arguments.clone());
}
}
merge_metadata(metadata, Value::Object(correlation))
}
fn event_agent_kind(event: &NormalizedEvent) -> AgentKind {
match event {
NormalizedEvent::AgentStarted(event)
| NormalizedEvent::AgentEnded(event)
| NormalizedEvent::TurnEnded(event)
| NormalizedEvent::PromptSubmitted(event)
| NormalizedEvent::Compaction(event)
| NormalizedEvent::Notification(event)
| NormalizedEvent::HookMark(event) => event.agent_kind,
NormalizedEvent::LlmHint(event) => event.agent_kind,
NormalizedEvent::SubagentStarted(event) | NormalizedEvent::SubagentEnded(event) => {
event.agent_kind
}
NormalizedEvent::LlmStarted(event) | NormalizedEvent::LlmEnded(event) => event.agent_kind,
NormalizedEvent::ToolStarted(event) | NormalizedEvent::ToolEnded(event) => event.agent_kind,
}
}
fn single_active_session_id(sessions: &HashMap<String, Session>) -> Option<String> {
(sessions.len() == 1)
.then(|| sessions.keys().next().cloned())
.flatten()
}
#[cfg(test)]
#[path = "../tests/coverage/session_tests.rs"]
mod tests;