use std::fmt::Write;
use std::path::Path;
use std::sync::Arc;
use std::time::Instant;
use anyhow::Context;
use tracing::Instrument;
use crate::providers::reasoning_roundtrip::assistant_replay_payload;
use crate::session::Session;
use crate::tools::{
ImagePayload, ToolExecutionOutcome, find_tool, format_tool_failure_feedback,
normalize_tool_call, scrub_tool_output,
};
use crate::util::{MEDIA_MARKER_RE, UnwrapPoison, parse_media_marker, scrub_credentials};
use crate::{Agent, ChatMessage, ChatRequest, ChatResponse, Tool, ToolCall};
tokio::task_local! {
pub(crate) static CURRENT_TOOL_USER_NAME: String;
pub(crate) static CURRENT_TOOL_CHANNEL: String;
pub(crate) static CURRENT_TOOL_PARENT_KEY: Option<crate::registry::ParentKey>;
pub(crate) static CURRENT_TOOL_PARENT_LABEL: Option<String>;
pub(crate) static CURRENT_TOOL_BACKGROUND_SESSIONS:
Option<std::sync::Arc<crate::tools::shell::BackgroundSessions>>;
pub(crate) static CURRENT_TOOL_AGENT_ID: Option<String>;
pub(crate) static CURRENT_TOOL_AGENT_TRACKING:
Option<crate::registry::AgentTracking>;
}
const MAX_LLM_ITERATIONS: usize = 1000;
const MAX_STATS_ARG_LENGTH: usize = 500;
pub(crate) const RETRY_EXHAUSTION_MARKER: &str = "exhausted retry budget";
fn extract_media_from_outcomes(
tools: &[Box<dyn Tool>],
tool_calls: &[ToolCall],
outcomes: &[ToolExecutionOutcome],
) -> Vec<(&'static str, String)> {
let mut paths = Vec::new();
for (call, outcome) in tool_calls.iter().zip(outcomes.iter()) {
if outcome.success
&& let Some(marker_prefix) = find_tool(tools, &call.name).and_then(Tool::media_marker)
{
let kind = &marker_prefix[1..marker_prefix.len() - 1];
let mut matched = false;
for caps in MEDIA_MARKER_RE.captures_iter(&outcome.output) {
let (captured_kind, path) = parse_media_marker(&caps);
if captured_kind == kind {
matched = true;
paths.push((marker_prefix, path.to_string()));
}
}
if !matched {
tracing::warn!(
media_tool = %call.name,
marker = %marker_prefix,
"Could not parse media path from tool output — skipping media marker",
);
}
}
}
paths
}
fn existing_image_marker_values(history: &[ChatMessage]) -> std::collections::HashSet<String> {
let mut set = std::collections::HashSet::new();
for msg in history {
if msg.role != crate::ChatRole::User {
continue;
}
for caps in MEDIA_MARKER_RE.captures_iter(&msg.content) {
let (kind, path) = parse_media_marker(&caps);
if kind == "IMAGE" {
set.insert(path.to_string());
}
}
}
set
}
async fn derive_image_payload_from_marker(output: &str) -> Option<ImagePayload> {
if !output.contains("[IMAGE:") {
return None;
}
for caps in MEDIA_MARKER_RE.captures_iter(output) {
let (kind, path) = parse_media_marker(&caps);
if kind != "IMAGE" {
continue;
}
let p = Path::new(path);
if !p.is_absolute() {
continue;
}
let Ok(meta) = crate::util::local_image_to_compressed_data_uri_with_meta(p).await else {
continue;
};
return Some(ImagePayload {
path: p.display().to_string(),
data_uri: meta.data_uri,
width: meta.width,
height: meta.height,
format: meta.format,
recovery_note: None,
source: crate::tools::ImagePayloadSource::Generated,
});
}
None
}
#[must_use]
pub(crate) fn role_tools_and_specs(
role: crate::Role,
ws: &crate::Workspace,
) -> (Vec<Box<dyn Tool>>, Vec<crate::ToolSpec>) {
let tools: Vec<Box<dyn Tool>> = role
.tools(ws)
.into_iter()
.filter(|t| t.is_advertised())
.collect();
let tool_specs = tools.iter().map(|t| t.spec()).collect();
(tools, tool_specs)
}
#[must_use]
pub(crate) fn chat_request(
role: crate::Role,
tool_specs: Option<Vec<crate::ToolSpec>>,
messages: Vec<ChatMessage>,
) -> ChatRequest {
let model = crate::config::CONFIG.role_model(role);
let routing = crate::config::CONFIG.model_routing(&model);
ChatRequest {
messages,
tools: tool_specs,
model,
max_tokens: Some(crate::DEFAULT_MAX_TOKENS),
reasoning_effort: Some(
crate::role::role_info(&role)
.default_reasoning_effort
.to_string(),
),
provider_order: routing.provider_order,
meta: None,
}
}
impl Agent {
#[must_use]
#[expect(clippy::too_many_arguments)] pub fn new(
agent_id: String,
role: crate::Role,
ws: &crate::Workspace,
ticket: Option<crate::board::Ticket>,
user_name: String,
channel: String,
parent_key: Option<crate::registry::ParentKey>,
parent_label: Option<String>,
) -> Self {
let (tools, tool_specs) = role_tools_and_specs(role, ws);
let cancel_token = tokio_util::sync::CancellationToken::new();
let label = if let Some(ref t) = ticket {
format!("{}: {}", role.as_str(), t.title)
} else {
role.to_string()
};
let parent_key = parent_key.or_else(|| {
ticket
.as_ref()
.map(|t| crate::registry::ParentKey::Ticket(t.id.clone()))
});
let parent_label = parent_label.or_else(|| match parent_key {
Some(crate::registry::ParentKey::Ticket(_)) => ticket.as_ref().map(|t| t.title.clone()),
_ => None,
});
if let Some(crate::registry::ParentKey::Research(run_id)) = &parent_key
&& crate::research_cancel::is_cancelled(run_id)
{
cancel_token.cancel();
}
let generation = crate::registry::AGENT_REGISTRY.register(
agent_id.clone(),
role.to_string(),
ticket.as_ref().map(|t| t.id.clone()),
ws,
label,
cancel_token.clone(),
parent_key.clone(),
parent_label.clone(),
);
Self {
agent_id,
role,
session: Session::default(),
workspace: Arc::new(ws.clone()),
tools,
tool_specs,
cancel_token,
ticket,
generation,
tool_stats: std::sync::Mutex::new(Vec::new()),
user_name,
channel,
parent_key,
parent_label,
incoming_rx: None,
round_ts: None,
first_call_notify: None,
failure: None,
failure_class: None,
background_sessions: std::sync::Arc::new(
crate::tools::shell::BackgroundSessions::default(),
),
}
}
}
impl Drop for Agent {
fn drop(&mut self) {
if self.generation > 0 {
crate::registry::AGENT_REGISTRY.deregister(&self.agent_id, self.generation);
}
self.background_sessions.terminate_all();
}
}
impl Agent {
pub async fn finalize_session(&mut self) -> anyhow::Result<()> {
let stats = {
let mut guard = self.tool_stats.lock().unwrap_poison();
std::mem::take(&mut *guard)
};
if !stats.is_empty()
&& let Some(store) = crate::logs::LOG_STORE.get()
&& let Err(e) = store
.flush_batch(
&self.agent_id,
self.role.as_str(),
&self.workspace.path,
&stats,
)
.await
{
tracing::warn!(
agent_id = %self.agent_id,
role = %self.role.as_str(),
error = %e,
"Failed to flush tool usage stats"
);
}
if self.cancel_token.is_cancelled() || crate::shutdown::shutdown_token().is_cancelled() {
tracing::debug!(
agent_id = %self.agent_id,
role = %self.role,
workspace = %self.workspace.name,
ticket = self.ticket.as_ref().map(|t| t.id.as_str()),
"Session finalize skipped (agent cancelled or shutdown)"
);
return Ok(());
}
match self.session.finalize(&self.agent_id).await? {
crate::session::FinalizeOutcome::Flushed => {}
crate::session::FinalizeOutcome::NoUnpersistedTail => {
if crate::shutdown::is_draining() {
tracing::info!(
agent_id = %self.agent_id,
role = %self.role,
workspace = %self.workspace.name,
ticket = self.ticket.as_ref().map(|t| t.id.as_str()),
"Session finalize no-op: turn cut by graceful drain — \
committed frames are durable; resumes at boot or on the next user message"
);
} else {
tracing::info!(
agent_id = %self.agent_id,
role = %self.role,
workspace = %self.workspace.name,
ticket = self.ticket.as_ref().map(|t| t.id.as_str()),
"finalize called but no new assistant message in history"
);
}
}
}
Ok(())
}
#[must_use]
pub fn is_cancelled(&self) -> bool {
self.cancel_token.is_cancelled()
}
#[must_use]
pub(crate) fn failure_reason(&self, fallback: &str) -> String {
if crate::shutdown::shutdown_token().is_cancelled() {
"service shutting down".to_string()
} else if self.is_cancelled() {
"agent cancelled by user".to_string()
} else {
self.failure.clone().unwrap_or_else(|| fallback.to_string())
}
}
pub async fn work(&mut self, msg: &str, resume: bool) -> anyhow::Result<String> {
self.session
.init(
&self.agent_id,
msg,
&self.workspace,
&self.role,
self.ticket.as_ref(),
&self.channel,
&self.user_name,
self.round_ts.as_deref(),
)
.await?;
if let Some(token_length) = self.session.token_length() {
crate::registry::AGENT_REGISTRY.set_session_tokens(
&self.agent_id,
self.generation,
token_length,
);
}
if crate::shutdown::aborting() {
anyhow::bail!("Agent round cut short by shutdown/drain — resumes at boot");
}
if !resume {
self.maybe_summarize().await;
}
let shutdown = crate::shutdown::shutdown_token();
let response_result = tokio::select! {
() = shutdown.cancelled() => {
Err(anyhow::anyhow!("Shutting down"))
}
result = self.llm_loop() => result,
};
if let Err(e) = self.finalize_session().await {
tracing::error!(error = %e, "Session finalize failed");
}
let response = response_result?;
Ok(response)
}
async fn llm_loop(&mut self) -> anyhow::Result<String> {
let span = tracing::info_span!("agent", agent_id = %self.agent_id, role = %self.role, workspace = %self.workspace.path);
async {
let mut iteration = 0usize;
let mut accumulated_media_paths: Vec<(&'static str, String)> = Vec::new();
loop {
if self.cancel_token.is_cancelled() {
anyhow::bail!("Agent cancelled by user");
}
if crate::shutdown::aborting() {
anyhow::bail!("Agent round cut short by shutdown/drain — resumes at boot");
}
if iteration >= MAX_LLM_ITERATIONS {
anyhow::bail!(
"Agent exceeded maximum of {MAX_LLM_ITERATIONS} tool rounds \
— model may be stuck in a tool-calling loop"
);
}
self.drain_incoming_messages().await?;
let llm_result = self.llm_call().await;
if iteration == 0
&& let Some(notify) = &self.first_call_notify
{
notify.notify_one();
}
let image_rejection = llm_result.as_ref().err().and_then(|e| {
e.chain()
.find_map(|cause| cause.downcast_ref::<crate::retry::RetryExhausted>())
});
if let Some(exhausted) = image_rejection
&& self.strip_rejected_input_image(exhausted).await
{
tracing::info!(
agent_id = %self.agent_id,
role = %self.role,
iteration,
"Stripped provider-rejected input image from the most recent user \
message — continuing the normal loop"
);
continue;
}
let PreparedAssistantTurn {
mut display_text,
tool_calls,
history_content,
} = prepare_assistant_turn(
llm_result
.with_context(|| format!("LLM step failed at iteration {iteration}"))?,
);
if tool_calls.is_empty() {
self.session.push_assistant(history_content);
for (marker_prefix, path) in &accumulated_media_paths {
let marker = format!("{marker_prefix}{path}]");
if !display_text.contains(&marker) {
let _ = write!(display_text, "\n{marker}");
}
}
return Ok(display_text);
}
let all_outcomes = self.execute_tool_group(&tool_calls).await;
accumulated_media_paths.extend(extract_media_from_outcomes(
&self.tools,
&tool_calls,
&all_outcomes,
));
self.commit_tool_results(&tool_calls, &all_outcomes, &history_content)
.await?;
iteration += 1;
}
}
.instrument(span)
.await
}
async fn execute_tool_group(&self, tool_calls: &[ToolCall]) -> Vec<ToolExecutionOutcome> {
let side_flags: Vec<bool> = tool_calls
.iter()
.map(|call| find_tool(&self.tools, &call.name).is_none_or(super::Tool::side_effects))
.collect();
let mut outcomes: Vec<ToolExecutionOutcome> = Vec::with_capacity(tool_calls.len());
let mut i = 0usize;
let user_name = self.user_name.clone();
let channel = self.channel.clone();
let parent_key = self.parent_key.clone();
let parent_label = self.parent_label.clone();
let background_sessions = Some(self.background_sessions.clone());
let agent_id = Some(self.agent_id.clone());
let agent_tracking = Some(crate::registry::AgentTracking {
agent_id: self.agent_id.clone(),
generation: self.generation,
role: self.role.as_str().to_string(),
workspace: self.workspace.name.clone(),
});
CURRENT_TOOL_USER_NAME
.scope(user_name, async {
CURRENT_TOOL_CHANNEL
.scope(channel, async {
CURRENT_TOOL_PARENT_KEY
.scope(parent_key, async {
CURRENT_TOOL_PARENT_LABEL
.scope(parent_label, async {
CURRENT_TOOL_BACKGROUND_SESSIONS
.scope(background_sessions, async {
CURRENT_TOOL_AGENT_ID
.scope(agent_id, async {
CURRENT_TOOL_AGENT_TRACKING
.scope(agent_tracking, async {
while i < tool_calls.len() {
if side_flags[i] {
let outcome = self
.execute_tool(
&tool_calls[i].name,
tool_calls[i]
.arguments
.clone(),
)
.await;
outcomes.push(outcome);
i += 1;
} else {
let group_start = i;
while i < tool_calls.len()
&& !side_flags[i]
{
i += 1;
}
let group_calls =
&tool_calls
[group_start..i];
let group_outcomes: Vec<_> =
futures_util::future::join_all(
group_calls.iter().map(
|call| {
self.execute_tool(
&call.name,
call.arguments
.clone(),
)
},
),
)
.await;
outcomes
.extend(group_outcomes);
}
}
})
.await;
})
.await;
})
.await;
})
.await;
})
.await;
})
.await;
})
.await;
outcomes
}
#[must_use]
fn failure_outcome(
call_name: &str,
call_arguments: &serde_json::Value,
reason: &str,
) -> (ToolExecutionOutcome, String) {
let reason = scrub_credentials(reason);
(
ToolExecutionOutcome {
output: format_tool_failure_feedback(call_name, call_arguments, &reason),
success: false,
image_payload: None,
},
reason,
)
}
async fn execute_tool(
&self,
call_name: &str,
call_arguments: serde_json::Value,
) -> ToolExecutionOutcome {
if self.cancel_token.is_cancelled() {
let reason = "Agent cancelled — tool execution skipped";
tracing::debug!(
tool = %call_name,
"Agent cancelled — skipping tool execution"
);
return Self::failure_outcome(call_name, &call_arguments, reason).0;
}
let start = Instant::now();
let (tool_name, tool_arguments) = normalize_tool_call(call_name, call_arguments);
if tool_name != call_name {
tracing::debug!(
original = %call_name,
normalized = %tool_name,
"Repaired tool call name"
);
}
let (outcome, error_reason) = match find_tool(&self.tools, &tool_name) {
None => {
let reason = format!("Unknown tool: {tool_name}");
let duration = start.elapsed();
tracing::info!(
tool = %tool_name,
duration_ms = duration.as_millis(),
success = false,
"Unknown tool call"
);
Self::failure_outcome(&tool_name, &tool_arguments, &reason)
}
Some(tool) => {
let _live_tool = crate::registry::AGENT_REGISTRY.tool_started(
&self.agent_id,
self.generation,
&tool_name,
&tool_arguments,
);
let exec_result = tool.execute(&self.workspace, tool_arguments.clone()).await;
let duration = start.elapsed();
match exec_result {
Ok(output) => {
let output_text = if output.is_empty() {
String::from("(no output)")
} else {
output
};
tracing::debug!(
tool = %tool_name,
duration_ms = duration.as_millis(),
"Tool execution completed"
);
let image_payload =
tool.image_payload(&self.workspace, &tool_arguments).await;
(
ToolExecutionOutcome {
output: scrub_tool_output(tool, &tool_arguments, &output_text),
success: true,
image_payload,
},
String::new(),
)
}
Err(e) => {
let (outcome, error_reason) = Self::failure_outcome(
&tool_name,
&tool_arguments,
&format!("Error executing {tool_name}: {e}"),
);
tracing::debug!(
tool = %tool_name,
duration_ms = duration.as_millis(),
success = false,
"Tool execution error: {error_reason}"
);
(outcome, error_reason)
}
}
}
};
{
let elapsed_ms = start.elapsed().as_millis();
let duration_ms = i64::try_from(elapsed_ms).unwrap_or(0);
let args_str =
serde_json::to_string(&tool_arguments).expect("Value is always serializable");
let args_scrubbed = scrub_credentials(&args_str);
let arguments =
crate::util::truncate_bytes(&args_scrubbed, MAX_STATS_ARG_LENGTH).to_string();
let mut guard = self.tool_stats.lock().unwrap_poison();
guard.push(crate::ToolCallRecord {
tool_name,
arguments,
duration_ms,
success: outcome.success,
error_message: (!error_reason.is_empty()).then_some(error_reason),
});
}
outcome
}
async fn llm_call(&mut self) -> anyhow::Result<ChatResponse> {
let messages = self.session.history().to_vec();
let request = self.build_chat_request(messages.clone(), "agent");
let policy = crate::retry::RetryPolicy::current();
let response = crate::retry::agent_chat(request, &policy)
.await
.with_context(|| format!("LLM call {RETRY_EXHAUSTION_MARKER}"))?;
let response = self
.recover_if_reasoning_only_stop(messages, response, Self::AGENT_REASONING_RECOVERY)
.await?;
self.record_session_usage(&response).await;
Ok(response)
}
async fn strip_rejected_input_image(
&mut self,
exhausted: &crate::retry::RetryExhausted,
) -> bool {
let Some(idx) =
crate::image_strip::detect_input_image_rejection(exhausted, self.session.history())
else {
return false;
};
let reason = crate::image_strip::extract_provider_reason(exhausted);
let content = {
let original = &self.session.history()[idx].content;
let stripped = crate::image_strip::strip_image_markers(original, reason.as_deref());
if stripped == *original {
tracing::warn!(
agent_id = %self.agent_id,
role = %self.role,
"Input-image rejection detected but stripping produced no change — \
treating as a non-strip (normal failure path)"
);
return false;
}
stripped
};
match self
.session
.rewrite_last_user_message(&self.agent_id, content)
.await
{
Ok(crate::session::RewriteOutcome::Rewritten) => true,
Ok(crate::session::RewriteOutcome::UnpersistedTailNoop) => {
tracing::info!(
agent_id = %self.agent_id,
role = %self.role,
"Input-image rejection detected but the most recent user message is in the \
unpersisted tail — conservative no-op, normal failure path applies"
);
false
}
Err(e) => {
tracing::error!(
agent_id = %self.agent_id,
role = %self.role,
error = %e,
"Failed to persist stripped user message — keeping the original error path"
);
false
}
}
}
async fn recover_reasoning_only_stop(
&self,
base: Vec<ChatMessage>,
first: ChatResponse,
purpose: &'static str,
) -> Result<ChatResponse, crate::retry::RetryExhausted> {
let policy = crate::retry::RetryPolicy::continuation();
let deadline = Instant::now() + policy.operation_timeout;
let operation_started = Instant::now();
let nudge = crate::prompt::load_prompt("resume_unfinished_turn.md")
.trim()
.to_string();
let mut failures: Vec<crate::retry::RetryFailureRecord> = Vec::new();
let mut last_request: Option<ChatRequest> = None;
let mut tail: Vec<ChatMessage> = vec![
ChatMessage::assistant(
assistant_replay_payload(None, &[], first.reasoning.as_ref()).to_string(),
),
ChatMessage::user(nudge.clone()),
];
let mut tail_grew = true;
for attempt in 1..=policy.max_attempts {
if crate::shutdown::aborting() {
failures.push(crate::retry::RetryFailureRecord::new_simple(
crate::retry::FailureClass::Shutdown,
&anyhow::anyhow!("global shutdown or drain during continuation recovery"),
None,
));
break;
}
if self.cancel_token.is_cancelled() {
break;
}
if Instant::now() >= deadline {
failures.push(crate::retry::RetryFailureRecord::new_simple(
crate::retry::FailureClass::WallClockExceeded,
&anyhow::anyhow!("continuation wall-clock budget exceeded"),
None,
));
break;
}
let request = if tail_grew {
tail_grew = false;
let mut messages = base.clone();
messages.extend(tail.iter().cloned());
let built = self.build_chat_request(messages, purpose);
last_request = Some(built.clone());
built
} else {
last_request
.clone()
.expect("the first iteration always builds a request")
};
match crate::providers::chat_scoped(request.clone(), policy.idle_timeout, deadline)
.await
{
Ok(resp) if is_reasoning_only_stop(&resp) => {
failures.push(crate::retry::RetryFailureRecord::with_metadata(
crate::retry::FailureClass::NoResponse,
&anyhow::anyhow!(
"model returned only reasoning with no answer \
(continuation attempt {attempt})"
),
resp.finish_reason.clone(),
None,
));
tail.push(ChatMessage::assistant(
assistant_replay_payload(None, &[], resp.reasoning.as_ref()).to_string(),
));
tail.push(ChatMessage::user(nudge.clone()));
tail_grew = true;
}
Ok(resp) => {
crate::stats::record_llm_success(&request, operation_started, attempt, &resp)
.await;
return Ok(resp);
}
Err(err) => {
failures.push(err.record);
if !err.class.is_retryable() {
break;
}
}
}
}
let final_class = failures
.last()
.map_or(crate::retry::FailureClass::NoResponse, |r| r.class);
let exhausted = crate::retry::RetryExhausted::with_last_raw(failures, final_class, None);
match last_request {
Some(request) => {
crate::retry::fail_exhausted(&request, operation_started, exhausted).await
}
None => Err(exhausted),
}
}
const AGENT_REASONING_RECOVERY: ReasoningOnlyStopRecovery = ReasoningOnlyStopRecovery {
purpose: "agent-continuation",
exhausted_ctx: "model returned only reasoning without an answer after continuation attempts",
};
const SUMMARIZE_REASONING_RECOVERY: ReasoningOnlyStopRecovery = ReasoningOnlyStopRecovery {
purpose: "summarize-continuation",
exhausted_ctx: "summarization continuation exhausted — failing open with full history",
};
async fn recover_if_reasoning_only_stop(
&self,
messages: Vec<ChatMessage>,
response: ChatResponse,
recovery: ReasoningOnlyStopRecovery,
) -> anyhow::Result<ChatResponse> {
if !is_reasoning_only_stop(&response) {
return Ok(response);
}
self.recover_reasoning_only_stop(messages, response, recovery.purpose)
.await
.map_err(|e| anyhow::Error::new(e).context(recovery.exhausted_ctx))
}
async fn record_session_usage(&mut self, response: &ChatResponse) {
let Some(usage) = &response.usage else {
return;
};
let (Some(input), Some(output)) = (usage.input_tokens, usage.output_tokens) else {
return;
};
let token_length = input.saturating_add(output);
self.session.set_token_length(Some(token_length));
if let Err(e) = crate::session::store()
.set_token_length(&self.agent_id, Some(token_length))
.await
{
tracing::warn!(
agent_id = %self.agent_id,
error = %e,
"Failed to persist session token length — in-memory value may drift from the store until the next successful call"
);
}
crate::registry::AGENT_REGISTRY.set_session_tokens(
&self.agent_id,
self.generation,
token_length,
);
}
async fn commit_tool_results(
&mut self,
tool_calls: &[ToolCall],
outcomes: &[ToolExecutionOutcome],
history_content: &str,
) -> anyhow::Result<()> {
let tools = &self.tools;
let assistant_call = ChatMessage::assistant(history_content.to_string());
let mut db_messages = Vec::with_capacity(1 + outcomes.len());
db_messages.push(assistant_call);
let mut seen_data_uris: Option<std::collections::HashSet<String>> = None;
let mut fresh_image_payloads: Vec<ImagePayload> = Vec::new();
for (call, outcome) in tool_calls.iter().zip(outcomes.iter()) {
let tool = find_tool(tools, &call.name);
let mut output = match tool {
Some(t) => t.format_output(&outcome.output),
None => crate::util::truncate_tool_output(&outcome.output),
};
let derived_payload = if outcome.image_payload.is_none() && outcome.success {
derive_image_payload_from_marker(&outcome.output).await
} else {
None
};
let payload = outcome.image_payload.as_ref().or(derived_payload.as_ref());
if let Some(payload) = payload {
let seen = seen_data_uris
.get_or_insert_with(|| existing_image_marker_values(self.session.history()));
if seen.insert(payload.data_uri.clone()) {
fresh_image_payloads.push(payload.clone());
output = payload.attached_annotation();
} else {
output = payload.already_attached_annotation();
}
}
db_messages.push(ChatMessage::tool_result(&call.id, &output));
}
for payload in fresh_image_payloads {
tracing::debug!(
agent_id = %self.agent_id,
role = %self.role,
path = %payload.path,
"Injecting tool image as a synthetic user message"
);
db_messages.push(ChatMessage::user(crate::util::injected_image_user_message(
&payload.data_uri,
)));
}
self.session
.persist_messages(&self.agent_id, &db_messages)
.await
.map_err(|e| anyhow::anyhow!("Failed to persist tool results: {e}"))?;
Ok(())
}
async fn drain_incoming_messages(&mut self) -> anyhow::Result<()> {
let Some(rx) = &mut self.incoming_rx else {
return Ok(());
};
let mut messages = Vec::new();
loop {
match rx.try_recv() {
Ok(job) => {
let content = match job.kind {
crate::message_router::JobKind::TicketComment => {
format!(
"[Comment from {} on ticket]: {}",
job.user_name, job.content
)
}
_ => job.content,
};
messages.push(crate::session::user_msg_with_ts(&content, None));
}
Err(tokio::sync::mpsc::error::TryRecvError::Empty) => break,
Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => {
self.incoming_rx = None;
break;
}
}
}
if messages.is_empty() {
return Ok(());
}
if let Err(e) = self
.session
.persist_messages(&self.agent_id, &messages)
.await
{
tracing::warn!(
agent_id = %self.agent_id,
error = %e,
"Failed to persist incoming messages to session DB — continuing without persistence",
);
self.session.push_messages_unpersisted(&messages);
}
Ok(())
}
fn build_chat_request(&self, messages: Vec<ChatMessage>, purpose: &'static str) -> ChatRequest {
ChatRequest {
meta: Some(crate::ChatRequestMeta {
purpose,
agent_id: self.agent_id.clone(),
role: self.role.as_str().to_string(),
workspace: self.workspace.name.clone(),
ticket_id: self.ticket.as_ref().map(|t| t.id.clone()),
}),
..chat_request(self.role, Some(self.tool_specs.clone()), messages)
}
}
pub(crate) async fn extract_verdict<T: serde::de::DeserializeOwned>(
&self,
extraction_prompt: &str,
validate: Option<&crate::ExtractionValidator<T>>,
policy_override: Option<&crate::retry::RetryPolicy>,
) -> Result<T, crate::retry::RetryExhausted> {
let _activity = crate::registry::AGENT_REGISTRY.activity_started(
&self.agent_id,
self.generation,
"extracting",
);
let params = self.build_chat_request(vec![], "extraction");
crate::extraction::retry_extract_structured_scoped(
self.session.history(),
extraction_prompt,
¶ms,
validate,
policy_override,
)
.await
}
pub(crate) async fn summarize(&self) -> anyhow::Result<String> {
let _activity = crate::registry::AGENT_REGISTRY.activity_started(
&self.agent_id,
self.generation,
"summarizing",
);
let mut history = self.session.history().to_vec();
history.push(crate::ChatMessage::user(self.role.summary_prompt()));
let policy = crate::retry::RetryPolicy::current();
let chat_resp = crate::retry::agent_chat(
self.build_chat_request(history.clone(), "summarize"),
&policy,
)
.await
.with_context(|| format!("summarization LLM call {RETRY_EXHAUSTION_MARKER}"))?;
let chat_resp = self
.recover_if_reasoning_only_stop(history, chat_resp, Self::SUMMARIZE_REASONING_RECOVERY)
.await?;
if let Some(ref u) = chat_resp.usage {
tracing::debug!(
input_tokens = u.input_tokens,
cached_input_tokens = u.cached_input_tokens,
output_tokens = u.output_tokens,
"Summarization token usage",
);
}
let summary_text = chat_resp
.text
.filter(|t| !t.trim().is_empty())
.ok_or_else(|| anyhow::anyhow!("summarization produced empty response"))?;
Ok(crate::util::truncate(&summary_text, 32_000))
}
async fn maybe_summarize(&mut self) {
let Some(token_length) = self.session.token_length() else {
return;
};
tracing::debug!(
agent_id = %self.agent_id,
role = %self.role,
token_length,
"Session token length",
);
if token_length > crate::session::SUMMARIZATION_THRESHOLD {
tracing::info!(
agent_id = %self.agent_id,
role = %self.role,
token_length,
"Session exceeded summarization threshold",
);
match self.summarize().await {
Ok(summary) => {
self.session
.apply_summary(
&self.agent_id,
&summary,
&self.workspace,
&self.role,
self.ticket.as_ref(),
)
.await;
}
Err(e) => {
tracing::warn!(
error_chain = %crate::util::truncate_sandwich(
&crate::util::scrub_credentials(&format!("{e:#}")),
crate::util::FAILURE_DETAIL_CAP,
"summarization failure",
),
"Summarization failed — continuing with full history"
);
}
}
}
}
}
struct ReasoningOnlyStopRecovery {
purpose: &'static str,
exhausted_ctx: &'static str,
}
struct PreparedAssistantTurn {
display_text: String,
tool_calls: Vec<ToolCall>,
history_content: String,
}
#[must_use]
fn is_reasoning_only_stop(response: &ChatResponse) -> bool {
response.tool_calls.is_empty() && response.text.as_deref().is_none_or(|t| t.trim().is_empty())
}
fn prepare_assistant_turn(response: ChatResponse) -> PreparedAssistantTurn {
let response_text = response.text_or_empty().to_string();
let tool_calls = response.tool_calls;
let reasoning = response.reasoning.as_ref();
let json_payload =
assistant_replay_payload(Some(&response_text), &tool_calls, reasoning).to_string();
let (display_text, history_content) = match (tool_calls.is_empty(), reasoning.is_some()) {
(true, false) => (response_text.clone(), response_text),
(true, true) => (response_text, json_payload),
(false, _) => (String::new(), json_payload),
};
PreparedAssistantTurn {
display_text,
tool_calls,
history_content,
}
}
#[derive(Clone)]
pub(crate) struct RoundOpts {
pub(crate) round_ts: String,
pub(crate) first_call_notify: Option<std::sync::Arc<tokio::sync::Notify>>,
}
const DEFAULT_STAGGER_WAIT_SECS: u64 = 8;
fn leader_stagger_wait() -> std::time::Duration {
crate::util::env_duration_secs("MAHBOT_STAGGER_WAIT_SECS", DEFAULT_STAGGER_WAIT_SECS)
}
pub(crate) async fn spawn_staggered_round<T, Fut, F>(
members: Vec<F>,
resume: bool,
) -> Vec<tokio::task::JoinHandle<T>>
where
T: Send + 'static,
Fut: std::future::Future<Output = T> + Send + 'static,
F: FnOnce(RoundOpts) -> Fut + Send + 'static,
{
let mut members = members.into_iter();
let Some(leader) = members.next() else {
return Vec::new();
};
let followers: Vec<F> = members.collect();
let round_ts = crate::session::render_timestamp();
if resume || followers.is_empty() {
let opts = RoundOpts {
round_ts,
first_call_notify: None,
};
let mut handles = Vec::with_capacity(followers.len() + 1);
handles.push(tokio::spawn(leader(opts.clone())));
handles.extend(followers.into_iter().map(|m| tokio::spawn(m(opts.clone()))));
return handles;
}
let notify = std::sync::Arc::new(tokio::sync::Notify::new());
let mut handles = vec![tokio::spawn(leader(RoundOpts {
round_ts: round_ts.clone(),
first_call_notify: Some(notify.clone()),
}))];
let follower_opts = RoundOpts {
round_ts,
first_call_notify: None,
};
let shutdown = crate::shutdown::shutdown_token();
tokio::select! {
() = notify.notified() => {}
() = tokio::time::sleep(leader_stagger_wait()) => {}
() = shutdown.cancelled() => {}
() = crate::shutdown::drain_wait() => {}
}
handles.extend(
followers
.into_iter()
.map(|m| tokio::spawn(m(follower_opts.clone()))),
);
handles
}
#[expect(clippy::too_many_arguments)]
pub(crate) async fn run_agent(
agent_id: String,
role: crate::Role,
ws: &crate::Workspace,
ticket: Option<&crate::board::Ticket>,
message: &str,
user_name: String,
channel: String,
incoming_rx: Option<tokio::sync::mpsc::UnboundedReceiver<crate::message_router::AgentJob>>,
resume: bool,
round: Option<RoundOpts>,
parent_key: Option<crate::registry::ParentKey>,
parent_label: Option<String>,
) -> (Agent, Option<String>) {
struct UnregisterOnDrop(String);
impl Drop for UnregisterOnDrop {
fn drop(&mut self) {
crate::message_router::unregister_agent(&self.0);
}
}
let _router_guard = incoming_rx
.is_some()
.then(|| UnregisterOnDrop(agent_id.clone()));
let agent_id_for_cleanup = agent_id.clone();
let mut agent = Agent::new(
agent_id,
role,
ws,
ticket.cloned(),
user_name,
channel,
parent_key,
parent_label,
);
agent.incoming_rx = incoming_rx;
if let Some(round) = round {
agent.round_ts = Some(round.round_ts);
agent.first_call_notify = round.first_call_notify;
}
let result = agent.work(message, resume).await;
let outcome = if agent.is_cancelled() {
tracing::debug!(
agent_id = %agent.agent_id,
workspace = %ws.name,
role = %role,
ticket = ticket.map(|t| t.id.as_str()),
classification = failure_classification(&agent, None),
"Agent cancelled"
);
(agent, None)
} else {
match result {
Ok(response) => (agent, Some(response)),
Err(e) => {
agent.failure = Some(format!("{e:#}"));
agent.failure_class = failure_class_from_error(&e);
let classification = failure_classification(&agent, Some(&e));
let error_chain = crate::util::truncate_sandwich(
&crate::util::scrub_credentials(&format!("{e:#}")),
crate::util::FAILURE_DETAIL_CAP,
"agent failure log",
);
if classification == "shutdown" || classification == "drain" {
tracing::debug!(
agent_id = %agent.agent_id,
workspace = %ws.name,
role = %role,
ticket = ticket.map(|t| t.id.as_str()),
classification,
error_chain,
"Agent failed during shutdown"
);
} else {
tracing::error!(
agent_id = %agent.agent_id,
workspace = %ws.name,
role = %role,
ticket = ticket.map(|t| t.id.as_str()),
classification,
error_chain,
"Agent failed"
);
}
(agent, None)
}
}
};
crate::tools::shell::cleanup_agent_spills(&agent_id_for_cleanup);
outcome
}
pub(crate) async fn run_default_agent(
agent_id: &str,
role: crate::Role,
ws: &crate::Workspace,
message: &str,
round: Option<RoundOpts>,
parent_key: Option<crate::registry::ParentKey>,
parent_label: Option<String>,
) -> (Agent, Option<String>) {
run_agent(
agent_id.to_string(),
role,
ws,
None,
message,
String::new(),
String::new(),
None,
false,
round,
parent_key,
parent_label,
)
.await
}
pub(crate) fn failure_class_from_error(
error: &anyhow::Error,
) -> Option<crate::retry::FailureClass> {
error
.chain()
.find_map(|cause| cause.downcast_ref::<crate::retry::RetryExhausted>())
.map(|exhausted| exhausted.final_class)
}
fn failure_classification(agent: &Agent, error: Option<&anyhow::Error>) -> &'static str {
if crate::shutdown::is_draining() {
"drain"
} else if crate::shutdown::shutdown_token().is_cancelled() {
"shutdown"
} else if agent.is_cancelled() {
"cancelled"
} else if let Some(class) = error.and_then(failure_class_from_error) {
class.label()
} else {
"runtime"
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::Tool;
use async_trait::async_trait;
use tokio_util::sync::CancellationToken;
struct TestTool {
output: String,
scrub: bool,
}
#[async_trait]
impl Tool for TestTool {
fn name(&self) -> &'static str {
if self.scrub {
"always_scrub"
} else {
"never_scrub"
}
}
fn description(&self) -> String {
"test".into()
}
fn parameters_schema(&self) -> serde_json::Value {
serde_json::json!({})
}
async fn execute(
&self,
_ws: &crate::Workspace,
_args: serde_json::Value,
) -> anyhow::Result<String> {
Ok(self.output.clone())
}
fn should_scrub_output(&self, _args: &serde_json::Value) -> bool {
self.scrub
}
}
const SCRUBBABLE_LINE: &str = "API_KEY=sk-1234567890abcdef";
fn make_agent(tools: Vec<Box<dyn Tool>>) -> Agent {
make_agent_with_role(tools, crate::Role::Engineer)
}
fn make_agent_with_role(tools: Vec<Box<dyn Tool>>, role: crate::Role) -> Agent {
let tool_specs = tools.iter().map(|t| t.spec()).collect();
Agent {
agent_id: "test-agent".into(),
role,
session: Session::default(),
workspace: std::sync::Arc::new(crate::Workspace::default()),
tools,
tool_specs,
cancel_token: CancellationToken::new(),
ticket: None,
generation: 0,
tool_stats: std::sync::Mutex::new(Vec::new()),
user_name: String::new(),
channel: String::new(),
parent_key: None,
parent_label: None,
incoming_rx: None,
round_ts: None,
first_call_notify: None,
failure: None,
failure_class: None,
background_sessions: std::sync::Arc::new(
crate::tools::shell::BackgroundSessions::default(),
),
}
}
#[tokio::test]
async fn tool_with_scrub_disabled_preserves_output() {
assert_scrubbed(false).await;
}
#[test]
fn failure_classification_recovers_retry_exhaustion() {
let _guard = crate::util::test::retry_tests_lock();
let exhausted = crate::retry::RetryExhausted::with_last_raw(
vec![],
crate::retry::FailureClass::NonRetryable,
None,
);
let err = anyhow::Error::new(exhausted).context("LLM call exhausted retry budget");
assert!(format!("{err:#}").contains("exhausted retry budget"));
let agent = make_agent(vec![]);
assert_eq!(
failure_classification(&agent, Some(&err)),
"non_retryable",
"RetryExhausted final_class must be recovered from the chain",
);
let runtime_err = anyhow::anyhow!("tool panicked");
assert_eq!(
failure_classification(&agent, Some(&runtime_err)),
"runtime"
);
let cancelled = make_agent(vec![]);
cancelled.cancel_token.cancel();
assert_eq!(
failure_classification(&cancelled, Some(&runtime_err)),
"cancelled"
);
assert_eq!(failure_classification(&cancelled, None), "cancelled");
}
#[tokio::test]
async fn failure_classification_recognizes_drain() {
let _guard = crate::util::test::retry_tests_lock();
let agent = make_agent(vec![]);
assert_eq!(failure_classification(&agent, None), "runtime");
crate::shutdown::drain_begin();
assert_eq!(
failure_classification(&agent, None),
"drain",
"a drained agent must classify as drain, never failure",
);
agent.cancel_token.cancel();
assert_eq!(failure_classification(&agent, None), "drain");
crate::shutdown::drain_clear();
}
async fn assert_scrubbed(should_scrub: bool) {
let tool: Box<dyn Tool> = Box::new(TestTool {
output: SCRUBBABLE_LINE.into(),
scrub: should_scrub,
});
let name = tool.name();
let agent = make_agent(vec![tool]);
let out = agent.execute_tool(name, serde_json::json!({})).await;
assert!(out.success, "{name} should succeed");
if should_scrub {
assert!(out.output.contains("[REDACTED]"), "{name} should redact");
assert!(
!out.output.contains("abcdef"),
"{name} should not leak original"
);
} else {
assert!(
!out.output.contains("[REDACTED]"),
"{name} should not redact"
);
assert!(
out.output.contains(SCRUBBABLE_LINE),
"{name} should preserve output"
);
}
}
#[tokio::test]
async fn tool_with_scrub_enabled_scrubs_sensitive_output() {
assert_scrubbed(true).await;
}
struct MediaTestTool {
name: &'static str,
marker: &'static str,
}
#[async_trait]
impl Tool for MediaTestTool {
fn name(&self) -> &'static str {
self.name
}
fn description(&self) -> String {
"media test tool".into()
}
fn parameters_schema(&self) -> serde_json::Value {
serde_json::json!({})
}
async fn execute(
&self,
_ws: &crate::Workspace,
_args: serde_json::Value,
) -> anyhow::Result<String> {
Ok(String::new())
}
fn media_marker(&self) -> Option<&'static str> {
Some(self.marker)
}
}
#[test]
#[expect(clippy::too_many_lines)]
fn extract_media_outcomes_consolidated() {
enum ToolDef {
Media {
name: &'static str,
marker: &'static str,
},
NonMedia,
}
struct OutcomeDef {
output: &'static str,
success: bool,
}
struct TestCase {
name: &'static str,
msg: &'static str,
tools: Vec<ToolDef>,
outcomes: Vec<OutcomeDef>,
expected: Vec<(&'static str, &'static str)>,
}
let cases = vec![
TestCase {
name: "parses_valid_marker",
msg: "valid marker with success=true should extract the path",
tools: vec![ToolDef::Media {
name: "image_gen",
marker: "[IMAGE:",
}],
outcomes: vec![OutcomeDef {
output: "[IMAGE:/tmp/img.png]",
success: true,
}],
expected: vec![("[IMAGE:", "/tmp/img.png")],
},
TestCase {
name: "skips_malformed_marker",
msg: "malformed marker should be skipped",
tools: vec![ToolDef::Media {
name: "image_gen",
marker: "[IMAGE:",
}],
outcomes: vec![OutcomeDef {
output: "description text [IMAGE:",
success: true,
}],
expected: vec![],
},
TestCase {
name: "skips_empty_marker",
msg: "empty marker '[IMAGE:]' should be skipped",
tools: vec![ToolDef::Media {
name: "image_gen",
marker: "[IMAGE:",
}],
outcomes: vec![OutcomeDef {
output: "[IMAGE:]",
success: true,
}],
expected: vec![],
},
TestCase {
name: "skips_no_closing_bracket_non_empty_path",
msg: "output with '[IMAGE:bogus' (no closing bracket, non-empty path) should be skipped",
tools: vec![ToolDef::Media {
name: "image_gen",
marker: "[IMAGE:",
}],
outcomes: vec![OutcomeDef {
output: "oops [IMAGE:bogus",
success: true,
}],
expected: vec![],
},
TestCase {
name: "skips_non_media_tool",
msg: "non-media tool should not be inspected for media markers",
tools: vec![ToolDef::NonMedia],
outcomes: vec![OutcomeDef {
output: "[IMAGE:path]",
success: true,
}],
expected: vec![],
},
TestCase {
name: "skips_failed_outcome",
msg: "failed outcomes should not produce media paths",
tools: vec![ToolDef::Media {
name: "image_gen",
marker: "[IMAGE:",
}],
outcomes: vec![OutcomeDef {
output: "[IMAGE:/tmp/img.png]",
success: false,
}],
expected: vec![],
},
TestCase {
name: "handles_mixed_tools",
msg: "mixed tools with valid outcomes should extract only media paths",
tools: vec![
ToolDef::Media {
name: "image_gen",
marker: "[IMAGE:",
},
ToolDef::NonMedia,
ToolDef::Media {
name: "video_gen",
marker: "[VIDEO:",
},
],
outcomes: vec![
OutcomeDef {
output: "[IMAGE:/tmp/img.png]",
success: true,
},
OutcomeDef {
output: "non-media output",
success: true,
},
OutcomeDef {
output: "[VIDEO:/tmp/vid.mp4]",
success: true,
},
],
expected: vec![("[IMAGE:", "/tmp/img.png"), ("[VIDEO:", "/tmp/vid.mp4")],
},
];
for case in cases {
let tools: Vec<Box<dyn Tool>> = case
.tools
.iter()
.map(|t| match t {
ToolDef::Media { name, marker } => {
Box::new(MediaTestTool { name, marker }) as Box<dyn Tool>
}
ToolDef::NonMedia => Box::new(TestTool {
output: String::new(),
scrub: false,
}) as Box<dyn Tool>,
})
.collect();
let calls: Vec<ToolCall> = case
.tools
.iter()
.enumerate()
.map(|(i, t)| {
let name = match t {
ToolDef::Media { name, .. } => *name,
ToolDef::NonMedia => "never_scrub",
};
ToolCall {
id: (i + 1).to_string(),
name: name.to_string(),
arguments: serde_json::json!({}),
}
})
.collect();
let outcomes: Vec<ToolExecutionOutcome> = case
.outcomes
.iter()
.map(|o| ToolExecutionOutcome {
output: o.output.to_string(),
success: o.success,
image_payload: None,
})
.collect();
let expected: Vec<(&'static str, String)> = case
.expected
.iter()
.map(|(n, p)| (*n, p.to_string()))
.collect();
let result = extract_media_from_outcomes(&tools, &calls, &outcomes);
assert_eq!(result, expected, "case '{}': {}", case.name, case.msg);
}
}
#[tokio::test]
async fn finalize_session_skipped_when_cancelled() {
let mut agent = make_agent(vec![]);
agent.cancel_token.cancel();
let result = agent.finalize_session().await;
assert!(
result.is_ok(),
"finalize_session should return Ok when cancelled, \
skipping the 'no assistant message' warning"
);
}
#[tokio::test]
async fn execute_tool_skips_when_cancelled() {
let tool: Box<dyn Tool> = Box::new(TestTool {
output: "should not run".into(),
scrub: false,
});
let name = tool.name();
let agent = make_agent(vec![tool]);
agent.cancel_token.cancel();
let out = agent.execute_tool(name, serde_json::json!({})).await;
assert!(!out.success, "cancelled agent should not execute tool");
assert!(
out.output.contains("Agent cancelled"),
"failure output should mention cancellation: {}",
out.output
);
}
#[tokio::test]
async fn task_locals_propagate_to_parallel_tool_execution() {
struct ReadTaskLocalsTool;
#[async_trait]
impl Tool for ReadTaskLocalsTool {
fn name(&self) -> &'static str {
"read_task_locals"
}
fn description(&self) -> String {
"test tool that reads task-local user context".into()
}
fn parameters_schema(&self) -> serde_json::Value {
serde_json::json!({})
}
async fn execute(
&self,
_ws: &crate::Workspace,
_args: serde_json::Value,
) -> anyhow::Result<String> {
let user_name = CURRENT_TOOL_USER_NAME
.try_with(String::clone)
.unwrap_or_default();
let channel = CURRENT_TOOL_CHANNEL
.try_with(String::clone)
.unwrap_or_default();
Ok(format!("user={user_name},channel={channel}"))
}
}
let tool: Box<dyn Tool> = Box::new(ReadTaskLocalsTool);
let name = tool.name();
let mut agent = make_agent(vec![tool]);
agent.user_name = "alice".into();
agent.channel = "gui".into();
let call = ToolCall {
id: "call_1".into(),
name: name.to_string(),
arguments: serde_json::json!({}),
};
let outcomes = agent.execute_tool_group(&[call]).await;
assert_eq!(
outcomes.len(),
1,
"one tool call should produce one outcome"
);
assert!(outcomes[0].success, "ReadTaskLocalsTool should succeed");
assert!(
outcomes[0].output.contains("user=alice"),
"should propagate user_name: {}",
outcomes[0].output
);
assert!(
outcomes[0].output.contains("channel=gui"),
"should propagate channel: {}",
outcomes[0].output
);
}
#[tokio::test]
async fn test_drain_incoming_messages_injects_ticket_comment() {
crate::util::test::init_management_test_stores().await;
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
let mut agent = make_agent(vec![]);
agent.incoming_rx = Some(rx);
agent.agent_id = "_test_drain_ticket_comment".into();
let job = crate::message_router::AgentJob {
content: "Please fix the formatting".to_string(),
workspace_name: "test_ws".to_string(),
user_name: "manager".to_string(),
channel: String::new(),
kind: crate::message_router::JobKind::TicketComment,
role: crate::Role::Manager,
reply_target: None,
pending_job_id: None,
};
let _ = tx.send(job);
agent
.drain_incoming_messages()
.await
.expect("drain should succeed");
let history = agent.session.history();
assert!(!history.is_empty(), "should have at least one message");
let last = history.last().unwrap();
assert_eq!(last.role, crate::ChatRole::User, "should be a user message");
assert!(
last.content.contains("Please fix the formatting"),
"should contain the comment content: {}",
last.content,
);
assert!(
last.content.contains("[Comment from manager on ticket]"),
"should include comment prefix: {}",
last.content,
);
}
#[tokio::test]
async fn test_drain_incoming_messages_non_comment() {
crate::util::test::init_management_test_stores().await;
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
let mut agent = make_agent(vec![]);
agent.incoming_rx = Some(rx);
agent.agent_id = "_test_drain_non_comment".into();
let job = crate::message_router::AgentJob {
content: "Hello agent".to_string(),
workspace_name: "test_ws".to_string(),
user_name: "user".to_string(),
channel: String::new(),
kind: crate::message_router::JobKind::UserMessage,
role: crate::Role::Assistant,
reply_target: None,
pending_job_id: None,
};
let _ = tx.send(job);
agent
.drain_incoming_messages()
.await
.expect("drain should succeed");
let history = agent.session.history();
assert!(!history.is_empty(), "should have at least one message");
let last = history.last().unwrap();
assert!(
last.content.contains("Hello agent"),
"should contain the raw content: {}",
last.content,
);
}
#[tokio::test]
async fn test_drain_incoming_messages_disconnected() {
let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<crate::message_router::AgentJob>();
let mut agent = make_agent(vec![]);
agent.incoming_rx = Some(rx);
agent.agent_id = "_test_drain_disconnected".into();
drop(tx);
agent
.drain_incoming_messages()
.await
.expect("drain should succeed on disconnected channel");
assert!(
agent.incoming_rx.is_none(),
"incoming_rx should be set to None after disconnect",
);
}
#[tokio::test]
async fn spawn_staggered_round_single_member_is_noop() {
let handles = crate::agent::spawn_staggered_round(
vec![move |round: crate::agent::RoundOpts| async move {
assert!(
round.first_call_notify.is_none(),
"sole member must not receive a signal"
);
1u8
}],
false,
)
.await;
assert_eq!(handles.len(), 1);
assert_eq!(handles.into_iter().next().unwrap().await.unwrap(), 1);
}
#[tokio::test]
async fn spawn_staggered_round_resume_skips_stagger() {
let handles = crate::agent::spawn_staggered_round(
(0..3)
.map(|i| {
move |round: crate::agent::RoundOpts| async move {
assert!(
round.first_call_notify.is_none(),
"resume must not stagger (member {i})"
);
i
}
})
.collect(),
true,
)
.await;
assert_eq!(handles.len(), 3);
let mut out = Vec::new();
for h in handles {
out.push(h.await.unwrap());
}
assert_eq!(out, vec![0, 1, 2]);
}
#[tokio::test]
async fn spawn_staggered_round_leader_notify_releases_followers() {
let handles = crate::agent::spawn_staggered_round(
(0..3)
.map(|i| {
move |round: crate::agent::RoundOpts| async move {
let tag = if i == 0 {
round
.first_call_notify
.expect("leader receives the signal")
.notify_one();
"leader".to_string()
} else {
assert!(
round.first_call_notify.is_none(),
"followers must not receive the signal"
);
"follower".to_string()
};
(tag, round.round_ts)
}
})
.collect(),
false,
)
.await;
assert_eq!(handles.len(), 3);
let mut out = Vec::new();
for h in handles {
out.push(h.await.unwrap());
}
assert!(out.iter().any(|(tag, _)| tag == "leader"));
assert_eq!(out.iter().filter(|(tag, _)| tag == "follower").count(), 2);
assert_eq!(
out.iter()
.map(|(_, ts)| ts)
.collect::<std::collections::HashSet<_>>()
.len(),
1,
"all members share one round-fixed timestamp"
);
}
#[tokio::test]
async fn spawn_staggered_round_leader_timeout_fail_open() {
let _guard = crate::util::test::set_env_var("MAHBOT_STAGGER_WAIT_SECS", Some("0"));
let handles = crate::agent::spawn_staggered_round(
(0..2)
.map(|i| {
move |round: crate::agent::RoundOpts| async move {
if i == 0 {
let _ = round; "stuck-leader".to_string()
} else {
assert!(
round.first_call_notify.is_none(),
"released follower gets no signal"
);
"released".to_string()
}
}
})
.collect(),
false,
)
.await;
assert_eq!(
handles.len(),
2,
"followers must be released despite the stuck leader"
);
for h in handles {
h.await.unwrap();
}
}
#[tokio::test]
#[expect(clippy::await_holding_lock)] async fn leader_first_call_failure_still_fires_signal() {
use crate::util::test::{
FakeProvider, install_fake_provider, install_test_retry_policy, retry_tests_lock,
};
let _lock = retry_tests_lock();
crate::util::test::init_test_stores().await;
let _policy_guard = install_test_retry_policy(crate::retry::tiny_test_policy());
let ws = crate::workspace::test_ws_named("/tmp/ws_leader_fail", "leader_fail");
let notify = std::sync::Arc::new(tokio::sync::Notify::new());
let provider = FakeProvider::new()
.err(crate::retry::FailureClass::Transport, "boom")
.err(crate::retry::FailureClass::Transport, "boom")
.err(crate::retry::FailureClass::Transport, "boom");
let _provider = install_fake_provider(std::sync::Arc::new(provider));
let (_agent, response) = run_agent(
"leader_fail_agent".to_string(),
crate::Role::Analyst,
&ws,
None,
"task",
String::new(),
String::new(),
None,
false,
Some(RoundOpts {
round_ts: crate::session::render_timestamp(),
first_call_notify: Some(notify.clone()),
}),
None,
None,
)
.await;
assert!(
response.is_none(),
"leader round must fail with an exhausted budget"
);
tokio::time::timeout(std::time::Duration::from_secs(5), notify.notified())
.await
.expect("first-call signal must fire even when the call fails");
}
#[test]
fn reasoning_only_stop_classification() {
let reasoning = || {
Some(crate::Reasoning {
reasoning: Some("thinking".into()),
reasoning_content: Some("thinking".into()),
reasoning_details: None,
})
};
for finish in [None, Some("stop"), Some("length"), Some("tool_calls")] {
let resp = crate::ChatResponse {
text: None,
reasoning: reasoning(),
finish_reason: finish.map(str::to_string),
..crate::ChatResponse::default()
};
assert!(is_reasoning_only_stop(&resp), "finish_reason={finish:?}");
}
let resp = crate::ChatResponse {
text: Some(String::new()),
reasoning: reasoning(),
..crate::ChatResponse::default()
};
assert!(is_reasoning_only_stop(&resp));
let resp = crate::ChatResponse {
text: Some(" \n ".into()),
..crate::ChatResponse::default()
};
assert!(is_reasoning_only_stop(&resp));
let resp = crate::ChatResponse::default();
assert!(is_reasoning_only_stop(&resp));
let resp = crate::ChatResponse {
text: None,
tool_calls: vec![crate::ToolCall {
id: "t1".into(),
name: "read".into(),
arguments: serde_json::json!({}),
}],
..crate::ChatResponse::default()
};
assert!(!is_reasoning_only_stop(&resp));
let resp = crate::ChatResponse {
text: Some("real answer".into()),
reasoning: reasoning(),
..crate::ChatResponse::default()
};
assert!(!is_reasoning_only_stop(&resp));
}
#[tokio::test]
#[expect(clippy::await_holding_lock)] async fn llm_call_recovers_reasoning_only_stop_via_continuation() {
use crate::util::test::{
FakeProvider, install_fake_provider, install_test_retry_policy, retry_tests_lock,
};
let _lock = retry_tests_lock();
crate::util::test::init_test_stores().await;
let _policy_guard = install_test_retry_policy(crate::retry::tiny_test_policy());
let fake = std::sync::Arc::new(
FakeProvider::new()
.ok_reasoning_only("draft plan then execute tool", Some("stop"))
.ok("final answer"),
);
let provider: std::sync::Arc<dyn crate::Provider> = fake.clone();
let _provider_guard = install_fake_provider(provider);
let mut agent = make_agent(vec![]);
let resp = agent.llm_call().await.expect("continuation must resolve");
assert_eq!(resp.text_or_empty(), "final answer");
let fingerprints = fake.request_fingerprints.lock().unwrap().clone();
assert_eq!(fingerprints.len(), 2, "original call + one continuation");
assert!(
!fingerprints[0].contains("Resume your unfinished turn"),
"original request must not carry the continuation tail"
);
assert!(
fingerprints[1].contains("Resume your unfinished turn"),
"continuation request must carry the appended nudge"
);
assert!(
fingerprints[1].contains("draft plan then execute tool"),
"continuation must echo the previous reasoning as the assistant turn"
);
}
#[tokio::test]
#[expect(clippy::await_holding_lock)] async fn llm_call_continuation_accumulates_tail_until_answer() {
use crate::util::test::{
FakeProvider, install_fake_provider, install_test_retry_policy, retry_tests_lock,
};
let _lock = retry_tests_lock();
crate::util::test::init_test_stores().await;
let _policy_guard = install_test_retry_policy(crate::retry::tiny_test_policy());
let fake = std::sync::Arc::new(
FakeProvider::new()
.ok_reasoning_only("thinking 1", Some("stop"))
.ok_reasoning_only("thinking 2", Some("stop"))
.ok("answer after two continuations"),
);
let provider: std::sync::Arc<dyn crate::Provider> = fake.clone();
let _provider_guard = install_fake_provider(provider);
let mut agent = make_agent(vec![]);
let resp = agent.llm_call().await.expect("continuation must resolve");
assert_eq!(resp.text_or_empty(), "answer after two continuations");
let fingerprints = fake.request_fingerprints.lock().unwrap().clone();
assert_eq!(fingerprints.len(), 3);
assert_eq!(
fingerprints[1]
.matches("Resume your unfinished turn")
.count(),
1,
"attempt 2 carries exactly the first appended pair"
);
assert_eq!(
fingerprints[2]
.matches("Resume your unfinished turn")
.count(),
2,
"attempt 3 carries both appended pairs"
);
assert!(fingerprints[2].contains("thinking 1"));
assert!(fingerprints[2].contains("thinking 2"));
let messages = fake.request_messages.lock().unwrap().clone();
assert_eq!(messages.len(), 3);
assert!(
messages[2].starts_with(&messages[1]),
"byte-stable prefix: attempt 3's messages begin with attempt 2's verbatim"
);
}
#[tokio::test]
#[expect(clippy::await_holding_lock)] async fn llm_call_continuation_exhaustion_fails_safely_without_leaking() {
use crate::util::test::{
FakeProvider, install_fake_provider, install_test_retry_policy, retry_tests_lock,
};
let _lock = retry_tests_lock();
crate::util::test::init_test_stores().await;
let _policy_guard = install_test_retry_policy(crate::retry::tiny_test_policy());
let fake = std::sync::Arc::new(
FakeProvider::new()
.ok_reasoning_only("secret thinking alpha", Some("stop"))
.ok_reasoning_only("secret thinking beta", Some("stop"))
.ok_reasoning_only("secret thinking gamma", Some("stop"))
.ok_reasoning_only("secret thinking delta", Some("stop")),
);
let provider: std::sync::Arc<dyn crate::Provider> = fake.clone();
let _provider_guard = install_fake_provider(provider);
let mut agent = make_agent(vec![]);
let err = agent
.llm_call()
.await
.expect_err("continuation must exhaust");
let exhausted = err
.chain()
.find_map(|c| c.downcast_ref::<crate::retry::RetryExhausted>())
.expect("RetryExhausted must survive in the error chain");
assert_eq!(
exhausted.final_class,
crate::retry::FailureClass::NoResponse,
"granular no-response classification"
);
assert_eq!(
exhausted.last_raw, None,
"no raw text on the exhausted error"
);
let last_failure = exhausted
.failures
.last()
.expect("failure trail is non-empty");
assert_eq!(
last_failure.finish_reason.as_deref(),
Some("stop"),
"in-class NoResponse records carry the response finish_reason into the telemetry trail"
);
let rendered = format!("{err:#}");
assert!(
!rendered.contains("secret thinking"),
"the thinking must never leak into the failure error"
);
assert!(
!rendered.contains(RETRY_EXHAUSTION_MARKER),
"must not be misclassified as LLM provider retry exhaustion"
);
assert!(
agent.session.history().is_empty(),
"the continuation tail must never reach the session transcript"
);
assert_eq!(failure_classification(&agent, Some(&err)), "no_response");
}
#[tokio::test]
#[expect(clippy::await_holding_lock)] async fn llm_call_continuation_transport_error_does_not_duplicate_tail() {
use crate::util::test::{
FakeProvider, install_fake_provider, install_test_retry_policy, retry_tests_lock,
};
let _lock = retry_tests_lock();
crate::util::test::init_test_stores().await;
let _policy_guard = install_test_retry_policy(crate::retry::tiny_test_policy());
let fake = std::sync::Arc::new(
FakeProvider::new()
.ok_reasoning_only("thinking 1", Some("stop"))
.err(crate::retry::FailureClass::Transport, "connection reset")
.err(crate::retry::FailureClass::Transport, "connection reset")
.err(crate::retry::FailureClass::Transport, "connection reset"),
);
let provider: std::sync::Arc<dyn crate::Provider> = fake.clone();
let _provider_guard = install_fake_provider(provider);
let mut agent = make_agent(vec![]);
let err = agent
.llm_call()
.await
.expect_err("continuation must exhaust");
let exhausted = err
.chain()
.find_map(|c| c.downcast_ref::<crate::retry::RetryExhausted>())
.expect("RetryExhausted must survive in the error chain");
assert_eq!(
exhausted.final_class,
crate::retry::FailureClass::Transport,
"final class derives from the last recorded failure, not NoResponse"
);
let fingerprints = fake.request_fingerprints.lock().unwrap().clone();
assert_eq!(
fingerprints.len(),
4,
"original call + 3 continuation attempts"
);
assert_eq!(
fingerprints[1], fingerprints[2],
"attempt 2 re-sends the byte-identical request after a transport error"
);
assert_eq!(fingerprints[2], fingerprints[3]);
let rendered = format!("{err:#}");
assert!(
!rendered.contains("thinking"),
"the thinking must never leak into the failure error"
);
}
#[tokio::test]
#[expect(clippy::await_holding_lock)] async fn llm_call_continuation_non_retryable_error_breaks_immediately() {
use crate::util::test::{
FakeProvider, install_fake_provider, install_test_retry_policy, retry_tests_lock,
};
let _lock = retry_tests_lock();
crate::util::test::init_test_stores().await;
let _policy_guard = install_test_retry_policy(crate::retry::tiny_test_policy());
let fake = std::sync::Arc::new(
FakeProvider::new()
.ok_reasoning_only("thinking 1", Some("stop"))
.err(crate::retry::FailureClass::NonRetryable, "invalid model"),
);
let provider: std::sync::Arc<dyn crate::Provider> = fake.clone();
let _provider_guard = install_fake_provider(provider);
let mut agent = make_agent(vec![]);
let err = agent.llm_call().await.expect_err("continuation must fail");
let exhausted = err
.chain()
.find_map(|c| c.downcast_ref::<crate::retry::RetryExhausted>())
.expect("RetryExhausted must survive in the error chain");
assert_eq!(
exhausted.final_class,
crate::retry::FailureClass::NonRetryable,
"non-retryable class survives to the terminal error"
);
assert_eq!(
fake.request_fingerprints.lock().unwrap().len(),
2,
"original call + exactly one continuation attempt (no budget burn)"
);
}
#[tokio::test]
#[expect(clippy::await_holding_lock)] async fn recover_reasoning_only_stop_abort_classifies_as_shutdown() {
let _guard = crate::util::test::retry_tests_lock();
let agent = make_agent(vec![]);
let first = crate::ChatResponse {
text: None,
reasoning: Some(crate::Reasoning {
reasoning: Some("thinking".into()),
reasoning_content: Some("thinking".into()),
reasoning_details: None,
}),
finish_reason: Some("stop".into()),
..crate::ChatResponse::default()
};
crate::shutdown::drain_begin();
let exhausted = agent
.recover_reasoning_only_stop(vec![], first, "agent-continuation")
.await
.expect_err("the drain must break the continuation immediately");
crate::shutdown::drain_clear();
assert_eq!(
exhausted.final_class,
crate::retry::FailureClass::Shutdown,
"a global abort must classify as shutdown, never no_response"
);
}
#[tokio::test]
#[expect(clippy::await_holding_lock)] async fn llm_call_continuation_exhaustion_does_not_update_session_length() {
use crate::util::test::{
FakeProvider, install_fake_provider, install_test_retry_policy, retry_tests_lock,
};
let _lock = retry_tests_lock();
crate::util::test::init_test_stores().await;
let _policy_guard = install_test_retry_policy(crate::retry::tiny_test_policy());
let fake = std::sync::Arc::new(
FakeProvider::new()
.ok_reasoning_only_with_usage("thinking a", Some("stop"), 1_000, 500)
.ok_reasoning_only_with_usage("thinking b", Some("stop"), 1_000, 500)
.ok_reasoning_only_with_usage("thinking c", Some("stop"), 1_000, 500)
.ok_reasoning_only_with_usage("thinking d", Some("stop"), 1_000, 500),
);
let provider: std::sync::Arc<dyn crate::Provider> = fake.clone();
let _provider_guard = install_fake_provider(provider);
let mut agent = make_agent(vec![]);
assert_eq!(agent.session.token_length(), None);
let err = agent
.llm_call()
.await
.expect_err("continuation must exhaust");
assert_eq!(
agent.session.token_length(),
None,
"a failed turn (continuation exhaustion) must not update the session length"
);
let rendered = format!("{err:#}");
assert!(
!rendered.contains("thinking"),
"the thinking must never leak into the failure error"
);
}
#[tokio::test]
#[expect(clippy::await_holding_lock)] async fn llm_call_continuation_success_records_only_resolving_usage() {
use crate::util::test::{
FakeProvider, install_fake_provider, install_test_retry_policy, retry_tests_lock,
};
let _lock = retry_tests_lock();
crate::util::test::init_test_stores().await;
let _policy_guard = install_test_retry_policy(crate::retry::tiny_test_policy());
let fake = std::sync::Arc::new(
FakeProvider::new()
.ok_reasoning_only_with_usage("thinking a", Some("stop"), 1_000, 500)
.ok_with_usage("final answer", 200, 300),
);
let provider: std::sync::Arc<dyn crate::Provider> = fake.clone();
let _provider_guard = install_fake_provider(provider);
let mut agent = make_agent(vec![]);
let resp = agent.llm_call().await.expect("continuation must resolve");
assert_eq!(resp.text_or_empty(), "final answer");
assert_eq!(
agent.session.token_length(),
Some(500),
"only the resolving continuation response (200 + 300) updates the session length"
);
}
#[tokio::test]
#[expect(clippy::await_holding_lock)] async fn llm_call_skips_continuation_for_normal_and_tool_call_turns() {
use crate::util::test::{
FakeProvider, install_fake_provider, install_test_retry_policy, retry_tests_lock,
};
let _lock = retry_tests_lock();
crate::util::test::init_test_stores().await;
let _policy_guard = install_test_retry_policy(crate::retry::tiny_test_policy());
{
let fake = std::sync::Arc::new(FakeProvider::new().ok("normal answer"));
let provider: std::sync::Arc<dyn crate::Provider> = fake.clone();
let _provider_guard = install_fake_provider(provider);
let mut agent = make_agent(vec![]);
let resp = agent.llm_call().await.expect("normal answer");
assert_eq!(resp.text_or_empty(), "normal answer");
assert_eq!(
fake.request_fingerprints.lock().unwrap().len(),
1,
"normal answer must not trigger continuation"
);
}
{
let fake = std::sync::Arc::new(FakeProvider::new().ok_tool_call("read"));
let provider: std::sync::Arc<dyn crate::Provider> = fake.clone();
let _provider_guard = install_fake_provider(provider);
let mut agent = make_agent(vec![]);
let resp = agent.llm_call().await.expect("tool-call turn");
assert!(resp.text_or_empty().is_empty());
assert_eq!(resp.tool_calls.len(), 1);
assert_eq!(
fake.request_fingerprints.lock().unwrap().len(),
1,
"tool-call turn (empty text) must not trigger continuation"
);
}
}
#[tokio::test]
#[expect(clippy::await_holding_lock)] async fn summarize_recovers_reasoning_only_stop_via_continuation() {
use crate::util::test::{
FakeProvider, install_fake_provider, install_test_retry_policy, retry_tests_lock,
};
let _lock = retry_tests_lock();
crate::util::test::init_test_stores().await;
let _policy_guard = install_test_retry_policy(crate::retry::tiny_test_policy());
let fake = std::sync::Arc::new(
FakeProvider::new()
.ok_reasoning_only("thinking about the summary", Some("stop"))
.ok("the summary"),
);
let provider: std::sync::Arc<dyn crate::Provider> = fake.clone();
let _provider_guard = install_fake_provider(provider);
let agent = make_agent(vec![]);
let summary = agent.summarize().await.expect("continuation must resolve");
assert_eq!(summary, "the summary");
let fingerprints = fake.request_fingerprints.lock().unwrap().clone();
assert_eq!(
fingerprints.len(),
2,
"original summary call + one continuation"
);
assert!(fingerprints[1].contains("Resume your unfinished turn"));
}
#[tokio::test]
#[expect(clippy::await_holding_lock)] async fn summarize_continuation_exhaustion_fails_open_without_leaking() {
use crate::util::test::{
FakeProvider, install_fake_provider, install_test_retry_policy, retry_tests_lock,
};
let _lock = retry_tests_lock();
crate::util::test::init_test_stores().await;
let _policy_guard = install_test_retry_policy(crate::retry::tiny_test_policy());
let fake = std::sync::Arc::new(
FakeProvider::new()
.ok_reasoning_only("summary thinking", Some("stop"))
.ok_reasoning_only("summary thinking", Some("stop"))
.ok_reasoning_only("summary thinking", Some("stop"))
.ok_reasoning_only("summary thinking", Some("stop")),
);
let provider: std::sync::Arc<dyn crate::Provider> = fake.clone();
let _provider_guard = install_fake_provider(provider);
let agent = make_agent(vec![]);
let err = agent
.summarize()
.await
.expect_err("continuation must exhaust");
let rendered = format!("{err:#}");
assert!(
rendered.contains("summarization"),
"fail-open path surfaces the summarization error for maybe_summarize"
);
assert!(
!rendered.contains("summary thinking"),
"the thinking must never leak into the summarization error"
);
}
#[tokio::test]
async fn record_session_usage_keeps_last_on_missing_or_partial_usage() {
crate::util::test::init_test_stores().await;
let mut agent = make_agent(vec![]);
agent.session.set_token_length(Some(7_000));
agent
.record_session_usage(&crate::ChatResponse::default())
.await;
assert_eq!(agent.session.token_length(), Some(7_000));
let partial_input = crate::ChatResponse {
usage: Some(crate::ProviderUsage {
input_tokens: Some(1_000),
..crate::ProviderUsage::default()
}),
..crate::ChatResponse::default()
};
agent.record_session_usage(&partial_input).await;
assert_eq!(agent.session.token_length(), Some(7_000));
let partial_output = crate::ChatResponse {
usage: Some(crate::ProviderUsage {
output_tokens: Some(1_000),
..crate::ProviderUsage::default()
}),
..crate::ChatResponse::default()
};
agent.record_session_usage(&partial_output).await;
assert_eq!(agent.session.token_length(), Some(7_000));
let full = crate::ChatResponse {
usage: Some(crate::ProviderUsage {
input_tokens: Some(12_000),
output_tokens: Some(300),
..crate::ProviderUsage::default()
}),
..crate::ChatResponse::default()
};
agent.record_session_usage(&full).await;
assert_eq!(agent.session.token_length(), Some(12_300));
agent
.record_session_usage(&crate::ChatResponse::default())
.await;
assert_eq!(
agent.session.token_length(),
Some(12_300),
"a usage-less response must never reset the length to zero"
);
}
#[tokio::test]
async fn record_session_usage_overflow_saturates() {
crate::util::test::init_test_stores().await;
let mut agent = make_agent(vec![]);
let huge = crate::ChatResponse {
usage: Some(crate::ProviderUsage {
input_tokens: Some(u64::MAX),
output_tokens: Some(u64::MAX),
..crate::ProviderUsage::default()
}),
..crate::ChatResponse::default()
};
agent.record_session_usage(&huge).await;
assert_eq!(agent.session.token_length(), Some(u64::MAX));
}
#[tokio::test]
async fn maybe_summarize_none_and_below_threshold_are_noops() {
crate::util::test::init_test_stores().await;
let mut agent = make_agent(vec![]);
agent.maybe_summarize().await;
assert_eq!(agent.session.token_length(), None);
agent
.session
.set_token_length(Some(crate::session::SUMMARIZATION_THRESHOLD / 2));
agent.maybe_summarize().await;
}
const IMAGE_REJECTION_BODY: &str = r#"{"error":{"message":"Input image data may contain inappropriate content.","code":"data_inspection_failed","type":"invalid_request_error"}}"#;
const TEXT_REJECTION_BODY: &str = r#"{"error":{"message":"Input data may contain inappropriate content.","code":"data_inspection_failed","type":"invalid_request_error"}}"#;
fn seed_empty_catalogs() {
crate::tools::image_catalog::seed_cache(Some(std::sync::Arc::new(
crate::tools::image_catalog::ImageCatalog::default(),
)));
crate::tools::video_catalog::seed_cache(Some(std::sync::Arc::new(
crate::tools::video_catalog::VideoCatalog::default(),
)));
}
#[tokio::test]
#[serial_test::serial(active_models)] #[expect(clippy::await_holding_lock)] async fn rejected_input_image_is_stripped_and_run_continues() {
use crate::util::test::{
FakeProvider, install_fake_provider, install_test_retry_policy, retry_tests_lock,
};
let _lock = retry_tests_lock();
crate::util::test::init_test_stores().await;
seed_empty_catalogs();
let _policy_guard = install_test_retry_policy(crate::retry::tiny_test_policy());
let agent_id = "e2e_artist_image_reject";
let fake = std::sync::Arc::new(
FakeProvider::new()
.err_http(400, IMAGE_REJECTION_BODY)
.ok("The image was rejected by the provider's content check."),
);
let provider: std::sync::Arc<dyn crate::Provider> = fake.clone();
let _provider_guard = install_fake_provider(provider);
let mut agent = make_agent_with_role(vec![], crate::Role::Artist);
agent.agent_id = agent_id.to_string();
let resp = agent
.work("[IMAGE:/tmp/photo.png] describe this photo", false)
.await
.expect("the run must not fail wholesale on a rejected input image");
assert_eq!(
resp,
"The image was rejected by the provider's content check."
);
let messages = fake.request_messages.lock().unwrap().clone();
assert_eq!(messages.len(), 2, "rejected attempt + normal-loop retry");
assert!(
messages[0].contains("[IMAGE:/tmp/photo.png]"),
"first request carried the image"
);
let retried_user = messages[1]
.split('\u{0}')
.next_back()
.expect("retried request has a user segment");
assert!(
!retried_user.contains("[IMAGE:"),
"retried user message no longer carries the image"
);
assert!(
retried_user.contains("rejected by the provider's content-inspection check"),
"retried user message carries the explanatory phrase"
);
assert!(
retried_user.contains("Input image data may contain inappropriate content."),
"phrase embeds the provider reason"
);
let history = crate::session::store().load(agent_id).await;
let last_user = history
.iter()
.rev()
.find(|m| m.role == crate::ChatRole::User)
.expect("user message exists");
assert!(
!last_user.content.contains("[IMAGE:"),
"rejected image durably removed from the session"
);
assert!(
last_user.content.contains("describe this photo"),
"user's accompanying text preserved"
);
assert_eq!(
history
.iter()
.filter(|m| m.role == crate::ChatRole::User)
.count(),
1,
"no separate notification message was added"
);
}
#[tokio::test]
#[serial_test::serial(active_models)] #[expect(clippy::await_holding_lock)] async fn text_content_rejection_follows_normal_failure_path() {
use crate::util::test::{
FakeProvider, install_fake_provider, install_test_retry_policy, retry_tests_lock,
};
let _lock = retry_tests_lock();
crate::util::test::init_test_stores().await;
seed_empty_catalogs();
let _policy_guard = install_test_retry_policy(crate::retry::tiny_test_policy());
let agent_id = "e2e_artist_text_reject";
let fake = std::sync::Arc::new(FakeProvider::new().err_http(400, TEXT_REJECTION_BODY));
let provider: std::sync::Arc<dyn crate::Provider> = fake.clone();
let _provider_guard = install_fake_provider(provider);
let mut agent = make_agent_with_role(vec![], crate::Role::Artist);
agent.agent_id = agent_id.to_string();
let result = agent
.work("[IMAGE:/tmp/photo.png] describe this photo", false)
.await;
assert!(
result.is_err(),
"text-content rejection must fail the run normally"
);
let history = crate::session::store().load(agent_id).await;
let last_user = history
.iter()
.rev()
.find(|m| m.role == crate::ChatRole::User)
.expect("user message exists");
assert!(
last_user.content.contains("[IMAGE:/tmp/photo.png]"),
"image untouched on a text-content rejection"
);
}
#[tokio::test]
#[serial_test::serial(active_models)] #[expect(clippy::await_holding_lock)] async fn subsequent_failure_after_strip_takes_normal_failure_path() {
use crate::util::test::{
FakeProvider, install_fake_provider, install_test_retry_policy, retry_tests_lock,
};
let _lock = retry_tests_lock();
crate::util::test::init_test_stores().await;
seed_empty_catalogs();
let _policy_guard = install_test_retry_policy(crate::retry::tiny_test_policy());
let agent_id = "e2e_artist_second_failure";
let fake = std::sync::Arc::new(
FakeProvider::new()
.err_http(400, IMAGE_REJECTION_BODY)
.err_http(400, IMAGE_REJECTION_BODY),
);
let provider: std::sync::Arc<dyn crate::Provider> = fake.clone();
let _provider_guard = install_fake_provider(provider);
let mut agent = make_agent_with_role(vec![], crate::Role::Artist);
agent.agent_id = agent_id.to_string();
let result = agent
.work("[IMAGE:/tmp/photo.png] describe this photo", false)
.await;
assert!(
result.is_err(),
"a second failure after the strip must fail normally"
);
let messages = fake.request_messages.lock().unwrap().clone();
assert_eq!(
messages.len(),
2,
"exactly one strip, then the normal failure"
);
let retried_user = messages[1]
.split('\u{0}')
.next_back()
.expect("retried request has a user segment");
assert!(
!retried_user.contains("[IMAGE:"),
"retried user message is already stripped"
);
let history = crate::session::store().load(agent_id).await;
let last_user = history
.iter()
.rev()
.find(|m| m.role == crate::ChatRole::User)
.expect("user message exists");
assert!(!last_user.content.contains("[IMAGE:"));
assert!(
last_user
.content
.contains("rejected by the provider's content-inspection check"),
"phrase present after a single strip"
);
}
#[test]
fn existing_image_marker_values_extracts_image_markers() {
let history = vec![
ChatMessage::system("sys"),
ChatMessage::user("[IMAGE:data:image/jpeg;base64,aaa] see this"),
ChatMessage::assistant("ok"),
ChatMessage::user("plain text, no marker"),
ChatMessage::user("[AUDIO:data:audio/ogg;base64,zzz]"),
];
let set = existing_image_marker_values(&history);
assert_eq!(
set.len(),
1,
"only IMAGE markers are collected, got: {set:?}"
);
assert!(set.contains("data:image/jpeg;base64,aaa"));
}
#[tokio::test]
#[expect(clippy::too_many_lines)]
async fn commit_tool_results_injects_image_after_tool_results_with_dedup() {
crate::util::test::init_test_stores().await;
let mut agent = make_agent(vec![]);
agent.session.push_messages_unpersisted(&[ChatMessage::user(
"[IMAGE:data:image/jpeg;base64,prior]",
)]);
let tool_calls = vec![
ToolCall {
id: "call1".into(),
name: "read".into(),
arguments: serde_json::json!({"path": "a.png"}),
},
ToolCall {
id: "call2".into(),
name: "read".into(),
arguments: serde_json::json!({"path": "b.png"}),
},
];
let outcomes = vec![
ToolExecutionOutcome {
output: "Read image file /a.png (1x1, PNG).".into(),
success: true,
image_payload: Some(ImagePayload {
path: "/a.png".into(),
data_uri: "data:image/jpeg;base64,prior".into(),
width: 1,
height: 1,
format: "PNG".into(),
recovery_note: None,
source: crate::tools::ImagePayloadSource::Read,
}),
},
ToolExecutionOutcome {
output: "Read image file /b.png (1x1, PNG).".into(),
success: true,
image_payload: Some(ImagePayload {
path: "/b.png".into(),
data_uri: "data:image/jpeg;base64,fresh".into(),
width: 1,
height: 1,
format: "PNG".into(),
recovery_note: None,
source: crate::tools::ImagePayloadSource::Read,
}),
},
];
agent
.commit_tool_results(&tool_calls, &outcomes, "assistant with tool_calls")
.await
.expect("commit_tool_results must succeed");
let history = agent.session.history();
let roles: Vec<crate::ChatRole> = history.iter().map(|m| m.role).collect();
assert_eq!(
roles,
vec![
crate::ChatRole::User,
crate::ChatRole::Assistant,
crate::ChatRole::Tool,
crate::ChatRole::Tool,
crate::ChatRole::User,
],
"unexpected message ordering: {roles:?}"
);
let image_users: Vec<&str> = history
.iter()
.filter(|m| m.role == crate::ChatRole::User && m.content.contains("[IMAGE:data:"))
.map(|m| m.content.as_str())
.collect();
assert_eq!(
image_users.len(),
2,
"prior + fresh image messages, got: {image_users:?}"
);
assert!(
image_users.contains(&"[IMAGE:data:image/jpeg;base64,prior]"),
"prior image survives in history: {image_users:?}"
);
assert!(
image_users
.iter()
.any(|s| s.contains("[IMAGE:data:image/jpeg;base64,fresh]")),
"fresh image injected once: {image_users:?}"
);
let fresh_msg = image_users
.iter()
.find(|s| s.contains("base64,fresh"))
.expect("fresh image message present");
assert!(
fresh_msg.starts_with("<injected-tool-result-image>\n[IMAGE:data:image/jpeg;base64,"),
"injected image carries the provenance tag: {fresh_msg:?}"
);
let fresh_count = history
.iter()
.filter(|m| m.role == crate::ChatRole::User && m.content.contains("base64,fresh"))
.count();
assert_eq!(fresh_count, 1, "a fresh image is injected exactly once");
let tool_results: Vec<String> = history
.iter()
.filter(|m| m.role == crate::ChatRole::Tool)
.map(|m| {
let v: serde_json::Value =
serde_json::from_str(&m.content).expect("tool result is valid JSON");
crate::util::json::get_str(&v, "content")
.expect("tool result has content")
.to_string()
})
.collect();
assert_eq!(
tool_results,
vec![
"Read image file /a.png (1x1, PNG). Image content is already attached to the conversation as a native image.",
"Read image file /b.png (1x1, PNG). Image content attached to the conversation as a native image.",
],
"unexpected tool-result text: {tool_results:?}"
);
}
#[tokio::test]
async fn commit_tool_results_dedups_identical_reads_in_same_round() {
crate::util::test::init_test_stores().await;
let mut agent = make_agent(vec![]);
let tool_calls = vec![
ToolCall {
id: "callA".into(),
name: "read".into(),
arguments: serde_json::json!({"path": "a.png"}),
},
ToolCall {
id: "callB".into(),
name: "read".into(),
arguments: serde_json::json!({"path": "a.png"}),
},
];
let outcomes = vec![
ToolExecutionOutcome {
output: "Read image file /a.png (PNG).".into(),
success: true,
image_payload: Some(ImagePayload {
path: "/a.png".into(),
data_uri: "data:image/jpeg;base64,same".into(),
width: 1,
height: 1,
format: "PNG".into(),
recovery_note: None,
source: crate::tools::ImagePayloadSource::Read,
}),
},
ToolExecutionOutcome {
output: "Read image file /a.png (PNG).".into(),
success: true,
image_payload: Some(ImagePayload {
path: "/a.png".into(),
data_uri: "data:image/jpeg;base64,same".into(),
width: 1,
height: 1,
format: "PNG".into(),
recovery_note: None,
source: crate::tools::ImagePayloadSource::Read,
}),
},
];
agent
.commit_tool_results(&tool_calls, &outcomes, "assistant with tool_calls")
.await
.expect("commit_tool_results must succeed");
let history = agent.session.history();
let image_users: Vec<&str> = history
.iter()
.filter(|m| {
m.role == crate::ChatRole::User
&& m.content.contains("[IMAGE:data:image/jpeg;base64,same]")
})
.map(|m| m.content.as_str())
.collect();
assert_eq!(
image_users.len(),
1,
"identical reads in one round inject exactly once, got: {image_users:?}"
);
let tool_results: Vec<String> = history
.iter()
.filter(|m| m.role == crate::ChatRole::Tool)
.map(|m| {
let v: serde_json::Value =
serde_json::from_str(&m.content).expect("tool result is valid JSON");
crate::util::json::get_str(&v, "content")
.expect("tool result has content")
.to_string()
})
.collect();
assert_eq!(tool_results.len(), 2, "two tool results persisted");
assert!(
tool_results[0].contains("attached to the conversation as a native image")
&& !tool_results[0].contains("already"),
"first read claims a fresh attachment: {tool_results:?}"
);
assert!(
tool_results[1].contains("already attached to the conversation"),
"second read returns a reference: {tool_results:?}"
);
}
#[tokio::test]
async fn commit_tool_results_derives_generated_image_and_tags_it() {
crate::util::test::init_test_stores().await;
let dir = tempfile::TempDir::new().expect("tempdir");
let png_path = dir.path().join("generated.png");
std::fs::write(&png_path, crate::util::test::noisy_png(4, 4)).expect("write png");
let abs = std::fs::canonicalize(&png_path).expect("canonicalize");
let tool = Box::new(MediaTestTool {
name: "image_gen",
marker: "[IMAGE:",
}) as Box<dyn Tool>;
let mut agent = make_agent(vec![tool]);
let marker = format!("[IMAGE:{}]", abs.display());
let tool_calls = vec![ToolCall {
id: "callgen".into(),
name: "image_gen".into(),
arguments: serde_json::json!({}),
}];
let outcomes = vec![ToolExecutionOutcome {
output: marker.clone(),
success: true,
image_payload: None,
}];
agent
.commit_tool_results(&tool_calls, &outcomes, "assistant with tool_call")
.await
.expect("commit_tool_results must succeed");
let history = agent.session.history();
let image_users: Vec<&str> = history
.iter()
.filter(|m| {
m.role == crate::ChatRole::User
&& m.content.contains("[IMAGE:data:image/jpeg;base64,")
})
.map(|m| m.content.as_str())
.collect();
assert_eq!(
image_users.len(),
1,
"generated image injected once: {image_users:?}"
);
assert!(
image_users[0]
.starts_with("<injected-tool-result-image>\n[IMAGE:data:image/jpeg;base64,"),
"injected image carries the provenance tag: {image_users:?}"
);
let tool_results: Vec<String> = history
.iter()
.filter(|m| m.role == crate::ChatRole::Tool)
.map(|m| {
let v: serde_json::Value =
serde_json::from_str(&m.content).expect("tool result JSON");
crate::util::json::get_str(&v, "content")
.expect("content")
.to_string()
})
.collect();
assert_eq!(tool_results.len(), 1);
assert!(
tool_results[0].starts_with("Generated image file"),
"generated annotation: {tool_results:?}"
);
assert!(
!tool_results[0].starts_with("Read image file"),
"must not be a Read annotation: {tool_results:?}"
);
let media = extract_media_from_outcomes(&agent.tools, &tool_calls, &outcomes);
assert_eq!(
media,
vec![("[IMAGE:", abs.display().to_string())],
"marker preserved for user delivery: {media:?}"
);
}
}