use std::collections::HashMap;
use std::future;
use std::sync::{Arc, Mutex, PoisonError};
use std::time::{Duration, Instant};
use magi_core::orchestrator::Magi;
use magi_core::schema::Mode;
use serde_json::Value;
use tokio_util::sync::CancellationToken;
use magi_rs::headless::log::{LogEvent, LogLevel, RunLog};
use magi_rs::headless::output::sanitize_error_message;
use magi_rs::headless::policy::{Policy, Tier};
use magi_rs::headless::resolution::Resolved;
use magi_rs::headless::types::{
ErrorKind, ErrorPayload, RunOutcome, StopReason, Timings, ToolCallRecord, TranscriptEntry,
Usage,
};
use crate::agent::messages::{Content, Message, Role};
use crate::agent::{Agent, AgentRunConfig, RunObserver, StreamPiece, MAX_TOOL_CALLS_ERROR};
use crate::tools::consult::{report_to_consult_json, AbortOnDrop, MAX_QUERY_LEN};
const CHUNK_CHANNEL_CAPACITY: usize = 100;
const ROLE_USER: &str = "user";
const ROLE_ASSISTANT: &str = "assistant";
const TIMEOUT_MESSAGE: &str = "run exceeded the --timeout wall-clock limit";
const CONSULT_TOOL: &str = "consult";
const CONSULT_INPUT_INVALID_MESSAGE: &str = "consult prompt is empty or exceeds the maximum length";
enum ConsultRunError {
InputInvalid,
Timeout,
Runtime(String),
}
async fn analyze_direct(
magi: &Arc<Magi>,
prompt: &str,
cancel: &CancellationToken,
timeout: Option<Duration>,
) -> Result<Value, ConsultRunError> {
if prompt.trim().is_empty() || prompt.len() > MAX_QUERY_LEN {
return Err(ConsultRunError::InputInvalid);
}
let magi = Arc::clone(magi);
let owned = prompt.to_string();
let handle = tokio::spawn(async move { magi.analyze(&Mode::Analysis, &owned).await });
let abort_guard = AbortOnDrop::new(handle.abort_handle());
let deadline = async {
match timeout {
Some(dur) => tokio::time::sleep(dur).await,
None => future::pending::<()>().await,
}
};
tokio::pin!(deadline);
let joined = tokio::select! {
biased;
() = cancel.cancelled() => {
abort_guard.abort();
return Err(ConsultRunError::Timeout);
}
() = &mut deadline => {
abort_guard.abort();
return Err(ConsultRunError::Timeout);
}
joined = handle => joined,
};
match joined {
Ok(Ok(report)) => Ok(report_to_consult_json(&report)),
Ok(Err(e)) => Err(ConsultRunError::Runtime(e.to_string())),
Err(join_err) => Err(ConsultRunError::Runtime(format!(
"consult crashed: {join_err}"
))),
}
}
fn consult_error_outcome(
err: ConsultRunError,
) -> (
Option<String>,
Option<Value>,
StopReason,
Option<ErrorPayload>,
) {
let payload = match err {
ConsultRunError::InputInvalid => ErrorPayload {
message: CONSULT_INPUT_INVALID_MESSAGE.to_string(),
kind: ErrorKind::InputInvalid,
},
ConsultRunError::Timeout => ErrorPayload {
message: TIMEOUT_MESSAGE.to_string(),
kind: ErrorKind::Timeout,
},
ConsultRunError::Runtime(message) => ErrorPayload {
message: sanitize_error_message(&message),
kind: ErrorKind::Runtime,
},
};
(None, None, StopReason::Error, Some(payload))
}
fn build_consult_transcript(prompt: &str, report: Option<&str>) -> Vec<TranscriptEntry> {
let mut transcript = vec![TranscriptEntry {
role: ROLE_USER.to_string(),
content: prompt.to_string(),
tool_calls: None,
}];
if let Some(text) = report {
transcript.push(TranscriptEntry {
role: ROLE_ASSISTANT.to_string(),
content: text.to_string(),
tool_calls: None,
});
}
transcript
}
fn extract_consult_value(calls: &[(String, ToolCallRecord)]) -> Option<Value> {
calls
.iter()
.rev()
.find(|(_, rec)| rec.name == CONSULT_TOOL && rec.ok)
.and_then(|(_, rec)| serde_json::from_str::<Value>(&rec.result).ok())
}
#[derive(Default)]
struct TrackerState {
calls: Vec<(String, ToolCallRecord)>,
tier_denials: usize,
final_turn_text_blocks: usize,
usage_total: (u64, u64),
}
struct RunTracker {
policy: Policy,
state: Mutex<TrackerState>,
}
fn with_state<R>(state: &Mutex<TrackerState>, f: impl FnOnce(&mut TrackerState) -> R) -> R {
let mut guard = state.lock().unwrap_or_else(PoisonError::into_inner);
f(&mut guard)
}
impl RunObserver for RunTracker {
fn authorize(&self, tool_name: &str) -> bool {
let allowed = self.policy.approves(tool_name);
if !allowed {
with_state(&self.state, |s| s.tier_denials += 1);
}
allowed
}
fn on_tool_call(
&self,
id: &str,
name: &str,
input: &serde_json::Value,
result: &str,
ok: bool,
ms: u64,
) {
let record = ToolCallRecord {
name: name.to_string(),
input: input.clone(),
result: result.to_string(),
ms,
ok,
};
with_state(&self.state, |s| s.calls.push((id.to_string(), record)));
}
fn on_final_turn(&self, text_block_count: usize) {
with_state(&self.state, |s| s.final_turn_text_blocks = text_block_count);
}
fn on_usage(&self, input_tokens: u64, output_tokens: u64) {
with_state(&self.state, |s| {
s.usage_total.0 = s.usage_total.0.saturating_add(input_tokens);
s.usage_total.1 = s.usage_total.1.saturating_add(output_tokens);
});
}
}
#[must_use]
pub fn resolve_run_timeout(policy: &Policy, full_auto_timeout_secs: u64) -> Option<Duration> {
if let Some(secs) = policy.timeout() {
return Some(Duration::from_secs(secs));
}
match policy.tier() {
Tier::Auto | Tier::FullAuto => Some(Duration::from_secs(full_auto_timeout_secs)),
Tier::Default => None,
}
}
fn elapsed_ms(start: Instant) -> u64 {
u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX)
}
fn timeout_outcome() -> (Option<String>, StopReason, Option<ErrorPayload>) {
(
None,
StopReason::Error,
Some(ErrorPayload {
message: TIMEOUT_MESSAGE.to_string(),
kind: ErrorKind::Timeout,
}),
)
}
fn log_event(run_log: Option<&mut &mut RunLog>, event: &LogEvent<'_>) {
if let Some(log) = run_log {
let _ = log.event(event);
}
}
fn join_text(msg: &Message) -> String {
msg.content
.iter()
.filter_map(|c| match c {
Content::Text { text } => Some(text.as_str()),
_ => None,
})
.collect::<Vec<_>>()
.join("")
}
fn build_transcript(
history: &[Message],
records: &HashMap<&str, &ToolCallRecord>,
) -> Vec<TranscriptEntry> {
let mut transcript = Vec::new();
for msg in history {
match msg.role {
Role::User => {
let is_tool_result = msg
.content
.iter()
.any(|c| matches!(c, Content::ToolResult { .. }));
if is_tool_result {
continue;
}
transcript.push(TranscriptEntry {
role: ROLE_USER.to_string(),
content: join_text(msg),
tool_calls: None,
});
}
Role::Assistant => {
let calls: Vec<ToolCallRecord> = msg
.content
.iter()
.filter_map(|c| match c {
Content::ToolUse { id, name, input } => Some(
records
.get(id.as_str())
.map(|r| (*r).clone())
.unwrap_or_else(|| ToolCallRecord {
name: name.clone(),
input: input.clone(),
result: String::new(),
ms: 0,
ok: false,
}),
),
_ => None,
})
.collect();
let tool_calls = if calls.is_empty() { None } else { Some(calls) };
transcript.push(TranscriptEntry {
role: ROLE_ASSISTANT.to_string(),
content: join_text(msg),
tool_calls,
});
}
}
}
transcript
}
pub async fn run_consult(
resolved: Resolved,
magi: Arc<Magi>,
prompt: &str,
timeout: Option<Duration>,
mut run_log: Option<&mut RunLog>,
) -> RunOutcome {
let cancel = CancellationToken::new();
let run_start = Instant::now();
let result = analyze_direct(&magi, prompt, &cancel, timeout).await;
let total_ms = elapsed_ms(run_start);
let (response, consult, stop_reason, error) = match result {
Ok(value) => {
let response = value
.get("report")
.and_then(Value::as_str)
.map(str::to_string);
(response, Some(value), StopReason::Done, None)
}
Err(err) => consult_error_outcome(err),
};
let transcript = build_consult_transcript(prompt, response.as_deref());
let summary = format!("consult complete: stop_reason={stop_reason:?}");
log_event(
run_log.as_mut(),
&LogEvent::Message {
level: LogLevel::Info,
text: &summary,
},
);
RunOutcome {
response,
model: resolved.model,
provider: resolved.provider,
usage: Usage {
input_tokens: 0,
output_tokens: 0,
},
timings: Timings {
total_ms,
ttfb_ms: None,
per_turn_ms: Vec::new(),
},
stop_reason,
tool_calls: Vec::new(),
transcript,
consult,
applied_caps: resolved.applied_caps,
error,
}
}
pub async fn run_query(
resolved: Resolved,
policy: Policy,
agent: &mut Agent,
prompt: &str,
timeout: Option<Duration>,
mut run_log: Option<&mut RunLog>,
) -> RunOutcome {
for warning in policy.warnings() {
log_event(
run_log.as_mut(),
&LogEvent::Message {
level: LogLevel::Warn,
text: &warning,
},
);
}
let tracker = Arc::new(RunTracker {
policy: policy.clone(),
state: Mutex::new(TrackerState::default()),
});
let cancel = CancellationToken::new();
let system = resolved.system.text();
let config = AgentRunConfig {
max_tool_calls: usize::try_from(policy.max_tool_calls()).unwrap_or(usize::MAX),
disable_repetitive_guard: policy.silences_soft_guards(),
observer: Some(Arc::clone(&tracker) as Arc<dyn RunObserver>),
cancel: cancel.clone(),
system: (!system.is_empty()).then(|| system.to_string()),
force_consult: resolved.consult == Some(true),
};
let (chunk_tx, mut chunk_rx) =
tokio::sync::mpsc::channel::<StreamPiece>(CHUNK_CHANNEL_CAPACITY);
let run_start = Instant::now();
let drain = tokio::spawn(async move {
let mut ttfb_ms: Option<u64> = None;
while let Some(piece) = chunk_rx.recv().await {
if ttfb_ms.is_none() {
if let StreamPiece::Content(_) = piece {
ttfb_ms = Some(elapsed_ms(run_start));
}
}
}
ttfb_ms
});
let query_fut = agent.query_streaming(prompt, chunk_tx, config);
let outcome_result: Option<Result<String, anyhow::Error>> = match timeout {
Some(dur) => match tokio::time::timeout(dur, query_fut).await {
Ok(r) => Some(r),
Err(_elapsed) => {
cancel.cancel();
None
}
},
None => Some(query_fut.await),
};
let ttfb_ms = drain.await.ok().flatten();
let total_ms = elapsed_ms(run_start);
let (calls, tier_denied, response_empty, usage_total) = with_state(&tracker.state, |s| {
(
s.calls.clone(),
s.tier_denials > 0,
s.final_turn_text_blocks == 0,
s.usage_total,
)
});
let tool_calls: Vec<ToolCallRecord> = calls.iter().map(|(_, rec)| rec.clone()).collect();
let records_by_id: HashMap<&str, &ToolCallRecord> =
calls.iter().map(|(id, rec)| (id.as_str(), rec)).collect();
let transcript = build_transcript(agent.history(), &records_by_id);
let (response, stop_reason, error) = match outcome_result {
None => timeout_outcome(),
Some(Ok(text)) => {
let stop_reason = if tier_denied && response_empty {
StopReason::Denied
} else {
StopReason::Done
};
(Some(text), stop_reason, None)
}
Some(Err(e)) => {
let message = e.to_string();
if message == MAX_TOOL_CALLS_ERROR {
(None, StopReason::MaxToolCalls, None)
} else {
(
None,
StopReason::Error,
Some(ErrorPayload {
message: sanitize_error_message(&message),
kind: ErrorKind::Runtime,
}),
)
}
}
};
for record in &tool_calls {
let input_str = record.input.to_string();
log_event(
run_log.as_mut(),
&LogEvent::ToolCall {
level: LogLevel::Info,
name: &record.name,
ok: record.ok,
ms: record.ms,
input: &input_str,
},
);
}
let summary = format!("run complete: stop_reason={stop_reason:?}");
log_event(
run_log.as_mut(),
&LogEvent::Message {
level: LogLevel::Info,
text: &summary,
},
);
RunOutcome {
response,
model: resolved.model,
provider: resolved.provider,
usage: Usage {
input_tokens: usage_total.0,
output_tokens: usage_total.1,
},
timings: Timings {
total_ms,
ttfb_ms,
per_turn_ms: Vec::new(),
},
stop_reason,
tool_calls,
transcript,
consult: extract_consult_value(&calls),
applied_caps: resolved.applied_caps,
error,
}
}
#[cfg(test)]
mod tests {
use super::*;
use anyhow::Result;
use async_trait::async_trait;
use futures::stream::{self, BoxStream};
use magi_rs::headless::limits::FULL_AUTO_TIMEOUT_SECS;
use serde_json::{json, Value};
use std::collections::VecDeque;
use std::sync::atomic::{AtomicBool, Ordering};
use magi_core::schema::AgentName;
use magi_core::test_support::RoutingMockProvider;
use magi_rs::headless::policy::Tier;
use magi_rs::headless::types::{AppliedCaps, SystemPolicy};
use crate::agent::provider::{Provider, ResponseChunk};
use crate::tools::consult::ConsultTool;
use crate::tools::{Tool, ToolResult};
fn canned_magi() -> Arc<Magi> {
fn agent_json(agent: &str) -> String {
format!(
r#"{{"agent":"{agent}","verdict":"approve","confidence":0.9,"summary":"s","reasoning":"r","findings":[],"recommendation":"rec"}}"#
)
}
let provider = RoutingMockProvider::new()
.with_agent_responses(AgentName::Melchior, vec![Ok(agent_json("melchior"))])
.with_agent_responses(AgentName::Balthasar, vec![Ok(agent_json("balthasar"))])
.with_agent_responses(AgentName::Caspar, vec![Ok(agent_json("caspar"))]);
Arc::new(Magi::new(Arc::new(provider)))
}
fn slow_magi(delay: Duration) -> Arc<Magi> {
use magi_core::error::ProviderError;
use magi_core::provider::{CompletionConfig, LlmProvider};
struct SlowLlm {
delay: Duration,
}
#[async_trait]
impl LlmProvider for SlowLlm {
async fn complete(
&self,
_system_prompt: &str,
_user_prompt: &str,
_config: &CompletionConfig,
) -> Result<String, ProviderError> {
tokio::time::sleep(self.delay).await;
Ok(r#"{"agent":"melchior","verdict":"approve","confidence":0.9,"summary":"s","reasoning":"r","findings":[],"recommendation":"rec"}"#.to_string())
}
fn name(&self) -> &str {
"slow"
}
fn model(&self) -> &str {
"slow-model"
}
}
Arc::new(Magi::new(Arc::new(SlowLlm { delay })))
}
fn slow_droppy_magi(delay: Duration, dropped: Arc<AtomicBool>) -> Arc<Magi> {
use magi_core::error::ProviderError;
use magi_core::provider::{CompletionConfig, LlmProvider};
struct DropFlag(Arc<AtomicBool>);
impl Drop for DropFlag {
fn drop(&mut self) {
self.0.store(true, Ordering::SeqCst);
}
}
struct SlowDroppyLlm {
delay: Duration,
dropped: Arc<AtomicBool>,
}
#[async_trait]
impl LlmProvider for SlowDroppyLlm {
async fn complete(
&self,
_system_prompt: &str,
_user_prompt: &str,
_config: &CompletionConfig,
) -> Result<String, ProviderError> {
let _spy = DropFlag(self.dropped.clone());
tokio::time::sleep(self.delay).await;
Ok(r#"{"agent":"melchior","verdict":"approve","confidence":0.9,"summary":"s","reasoning":"r","findings":[],"recommendation":"rec"}"#.to_string())
}
fn name(&self) -> &str {
"slow-droppy"
}
fn model(&self) -> &str {
"slow-droppy-model"
}
}
Arc::new(Magi::new(Arc::new(SlowDroppyLlm { delay, dropped })))
}
fn forced_resolved() -> Resolved {
Resolved {
consult: Some(true),
..resolved_stub()
}
}
enum Turn {
Tool {
id: String,
name: String,
input: Value,
},
ToolWithUsage {
id: String,
name: String,
input: Value,
input_tokens: u64,
output_tokens: u64,
},
Text(String),
TextWithUsage {
text: String,
input_tokens: u64,
output_tokens: u64,
},
Empty,
Fail,
}
struct ScriptedProvider {
turns: Mutex<VecDeque<Turn>>,
}
impl ScriptedProvider {
fn new(turns: Vec<Turn>) -> Arc<Self> {
Arc::new(Self {
turns: Mutex::new(turns.into_iter().collect()),
})
}
}
#[async_trait]
impl Provider for ScriptedProvider {
async fn stream_messages(
&self,
_messages: &[Message],
_tools: &[Box<dyn Tool>],
_system: Option<&str>,
) -> Result<BoxStream<'static, Result<ResponseChunk>>> {
let turn = self
.turns
.lock()
.unwrap_or_else(PoisonError::into_inner)
.pop_front();
match turn {
Some(Turn::Tool { id, name, input }) => {
let msg = Message {
role: Role::Assistant,
content: vec![Content::ToolUse { id, name, input }],
};
Ok(Box::pin(stream::iter(vec![Ok(
ResponseChunk::MessageDone(msg),
)])))
}
Some(Turn::ToolWithUsage {
id,
name,
input,
input_tokens,
output_tokens,
}) => {
let msg = Message {
role: Role::Assistant,
content: vec![Content::ToolUse { id, name, input }],
};
Ok(Box::pin(stream::iter(vec![
Ok(ResponseChunk::Usage {
input_tokens,
output_tokens,
}),
Ok(ResponseChunk::MessageDone(msg)),
])))
}
Some(Turn::Text(text)) => Ok(Box::pin(stream::iter(vec![
Ok(ResponseChunk::TextDelta(text.clone())),
Ok(ResponseChunk::MessageDone(Message::assistant(&text))),
]))),
Some(Turn::TextWithUsage {
text,
input_tokens,
output_tokens,
}) => Ok(Box::pin(stream::iter(vec![
Ok(ResponseChunk::TextDelta(text.clone())),
Ok(ResponseChunk::Usage {
input_tokens,
output_tokens,
}),
Ok(ResponseChunk::MessageDone(Message::assistant(&text))),
]))),
Some(Turn::Empty) | None => Ok(Box::pin(stream::iter(vec![Ok(
ResponseChunk::MessageDone(Message {
role: Role::Assistant,
content: vec![],
}),
)]))),
Some(Turn::Fail) => Err(anyhow::anyhow!("provider network failure")),
}
}
}
struct SystemCapturingProvider {
seen: Mutex<Vec<Option<String>>>,
}
impl SystemCapturingProvider {
fn new() -> Arc<Self> {
Arc::new(Self {
seen: Mutex::new(Vec::new()),
})
}
}
#[async_trait]
impl Provider for SystemCapturingProvider {
async fn stream_messages(
&self,
_messages: &[Message],
_tools: &[Box<dyn Tool>],
system: Option<&str>,
) -> Result<BoxStream<'static, Result<ResponseChunk>>> {
self.seen
.lock()
.unwrap_or_else(PoisonError::into_inner)
.push(system.map(str::to_string));
Ok(Box::pin(stream::iter(vec![Ok(
ResponseChunk::MessageDone(Message::assistant("ok")),
)])))
}
}
#[tokio::test]
async fn test_run_query_sets_system_from_caller_override_policy() {
let provider = SystemCapturingProvider::new();
let mut agent = Agent::new(provider.clone());
let resolved = Resolved {
system: SystemPolicy::CallerOverride("caller system prompt".to_string()),
..resolved_stub()
};
let policy = Policy::new(Tier::Default, 15, None);
run_query(resolved, policy, &mut agent, "prompt", None, None).await;
assert_eq!(
provider.seen.lock().unwrap().as_slice(),
&[Some("caller system prompt".to_string())]
);
}
#[tokio::test]
async fn test_run_query_sends_no_system_when_operator_default_is_empty() {
let provider = SystemCapturingProvider::new();
let mut agent = Agent::new(provider.clone());
let policy = Policy::new(Tier::Default, 15, None);
run_query(resolved_stub(), policy, &mut agent, "prompt", None, None).await;
assert_eq!(provider.seen.lock().unwrap().as_slice(), &[None]);
}
struct EchoTool {
name: String,
}
#[async_trait]
impl Tool for EchoTool {
fn name(&self) -> &str {
&self.name
}
fn description(&self) -> &str {
"echo tool for runner tests"
}
fn input_schema(&self) -> Value {
json!({"type": "object", "properties": {}})
}
async fn execute(&self, _args: Value, _cancel: &CancellationToken) -> ToolResult<Value> {
Ok(json!(format!("{}-ran", self.name)))
}
}
fn tool_turn(id: &str, name: &str) -> Turn {
Turn::Tool {
id: id.to_string(),
name: name.to_string(),
input: json!({}),
}
}
fn register_echo(agent: &mut Agent, name: &str) {
agent.register_tool(Box::new(EchoTool {
name: name.to_string(),
}));
}
fn resolved_stub() -> Resolved {
Resolved {
model: "test-model".to_string(),
provider: "test-provider".to_string(),
system: SystemPolicy::Operator(String::new()),
max_tool_calls: 15,
consult: None,
applied_caps: AppliedCaps {
max_tool_calls: 15,
max_tool_calls_clamped: false,
timeout_secs: None,
system_override_applied: false,
},
}
}
fn identical_edit_turns(n: usize) -> Vec<Turn> {
(0..n)
.map(|i| Turn::Tool {
id: format!("r{i}"),
name: "edit".to_string(),
input: json!({"same": "input"}),
})
.collect()
}
#[tokio::test]
async fn test_run_query_sums_usage_across_turns_into_run_outcome() {
let provider = ScriptedProvider::new(vec![
Turn::ToolWithUsage {
id: "t1".to_string(),
name: "edit".to_string(),
input: json!({}),
input_tokens: 10,
output_tokens: 2,
},
Turn::TextWithUsage {
text: "answered".to_string(),
input_tokens: 5,
output_tokens: 3,
},
]);
let mut agent = Agent::new(provider);
register_echo(&mut agent, "edit");
let policy = Policy::new(Tier::Auto, 15, None);
let outcome = run_query(resolved_stub(), policy, &mut agent, "prompt", None, None).await;
assert_eq!(outcome.usage.input_tokens, 15);
assert_eq!(outcome.usage.output_tokens, 5);
}
#[tokio::test]
async fn test_run_query_usage_defaults_to_zero_when_provider_reports_none() {
let provider = ScriptedProvider::new(vec![Turn::Text("hi".to_string())]);
let mut agent = Agent::new(provider);
let policy = Policy::new(Tier::Default, 15, None);
let outcome = run_query(resolved_stub(), policy, &mut agent, "prompt", None, None).await;
assert_eq!(outcome.usage.input_tokens, 0);
assert_eq!(outcome.usage.output_tokens, 0);
}
#[tokio::test]
async fn test_runner_default_denies_edit_and_reports_without_aborting() {
let provider = ScriptedProvider::new(vec![
tool_turn("t1", "edit"),
Turn::Text("answered anyway".to_string()),
]);
let mut agent = Agent::new(provider);
register_echo(&mut agent, "edit");
let policy = Policy::new(Tier::Default, 15, None);
let outcome = run_query(resolved_stub(), policy, &mut agent, "prompt", None, None).await;
assert!(
outcome.tool_calls.iter().any(|t| t.name == "edit" && !t.ok),
"edit must be recorded as denied (ok=false) in default tier"
);
assert_eq!(
outcome.stop_reason,
StopReason::Done,
"the agent answered despite the denial ⇒ Done (exit 0)"
);
assert_eq!(outcome.response.as_deref(), Some("answered anyway"));
assert_eq!(outcome.model, "test-model");
assert_eq!(outcome.provider, "test-provider");
}
#[tokio::test]
async fn test_runner_auto_approves_edit_and_bash() {
let provider = ScriptedProvider::new(vec![
tool_turn("t1", "edit"),
tool_turn("t2", "bash"),
Turn::Text("done".to_string()),
]);
let mut agent = Agent::new(provider);
register_echo(&mut agent, "edit");
register_echo(&mut agent, "bash");
let policy = Policy::new(Tier::Auto, 15, None);
let outcome = run_query(resolved_stub(), policy, &mut agent, "prompt", None, None).await;
assert!(outcome.tool_calls.iter().any(|t| t.name == "edit" && t.ok));
assert!(outcome.tool_calls.iter().any(|t| t.name == "bash" && t.ok));
assert_eq!(outcome.stop_reason, StopReason::Done);
}
#[tokio::test]
async fn test_runner_denied_when_response_empty_and_tool_denied() {
let provider = ScriptedProvider::new(vec![tool_turn("t1", "edit"), Turn::Empty]);
let mut agent = Agent::new(provider);
register_echo(&mut agent, "edit");
let policy = Policy::new(Tier::Default, 15, None);
let outcome = run_query(resolved_stub(), policy, &mut agent, "prompt", None, None).await;
assert!(
outcome.tool_calls.iter().any(|t| t.name == "edit" && !t.ok),
"edit denied by the default tier"
);
assert_eq!(
outcome.stop_reason,
StopReason::Denied,
"denial + empty final turn ⇒ Denied"
);
}
#[tokio::test]
async fn test_runner_max_tool_calls_when_cap_exhausted() {
let provider = ScriptedProvider::new(vec![
tool_turn("t1", "ls"),
tool_turn("t2", "ls"),
tool_turn("t3", "ls"),
tool_turn("t4", "ls"),
]);
let mut agent = Agent::new(provider);
register_echo(&mut agent, "ls");
let policy = Policy::new(Tier::Auto, 2, None);
let outcome = run_query(resolved_stub(), policy, &mut agent, "prompt", None, None).await;
assert_eq!(outcome.stop_reason, StopReason::MaxToolCalls);
assert!(
outcome.error.is_none(),
"reaching the cap is a terminal state, not an error"
);
}
#[tokio::test]
async fn test_runner_provider_error_maps_to_error() {
let provider = ScriptedProvider::new(vec![Turn::Fail]);
let mut agent = Agent::new(provider);
let policy = Policy::new(Tier::Auto, 15, None);
let outcome = run_query(resolved_stub(), policy, &mut agent, "prompt", None, None).await;
assert_eq!(outcome.stop_reason, StopReason::Error);
assert!(outcome.error.is_some(), "error payload must be populated");
assert_eq!(outcome.response, None);
}
#[tokio::test]
async fn test_runner_error_priority_over_denied() {
let provider = ScriptedProvider::new(vec![tool_turn("t1", "edit"), Turn::Fail]);
let mut agent = Agent::new(provider);
register_echo(&mut agent, "edit");
let policy = Policy::new(Tier::Default, 15, None);
let outcome = run_query(resolved_stub(), policy, &mut agent, "prompt", None, None).await;
assert!(
outcome.tool_calls.iter().any(|t| t.name == "edit" && !t.ok),
"edit was denied before the provider failed"
);
assert_eq!(
outcome.stop_reason,
StopReason::Error,
"a run error dominates a tier denial (Error > Denied)"
);
}
#[tokio::test]
async fn test_runner_max_tool_calls_priority_over_denied() {
let provider = ScriptedProvider::new(vec![
tool_turn("t1", "edit"),
tool_turn("t2", "edit"),
tool_turn("t3", "edit"),
]);
let mut agent = Agent::new(provider);
register_echo(&mut agent, "edit");
let policy = Policy::new(Tier::Default, 2, None);
let outcome = run_query(resolved_stub(), policy, &mut agent, "prompt", None, None).await;
assert_eq!(
outcome.stop_reason,
StopReason::MaxToolCalls,
"the cap dominates the tier denials (MaxToolCalls > Denied)"
);
}
#[tokio::test]
async fn test_runner_full_auto_allows_repeated_identical_calls() {
let mut turns = identical_edit_turns(5);
turns.push(Turn::Text("done".to_string()));
let provider = ScriptedProvider::new(turns);
let mut agent = Agent::new(provider);
register_echo(&mut agent, "edit");
let policy = Policy::new(Tier::FullAuto, 50, None);
let outcome = run_query(resolved_stub(), policy, &mut agent, "prompt", None, None).await;
assert_eq!(
outcome.stop_reason,
StopReason::Done,
"--full-auto silences the repetitive guard ⇒ no abort"
);
assert_eq!(
outcome.tool_calls.len(),
5,
"all five identical calls executed under --full-auto"
);
assert!(outcome.tool_calls.iter().all(|t| t.ok));
}
#[tokio::test]
async fn test_runner_auto_aborts_on_repeated_identical_calls() {
let mut turns = identical_edit_turns(5);
turns.push(Turn::Text("unreached".to_string()));
let provider = ScriptedProvider::new(turns);
let mut agent = Agent::new(provider);
register_echo(&mut agent, "edit");
let policy = Policy::new(Tier::Auto, 15, None);
let outcome = run_query(resolved_stub(), policy, &mut agent, "prompt", None, None).await;
assert_eq!(
outcome.stop_reason,
StopReason::Error,
"the repetitive-guard abort collapses to Error under --auto"
);
}
#[tokio::test]
async fn test_runner_transcript_folds_tool_call_into_assistant_entry() {
let provider = ScriptedProvider::new(vec![
tool_turn("call-1", "ls"),
Turn::Text("here you go".to_string()),
]);
let mut agent = Agent::new(provider);
register_echo(&mut agent, "ls");
let policy = Policy::new(Tier::Auto, 15, None);
let outcome = run_query(resolved_stub(), policy, &mut agent, "prompt", None, None).await;
let user = outcome
.transcript
.iter()
.find(|e| e.role == "user")
.expect("a user entry");
assert_eq!(user.content, "prompt");
let with_tool = outcome
.transcript
.iter()
.find(|e| e.tool_calls.is_some())
.expect("an assistant entry carrying the tool call");
let calls = with_tool.tool_calls.as_ref().expect("tool_calls present");
assert_eq!(calls.len(), 1);
assert_eq!(calls[0].name, "ls");
assert!(calls[0].ok);
}
#[test]
fn test_resolve_run_timeout_applies_tier_default() {
assert_eq!(
resolve_run_timeout(
&Policy::new(Tier::Default, 15, None),
FULL_AUTO_TIMEOUT_SECS
),
None,
"the read-only default tier carries no wall-clock ceiling"
);
assert_eq!(
resolve_run_timeout(&Policy::new(Tier::Auto, 15, None), FULL_AUTO_TIMEOUT_SECS),
Some(Duration::from_secs(FULL_AUTO_TIMEOUT_SECS))
);
assert_eq!(
resolve_run_timeout(
&Policy::new(Tier::FullAuto, 50, None),
FULL_AUTO_TIMEOUT_SECS
),
Some(Duration::from_secs(FULL_AUTO_TIMEOUT_SECS))
);
assert_eq!(
resolve_run_timeout(
&Policy::new(Tier::Auto, 15, Some(5)),
FULL_AUTO_TIMEOUT_SECS
),
Some(Duration::from_secs(5))
);
assert_eq!(
resolve_run_timeout(
&Policy::new(Tier::Default, 15, Some(7)),
FULL_AUTO_TIMEOUT_SECS
),
Some(Duration::from_secs(7))
);
}
#[test]
fn test_resolve_run_timeout_respects_custom_effective_full_auto_default() {
let custom_default = 42u64;
assert_ne!(
custom_default, FULL_AUTO_TIMEOUT_SECS,
"fixture must differ from the module constant to prove it is not used"
);
assert_eq!(
resolve_run_timeout(&Policy::new(Tier::Auto, 15, None), custom_default),
Some(Duration::from_secs(custom_default)),
"a custom (smaller) effective default must apply, not the module constant"
);
assert_eq!(
resolve_run_timeout(&Policy::new(Tier::FullAuto, 50, None), custom_default),
Some(Duration::from_secs(custom_default))
);
assert_eq!(
resolve_run_timeout(&Policy::new(Tier::Default, 15, None), custom_default),
None
);
}
#[tokio::test]
async fn test_run_query_none_timeout_does_not_cancel() {
let provider = ScriptedProvider::new(vec![
tool_turn("t1", "ls"),
Turn::Text("finished".to_string()),
]);
let mut agent = Agent::new(provider);
register_echo(&mut agent, "ls");
let policy = Policy::new(Tier::Auto, 15, None);
let outcome = run_query(resolved_stub(), policy, &mut agent, "prompt", None, None).await;
assert_eq!(
outcome.stop_reason,
StopReason::Done,
"an unbounded run completes normally"
);
assert!(outcome.error.is_none(), "no timeout ⇒ no error payload");
assert_eq!(outcome.response.as_deref(), Some("finished"));
assert!(outcome.tool_calls.iter().any(|t| t.name == "ls" && t.ok));
}
#[cfg(windows)]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_run_query_timeout_kills_bash_and_reports_timeout_windows() {
run_query_timeout_kills_bash().await;
}
#[cfg(unix)]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_run_query_timeout_kills_bash_and_reports_timeout_unix() {
run_query_timeout_kills_bash().await;
}
#[cfg(any(windows, unix))]
async fn run_query_timeout_kills_bash() {
use crate::tools::bash::BashTool;
use crate::tools::proc_group::test_support::{
python_available, tree_kill_worker, CANCEL_FIRE_DELAY_MS, POST_KILL_WAIT_MS,
};
if !python_available() {
eprintln!("skipping: python interpreter not found — cannot spawn a real child");
return;
}
let dir = tempfile::tempdir().expect("tempdir");
let root = dir.path().canonicalize().expect("canonicalize");
std::fs::write(root.join("worker.py"), tree_kill_worker()).expect("write worker");
let provider = ScriptedProvider::new(vec![
Turn::Tool {
id: "b1".to_string(),
name: "bash".to_string(),
input: json!({ "command": "python worker.py" }),
},
Turn::Text("unreached".to_string()),
]);
let mut agent = Agent::new(provider);
agent.register_tool(Box::new(BashTool::new(root.clone()).expect("BashTool")));
let policy = Policy::new(Tier::Auto, 15, None);
let outcome = run_query(
resolved_stub(),
policy,
&mut agent,
"go",
Some(Duration::from_millis(CANCEL_FIRE_DELAY_MS)),
None,
)
.await;
assert_eq!(
outcome.stop_reason,
StopReason::Error,
"a wall-clock timeout maps to Error"
);
let err = outcome
.error
.expect("a timeout populates the error payload");
assert_eq!(
err.kind,
ErrorKind::Timeout,
"error.kind must be timeout (first-class)"
);
let done = root.join("done.marker");
tokio::time::sleep(Duration::from_millis(POST_KILL_WAIT_MS)).await;
assert!(
root.join("start.marker").exists(),
"the child really ran (START present)"
);
assert!(
!done.exists(),
"the bash subprocess tree must be dead — no DONE marker"
);
}
#[tokio::test]
async fn test_run_consult_direct_runs_three_perspectives_and_populates_consult() {
let outcome = run_consult(
resolved_stub(),
canned_magi(),
"should we migrate X to Y?",
None,
None,
)
.await;
assert_eq!(outcome.stop_reason, StopReason::Done);
assert!(outcome.error.is_none(), "a healthy consult has no error");
let consult = outcome.consult.expect("the MAGI object must be present");
assert_eq!(consult["degraded"], json!(false), "3 agents ⇒ not degraded");
assert!(
consult["report"].as_str().is_some_and(|s| !s.is_empty()),
"the consult object carries the MAGI report"
);
assert!(
outcome.response.as_deref().is_some_and(|s| !s.is_empty()),
"response reflects the direct report text"
);
assert!(
outcome.tool_calls.is_empty(),
"the direct path runs no agent tool-loop ⇒ no tool calls"
);
}
#[tokio::test]
async fn test_run_consult_over_cap_is_input_invalid_not_truncated() {
let big = "x".repeat(MAX_QUERY_LEN + 1);
let outcome = run_consult(resolved_stub(), canned_magi(), &big, None, None).await;
assert_eq!(outcome.stop_reason, StopReason::Error);
let err = outcome
.error
.expect("an over-cap prompt populates the error");
assert_eq!(
err.kind,
ErrorKind::InputInvalid,
"over-cap ⇒ input_invalid (→ exit 2)"
);
assert!(outcome.consult.is_none(), "rejected input ⇒ no MAGI object");
assert!(outcome.response.is_none(), "rejected, not truncated");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_run_consult_future_drop_aborts_spawned_analysis() {
let dropped = Arc::new(AtomicBool::new(false));
let magi = slow_droppy_magi(Duration::from_secs(3_600), dropped.clone());
let started = Instant::now();
{
let fut = run_consult(
resolved_stub(),
magi,
"should we migrate X to Y?",
None,
None,
);
let _ = tokio::time::timeout(Duration::from_millis(20), fut).await;
}
tokio::time::sleep(Duration::from_millis(100)).await;
let elapsed = started.elapsed();
assert!(
dropped.load(Ordering::SeqCst),
"dropping the run_consult future must abort the spawned MAGI \
analysis, not merely detach it"
);
assert!(
elapsed < Duration::from_secs(2),
"the abort must happen promptly, not after the analysis's 3600s \
delay (took {elapsed:?})"
);
}
#[tokio::test]
async fn test_query_forced_consult_auto_runs_exactly_once_and_populates() {
let provider = ScriptedProvider::new(vec![Turn::Text("here is my answer".to_string())]);
let mut agent = Agent::new(provider);
agent.register_tool(Box::new(ConsultTool::new(canned_magi(), true)));
let policy = Policy::new(Tier::Auto, 15, None);
let outcome = run_query(
forced_resolved(),
policy,
&mut agent,
"decide X vs Y",
None,
None,
)
.await;
let consult_calls = outcome
.tool_calls
.iter()
.filter(|t| t.name == "consult")
.count();
assert_eq!(consult_calls, 1, "forced consult runs exactly once");
assert!(
outcome
.tool_calls
.iter()
.any(|t| t.name == "consult" && t.ok),
"the forced consult succeeded under --auto"
);
assert!(
outcome.consult.is_some(),
"the forced consult result is captured into RunOutcome.consult"
);
}
#[tokio::test]
async fn test_query_forced_consult_default_denied_and_not_elevated() {
let provider = ScriptedProvider::new(vec![Turn::Text("read-only answer".to_string())]);
let mut agent = Agent::new(provider);
agent.register_tool(Box::new(ConsultTool::new(canned_magi(), true)));
let policy = Policy::new(Tier::Default, 15, None);
let outcome = run_query(
forced_resolved(),
policy,
&mut agent,
"decide X vs Y",
None,
None,
)
.await;
assert!(
outcome
.tool_calls
.iter()
.any(|t| t.name == "consult" && !t.ok),
"the forced consult is denied by the default tier (ok=false)"
);
assert!(
outcome.consult.is_none(),
"a denied consult leaves RunOutcome.consult null — not elevated"
);
}
#[tokio::test]
async fn test_query_forced_consult_does_not_double_fire() {
let provider = ScriptedProvider::new(vec![
Turn::Tool {
id: "c1".to_string(),
name: "consult".to_string(),
input: json!({ "query": "decide X vs Y" }),
},
Turn::Text("synthesized".to_string()),
]);
let mut agent = Agent::new(provider);
agent.register_tool(Box::new(ConsultTool::new(canned_magi(), true)));
let policy = Policy::new(Tier::Auto, 15, None);
let outcome = run_query(
forced_resolved(),
policy,
&mut agent,
"decide X vs Y",
None,
None,
)
.await;
let consult_calls = outcome
.tool_calls
.iter()
.filter(|t| t.name == "consult")
.count();
assert_eq!(
consult_calls, 1,
"the forced pass runs once; the model's own later request is blocked \
without a second observer-recorded call"
);
assert!(
outcome.consult.is_some(),
"the single forced consult populates RunOutcome.consult"
);
}
#[tokio::test]
async fn test_query_forced_consult_counts_against_max_tool_calls() {
let provider = ScriptedProvider::new(vec![tool_turn("t1", "edit")]);
let mut agent = Agent::new(provider);
agent.register_tool(Box::new(ConsultTool::new(canned_magi(), true)));
register_echo(&mut agent, "edit");
let policy = Policy::new(Tier::Auto, 1, None);
let outcome = run_query(
forced_resolved(),
policy,
&mut agent,
"decide X vs Y",
None,
None,
)
.await;
assert_eq!(
outcome.stop_reason,
StopReason::MaxToolCalls,
"the forced consult already spent the single available slot, so the \
model's next tool request trips the cap"
);
let consult_calls = outcome
.tool_calls
.iter()
.filter(|t| t.name == "consult")
.count();
assert_eq!(
consult_calls, 1,
"the forced consult itself did run — it occupied the slot"
);
assert!(
outcome.tool_calls.iter().all(|t| t.name != "edit"),
"the cap trips before the model's tool is ever authorized/executed"
);
}
#[tokio::test]
async fn test_forced_consult_does_not_seed_repetitive_guard_for_model_calls() {
let consult_call = || Turn::Tool {
id: "c".to_string(),
name: "consult".to_string(),
input: json!({ "query": "decide X vs Y" }),
};
let provider = ScriptedProvider::new(vec![
consult_call(),
consult_call(),
consult_call(),
consult_call(),
]);
let mut agent = Agent::new(provider);
agent.register_tool(Box::new(ConsultTool::new(canned_magi(), true)));
let policy = Policy::new(Tier::Auto, 4, None);
let outcome = run_query(
forced_resolved(),
policy,
&mut agent,
"decide X vs Y",
None,
None,
)
.await;
assert_eq!(
outcome.stop_reason,
StopReason::MaxToolCalls,
"the cap, not the repetitive guard, must be what stops the run — \
the forced consult must not have seeded the model's repeat \
tracking (got error: {:?})",
outcome.error,
);
}
struct ConsultReactingProvider;
#[async_trait]
impl Provider for ConsultReactingProvider {
async fn stream_messages(
&self,
messages: &[Message],
_tools: &[Box<dyn Tool>],
_system: Option<&str>,
) -> Result<BoxStream<'static, Result<ResponseChunk>>> {
let saw_consult = messages.iter().any(|m| {
m.content.iter().any(|c| {
matches!(c, Content::ToolResult { content, .. } if content.contains("degraded"))
})
});
let text = if saw_consult {
"reacted-to-consult"
} else {
"no-consult-seen"
};
Ok(Box::pin(stream::iter(vec![
Ok(ResponseChunk::TextDelta(text.to_string())),
Ok(ResponseChunk::MessageDone(Message::assistant(text))),
])))
}
}
#[tokio::test]
async fn test_query_forced_consult_result_reaches_the_models_next_turn() {
let mut agent = Agent::new(Arc::new(ConsultReactingProvider));
agent.register_tool(Box::new(ConsultTool::new(canned_magi(), true)));
let policy = Policy::new(Tier::Auto, 15, None);
let outcome = run_query(
forced_resolved(),
policy,
&mut agent,
"decide X vs Y",
None,
None,
)
.await;
assert_eq!(
outcome.response.as_deref(),
Some("reacted-to-consult"),
"the model's first provider turn must already see the forced \
consult's ToolResult content"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_forced_consult_shares_single_wall_clock_budget() {
let provider = ScriptedProvider::new(vec![Turn::Text("fast loop answer".to_string())]);
let mut agent = Agent::new(provider);
let magi = slow_magi(Duration::from_secs(30));
agent.register_tool(Box::new(ConsultTool::new(magi, true)));
let started = Instant::now();
let policy = Policy::new(Tier::Auto, 15, None);
let outcome = run_query(
forced_resolved(),
policy,
&mut agent,
"decide X vs Y",
Some(Duration::from_millis(300)),
None,
)
.await;
let elapsed = started.elapsed();
assert_eq!(
outcome.stop_reason,
StopReason::Error,
"loop + forced consult over the shared budget ⇒ Error"
);
assert_eq!(
outcome
.error
.expect("a wall-clock timeout populates the error payload")
.kind,
ErrorKind::Timeout,
"the combined wall-clock overrun is a first-class Timeout"
);
assert!(
outcome.consult.is_none(),
"a timed-out forced consult yields no MAGI object"
);
assert!(
elapsed < Duration::from_secs(5),
"the forced consult must be bounded by the deadline, not run to completion"
);
}
}