use std::{
collections::BTreeMap,
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
};
use async_trait::async_trait;
use serde_json::{Value, json};
use tokio::{
sync::Mutex as TokioMutex,
time::{Duration, sleep},
};
use crate::{
BuiltinProvider, ContentBlock, Role,
provider::{ContentBlockDelta, ContentBlockStart, ProviderEvent},
runtime::{RunOptions, Runtime, RuntimeError, RuntimeHookEvent},
tool::{
ParallelToolContext, ToolContext, ToolDefinition, ToolDurability, ToolExecutionCategory,
ToolExecutor, ToolOutput, ToolResult, ToolResultContent, ToolSideEffectLevel, ToolSpec,
},
};
use super::support::{ScriptedProvider, StaticTool, StreamScript, model_info, ok_stream};
struct StructuredDetailsTool;
#[async_trait]
impl ToolDefinition for StructuredDetailsTool {
fn descriptor(&self) -> ToolSpec {
ToolSpec::builder("structured_details_tool")
.description("test tool: returns structured content plus opaque details")
.input_schema(json!({ "type": "object", "properties": {} }))
.side_effect_level(ToolSideEffectLevel::None)
.durability(ToolDurability::ReplaySafe)
.build()
}
}
#[async_trait]
impl ToolExecutor for StructuredDetailsTool {
async fn execute_mut_output(
&self,
_ctx: ToolContext<'_>,
_input: Value,
) -> Result<ToolOutput, String> {
Ok(
ToolOutput::structured(json!({ "answer": 42 }))
.with_details(json!({ "secret": "shh" })),
)
}
}
struct DetailsToolA;
#[async_trait]
impl ToolDefinition for DetailsToolA {
fn descriptor(&self) -> ToolSpec {
ToolSpec::builder("details_tool_a")
.description("test tool: returns details keyed to call A")
.input_schema(json!({ "type": "object", "properties": {} }))
.side_effect_level(ToolSideEffectLevel::None)
.durability(ToolDurability::ReplaySafe)
.execution_category(ToolExecutionCategory::ReadOnlyParallel)
.build()
}
}
#[async_trait]
impl ToolExecutor for DetailsToolA {
async fn execute_output(
&self,
_ctx: ParallelToolContext,
_input: Value,
) -> Result<ToolOutput, String> {
Ok(ToolOutput::text("a-result").with_details(json!({ "who": "a" })))
}
}
struct DetailsToolB;
#[async_trait]
impl ToolDefinition for DetailsToolB {
fn descriptor(&self) -> ToolSpec {
ToolSpec::builder("details_tool_b")
.description("test tool: returns details keyed to call B")
.input_schema(json!({ "type": "object", "properties": {} }))
.side_effect_level(ToolSideEffectLevel::None)
.durability(ToolDurability::ReplaySafe)
.execution_category(ToolExecutionCategory::ReadOnlyParallel)
.build()
}
}
#[async_trait]
impl ToolExecutor for DetailsToolB {
async fn execute_output(
&self,
_ctx: ParallelToolContext,
_input: Value,
) -> Result<ToolOutput, String> {
Ok(ToolOutput::text("b-result").with_details(json!({ "who": "b" })))
}
}
struct TerminatingTool;
#[async_trait]
impl ToolDefinition for TerminatingTool {
fn descriptor(&self) -> ToolSpec {
ToolSpec::builder("terminating_tool")
.description("test tool: ends the run via ToolOutput::terminate")
.input_schema(json!({ "type": "object", "properties": {} }))
.side_effect_level(ToolSideEffectLevel::None)
.durability(ToolDurability::ReplaySafe)
.build()
}
}
#[async_trait]
impl ToolExecutor for TerminatingTool {
async fn execute_mut_output(
&self,
_ctx: ToolContext<'_>,
_input: Value,
) -> Result<ToolOutput, String> {
Ok(ToolOutput::text("final answer").terminating())
}
}
struct FailingOutputTool;
#[async_trait]
impl ToolDefinition for FailingOutputTool {
fn descriptor(&self) -> ToolSpec {
ToolSpec::builder("failing_output_tool")
.description("test tool: fails via the new structured surface")
.input_schema(json!({ "type": "object", "properties": {} }))
.side_effect_level(ToolSideEffectLevel::None)
.durability(ToolDurability::ReplaySafe)
.build()
}
}
#[async_trait]
impl ToolExecutor for FailingOutputTool {
async fn execute_mut_output(
&self,
_ctx: ToolContext<'_>,
_input: Value,
) -> Result<ToolOutput, String> {
Err("boom".to_string())
}
}
struct TerminalParallelProbe {
name: &'static str,
log: Arc<TokioMutex<Vec<String>>>,
}
#[async_trait]
impl ToolDefinition for TerminalParallelProbe {
fn descriptor(&self) -> ToolSpec {
ToolSpec::builder(self.name)
.description("test tool: declares ReadOnlyParallel but is terminal")
.input_schema(json!({ "type": "object", "properties": {} }))
.side_effect_level(ToolSideEffectLevel::None)
.durability(ToolDurability::ReplaySafe)
.execution_category(ToolExecutionCategory::ReadOnlyParallel)
.terminal()
.build()
}
}
#[async_trait]
impl ToolExecutor for TerminalParallelProbe {
async fn execute(&self, _ctx: ParallelToolContext, _input: Value) -> ToolResult {
self.log.lock().await.push(format!("{}:start", self.name));
sleep(Duration::from_millis(15)).await;
self.log.lock().await.push(format!("{}:end", self.name));
Ok(format!("{} complete", self.name))
}
}
struct MisbehavingParallelTerminateTool;
#[async_trait]
impl ToolDefinition for MisbehavingParallelTerminateTool {
fn descriptor(&self) -> ToolSpec {
ToolSpec::builder("misbehaving_parallel_terminate")
.description("test tool: wrongly requests termination from a parallel execution")
.input_schema(json!({ "type": "object", "properties": {} }))
.side_effect_level(ToolSideEffectLevel::None)
.durability(ToolDurability::ReplaySafe)
.execution_category(ToolExecutionCategory::ReadOnlyParallel)
.build()
}
}
#[async_trait]
impl ToolExecutor for MisbehavingParallelTerminateTool {
async fn execute_output(
&self,
_ctx: ParallelToolContext,
_input: Value,
) -> Result<ToolOutput, String> {
Ok(ToolOutput::text("i should not be able to stop the run").terminating())
}
}
fn tool_use_stream(model: &str, id: &str, name: &str, input_json: &str) -> StreamScript {
ok_stream(vec![
ProviderEvent::MessageStarted {
id: format!("msg-{id}"),
model: model.to_string(),
role: Role::Assistant,
},
ProviderEvent::ContentBlockStarted {
index: 0,
kind: ContentBlockStart::ToolUse {
id: id.to_string(),
name: name.to_string(),
},
},
ProviderEvent::ContentBlockDelta {
index: 0,
delta: ContentBlockDelta::ToolUseInputJson(input_json.to_string()),
},
ProviderEvent::ContentBlockStopped { index: 0 },
ProviderEvent::MessageStopped,
])
}
fn multi_tool_use_stream(model: &str, calls: &[(&str, &str, &str)]) -> StreamScript {
let mut events = vec![ProviderEvent::MessageStarted {
id: "msg-multi-tool".to_string(),
model: model.to_string(),
role: Role::Assistant,
}];
for (index, (id, name, input_json)) in calls.iter().enumerate() {
events.push(ProviderEvent::ContentBlockStarted {
index,
kind: ContentBlockStart::ToolUse {
id: (*id).to_string(),
name: (*name).to_string(),
},
});
events.push(ProviderEvent::ContentBlockDelta {
index,
delta: ContentBlockDelta::ToolUseInputJson((*input_json).to_string()),
});
events.push(ProviderEvent::ContentBlockStopped { index });
}
events.push(ProviderEvent::MessageStopped);
ok_stream(events)
}
fn text_stream(model: &str, text: &str) -> StreamScript {
ok_stream(vec![
ProviderEvent::MessageStarted {
id: format!("msg-{text}"),
model: model.to_string(),
role: Role::Assistant,
},
ProviderEvent::ContentBlockStarted {
index: 0,
kind: ContentBlockStart::Text,
},
ProviderEvent::ContentBlockDelta {
index: 0,
delta: ContentBlockDelta::Text(text.to_string()),
},
ProviderEvent::ContentBlockStopped { index: 0 },
ProviderEvent::MessageStopped,
])
}
fn tool_result_blocks(messages: &[crate::Message]) -> Vec<ContentBlock> {
messages
.iter()
.filter(|message| message.role == Role::User)
.flat_map(|message| message.content.iter().cloned())
.filter(|block| matches!(block, ContentBlock::ToolResult { .. }))
.collect()
}
#[derive(Clone, Default)]
struct RecordingHook {
events: Arc<std::sync::Mutex<Vec<RuntimeHookEvent>>>,
}
impl crate::runtime::control::RuntimeHook for RecordingHook {
fn on_event(
&self,
_store: &dyn crate::runtime::AuditStore,
event: &RuntimeHookEvent,
) -> Result<(), RuntimeError> {
self.events
.lock()
.expect("hook events poisoned")
.push(event.clone());
Ok(())
}
}
#[tokio::test]
async fn string_tool_bridges_to_text_output_unchanged() {
let model = model_info("model", BuiltinProvider::Anthropic);
let provider = ScriptedProvider::new(
BuiltinProvider::Anthropic,
vec![model.clone()],
vec![
tool_use_stream(&model.id, "call-1", "echo_tool", r#"{}"#),
text_stream(&model.id, "done"),
],
);
let runtime = Runtime::empty_builder()
.with_provider_instance(provider)
.with_tool(StaticTool::success("echo_tool", "echoed"))
.build()
.expect("build runtime");
let mut agent = runtime.spawn("agent", model).expect("spawn agent");
agent
.send(vec![ContentBlock::text("run the echo tool")])
.await
.expect("send");
let blocks = tool_result_blocks(agent.history());
assert_eq!(blocks.len(), 1);
assert_eq!(
blocks[0],
ContentBlock::ToolResult {
tool_use_id: "call-1".to_string(),
content: ToolResultContent::Text("echoed".to_string()),
is_error: false,
}
);
}
#[tokio::test]
async fn err_string_behaves_identically_through_bridge_and_new_surface() {
let model = model_info("model", BuiltinProvider::Anthropic);
let provider = ScriptedProvider::new(
BuiltinProvider::Anthropic,
vec![model.clone()],
vec![
multi_tool_use_stream(
&model.id,
&[
("call-1", "bridged_failure", r#"{}"#),
("call-2", "failing_output_tool", r#"{}"#),
],
),
text_stream(&model.id, "done"),
],
);
let runtime = Runtime::empty_builder()
.with_provider_instance(provider)
.with_tool(StaticTool::failure("bridged_failure", "old bridge error"))
.with_tool(FailingOutputTool)
.build()
.expect("build runtime");
let mut agent = runtime.spawn("agent", model).expect("spawn agent");
let message = agent
.send(vec![ContentBlock::text("run the failing tools")])
.await
.expect("send should still succeed: tool errors don't fail the run");
assert_eq!(message.text(), "done");
let blocks = tool_result_blocks(agent.history());
assert_eq!(
blocks[0],
ContentBlock::ToolResult {
tool_use_id: "call-1".to_string(),
content: ToolResultContent::Text("old bridge error".to_string()),
is_error: true,
}
);
assert_eq!(
blocks[1],
ContentBlock::ToolResult {
tool_use_id: "call-2".to_string(),
content: ToolResultContent::Text("boom".to_string()),
is_error: true,
}
);
}
#[tokio::test]
async fn structured_tool_projects_content_and_hides_details_from_provider() {
let model = model_info("model", BuiltinProvider::Anthropic);
let provider = ScriptedProvider::new(
BuiltinProvider::Anthropic,
vec![model.clone()],
vec![
tool_use_stream(&model.id, "call-1", "structured_details_tool", r#"{}"#),
text_stream(&model.id, "done"),
],
);
let hook = RecordingHook::default();
let runtime = Runtime::empty_builder()
.with_provider_instance(provider.clone())
.with_tool(StructuredDetailsTool)
.with_hook(hook.clone())
.build()
.expect("build runtime");
let mut agent = runtime.spawn("agent", model).expect("spawn agent");
agent
.send(vec![ContentBlock::text("run the structured tool")])
.await
.expect("send");
let blocks = tool_result_blocks(agent.history());
assert_eq!(
blocks[0],
ContentBlock::ToolResult {
tool_use_id: "call-1".to_string(),
content: ToolResultContent::Structured(json!({ "answer": 42 })),
is_error: false,
}
);
let requests = provider.recorded_requests().await;
assert_eq!(requests.len(), 2, "tool round, then the follow-up round");
let follow_up = requests[1]
.messages
.iter()
.map(|message| format!("{message:?}"))
.collect::<Vec<_>>()
.join("\n");
assert!(follow_up.contains("answer"));
assert!(!follow_up.contains("secret"));
assert!(!follow_up.contains("shh"));
let events = hook.events.lock().expect("hook events poisoned").clone();
let details = events.iter().find_map(|event| match event {
RuntimeHookEvent::ToolExecutionFinished {
tool_name, details, ..
} if tool_name == "structured_details_tool" => Some(details.clone()),
_ => None,
});
assert_eq!(details, Some(Some(json!({ "secret": "shh" }))));
}
#[tokio::test]
async fn parallel_round_maps_each_results_details_to_its_own_tool_use_id() {
let model = model_info("model", BuiltinProvider::Anthropic);
let provider = ScriptedProvider::new(
BuiltinProvider::Anthropic,
vec![model.clone()],
vec![
multi_tool_use_stream(
&model.id,
&[
("call-a", "details_tool_a", r#"{}"#),
("call-b", "details_tool_b", r#"{}"#),
],
),
text_stream(&model.id, "done"),
],
);
let runtime = Runtime::empty_builder()
.with_provider_instance(provider)
.with_tool(DetailsToolA)
.with_tool(DetailsToolB)
.build()
.expect("build runtime");
let mut agent = runtime.spawn("agent", model).expect("spawn agent");
agent
.send(vec![ContentBlock::text("run both details tools")])
.await
.expect("send");
let item = agent
.transcript()
.items()
.iter()
.rev()
.find(|item| matches!(item.kind, crate::TranscriptKind::ToolExchange { .. }))
.expect("tool exchange item committed");
let expected: BTreeMap<String, Value> = BTreeMap::from([
("call-a".to_string(), json!({ "who": "a" })),
("call-b".to_string(), json!({ "who": "b" })),
]);
assert_eq!(item.details(), Some(&expected));
assert_eq!(item.detail("call-a"), Some(&json!({ "who": "a" })));
assert_eq!(item.detail("call-b"), Some(&json!({ "who": "b" })));
}
#[tokio::test]
async fn terminating_tool_commits_transcript_without_rollback() {
let model = model_info("model", BuiltinProvider::Anthropic);
let provider = ScriptedProvider::new(
BuiltinProvider::Anthropic,
vec![model.clone()],
vec![
tool_use_stream(&model.id, "call-1", "terminating_tool", r#"{}"#),
text_stream(&model.id, "must not run"),
],
);
let runtime = Runtime::empty_builder()
.with_provider_instance(provider)
.with_tool(TerminatingTool)
.build()
.expect("build runtime");
let mut agent = runtime.spawn("agent", model).expect("spawn agent");
let result = agent
.run(
vec![ContentBlock::text("go")],
RunOptions {
..Default::default()
},
)
.await;
assert!(matches!(result, Err(RuntimeError::EmptyAssistantResponse)));
assert_eq!(
agent.history().len(),
3,
"user, the assistant tool call, and the committed tool result"
);
let blocks = tool_result_blocks(agent.history());
assert_eq!(
blocks[0],
ContentBlock::ToolResult {
tool_use_id: "call-1".to_string(),
content: ToolResultContent::Text("final answer".to_string()),
is_error: false,
}
);
}
#[tokio::test]
async fn terminating_call_creates_a_barrier_and_skips_the_scheduled_parallel_batch() {
let model = model_info("model", BuiltinProvider::Anthropic);
let provider = ScriptedProvider::new(
BuiltinProvider::Anthropic,
vec![model.clone()],
vec![multi_tool_use_stream(
&model.id,
&[
("call-1", "probe_one", r#"{}"#),
("call-2", "probe_two", r#"{}"#),
("call-3", "terminating_tool", r#"{}"#),
("call-4", "probe_three", r#"{}"#),
],
)],
);
let log = Arc::new(TokioMutex::new(Vec::new()));
let active = Arc::new(AtomicUsize::new(0));
let max_active = Arc::new(AtomicUsize::new(0));
let runtime = Runtime::empty_builder()
.with_provider_instance(provider)
.with_tool(super::support::ProbeTool::new(
"probe_one",
true,
Duration::from_millis(30),
Arc::clone(&log),
Arc::clone(&active),
Arc::clone(&max_active),
))
.with_tool(super::support::ProbeTool::new(
"probe_two",
true,
Duration::from_millis(30),
Arc::clone(&log),
Arc::clone(&active),
Arc::clone(&max_active),
))
.with_tool(TerminatingTool)
.with_tool(super::support::ProbeTool::new(
"probe_three",
true,
Duration::from_millis(30),
Arc::clone(&log),
Arc::clone(&active),
Arc::clone(&max_active),
))
.build()
.expect("build runtime");
let mut agent = runtime.spawn("agent", model).expect("spawn agent");
let result = agent
.run(vec![ContentBlock::text("go")], RunOptions::default())
.await;
assert!(matches!(result, Err(RuntimeError::EmptyAssistantResponse)));
let log = log.lock().await.clone();
assert!(!log.contains(&"probe_three:start".to_string()));
assert!(!log.contains(&"probe_three:end".to_string()));
assert!(
log.contains(&"probe_one:start".to_string())
&& log.contains(&"probe_two:start".to_string()),
"the earlier parallel batch still ran before the barrier"
);
assert!(max_active.load(Ordering::SeqCst) >= 2);
let blocks = tool_result_blocks(agent.history());
assert_eq!(blocks.len(), 4);
assert_eq!(
blocks[2],
ContentBlock::ToolResult {
tool_use_id: "call-3".to_string(),
content: ToolResultContent::Text("final answer".to_string()),
is_error: false,
}
);
let ContentBlock::ToolResult {
tool_use_id,
content,
is_error,
} = &blocks[3]
else {
panic!("expected a tool result block");
};
assert_eq!(tool_use_id, "call-4");
assert!(*is_error);
let text = content.to_display_string();
assert!(text.contains("not executed"));
assert!(text.contains("terminating_tool"));
}
#[tokio::test]
async fn terminal_marked_tool_declared_parallel_is_coerced_to_exclusive() {
let model = model_info("model", BuiltinProvider::Anthropic);
let provider = ScriptedProvider::new(
BuiltinProvider::Anthropic,
vec![model.clone()],
vec![
multi_tool_use_stream(
&model.id,
&[
("call-1", "probe_one", r#"{}"#),
("call-2", "probe_two", r#"{}"#),
("call-3", "terminal_parallel_probe", r#"{}"#),
("call-4", "probe_three", r#"{}"#),
],
),
text_stream(&model.id, "done"),
],
);
let log = Arc::new(TokioMutex::new(Vec::new()));
let active = Arc::new(AtomicUsize::new(0));
let max_active = Arc::new(AtomicUsize::new(0));
let runtime = Runtime::empty_builder()
.with_provider_instance(provider)
.with_tool(super::support::ProbeTool::new(
"probe_one",
true,
Duration::from_millis(30),
Arc::clone(&log),
Arc::clone(&active),
Arc::clone(&max_active),
))
.with_tool(super::support::ProbeTool::new(
"probe_two",
true,
Duration::from_millis(30),
Arc::clone(&log),
Arc::clone(&active),
Arc::clone(&max_active),
))
.with_tool(TerminalParallelProbe {
name: "terminal_parallel_probe",
log: Arc::clone(&log),
})
.with_tool(super::support::ProbeTool::new(
"probe_three",
true,
Duration::from_millis(30),
Arc::clone(&log),
Arc::clone(&active),
Arc::clone(&max_active),
))
.build()
.expect("build runtime");
let mut agent = runtime.spawn("agent", model).expect("spawn agent");
agent
.send(vec![ContentBlock::text(
"run probes with a terminal barrier",
)])
.await
.expect("send");
let log = log.lock().await.clone();
let position = |entry: &str| {
log.iter()
.position(|logged| logged == entry)
.unwrap_or_else(|| panic!("missing log entry: {entry}"))
};
let terminal_start = position("terminal_parallel_probe:start");
let terminal_end = position("terminal_parallel_probe:end");
let probe_one_end = position("probe_one:end");
let probe_two_end = position("probe_two:end");
let probe_three_start = position("probe_three:start");
assert!(
probe_one_end < terminal_start,
"terminal probe must not start until probe_one has fully finished"
);
assert!(
probe_two_end < terminal_start,
"terminal probe must not start until probe_two has fully finished"
);
assert!(
terminal_end < probe_three_start,
"probe_three must not start until the terminal probe has fully finished"
);
assert!(
max_active.load(Ordering::SeqCst) >= 2,
"the surrounding probes still ran in parallel with each other"
);
}
#[tokio::test]
async fn parallel_lane_terminate_is_rejected_as_misuse_and_run_continues() {
let model = model_info("model", BuiltinProvider::Anthropic);
let provider = ScriptedProvider::new(
BuiltinProvider::Anthropic,
vec![model.clone()],
vec![
multi_tool_use_stream(
&model.id,
&[
("call-1", "misbehaving_parallel_terminate", r#"{}"#),
("call-2", "probe_one", r#"{}"#),
],
),
text_stream(&model.id, "done"),
],
);
let log = Arc::new(TokioMutex::new(Vec::new()));
let active = Arc::new(AtomicUsize::new(0));
let max_active = Arc::new(AtomicUsize::new(0));
let runtime = Runtime::empty_builder()
.with_provider_instance(provider)
.with_tool(MisbehavingParallelTerminateTool)
.with_tool(super::support::ProbeTool::new(
"probe_one",
true,
Duration::from_millis(10),
Arc::clone(&log),
Arc::clone(&active),
Arc::clone(&max_active),
))
.build()
.expect("build runtime");
let mut agent = runtime.spawn("agent", model).expect("spawn agent");
let message = agent
.send(vec![ContentBlock::text("run the misbehaving tool")])
.await
.expect("send should succeed: the misuse is a tool error, not a run failure");
assert_eq!(message.text(), "done");
let blocks = tool_result_blocks(agent.history());
let ContentBlock::ToolResult {
tool_use_id,
content,
is_error,
} = &blocks[0]
else {
panic!("expected a tool result block");
};
assert_eq!(tool_use_id, "call-1");
assert!(*is_error);
let text = content.to_display_string();
assert!(text.contains("not honored"));
assert!(text.contains("parallel"));
}