use std::collections::HashMap;
use crate::agent::{
CompletionCallAction, CompletionCallEvent, DispatchAction, DispatchEvent, HookStack,
InvalidToolCallAction, InvalidToolCallContext, ModelTurnAction, ModelTurnFinished,
ObservationAction, OutcomeAction, OutcomeEvent, ReasoningDelta, TextDelta, ToolCallDelta,
};
use std::sync::{
Arc, Mutex,
atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering::SeqCst},
};
use futures::StreamExt;
use serde::Deserialize;
use serde_json::json;
use tokio::sync::{Barrier, Notify};
use crate::agent::AgentBuilder;
use crate::agent::hook::{AgentHook, HookContext, RequestPatch, StepEventKind};
use crate::agent::run::OutputMode;
use crate::agent::streaming::MultiTurnStreamItem;
use crate::completion::{Message, PromptError, Usage};
use crate::streaming::{Item, StreamEvent};
use crate::test_utils::{
MockAddTool, MockCompletionModel, MockOperationArgs, MockScript, MockStreamEvent,
MockSubtractTool, MockToolError, MockTurn,
};
use crate::tool::{
Tool, ToolContext, ToolExecutionError,
server::{ToolServer, ToolServerHandle},
};
use rig_core::driver::{Exchange, Opening};
use rig_core::message::{
AssistantContent, ToolCall as MessageToolCall, ToolChoice, ToolFunction, UserContent,
};
use rig_core::vector_store::{
VectorSearchIdResult, VectorSearchRequest, VectorSearchResult, VectorStoreError,
VectorStoreIndex, request::Filter,
};
use rig_core::wasm_compat::WasmCompatSend;
#[derive(Clone, Default)]
struct RecordingHook {
events: Arc<Mutex<Vec<StepEventKind>>>,
tool_results: Arc<Mutex<Vec<String>>>,
}
impl RecordingHook {
fn shared_events(&self) -> Vec<StepEventKind> {
self.events
.lock()
.expect("events lock")
.iter()
.copied()
.filter(|kind| {
matches!(
kind,
StepEventKind::CompletionCall
| StepEventKind::CompletionDispatch
| StepEventKind::ToolDispatch
| StepEventKind::InvalidToolCall
)
})
.collect()
}
fn tool_results(&self) -> Vec<String> {
self.tool_results.lock().expect("results lock").clone()
}
fn count(&self, kind: StepEventKind) -> usize {
self.events
.lock()
.expect("events lock")
.iter()
.filter(|recorded| **recorded == kind)
.count()
}
}
impl RecordingHook {
fn record(&self, kind: StepEventKind) {
self.events.lock().expect("events lock").push(kind);
}
}
impl AgentHook for RecordingHook {
async fn on_completion_call(
&self,
_: &HookContext,
_: CompletionCallEvent<'_>,
) -> CompletionCallAction {
self.record(StepEventKind::CompletionCall);
CompletionCallAction::continue_run()
}
async fn on_model_turn_finished(
&self,
_: &HookContext,
_: ModelTurnFinished<'_>,
) -> ModelTurnAction {
self.record(StepEventKind::ModelTurnFinished);
ModelTurnAction::continue_run()
}
async fn on_invalid_tool_call(
&self,
_: &HookContext,
_: &InvalidToolCallContext,
) -> Option<InvalidToolCallAction> {
self.record(StepEventKind::InvalidToolCall);
None
}
async fn on_dispatch(&self, _: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
if event.tool_name().is_some() {
self.record(StepEventKind::ToolDispatch);
}
DispatchAction::proceed()
}
async fn on_outcome(&self, _: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
if event.completion().is_some() {
self.record(StepEventKind::CompletionDispatch);
} else if let Some(result) = event.tool_result() {
self.record(StepEventKind::ToolDispatch);
self.tool_results
.lock()
.expect("results lock")
.push(result.output().render());
}
OutcomeAction::proceed()
}
async fn on_text_delta(&self, _: &HookContext, _: TextDelta<'_>) -> ObservationAction {
self.record(StepEventKind::TextDelta);
ObservationAction::continue_run()
}
async fn on_reasoning_delta(
&self,
_: &HookContext,
_: ReasoningDelta<'_>,
) -> ObservationAction {
self.record(StepEventKind::ReasoningDelta);
ObservationAction::continue_run()
}
async fn on_tool_call_delta(&self, _: &HookContext, _: ToolCallDelta<'_>) -> ObservationAction {
self.record(StepEventKind::ToolCallDelta);
ObservationAction::continue_run()
}
}
#[derive(Clone, Debug, PartialEq)]
struct CanonicalResponseSnapshot {
prompt: Message,
content: Vec<AssistantContent>,
usage: Usage,
}
#[derive(Clone, Default)]
struct FinishLifecycleHook {
snapshots: Arc<Mutex<Vec<CanonicalResponseSnapshot>>>,
model_turns: Arc<AtomicU32>,
stop: Arc<AtomicBool>,
pending_prompt: Arc<Mutex<Option<Message>>>,
}
impl AgentHook for FinishLifecycleHook {
async fn on_completion_call(
&self,
_ctx: &HookContext,
event: CompletionCallEvent<'_>,
) -> CompletionCallAction {
*self.pending_prompt.lock().expect("pending prompt") = Some(event.prompt.clone());
CompletionCallAction::continue_run()
}
async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
let Some(response) = event.completion() else {
return OutcomeAction::proceed();
};
let prompt = self
.pending_prompt
.lock()
.expect("pending prompt")
.clone()
.expect("a completion outcome follows its completion call");
self.snapshots
.lock()
.expect("finish snapshots")
.push(CanonicalResponseSnapshot {
prompt,
content: response.choice.clone(),
usage: response.usage,
});
if self.stop.load(SeqCst) {
OutcomeAction::stop("stop at stream EOF")
} else {
OutcomeAction::proceed()
}
}
async fn on_model_turn_finished(
&self,
_ctx: &HookContext,
_event: ModelTurnFinished<'_>,
) -> ModelTurnAction {
self.model_turns.fetch_add(1, SeqCst);
ModelTurnAction::continue_run()
}
}
fn canonical_usage() -> Usage {
Usage::new()
.input_tokens(11)
.output_tokens(7)
.total_tokens(18)
}
#[tokio::test]
async fn completion_response_hook_and_calls_carry_identity_metadata() {
type IdentityTriple = (Option<String>, Option<String>);
#[derive(Clone, Default)]
struct IdentityHook {
seen: Arc<Mutex<Vec<IdentityTriple>>>,
}
impl AgentHook for IdentityHook {
async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
if let Some(response) = event.completion() {
self.seen.lock().expect("identity snapshots").push((
response.response_id().map(str::to_owned),
response.provider_request_id.clone(),
));
}
OutcomeAction::proceed()
}
}
let hook = IdentityHook::default();
let response = AgentBuilder::new(MockCompletionModel::from_turns([MockTurn::text("reply")
.with_response_id("resp_1")
.with_provider_request_id("req_1")]))
.add_hook(hook.clone())
.build()
.prompt(Message::user("prompt"))
.run()
.await
.expect("blocking response");
assert_eq!(
*hook.seen.lock().expect("identity snapshots"),
[(Some("resp_1".to_string()), Some("req_1".to_string()),)]
);
let call = &response.completion_calls[0];
assert_eq!(call.response_id.as_deref(), Some("resp_1"));
assert_eq!(call.provider_request_id.as_deref(), Some("req_1"));
}
#[tokio::test]
async fn failed_attempt_error_carries_its_own_request_id() {
let hook = TurnIdentityHook::default();
let error = AgentBuilder::new(MockCompletionModel::from_turns([
MockTurn::tool_call("tc1", "add", serde_json::json!({"x": 2, "y": 3}))
.with_provider_request_id("req-success-1"),
MockTurn::provider_response_error(
http::StatusCode::TOO_MANY_REQUESTS,
r#"{"error":"rate limited"}"#,
"req-failed-2",
),
]))
.tool(crate::test_utils::MockAddTool)
.add_hook(hook.clone())
.build()
.prompt(Message::user("add 2 and 3"))
.max_turns(4)
.run()
.await
.expect_err("the second attempt fails");
let PromptError::Report(ref report) = error else {
panic!("expected the failing attempt's report, got {error:?}");
};
assert_eq!(
report.request_id.as_deref(),
Some("req-failed-2"),
"the surfaced error reports the failing attempt's id: {error:?}"
);
assert_eq!(
error.provider_request_id(),
Some("req-failed-2"),
"the run-surface accessor reads the wire report too"
);
assert_eq!(
error
.provider_response_status()
.map(|status| status.as_u16()),
report.http_status,
"and so does the status accessor"
);
let turns = hook.turns.lock().expect("turn identities").clone();
assert_eq!(turns.len(), 1, "only the successful call fired the event");
assert_eq!(
turns[0].provider_request_id.as_deref(),
Some("req-success-1"),
"the success keeps its own id — no cross-attribution"
);
}
#[derive(Clone, Default)]
struct TurnIdentityHook {
turns: Arc<Mutex<Vec<rig_core::completion::ResponseIdentity>>>,
responses: Arc<Mutex<Vec<rig_core::completion::ResponseIdentity>>>,
}
impl AgentHook for TurnIdentityHook {
async fn on_model_turn_finished(
&self,
_ctx: &HookContext,
event: ModelTurnFinished<'_>,
) -> ModelTurnAction {
self.turns
.lock()
.expect("turn identities")
.push(event.identity.clone());
ModelTurnAction::continue_run()
}
async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
if let Some(response) = event.completion() {
self.responses
.lock()
.expect("response identities")
.push(response.identity());
}
OutcomeAction::proceed()
}
}
fn stream_final_with_id(response_id: &str) -> MockStreamEvent {
MockStreamEvent::FinalResponse(rig_core::operation::Finish {
usage: Usage::default(),
response_id: Some(response_id.into()),
..rig_core::operation::Finish::default()
})
}
#[tokio::test]
async fn model_turn_finished_identity_blocking_tool_only_and_text() {
let hook = TurnIdentityHook::default();
let response = AgentBuilder::new(MockCompletionModel::from_turns([
MockTurn::tool_call("tc1", "add", json!({"x": 2, "y": 3}))
.with_provider_request_id("req-turn-1")
.with_response_id("resp-turn-1"),
MockTurn::text("5")
.with_provider_request_id("req-turn-2")
.with_response_id("resp-turn-2"),
]))
.tool(crate::test_utils::MockAddTool)
.add_hook(hook.clone())
.build()
.prompt(Message::user("add 2 and 3"))
.max_turns(3)
.run()
.await
.expect("blocking tool run");
let turns = hook.turns.lock().expect("turn identities").clone();
let request_ids: Vec<_> = turns
.iter()
.map(|identity| identity.provider_request_id.clone())
.collect();
assert_eq!(
request_ids,
[
Some("req-turn-1".to_string()),
Some("req-turn-2".to_string())
],
"each attempt reports its own transport id, in order"
);
let call_ids: Vec<_> = response
.completion_calls
.iter()
.map(|call| call.provider_request_id.clone())
.collect();
assert_eq!(request_ids, call_ids);
}
#[tokio::test]
async fn model_turn_finished_identity_streamed_tool_only_and_text() {
let hook = TurnIdentityHook::default();
let model = MockCompletionModel::from_stream_turns([
vec![
MockStreamEvent::tool_call_name_delta("tc1", "add"),
MockStreamEvent::tool_call_arguments_delta("tc1", "{\"x\":2,\"y\":3}"),
MockStreamEvent::tool_call("tc1", "add", json!({"x": 2, "y": 3})),
MockStreamEvent::RequestId("req-stream-1".to_owned()),
stream_final_with_id("resp-stream-1"),
],
vec![
MockStreamEvent::text("5"),
MockStreamEvent::RequestId("req-stream-2".to_owned()),
stream_final_with_id("resp-stream-2"),
],
]);
let mut stream = AgentBuilder::new(model)
.tool(crate::test_utils::MockAddTool)
.add_hook(hook.clone())
.build()
.prompt(Message::user("add 2 and 3"))
.max_turns(3)
.stream();
while let Some(item) = stream.next().await {
item.expect("stream item");
}
let turns = hook.turns.lock().expect("turn identities").clone();
let request_ids: Vec<_> = turns
.iter()
.map(|identity| identity.provider_request_id.clone())
.collect();
assert_eq!(
request_ids,
[
Some("req-stream-1".to_string()),
Some("req-stream-2".to_string())
],
"streamed tool-only and text turns each carry their own identity"
);
let responses = hook.responses.lock().expect("responses").clone();
let response_ids: Vec<_> = responses
.iter()
.map(|identity| identity.provider_request_id.clone())
.collect();
assert_eq!(
response_ids, request_ids,
"CompletionResponse fires for the tool-only turn too, each carrying its own identity"
);
}
#[tokio::test]
async fn retried_turn_reports_the_retried_attempts_own_identity() {
#[derive(Clone, Default)]
struct RetryOnceCapturingIdentity {
seen: Arc<Mutex<Vec<Option<String>>>>,
}
impl AgentHook for RetryOnceCapturingIdentity {
async fn on_model_turn_finished(
&self,
_ctx: &HookContext,
event: ModelTurnFinished<'_>,
) -> ModelTurnAction {
let mut seen = self.seen.lock().expect("retry identities");
seen.push(event.identity.provider_request_id.clone());
if seen.len() == 1 {
ModelTurnAction::repeat()
} else {
ModelTurnAction::continue_run()
}
}
}
let hook = RetryOnceCapturingIdentity::default();
AgentBuilder::new(MockCompletionModel::from_turns([
MockTurn::text("first attempt").with_provider_request_id("req-attempt-1"),
MockTurn::text("second attempt").with_provider_request_id("req-attempt-2"),
]))
.add_hook(hook.clone())
.build()
.prompt(Message::user("prompt"))
.max_turns(3)
.run()
.await
.expect("retried run");
assert_eq!(
*hook.seen.lock().expect("retry identities"),
[
Some("req-attempt-1".to_string()),
Some("req-attempt-2".to_string())
],
"each attempt's event carries that attempt's id — no stale leak"
);
}
fn relayed(
turns: impl IntoIterator<Item = impl IntoIterator<Item = MockStreamEvent>>,
) -> AgentBuilder {
let (dispatcher, registrar, mut driver) = crate::bus::Bus::channel();
driver
.register(
"model:relay",
rig_core::test_utils::MockRelay::new("relay", turns),
)
.expect("register");
tokio::spawn(driver);
AgentBuilder::over_bus(
dispatcher,
registrar,
"relay",
rig_core::effect::HandlerKey::from("model:relay"),
)
}
#[tokio::test]
async fn frames_after_the_providers_end_are_not_read() {
let cases = [
("error", MockStreamEvent::error("post-final failure")),
("text", MockStreamEvent::text("late text")),
("reasoning", MockStreamEvent::reasoning("late reasoning")),
(
"tool call",
MockStreamEvent::tool_call("late", "add", json!({"x": 1, "y": 2})),
),
("unknown", MockStreamEvent::unknown(json!({"type": "late"}))),
];
for (case, late) in cases {
let hook = FinishLifecycleHook::default();
let mut stream = relayed([vec![
MockStreamEvent::text("canonical response"),
MockStreamEvent::final_response(canonical_usage()),
late,
]])
.add_hook(hook.clone())
.build()
.prompt("canonical prompt")
.stream();
let mut texts = String::new();
let mut completion_calls = 0;
let mut response = None;
while let Some(item) = stream.next().await {
match item.unwrap_or_else(|error| panic!("{case}: {error:?}")) {
MultiTurnStreamItem::StreamAssistantItem(Item::Event(StreamEvent::Text {
text,
..
})) => texts.push_str(&text),
MultiTurnStreamItem::StreamAssistantItem(Item::Unknown(payload)) => {
panic!("{case}: a late item was read: {payload:?}")
}
MultiTurnStreamItem::CompletionCall(_) => completion_calls += 1,
MultiTurnStreamItem::FinalResponse(final_response) => {
response = Some(final_response)
}
_ => {}
}
}
assert_eq!(texts, "canonical response", "{case}");
assert_eq!(completion_calls, 1, "{case}");
assert_eq!(
response.expect("the run completes").output(),
"canonical response",
"{case}"
);
assert_eq!(
hook.snapshots.lock().expect("finish snapshots").len(),
1,
"{case}"
);
assert_eq!(hook.model_turns.load(SeqCst), 1, "{case}");
}
}
fn blocking_model() -> MockCompletionModel {
MockCompletionModel::from_turns([
MockTurn::tool_call("tc1", "add", json!({"x": 2, "y": 3})),
MockTurn::text("the answer is 5"),
])
}
fn streaming_model() -> MockCompletionModel {
MockCompletionModel::from_stream_turns([
vec![
MockStreamEvent::tool_call_name_delta("tc1", "add"),
MockStreamEvent::tool_call_arguments_delta("tc1", "{\"x\":2,\"y\":3}"),
MockStreamEvent::tool_call("tc1", "add", json!({"x": 2, "y": 3})),
MockStreamEvent::final_response_with_total_tokens(0),
],
vec![
MockStreamEvent::text("the answer is 5"),
MockStreamEvent::final_response_with_total_tokens(0),
],
])
}
#[tokio::test]
async fn prompt_surfaces_reject_second_tool_roundtrip_request_at_budget_one() {
let blocking_model = blocking_model();
let blocking_recorded = blocking_model.clone();
let blocking_agent = AgentBuilder::new(blocking_model).tool(MockAddTool).build();
let blocking_err = blocking_agent
.prompt("add 2 and 3")
.max_turns(1)
.await
.expect_err("blocking prompt should reject request two");
assert!(matches!(
blocking_err,
PromptError::MaxTurns { max_turns: 1, .. }
));
assert_eq!(blocking_recorded.request_count(), 1);
let streaming_model = streaming_model();
let streaming_recorded = streaming_model.clone();
let streaming_agent = AgentBuilder::new(streaming_model).tool(MockAddTool).build();
let mut stream = streaming_agent.prompt("add 2 and 3").max_turns(1).stream();
let mut streaming_err = None;
while let Some(item) = stream.next().await {
if let Err(err) = item {
streaming_err = Some(err);
break;
}
}
match streaming_err {
Some(err) => assert!(matches!(err, PromptError::MaxTurns { max_turns: 1, .. })),
other => panic!("expected streaming max-turns error, got {other:?}"),
}
assert_eq!(streaming_recorded.request_count(), 1);
}
mod structured_tool_results {
use std::sync::{Arc, Mutex};
use futures::StreamExt;
use serde_json::json;
use crate::agent::{
AgentBuilder, AgentHook, DispatchAction, DispatchEvent, HookContext, HookStack,
OutcomeAction, OutcomeEvent,
};
use crate::test_utils::{
MockAddTool, MockCompletionModel, MockDeniedTool, MockFailingTool, MockHandledFailureTool,
MockMetadataTool, MockRequestId, MockStreamEvent, MockTurn,
};
use crate::tool::{ToolErrorKind, ToolResult};
#[derive(Clone, Default)]
struct OutcomeHook {
outcomes: Arc<Mutex<Vec<String>>>,
results: Arc<Mutex<Vec<String>>>,
}
impl OutcomeHook {
fn outcomes(&self) -> Vec<String> {
self.outcomes.lock().expect("outcomes").clone()
}
fn results(&self) -> Vec<String> {
self.results.lock().expect("results").clone()
}
}
fn outcome_label(result: &ToolResult) -> String {
if result.is_skipped() {
"skipped".to_string()
} else if result.is_refused() {
"denied".to_string()
} else if let Some(error) = result.error() {
format!("error:{}", error.kind().as_str())
} else {
"success".to_string()
}
}
impl AgentHook for OutcomeHook {
async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
if let Some(result) = event.tool_result() {
self.outcomes
.lock()
.expect("outcomes")
.push(outcome_label(result));
self.results
.lock()
.expect("results")
.push(result.output().render());
}
OutcomeAction::proceed()
}
}
fn model_one_tool_then_text(tool: &str) -> MockCompletionModel {
MockCompletionModel::from_turns([
MockTurn::tool_call("tc1", tool, json!({})),
MockTurn::text("done"),
])
}
fn stream_model_one_tool_then_text(tool: &str) -> MockCompletionModel {
MockCompletionModel::from_stream_turns([
vec![
MockStreamEvent::tool_call_name_delta("tc1", tool),
MockStreamEvent::tool_call_arguments_delta("tc1", "{}"),
MockStreamEvent::tool_call("tc1", tool, json!({})),
MockStreamEvent::final_response_with_total_tokens(0),
],
vec![
MockStreamEvent::text("done"),
MockStreamEvent::final_response_with_total_tokens(0),
],
])
}
#[tokio::test]
async fn timeout_failure_surfaces_structured_outcome() {
let hook = OutcomeHook::default();
AgentBuilder::new(model_one_tool_then_text("flaky_tool"))
.tool(MockFailingTool::new(ToolErrorKind::Timeout))
.add_hook(hook.clone())
.build()
.prompt("go")
.max_turns(3)
.run()
.await
.expect("run should succeed; a tool timeout is model-visible feedback, not fatal");
assert_eq!(hook.outcomes(), vec!["error:timeout".to_string()]);
assert_eq!(hook.results(), vec!["mock tool call failed".to_string()]);
}
#[tokio::test]
async fn hook_terminates_after_repeated_timeouts() {
#[derive(Clone, Default)]
struct TimeoutCount(usize);
struct TimeoutTerminator;
impl AgentHook for TimeoutTerminator {
async fn on_outcome(
&self,
ctx: &HookContext,
event: OutcomeEvent<'_>,
) -> OutcomeAction {
if let Some(result) = event.tool_result()
&& result.is_error_kind(ToolErrorKind::Timeout)
{
let count = ctx.scratchpad().update(|c: &mut TimeoutCount| {
c.0 += 1;
c.0
});
if count >= 2 {
return OutcomeAction::stop("aborting after repeated tool timeouts");
}
}
OutcomeAction::proceed()
}
}
let observer = OutcomeHook::default();
let err = AgentBuilder::new(MockCompletionModel::from_turns([
MockTurn::tool_call("tc1", "flaky_tool", json!({})),
MockTurn::tool_call("tc2", "flaky_tool", json!({})),
MockTurn::text("unreachable"),
]))
.tool(MockFailingTool::new(ToolErrorKind::Timeout))
.add_hook(observer.clone())
.add_hook(TimeoutTerminator)
.build()
.prompt("go")
.max_turns(5)
.run()
.await
.expect_err("the run must terminate after two timeouts");
assert!(
err.to_string()
.contains("aborting after repeated tool timeouts"),
"unexpected error: {err}"
);
assert_eq!(
observer.outcomes(),
vec!["error:timeout".to_string(), "error:timeout".to_string()],
"both timeout outcomes must be observed before termination"
);
}
#[tokio::test]
async fn not_found_outcome_is_structured_and_non_fatal() {
let hook = OutcomeHook::default();
let status: Arc<Mutex<Option<u16>>> = Arc::new(Mutex::new(None));
struct StatusProbe(Arc<Mutex<Option<u16>>>);
impl AgentHook for StatusProbe {
async fn on_outcome(
&self,
_ctx: &HookContext,
event: OutcomeEvent<'_>,
) -> OutcomeAction {
if let Some(error) = event.tool_result().and_then(|result| result.error()) {
*self.0.lock().expect("status") = error.http_status();
}
OutcomeAction::proceed()
}
}
AgentBuilder::new(model_one_tool_then_text("flaky_tool"))
.tool(MockFailingTool::new(ToolErrorKind::NotFound))
.add_hook(hook.clone())
.add_hook(StatusProbe(status.clone()))
.build()
.prompt("go")
.max_turns(3)
.run()
.await
.expect("a 404 must not terminate the run by default");
assert_eq!(hook.outcomes(), vec!["error:not_found".to_string()]);
assert_eq!(
*status.lock().expect("status"),
Some(404),
"the structured failure must carry the HTTP status"
);
}
#[tokio::test]
async fn handled_failure_delivers_model_output_and_error_outcome() {
let hook = OutcomeHook::default();
AgentBuilder::new(model_one_tool_then_text("lookup"))
.tool(MockHandledFailureTool)
.add_hook(hook.clone())
.build()
.prompt("go")
.max_turns(3)
.run()
.await
.expect("a handled failure is not fatal");
assert_eq!(hook.outcomes(), vec!["error:not_found".to_string()]);
assert_eq!(
hook.results(),
vec!["no record found for id 42; try a different id".to_string()],
"the tool's model-visible output must survive alongside the error outcome"
);
}
#[tokio::test]
async fn flow_skip_produces_skipped_outcome() {
struct SkipHook;
impl AgentHook for SkipHook {
async fn on_dispatch(
&self,
_ctx: &HookContext,
event: DispatchEvent<'_>,
) -> DispatchAction {
if event.tool_name().is_some() {
DispatchAction::skip("not executed (denied by policy); do not retry")
} else {
DispatchAction::proceed()
}
}
}
let observer = OutcomeHook::default();
AgentBuilder::new(model_one_tool_then_text("flaky_tool"))
.tool(MockFailingTool::new(ToolErrorKind::Timeout))
.add_hook(SkipHook)
.add_hook(observer.clone())
.build()
.prompt("go")
.max_turns(3)
.run()
.await
.expect("run should succeed after skipping the tool");
assert_eq!(observer.outcomes(), vec!["skipped".to_string()]);
assert_eq!(
observer.results(),
vec!["not executed (denied by policy); do not retry".to_string()]
);
}
#[tokio::test]
async fn tool_authored_denial_produces_denied_outcome() {
let hook = OutcomeHook::default();
AgentBuilder::new(model_one_tool_then_text("guarded"))
.tool(MockDeniedTool)
.add_hook(hook.clone())
.build()
.prompt("go")
.max_turns(3)
.run()
.await
.expect("a tool-authored denial is not fatal");
assert_eq!(hook.outcomes(), vec!["denied".to_string()]);
assert_eq!(
hook.results(),
vec!["access to this resource is not permitted".to_string()],
"the model still receives the tool's denial message"
);
}
#[tokio::test]
async fn permission_denied_failure_is_not_a_tool_refusal() {
let hook = OutcomeHook::default();
AgentBuilder::new(model_one_tool_then_text("flaky_tool"))
.tool(MockFailingTool::new(ToolErrorKind::PermissionDenied))
.add_hook(hook.clone())
.build()
.prompt("go")
.max_turns(3)
.run()
.await
.expect("a permission failure is model-visible feedback, not fatal");
assert_eq!(hook.outcomes(), vec!["error:permission_denied".to_string()]);
assert_eq!(hook.results(), vec!["mock tool call failed".to_string()]);
}
#[tokio::test]
async fn rewrite_args_then_skip_reports_rewritten_args() {
struct RewriteHook;
impl AgentHook for RewriteHook {
async fn on_dispatch(
&self,
_ctx: &HookContext,
event: DispatchEvent<'_>,
) -> DispatchAction {
DispatchAction::rewrite_tool_args(event.kind, json!({ "x": 41, "y": 1 }))
}
}
struct SkipHook;
impl AgentHook for SkipHook {
async fn on_dispatch(
&self,
_ctx: &HookContext,
event: DispatchEvent<'_>,
) -> DispatchAction {
if event.tool_name().is_some() {
DispatchAction::skip("denied after rewrite")
} else {
DispatchAction::proceed()
}
}
}
#[derive(Clone, Default)]
struct ArgsProbe {
args: Arc<Mutex<Option<String>>>,
outcome: Arc<Mutex<Option<String>>>,
}
impl AgentHook for ArgsProbe {
async fn on_outcome(
&self,
_ctx: &HookContext,
event: OutcomeEvent<'_>,
) -> OutcomeAction {
if let (Some(args), Some(result)) = (event.tool_args(), event.tool_result()) {
*self.args.lock().expect("args") = Some(args.to_string());
*self.outcome.lock().expect("outcome") = Some(outcome_label(result));
}
OutcomeAction::proceed()
}
}
async fn run_surface(streaming: bool) -> (String, String) {
let probe = ArgsProbe::default();
if streaming {
let mut stream = AgentBuilder::new(stream_model_one_tool_then_text("add"))
.tool(MockAddTool)
.add_hook(RewriteHook)
.add_hook(SkipHook)
.add_hook(probe.clone())
.build()
.prompt("go")
.max_turns(3)
.stream();
while let Some(item) = stream.next().await {
if let Err(err) = item {
panic!("stream item errored: {err}");
}
}
} else {
AgentBuilder::new(model_one_tool_then_text("add"))
.tool(MockAddTool)
.add_hook(RewriteHook)
.add_hook(SkipHook)
.add_hook(probe.clone())
.build()
.prompt("go")
.max_turns(3)
.run()
.await
.expect("run should succeed after skipping the tool");
}
let args = probe.args.lock().expect("args").clone().expect("args seen");
let outcome = probe
.outcome
.lock()
.expect("outcome")
.clone()
.expect("outcome seen");
(args, outcome)
}
for streaming in [false, true] {
let (args, outcome) = run_surface(streaming).await;
assert_eq!(
outcome, "skipped",
"the skipped tool must produce a Skipped outcome (streaming={streaming})"
);
let parsed: serde_json::Value =
serde_json::from_str(&args).expect("ToolResult args are valid JSON");
assert_eq!(
parsed,
json!({ "x": 41, "y": 1 }),
"the skipped ToolResult must report the rewritten args, not the model's \
original {{}} (streaming={streaming}); got {args}"
);
}
}
#[tokio::test]
async fn nested_hook_stack_rewrite_then_skip_reports_rewritten_args() {
struct RewriteHook;
impl AgentHook for RewriteHook {
async fn on_dispatch(
&self,
_ctx: &HookContext,
event: DispatchEvent<'_>,
) -> DispatchAction {
DispatchAction::rewrite_tool_args(event.kind, json!({ "x": 41, "y": 1 }))
}
}
struct SkipHook;
impl AgentHook for SkipHook {
async fn on_dispatch(
&self,
_ctx: &HookContext,
event: DispatchEvent<'_>,
) -> DispatchAction {
if event.tool_name().is_some() {
DispatchAction::skip("denied after nested rewrite")
} else {
DispatchAction::proceed()
}
}
}
#[derive(Clone, Default)]
struct ArgsProbe {
args: Arc<Mutex<Option<String>>>,
outcome: Arc<Mutex<Option<String>>>,
}
impl AgentHook for ArgsProbe {
async fn on_outcome(
&self,
_ctx: &HookContext,
event: OutcomeEvent<'_>,
) -> OutcomeAction {
if let (Some(args), Some(result)) = (event.tool_args(), event.tool_result()) {
*self.args.lock().expect("args") = Some(args.to_string());
*self.outcome.lock().expect("outcome") = Some(outcome_label(result));
}
OutcomeAction::proceed()
}
}
fn nested_stack() -> HookStack {
let mut nested = HookStack::new();
nested.push(RewriteHook);
nested.push(SkipHook);
nested
}
for streaming in [false, true] {
let probe = ArgsProbe::default();
if streaming {
let mut stream = AgentBuilder::new(stream_model_one_tool_then_text("add"))
.tool(MockAddTool)
.add_hook(nested_stack())
.add_hook(probe.clone())
.build()
.prompt("go")
.max_turns(3)
.stream();
while let Some(item) = stream.next().await {
if let Err(err) = item {
panic!("stream item errored: {err}");
}
}
} else {
AgentBuilder::new(model_one_tool_then_text("add"))
.tool(MockAddTool)
.add_hook(nested_stack())
.add_hook(probe.clone())
.build()
.prompt("go")
.max_turns(3)
.run()
.await
.expect("run should succeed after the nested stack skips the tool");
}
assert_eq!(
probe.outcome.lock().expect("outcome").clone(),
Some("skipped".to_string()),
"streaming={streaming}"
);
let args = probe.args.lock().expect("args").clone().expect("args seen");
let parsed: serde_json::Value = serde_json::from_str(&args).expect("valid JSON args");
assert_eq!(
parsed,
json!({ "x": 41, "y": 1 }),
"the nested stack's rewrite must survive its skip and reach the ToolResult \
(streaming={streaming}); got {args}"
);
}
}
#[tokio::test]
async fn invalid_args_are_classified_as_invalid_args() {
let hook = OutcomeHook::default();
AgentBuilder::new(MockCompletionModel::from_turns([
MockTurn::tool_call("tc1", "add", json!({ "x": "not-a-number", "y": 1 })),
MockTurn::text("done"),
]))
.tool(MockAddTool)
.add_hook(hook.clone())
.build()
.prompt("go")
.max_turns(3)
.run()
.await
.expect("an invalid-args failure is model-visible feedback, not fatal");
assert_eq!(hook.outcomes(), vec!["error:invalid_args".to_string()]);
}
#[tokio::test]
async fn success_result_metadata_reaches_hook_but_not_model() {
struct MetadataProbe {
seen: Arc<Mutex<Option<String>>>,
model_output: Arc<Mutex<Option<String>>>,
}
impl AgentHook for MetadataProbe {
async fn on_outcome(
&self,
_ctx: &HookContext,
event: OutcomeEvent<'_>,
) -> OutcomeAction {
if let (Some(result), Some(tool_context)) =
(event.tool_result(), event.tool_context())
{
*self.seen.lock().expect("seen") = tool_context
.result::<MockRequestId>()
.expect("request id decodes")
.map(|id| id.0);
*self.model_output.lock().expect("model_output") =
Some(result.output().render());
}
OutcomeAction::proceed()
}
}
async fn run_surface(streaming: bool) -> (Option<String>, String) {
let seen: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None));
let model_output: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None));
let probe = MetadataProbe {
seen: seen.clone(),
model_output: model_output.clone(),
};
if streaming {
let mut stream = AgentBuilder::new(stream_model_one_tool_then_text("with_meta"))
.tool(MockMetadataTool)
.add_hook(probe)
.build()
.prompt("go")
.max_turns(3)
.stream();
while let Some(item) = stream.next().await {
if let Err(error) = item {
panic!("stream item errored: {error}");
}
}
} else {
AgentBuilder::new(model_one_tool_then_text("with_meta"))
.tool(MockMetadataTool)
.add_hook(probe)
.build()
.prompt("go")
.max_turns(3)
.run()
.await
.expect("run should succeed");
}
let seen_value = seen.lock().expect("seen").clone();
let output = model_output
.lock()
.expect("model_output")
.clone()
.expect("output");
(seen_value, output)
}
for streaming in [false, true] {
let (seen, output) = run_surface(streaming).await;
assert_eq!(
seen,
Some("req-7".to_string()),
"the tool's result metadata must reach the hook (streaming={streaming})"
);
assert_eq!(output, "done");
assert!(
!output.contains("req-7"),
"result metadata must never leak into model output (streaming={streaming})"
);
}
}
#[tokio::test]
async fn rewrite_result_does_not_mask_the_structured_outcome() {
struct Redact;
impl AgentHook for Redact {
async fn on_outcome(
&self,
_ctx: &HookContext,
event: OutcomeEvent<'_>,
) -> OutcomeAction {
OutcomeAction::rewrite_tool_result(&event, "[REDACTED]")
}
}
let observer = OutcomeHook::default();
AgentBuilder::new(model_one_tool_then_text("flaky_tool"))
.tool(MockFailingTool::new(ToolErrorKind::NotFound))
.add_hook(Redact)
.add_hook(observer.clone())
.build()
.prompt("go")
.max_turns(3)
.run()
.await
.expect("run should succeed");
assert_eq!(observer.outcomes(), vec!["error:not_found".to_string()]);
assert_eq!(observer.results(), vec!["[REDACTED]".to_string()]);
}
#[tokio::test]
async fn streaming_and_blocking_outcomes_match() {
let blocking = OutcomeHook::default();
AgentBuilder::new(model_one_tool_then_text("flaky_tool"))
.tool(MockFailingTool::new(ToolErrorKind::Timeout))
.add_hook(blocking.clone())
.build()
.prompt("go")
.max_turns(3)
.run()
.await
.expect("blocking run should succeed");
let streaming = OutcomeHook::default();
let mut stream = AgentBuilder::new(stream_model_one_tool_then_text("flaky_tool"))
.tool(MockFailingTool::new(ToolErrorKind::Timeout))
.add_hook(streaming.clone())
.build()
.prompt("go")
.max_turns(3)
.stream();
while let Some(item) = stream.next().await {
if let Err(err) = item {
panic!("stream item errored: {err}");
}
}
assert_eq!(blocking.outcomes(), vec!["error:timeout".to_string()]);
assert_eq!(blocking.outcomes(), streaming.outcomes());
assert_eq!(blocking.results(), streaming.results());
}
#[tokio::test]
async fn concurrent_tools_preserve_order_and_both_outcomes() {
use rig_core::message::{
AssistantContent, ToolCall as MessageToolCall, ToolFunction, UserContent,
};
let turn = MockTurn::from_contents([
AssistantContent::ToolCall(MessageToolCall::from_wire(
"tc_add",
ToolFunction::new(
rig_core::message::ToolName::new("add".to_string()).expect("tool name"),
json!({ "x": 2, "y": 3 }),
),
)),
AssistantContent::ToolCall(MessageToolCall::from_wire(
"tc_flaky",
ToolFunction::new(
rig_core::message::ToolName::new("flaky_tool".to_string()).expect("tool name"),
json!({}),
),
)),
]);
let observer = OutcomeHook::default();
let response = AgentBuilder::new(MockCompletionModel::from_turns([
turn,
MockTurn::text("done"),
]))
.tool(MockAddTool)
.tool(MockFailingTool::new(ToolErrorKind::Timeout))
.add_hook(observer.clone())
.build()
.prompt("go")
.max_turns(3)
.tool_concurrency(2)
.run()
.await
.expect("run should succeed");
let mut outcomes = observer.outcomes();
outcomes.sort();
assert_eq!(
outcomes,
vec!["error:timeout".to_string(), "success".to_string()]
);
let messages = response.messages;
let tool_result_ids: Vec<String> = messages
.iter()
.flat_map(|message| match message {
crate::completion::Message::User { content } => content
.iter()
.filter_map(|c| match c {
UserContent::ToolResult(result) => Some(
result
.call
.provider()
.map(|provider| provider.as_str())
.expect("explicit provider ID")
.to_owned(),
),
_ => None,
})
.collect::<Vec<_>>(),
_ => Vec::new(),
})
.collect();
assert_eq!(
tool_result_ids,
vec!["tc_add".to_string(), "tc_flaky".to_string()],
"tool results must be persisted in call order"
);
}
}
mod span_safety_net {
use serde_json::Value;
use crate::agent::{AgentBuilder, HookContext, OutcomeAction, OutcomeEvent};
use crate::completion::Usage;
use crate::test_utils::{MockAddTool, MockCompletionModel, MockTurn, TraceCapture};
use crate::tool::{ToolContext, ToolExecutionError};
fn usage(input: u64, output: u64) -> Usage {
Usage::new().input_tokens(input).output_tokens(output)
}
fn tool_then_text_model() -> MockCompletionModel {
MockCompletionModel::from_turns([
MockTurn::tool_call("tc1", "add", serde_json::json!({"x": 2, "y": 3}))
.with_usage(usage(7, 11)),
MockTurn::text("the answer is 5").with_usage(usage(13, 17)),
])
}
async fn warm_blocking_callsites() {
let agent = AgentBuilder::new(tool_then_text_model())
.record_content_telemetry(true)
.tool(MockAddTool)
.build();
let _ = agent.prompt("add 2 and 3").max_turns(3).run().await;
}
struct RawOutputTool;
impl crate::tool::Tool for RawOutputTool {
const NAME: &'static str = "raw_output";
type Error = rig::tool::ToolExecutionError;
type Args = serde_json::Value;
type Output = String;
fn description(&self) -> String {
"returns a raw output marker".to_string()
}
fn parameters(&self) -> serde_json::Value {
serde_json::json!({ "type": "object", "properties": {} })
}
async fn call(
&self,
_context: &mut ToolContext,
_args: Self::Args,
) -> Result<Self::Output, ToolExecutionError> {
Ok("RAW_EXECUTION_OUTPUT_42".to_string())
}
}
struct RedactResultHook;
impl crate::agent::AgentHook for RedactResultHook {
async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
OutcomeAction::rewrite_tool_result(&event, "[REDACTED]")
}
}
struct StopOnResultHook;
impl crate::agent::AgentHook for StopOnResultHook {
async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
if event.tool_result().is_some() {
OutcomeAction::stop("stop after raw result")
} else {
OutcomeAction::proceed()
}
}
}
#[tokio::test]
async fn tool_result_rewrite_redacts_span_output() {
let _isolation = crate::test_utils::scoped_tracing_subscriber_guard().await;
let capture = TraceCapture::default();
let _default = tracing::subscriber::set_default(capture.subscriber());
warm_blocking_callsites().await;
tracing::callsite::rebuild_interest_cache();
capture.clear();
let model = MockCompletionModel::from_turns([
MockTurn::tool_call("tc1", "raw_output", serde_json::json!({})),
MockTurn::text("ok"),
]);
let response = AgentBuilder::new(model)
.record_content_telemetry(true)
.tool(RawOutputTool)
.add_hook(RedactResultHook)
.build()
.prompt("go")
.max_turns(3)
.run()
.await
.expect("run should succeed");
assert_eq!(response.output(), "ok");
let captured = capture.values_of("gen_ai.tool.call.result");
let captured: Vec<&str> = captured.iter().filter_map(Value::as_str).collect();
assert!(
captured.iter().any(|v| v.contains("[REDACTED]")),
"the rewritten presentation must reach telemetry; captured: {captured:?}"
);
assert!(
!captured
.iter()
.any(|v| v.contains("RAW_EXECUTION_OUTPUT_42")),
"the raw tool output must not leak through telemetry; captured: {captured:?}"
);
}
#[tokio::test]
async fn tool_result_stop_omits_span_output() {
let _isolation = crate::test_utils::scoped_tracing_subscriber_guard().await;
let capture = TraceCapture::default();
let _default = tracing::subscriber::set_default(capture.subscriber());
warm_blocking_callsites().await;
tracing::callsite::rebuild_interest_cache();
capture.clear();
let result = AgentBuilder::new(MockCompletionModel::from_turns([MockTurn::tool_call(
"tc1",
"raw_output",
serde_json::json!({}),
)]))
.tool(RawOutputTool)
.add_hook(StopOnResultHook)
.build()
.prompt("go")
.max_turns(2)
.run()
.await;
assert!(result.is_err(), "the result hook should stop the run");
let captured = capture.values_of("gen_ai.tool.call.result");
let captured: Vec<&str> = captured.iter().filter_map(Value::as_str).collect();
assert!(
!captured
.iter()
.any(|value| value.contains("RAW_EXECUTION_OUTPUT_42")),
"a Stop must not leak raw execution telemetry; captured: {captured:?}"
);
}
}
fn tool_call_content(id: &str, args: serde_json::Value) -> AssistantContent {
AssistantContent::ToolCall(MessageToolCall::from_wire(
id,
ToolFunction::new(
rig_core::message::ToolName::new("add".to_string()).expect("tool name"),
args,
),
))
}
fn tool_result_text_in_history(messages: &[Message], expected: &str) -> bool {
messages.iter().any(|message| {
matches!(
message,
Message::User { content }
if content.iter().any(|item| matches!(
item,
UserContent::ToolResult(result)
if result.content.iter().any(|c| matches!(
c,
rig_core::message::ToolResultContent::Text(text)
if text.text == expected
))
))
)
})
}
fn tool_result_json_in_history(messages: &[Message], expected: &serde_json::Value) -> bool {
messages.iter().any(|message| {
matches!(
message,
Message::User { content }
if content.iter().any(|item| matches!(
item,
UserContent::ToolResult(result)
if result.content.iter().any(|content| matches!(
content,
rig_core::message::ToolResultContent::Json { value }
if value == expected
))
))
)
})
}
#[derive(Clone)]
struct OutOfOrderTool {
gate: Arc<tokio::sync::Notify>,
order: Arc<AtomicU32>,
}
impl Tool for OutOfOrderTool {
const NAME: &'static str = "add";
type Error = MockToolError;
type Args = MockOperationArgs;
type Output = i32;
fn description(&self) -> String {
MockAddTool.description()
}
fn parameters(&self) -> serde_json::Value {
MockAddTool.parameters()
}
async fn call(
&self,
_context: &mut ToolContext,
_args: Self::Args,
) -> Result<Self::Output, Self::Error> {
let nth = self.order.fetch_add(1, SeqCst);
if nth == 0 {
self.gate.notified().await;
} else {
self.gate.notify_one();
}
Ok(nth as i32)
}
}
#[tokio::test]
async fn run_preserves_tool_call_order_under_out_of_order_completion() {
let model = MockCompletionModel::from_turns([
MockTurn::from_contents([
tool_call_content("tc1", json!({"x": 1, "y": 0})),
tool_call_content("tc2", json!({"x": 2, "y": 0})),
]),
MockTurn::text("done"),
]);
let response = AgentBuilder::new(model)
.tool(OutOfOrderTool {
gate: Arc::new(tokio::sync::Notify::new()),
order: Arc::new(AtomicU32::new(0)),
})
.build()
.prompt("go")
.max_turns(3)
.tool_concurrency(4)
.run()
.await
.expect("run should succeed");
let messages = response.messages;
let result_ids: Vec<String> = messages
.iter()
.flat_map(|message| match message {
Message::User { content } => content
.iter()
.filter_map(|item| match item {
UserContent::ToolResult(result) => Some(
result
.call
.provider()
.map(|provider| provider.as_str())
.expect("explicit provider ID")
.to_owned(),
),
_ => None,
})
.collect::<Vec<_>>(),
_ => Vec::new(),
})
.collect();
assert_eq!(result_ids, vec!["tc1".to_string(), "tc2".to_string()]);
}
async fn drive_to_final_response(
mut stream: crate::agent::streaming::StreamingResult,
) -> crate::agent::PromptResponse {
let mut final_response = None;
while let Some(item) = stream.next().await {
if let MultiTurnStreamItem::FinalResponse(resp) =
item.unwrap_or_else(|err| panic!("stream item errored: {err}"))
{
final_response = Some(resp);
}
}
final_response.expect("stream should yield a final response")
}
fn tool_result_ids(messages: &[Message]) -> Vec<String> {
messages
.iter()
.flat_map(|message| match message {
Message::User { content } => content
.iter()
.filter_map(|item| match item {
UserContent::ToolResult(result) => Some(
result
.call
.provider()
.map(|provider| provider.as_str())
.expect("explicit provider ID")
.to_owned(),
),
_ => None,
})
.collect::<Vec<_>>(),
_ => Vec::new(),
})
.collect()
}
#[tokio::test]
async fn stream_preserves_history_order_under_out_of_order_completion() {
let model = MockCompletionModel::from_stream_turns([
vec![
MockStreamEvent::tool_call("tc1", "add", json!({"x": 1, "y": 0})),
MockStreamEvent::tool_call("tc2", "add", json!({"x": 2, "y": 0})),
MockStreamEvent::final_response_with_total_tokens(0),
],
vec![
MockStreamEvent::text("done"),
MockStreamEvent::final_response_with_total_tokens(0),
],
]);
let stream = AgentBuilder::new(model)
.tool(OutOfOrderTool {
gate: Arc::new(tokio::sync::Notify::new()),
order: Arc::new(AtomicU32::new(0)),
})
.build()
.prompt("go")
.max_turns(3)
.tool_concurrency(4)
.stream();
let final_response = tokio::time::timeout(
std::time::Duration::from_secs(5),
drive_to_final_response(stream),
)
.await
.expect("streamed tools must run concurrently, not deadlock on the first call");
let messages = final_response.messages().to_vec();
assert_eq!(
tool_result_ids(&messages),
vec!["tc1".to_string(), "tc2".to_string()]
);
}
#[tokio::test]
async fn stream_emits_tool_results_in_call_order_after_batch_settles_under_concurrency() {
let model = MockCompletionModel::from_stream_turns([
vec![
MockStreamEvent::tool_call("tc1", "add", json!({"x": 1, "y": 0})),
MockStreamEvent::tool_call("tc2", "add", json!({"x": 2, "y": 0})),
MockStreamEvent::final_response_with_total_tokens(0),
],
vec![
MockStreamEvent::text("done"),
MockStreamEvent::final_response_with_total_tokens(0),
],
]);
let mut stream = AgentBuilder::new(model)
.tool(OutOfOrderTool {
gate: Arc::new(tokio::sync::Notify::new()),
order: Arc::new(AtomicU32::new(0)),
})
.build()
.prompt("go")
.max_turns(3)
.tool_concurrency(4)
.stream();
let mut streamed_result_ids = Vec::new();
let mut final_response = None;
tokio::time::timeout(std::time::Duration::from_secs(5), async {
while let Some(item) = stream.next().await {
match item.unwrap_or_else(|err| panic!("stream item errored: {err}")) {
MultiTurnStreamItem::ToolResult { tool_result, .. } => streamed_result_ids.push(
tool_result
.call
.provider()
.map(|provider| provider.as_str())
.expect("explicit provider ID")
.to_owned(),
),
MultiTurnStreamItem::FinalResponse(resp) => final_response = Some(resp),
_ => {}
}
}
})
.await
.expect("streamed tools must run concurrently, not deadlock on the first call");
assert_eq!(
streamed_result_ids,
vec!["tc1".to_string(), "tc2".to_string()]
);
let final_response = final_response.expect("stream should yield a final response");
assert_eq!(
tool_result_ids(final_response.messages()),
vec!["tc1".to_string(), "tc2".to_string()]
);
}
struct OrderedTerminateHook {
gate: Arc<tokio::sync::Notify>,
}
impl AgentHook for OrderedTerminateHook {
async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
if let Some(args) = event.tool_args() {
let x = serde_json::from_str::<serde_json::Value>(args)
.ok()
.and_then(|v| v.get("x").and_then(serde_json::Value::as_i64));
match x {
Some(2) => {
self.gate.notify_one();
return DispatchAction::stop("terminated-by-tc2".to_string());
}
Some(1) => {
self.gate.notified().await;
return DispatchAction::stop("terminated-by-tc1".to_string());
}
_ => {}
}
}
DispatchAction::proceed()
}
}
fn two_terminating_tools_blocking_model() -> MockCompletionModel {
MockCompletionModel::from_turns([
MockTurn::from_contents([
tool_call_content("tc1", json!({"x": 1, "y": 1})),
tool_call_content("tc2", json!({"x": 2, "y": 2})),
]),
MockTurn::text("unreachable"),
])
}
fn two_terminating_tools_streaming_model() -> MockCompletionModel {
MockCompletionModel::from_stream_turns([
vec![
MockStreamEvent::tool_call("tc1", "add", json!({"x": 1, "y": 1})),
MockStreamEvent::tool_call("tc2", "add", json!({"x": 2, "y": 2})),
MockStreamEvent::final_response_with_total_tokens(0),
],
vec![
MockStreamEvent::text("unreachable"),
MockStreamEvent::final_response_with_total_tokens(0),
],
])
}
#[tokio::test]
async fn concurrent_simultaneous_tool_terminations_pick_call_order_on_both_drivers() {
let run_err = tokio::time::timeout(
std::time::Duration::from_secs(5),
AgentBuilder::new(two_terminating_tools_blocking_model())
.tool(MockAddTool)
.build()
.prompt("go")
.max_turns(3)
.tool_concurrency(2)
.add_hook(OrderedTerminateHook {
gate: Arc::new(tokio::sync::Notify::new()),
})
.run(),
)
.await
.expect("blocking run must not hang")
.expect_err("the run must terminate");
let mut stream = AgentBuilder::new(two_terminating_tools_streaming_model())
.tool(MockAddTool)
.build()
.prompt("go")
.max_turns(3)
.tool_concurrency(2)
.add_hook(OrderedTerminateHook {
gate: Arc::new(tokio::sync::Notify::new()),
})
.stream();
let stream_err = tokio::time::timeout(std::time::Duration::from_secs(5), async move {
while let Some(item) = stream.next().await {
if let Err(err) = item {
return Some(err);
}
}
None
})
.await
.expect("streamed run must not hang")
.expect("the stream must surface a terminate error");
let run_msg = run_err.to_string();
let stream_msg = stream_err.to_string();
assert!(
run_msg.contains("terminated-by-tc1"),
"blocking run should surface the first-called tool's reason, got: {run_msg}"
);
assert!(
stream_msg.contains("terminated-by-tc1"),
"stream should surface the first-called tool's reason, got: {stream_msg}"
);
assert!(
!run_msg.contains("terminated-by-tc2") && !stream_msg.contains("terminated-by-tc2"),
"neither driver should surface the later-completing tool's reason"
);
}
struct TerminateOnFirstToolHook;
impl AgentHook for TerminateOnFirstToolHook {
async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
if event
.tool_args()
.and_then(|args| serde_json::from_str::<serde_json::Value>(args).ok())
.and_then(|v| v.get("x").and_then(serde_json::Value::as_i64))
== Some(1)
{
return DispatchAction::stop("stop".to_string());
}
DispatchAction::proceed()
}
}
#[tokio::test]
async fn default_concurrency_terminate_skips_remaining_tools_on_both_drivers() {
let blocking_calls = Arc::new(AtomicU32::new(0));
AgentBuilder::new(two_terminating_tools_blocking_model())
.tool(CountingAddTool {
calls: blocking_calls.clone(),
})
.build()
.prompt("go")
.max_turns(3)
.add_hook(TerminateOnFirstToolHook)
.run()
.await
.expect_err("the run terminates");
assert_eq!(
blocking_calls.load(SeqCst),
0,
"fail-fast: blocking run() must not start the second tool after the first terminates"
);
let streaming_calls = Arc::new(AtomicU32::new(0));
let mut stream = AgentBuilder::new(two_terminating_tools_streaming_model())
.tool(CountingAddTool {
calls: streaming_calls.clone(),
})
.build()
.prompt("go")
.max_turns(3)
.add_hook(TerminateOnFirstToolHook)
.stream();
let mut saw_error = false;
while let Some(item) = stream.next().await {
if let Err(err) = item {
saw_error = true;
assert!(
err.to_string().contains("stop"),
"stream() should surface the terminate reason, got: {err}"
);
break;
}
}
assert!(saw_error, "stream() must surface the terminate error");
assert_eq!(
streaming_calls.load(SeqCst),
0,
"fail-fast: stream() must not start the second tool after the first terminates"
);
}
#[tokio::test]
async fn stream_hook_skip_surfaces_result_without_execution_commit() {
struct SkipHook;
impl AgentHook for SkipHook {
async fn on_dispatch(
&self,
_ctx: &HookContext,
event: DispatchEvent<'_>,
) -> DispatchAction {
if event.tool_name().is_some() {
DispatchAction::skip("blocked by policy")
} else {
DispatchAction::proceed()
}
}
}
let calls = Arc::new(AtomicU32::new(0));
let model = MockCompletionModel::from_stream_turns([
vec![
MockStreamEvent::tool_call("tc1", "add", json!({"x": 1, "y": 2})),
MockStreamEvent::final_response_with_total_tokens(0),
],
vec![
MockStreamEvent::text("done"),
MockStreamEvent::final_response_with_total_tokens(0),
],
]);
let stream = AgentBuilder::new(model)
.tool(CountingAddTool {
calls: calls.clone(),
})
.add_hook(SkipHook)
.build()
.prompt("go")
.max_turns(3)
.stream();
let mut exec_commits = 0;
let mut results = 0;
let mut final_response = None;
let mut stream = stream;
while let Some(item) = stream.next().await {
match item.unwrap_or_else(|err| panic!("stream item errored: {err}")) {
MultiTurnStreamItem::ToolExecutionCommitted { .. } => exec_commits += 1,
MultiTurnStreamItem::ToolResult { .. } => {
results += 1;
}
MultiTurnStreamItem::FinalResponse(resp) => final_response = Some(resp),
_ => {}
}
}
assert_eq!(calls.load(SeqCst), 0, "a skipped tool's body never runs");
assert_eq!(
exec_commits, 0,
"a hook-skipped tool produces no execution-commit"
);
assert_eq!(
results, 1,
"the skip result is still surfaced to the consumer"
);
let final_response = final_response.expect("stream should yield a final response");
let history = final_response.messages();
assert!(
history.iter().any(|m| serde_json::to_string(m)
.map(|s| s.contains("blocked by policy"))
.unwrap_or(false)),
"the skip result is committed to history"
);
}
#[tokio::test]
async fn required_with_empty_active_tools_errors_locally_without_provider_call() {
struct EmptyActiveToolsHook;
impl AgentHook for EmptyActiveToolsHook {
async fn on_completion_call(
&self,
_ctx: &HookContext,
event: CompletionCallEvent<'_>,
) -> CompletionCallAction {
if let CompletionCallEvent { .. } = event {
CompletionCallAction::patch(RequestPatch::new().active_tools(Vec::<String>::new()))
} else {
CompletionCallAction::continue_run()
}
}
}
let model = MockCompletionModel::from_turns([MockTurn::text("unreachable")]);
let probe = model.clone();
let err = AgentBuilder::new(model)
.tool(MockAddTool)
.tool_choice(ToolChoice::Required)
.add_hook(EmptyActiveToolsHook)
.build()
.prompt("go")
.run()
.await
.expect_err("Required with an empty active_tools filter must fail locally");
assert!(
probe.requests().is_empty(),
"the request must fail locally, with no provider round-trip"
);
let msg = err.to_string();
assert!(
msg.contains("Required"),
"error should mention Required: {msg}"
);
assert!(
msg.contains("active_tools"),
"error should name active_tools: {msg}"
);
}
#[tokio::test]
async fn specific_naming_filtered_out_tool_errors_locally_without_provider_call() {
struct FilterToAddHook;
impl AgentHook for FilterToAddHook {
async fn on_completion_call(
&self,
_ctx: &HookContext,
event: CompletionCallEvent<'_>,
) -> CompletionCallAction {
if let CompletionCallEvent { .. } = event {
CompletionCallAction::patch(RequestPatch::new().active_tools(["add"]))
} else {
CompletionCallAction::continue_run()
}
}
}
let model = MockCompletionModel::from_turns([MockTurn::text("unreachable")]);
let probe = model.clone();
let err = AgentBuilder::new(model)
.tool(MockAddTool)
.tool(MockSubtractTool)
.tool_choice(ToolChoice::Specific {
function_names: vec![rig_core::message::ToolName::new("subtract").expect("tool name")],
})
.add_hook(FilterToAddHook)
.build()
.prompt("go")
.run()
.await
.expect_err("Specific naming a filtered-out tool must fail locally");
assert!(
probe.requests().is_empty(),
"the request must fail locally, with no provider round-trip"
);
let msg = err.to_string();
assert!(
msg.contains("subtract"),
"error should name the missing tool: {msg}"
);
assert!(
msg.contains("active_tools"),
"error should name active_tools: {msg}"
);
}
#[derive(Clone)]
struct CountingAddTool {
calls: Arc<AtomicU32>,
}
impl Tool for CountingAddTool {
const NAME: &'static str = "add";
type Error = MockToolError;
type Args = MockOperationArgs;
type Output = i32;
fn description(&self) -> String {
MockAddTool.description()
}
fn parameters(&self) -> serde_json::Value {
MockAddTool.parameters()
}
async fn call(
&self,
_context: &mut ToolContext,
args: Self::Args,
) -> Result<Self::Output, Self::Error> {
self.calls.fetch_add(1, SeqCst);
MockAddTool.call(_context, args).await
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum TerminatePoint {
CompletionCall,
ToolDispatch,
ToolOutcome,
}
struct TerminateOn(TerminatePoint);
impl AgentHook for TerminateOn {
async fn on_completion_call(
&self,
_: &HookContext,
_: CompletionCallEvent<'_>,
) -> CompletionCallAction {
if self.0 == TerminatePoint::CompletionCall {
CompletionCallAction::stop("stop here")
} else {
CompletionCallAction::continue_run()
}
}
async fn on_dispatch(&self, _: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
if self.0 == TerminatePoint::ToolDispatch && event.tool_name().is_some() {
DispatchAction::stop("stop here")
} else {
DispatchAction::proceed()
}
}
async fn on_outcome(&self, _: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
if self.0 == TerminatePoint::ToolOutcome && event.tool_result().is_some() {
OutcomeAction::stop("stop here")
} else {
OutcomeAction::proceed()
}
}
}
struct RepairInvalidToHook(&'static str);
impl AgentHook for RepairInvalidToHook {
async fn on_invalid_tool_call(
&self,
_ctx: &HookContext,
event: &InvalidToolCallContext,
) -> Option<InvalidToolCallAction> {
Some(if let _ = event {
InvalidToolCallAction::repair(self.0)
} else {
InvalidToolCallAction::fail()
})
}
}
#[tokio::test]
async fn invalid_tool_call_repair_parity_across_run_and_stream() {
let blocking_model = MockCompletionModel::from_turns([
MockTurn::tool_call("tc1", "default_api", json!({"x": 2, "y": 3})),
MockTurn::text("the answer is 5"),
]);
let blocking_hook = RecordingHook::default();
let blocking = AgentBuilder::new(blocking_model)
.tool(MockAddTool)
.build()
.prompt("add 2 and 3")
.max_turns(3)
.add_hook(blocking_hook.clone())
.add_hook(RepairInvalidToHook("add"))
.run()
.await
.expect("blocking run should recover via repair");
let streaming_model = MockCompletionModel::from_stream_turns([
vec![
MockStreamEvent::tool_call("tc1", "default_api", json!({"x": 2, "y": 3})),
MockStreamEvent::final_response_with_total_tokens(0),
],
vec![
MockStreamEvent::text("the answer is 5"),
MockStreamEvent::final_response_with_total_tokens(0),
],
]);
let streaming_hook = RecordingHook::default();
let mut stream = AgentBuilder::new(streaming_model)
.tool(MockAddTool)
.build()
.prompt("add 2 and 3")
.max_turns(3)
.add_hook(streaming_hook.clone())
.add_hook(RepairInvalidToHook("add"))
.stream();
let mut final_response = None;
while let Some(item) = stream.next().await {
if let Ok(MultiTurnStreamItem::FinalResponse(resp)) =
item.map_err(|err| panic!("stream item errored: {err}"))
{
final_response = Some(resp);
}
}
let final_response = final_response.expect("stream should recover and yield a final response");
assert_eq!(blocking.output(), "the answer is 5");
assert_eq!(final_response.output(), blocking.output());
assert_eq!(
blocking_hook.shared_events(),
streaming_hook.shared_events()
);
assert!(
blocking_hook
.shared_events()
.contains(&StepEventKind::InvalidToolCall),
"the hook must observe the invalid tool call"
);
assert_eq!(blocking_hook.tool_results(), streaming_hook.tool_results());
assert_eq!(blocking_hook.tool_results(), vec!["5".to_string()]);
let blocking_messages = blocking.messages;
let streaming_messages = final_response.messages().to_vec();
assert_eq!(
serde_json::to_value(&blocking_messages).expect("serialize blocking"),
serde_json::to_value(&streaming_messages).expect("serialize streaming"),
);
}
#[derive(Clone)]
struct ScriptedToolCall {
id: &'static str,
name: &'static str,
args: serde_json::Value,
}
#[derive(Clone)]
enum ScriptedTurn {
Text(&'static str),
ToolCalls(Vec<ScriptedToolCall>),
}
#[derive(Clone, Copy)]
enum StreamShape {
Complete,
Chunked,
}
impl ScriptedTurn {
fn as_blocking_turn(&self) -> MockTurn {
match self {
ScriptedTurn::Text(text) => MockTurn::text(*text),
ScriptedTurn::ToolCalls(calls) => {
MockTurn::from_contents(calls.iter().map(|call| {
AssistantContent::ToolCall(MessageToolCall::from_wire(
call.id,
ToolFunction::new(
rig_core::message::ToolName::new(call.name.to_string())
.expect("tool name"),
call.args.clone(),
),
))
}))
}
}
}
fn as_stream_events(&self, shape: StreamShape) -> Vec<MockStreamEvent> {
let mut events = Vec::new();
match self {
ScriptedTurn::Text(text) => events.push(MockStreamEvent::text(*text)),
ScriptedTurn::ToolCalls(calls) => {
for call in calls {
if let StreamShape::Chunked = shape {
let args = serde_json::to_string(&call.args)
.expect("scripted args serialize to json");
events.push(MockStreamEvent::tool_call_name_delta(call.id, call.name));
events.push(MockStreamEvent::tool_call_arguments_delta(call.id, &args));
}
events.push(MockStreamEvent::tool_call(
call.id,
call.name,
call.args.clone(),
));
}
}
}
events.push(MockStreamEvent::final_response_with_total_tokens(0));
events
}
}
struct ParityOutcome {
output: String,
messages: Vec<Message>,
shared_events: Vec<StepEventKind>,
tool_results: Vec<String>,
}
async fn run_blocking_scenario(prompt: &'static str, turns: &[ScriptedTurn]) -> ParityOutcome {
let model = MockCompletionModel::from_turns(turns.iter().map(ScriptedTurn::as_blocking_turn));
let hook = RecordingHook::default();
let response = AgentBuilder::new(model)
.tool(MockAddTool)
.build()
.prompt(prompt)
.max_turns(8)
.add_hook(hook.clone())
.run()
.await
.expect("blocking scenario should succeed");
ParityOutcome {
output: response.output(),
messages: response.messages,
shared_events: hook.shared_events(),
tool_results: hook.tool_results(),
}
}
async fn run_streaming_scenario(
prompt: &'static str,
turns: &[ScriptedTurn],
shape: StreamShape,
) -> ParityOutcome {
let model = MockCompletionModel::from_stream_turns(
turns.iter().map(|turn| turn.as_stream_events(shape)),
);
let hook = RecordingHook::default();
let mut stream = AgentBuilder::new(model)
.tool(MockAddTool)
.build()
.prompt(prompt)
.max_turns(8)
.add_hook(hook.clone())
.stream();
let mut final_response = None;
while let Some(item) = stream.next().await {
if let Ok(MultiTurnStreamItem::FinalResponse(resp)) =
item.map_err(|err| panic!("stream item errored: {err}"))
{
final_response = Some(resp);
}
}
let final_response = final_response.expect("streaming scenario should yield a final response");
ParityOutcome {
output: final_response.output().to_string(),
messages: final_response.messages().to_vec(),
shared_events: hook.shared_events(),
tool_results: hook.tool_results(),
}
}
fn assert_outcomes_match(blocking: &ParityOutcome, streaming: &ParityOutcome, label: &str) {
assert_eq!(
blocking.output, streaming.output,
"{label}: final output diverged"
);
assert_eq!(
blocking.shared_events, streaming.shared_events,
"{label}: hook event sequence diverged"
);
assert_eq!(
blocking.tool_results, streaming.tool_results,
"{label}: tool-result content diverged"
);
assert_eq!(
serde_json::to_value(&blocking.messages).expect("serialize blocking"),
serde_json::to_value(&streaming.messages).expect("serialize streaming"),
"{label}: message history diverged"
);
}
async fn assert_run_stream_parity(prompt: &'static str, turns: &[ScriptedTurn]) {
let blocking = run_blocking_scenario(prompt, turns).await;
for (shape, label) in [
(StreamShape::Complete, "complete-stream"),
(StreamShape::Chunked, "chunked-stream"),
] {
let streaming = run_streaming_scenario(prompt, turns, shape).await;
assert_outcomes_match(&blocking, &streaming, label);
}
}
fn add_call(id: &'static str, x: i64, y: i64) -> ScriptedToolCall {
ScriptedToolCall {
id,
name: "add",
args: json!({ "x": x, "y": y }),
}
}
#[tokio::test]
async fn parity_text_only_run() {
assert_run_stream_parity("just say hi", &[ScriptedTurn::Text("hi there")]).await;
}
#[tokio::test]
async fn parity_single_tool_then_text() {
assert_run_stream_parity(
"add 2 and 3",
&[
ScriptedTurn::ToolCalls(vec![add_call("tc1", 2, 3)]),
ScriptedTurn::Text("the answer is 5"),
],
)
.await;
}
#[tokio::test]
async fn parity_multiple_tools_in_one_turn() {
assert_run_stream_parity(
"add two pairs",
&[
ScriptedTurn::ToolCalls(vec![add_call("tc1", 2, 3), add_call("tc2", 10, 20)]),
ScriptedTurn::Text("done"),
],
)
.await;
}
#[tokio::test]
async fn parity_multi_turn_sequential_tools() {
assert_run_stream_parity(
"chain two additions",
&[
ScriptedTurn::ToolCalls(vec![add_call("tc1", 1, 1)]),
ScriptedTurn::ToolCalls(vec![add_call("tc2", 2, 2)]),
ScriptedTurn::Text("chained"),
],
)
.await;
}
struct SkipInvalidHook(&'static str);
impl AgentHook for SkipInvalidHook {
async fn on_invalid_tool_call(
&self,
_ctx: &HookContext,
event: &InvalidToolCallContext,
) -> Option<InvalidToolCallAction> {
Some(if let _ = event {
InvalidToolCallAction::skip(self.0)
} else {
InvalidToolCallAction::fail()
})
}
}
#[tokio::test]
async fn invalid_tool_call_skip_parity_across_run_and_stream() {
let blocking_model = MockCompletionModel::from_turns([
MockTurn::tool_call("tc1", "default_api", json!({"x": 2, "y": 3})),
MockTurn::text("acknowledged"),
]);
let blocking_hook = RecordingHook::default();
let blocking = AgentBuilder::new(blocking_model)
.tool(MockAddTool)
.build()
.prompt("do the thing")
.max_turns(3)
.add_hook(blocking_hook.clone())
.add_hook(SkipInvalidHook("tool not permitted"))
.run()
.await
.expect("blocking run should recover via skip");
let streaming_model = MockCompletionModel::from_stream_turns([
vec![
MockStreamEvent::tool_call("tc1", "default_api", json!({"x": 2, "y": 3})),
MockStreamEvent::final_response_with_total_tokens(0),
],
vec![
MockStreamEvent::text("acknowledged"),
MockStreamEvent::final_response_with_total_tokens(0),
],
]);
let streaming_hook = RecordingHook::default();
let mut stream = AgentBuilder::new(streaming_model)
.tool(MockAddTool)
.build()
.prompt("do the thing")
.max_turns(3)
.add_hook(streaming_hook.clone())
.add_hook(SkipInvalidHook("tool not permitted"))
.stream();
let mut final_response = None;
while let Some(item) = stream.next().await {
if let Ok(MultiTurnStreamItem::FinalResponse(resp)) =
item.map_err(|err| panic!("stream item errored: {err}"))
{
final_response = Some(resp);
}
}
let final_response = final_response.expect("stream should recover and yield a final response");
assert_eq!(blocking.output(), "acknowledged");
assert_eq!(final_response.output(), blocking.output());
assert_eq!(
blocking_hook.shared_events(),
streaming_hook.shared_events()
);
assert!(
blocking_hook
.shared_events()
.contains(&StepEventKind::InvalidToolCall),
"the hook must observe the invalid tool call"
);
let blocking_messages = canonical_history(&blocking.messages);
let streaming_messages = canonical_history(final_response.messages());
assert_eq!(
serde_json::to_value(&blocking_messages).expect("serialize blocking"),
serde_json::to_value(&streaming_messages).expect("serialize streaming"),
);
assert!(
tool_result_text_in_history(&blocking_messages, "tool not permitted"),
"the verbatim invalid-tool skip reason must be the tool result content"
);
}
struct SkipToolCallHook(&'static str);
impl AgentHook for SkipToolCallHook {
async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
if event.tool_name().is_some() {
DispatchAction::skip(self.0)
} else {
DispatchAction::proceed()
}
}
}
#[tokio::test]
async fn valid_tool_call_skip_parity_across_run_and_stream() {
let turns = [
ScriptedTurn::ToolCalls(vec![add_call("tc1", 2, 3)]),
ScriptedTurn::Text("acknowledged"),
];
let blocking_model =
MockCompletionModel::from_turns(turns.iter().map(ScriptedTurn::as_blocking_turn));
let blocking_hook = RecordingHook::default();
let blocking = AgentBuilder::new(blocking_model)
.tool(MockAddTool)
.build()
.prompt("add 2 and 3")
.max_turns(3)
.add_hook(blocking_hook.clone())
.add_hook(SkipToolCallHook("skipped by policy"))
.run()
.await
.expect("blocking run should succeed with a skipped tool call");
let streaming_model = MockCompletionModel::from_stream_turns(
turns
.iter()
.map(|turn| turn.as_stream_events(StreamShape::Complete)),
);
let streaming_hook = RecordingHook::default();
let mut stream = AgentBuilder::new(streaming_model)
.tool(MockAddTool)
.build()
.prompt("add 2 and 3")
.max_turns(3)
.add_hook(streaming_hook.clone())
.add_hook(SkipToolCallHook("skipped by policy"))
.stream();
let mut final_response = None;
while let Some(item) = stream.next().await {
if let Ok(MultiTurnStreamItem::FinalResponse(resp)) =
item.map_err(|err| panic!("stream item errored: {err}"))
{
final_response = Some(resp);
}
}
let final_response = final_response.expect("stream should yield a final response");
assert_eq!(blocking.output(), "acknowledged");
assert_eq!(final_response.output(), blocking.output());
assert_eq!(
blocking_hook.shared_events(),
streaming_hook.shared_events()
);
assert_eq!(blocking_hook.tool_results(), streaming_hook.tool_results());
assert_eq!(
blocking_hook.tool_results(),
vec!["skipped by policy".to_string()],
"a skipped tool fires a ToolResult hook with the verbatim skip reason"
);
let blocking_messages = blocking.messages;
let streaming_messages = final_response.messages().to_vec();
assert_eq!(
serde_json::to_value(&blocking_messages).expect("serialize blocking"),
serde_json::to_value(&streaming_messages).expect("serialize streaming"),
);
assert!(
tool_result_text_in_history(&blocking_messages, "skipped by policy"),
"the verbatim skip reason must be the tool result content in the history"
);
}
struct RewriteToolArgsHook(serde_json::Value);
impl AgentHook for RewriteToolArgsHook {
async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
DispatchAction::rewrite_tool_args(event.kind, self.0.clone())
}
}
struct RenameBoundaryTool<const SAFE: bool>(Arc<AtomicU32>);
impl<const SAFE: bool> Tool for RenameBoundaryTool<SAFE> {
const NAME: &'static str = if SAFE {
"safe_target"
} else {
"original_target"
};
type Error = ToolExecutionError;
type Args = serde_json::Value;
type Output = String;
fn description(&self) -> String {
"Count dispatches to this target".into()
}
fn parameters(&self) -> serde_json::Value {
json!({"type": "object"})
}
async fn call(
&self,
_context: &mut ToolContext,
_args: Self::Args,
) -> Result<String, Self::Error> {
self.0.fetch_add(1, SeqCst);
Ok(Self::NAME.into())
}
}
struct RenameToolTargetHook;
impl AgentHook for RenameToolTargetHook {
async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
match event.kind {
rig_core::effect::EffectKind::ToolCall { args, .. } => {
DispatchAction::Patch(rig_core::effect::EffectKind::ToolCall {
name: RenameBoundaryTool::<true>::NAME.into(),
args: args.clone(),
})
}
_ => DispatchAction::Proceed,
}
}
}
#[derive(Clone, Default)]
struct ToolTargetPolicySpy(Arc<AtomicU32>);
impl AgentHook for ToolTargetPolicySpy {
async fn on_dispatch(&self, _: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
if event.tool_name().is_some() {
self.0.fetch_add(1, SeqCst);
}
DispatchAction::Proceed
}
}
async fn check_tool_target_patch_is_refused(streaming: bool) {
let original = Arc::new(AtomicU32::new(0));
let safe = Arc::new(AtomicU32::new(0));
let turns = [
ScriptedTurn::ToolCalls(vec![ScriptedToolCall {
id: "rename-call",
name: RenameBoundaryTool::<false>::NAME,
args: json!({}),
}]),
ScriptedTurn::Text("done"),
];
let model = if streaming {
MockCompletionModel::from_stream_turns(
turns
.iter()
.map(|turn| turn.as_stream_events(StreamShape::Complete)),
)
} else {
MockCompletionModel::from_turns(turns.iter().map(ScriptedTurn::as_blocking_turn))
};
let observed = RecordingHook::default();
let policy = ToolTargetPolicySpy::default();
let runner = AgentBuilder::new(model)
.tool(RenameBoundaryTool::<false>(original.clone()))
.tool(RenameBoundaryTool::<true>(safe.clone()))
.build()
.prompt("perform the requested operation")
.max_turns(3)
.add_hook(RenameToolTargetHook)
.add_hook(policy.clone())
.add_hook(observed.clone());
if streaming {
let mut stream = runner.stream();
let mut finished = false;
while let Some(item) = stream.next().await {
if let MultiTurnStreamItem::FinalResponse(response) =
item.expect("stream remains usable")
{
assert_eq!(response.output(), "done");
finished = true;
}
}
assert!(finished);
} else {
assert_eq!(
runner.run().await.expect("run remains usable").output(),
"done"
);
}
assert_eq!(
original.load(SeqCst),
0,
"a renamed call must not execute the original tool"
);
assert_eq!(
safe.load(SeqCst),
0,
"dispatch patches cannot authorize a new target"
);
assert_eq!(
policy.0.load(SeqCst),
0,
"later policy must not observe an invalid target patch"
);
assert!(
observed
.tool_results()
.iter()
.any(|result| result.contains("target")),
"the model must receive the refusal reason"
);
}
#[tokio::test]
async fn tool_target_patch_is_refused_on_run() {
check_tool_target_patch_is_refused(false).await;
}
#[tokio::test]
async fn tool_target_patch_is_refused_on_stream() {
check_tool_target_patch_is_refused(true).await;
}
#[derive(serde::Deserialize)]
struct FirstGenerationArgs {
old: String,
}
struct FirstGenerationTool(Arc<AtomicU32>);
impl Tool for FirstGenerationTool {
const NAME: &'static str = "generation_pinned";
type Error = rig::tool::ToolExecutionError;
type Args = FirstGenerationArgs;
type Output = String;
fn description(&self) -> String {
"first generation schema".to_string()
}
fn parameters(&self) -> serde_json::Value {
json!({
"type": "object",
"properties": {"old": {"type": "string"}},
"required": ["old"]
})
}
async fn call(
&self,
_context: &mut ToolContext,
args: Self::Args,
) -> Result<Self::Output, ToolExecutionError> {
self.0.fetch_add(1, SeqCst);
Ok(format!("first:{}", args.old))
}
}
#[derive(Clone)]
struct PausingScript {
inner: rig_core::test_utils::MockRuntime,
request_started: Arc<Notify>,
release_response: Arc<Notify>,
requests: Arc<AtomicU32>,
}
impl PausingScript {
fn model(
inner: MockCompletionModel,
) -> (rig_core::Model<MockScript, Self>, Arc<Notify>, Arc<Notify>) {
let request_started = Arc::new(Notify::new());
let release_response = Arc::new(Notify::new());
let script = Self {
inner: inner.transport,
request_started: request_started.clone(),
release_response: release_response.clone(),
requests: Arc::new(AtomicU32::new(0)),
};
(
rig_core::Model::new(inner.wire, script),
request_started,
release_response,
)
}
async fn inspect_and_pause(&self, request: &crate::completion::CompletionRequest) {
let request_index = self.requests.fetch_add(1, SeqCst);
let definition = request
.tools
.iter()
.find(|definition| definition.name == FirstGenerationTool::NAME)
.expect("generation tool must be advertised");
if request_index == 0 {
assert_eq!(definition.description, "first generation schema");
self.request_started.notify_one();
self.release_response.notified().await;
} else {
assert_eq!(definition.description, "second generation schema");
}
}
}
impl rig_core::driver::Transport<MockScript> for PausingScript {
fn send(
&self,
payload: crate::completion::CompletionRequest,
exchange: Exchange,
) -> Opening<rig_core::test_utils::MockFrame> {
let this = self.clone();
Opening::new(async move {
this.inspect_and_pause(&payload).await;
rig_core::driver::Transport::<MockScript>::send(&this.inner, payload, exchange).await
})
}
}
#[test]
fn one_hook_instance_attaches_to_distinct_completion_models() {
#[derive(Clone)]
struct ProviderIndependentHook;
impl AgentHook for ProviderIndependentHook {}
let hook = ProviderIndependentHook;
let _mock_agent = AgentBuilder::new(MockCompletionModel::from_turns([]))
.add_hook(hook.clone())
.build();
let (other_model, _, _) = PausingScript::model(MockCompletionModel::from_turns([]));
let _other_agent = AgentBuilder::new(other_model).add_hook(hook).build();
}
#[tokio::test]
async fn valid_tool_call_rewrite_args_parity_across_run_and_stream() {
let turns = [
ScriptedTurn::ToolCalls(vec![add_call("tc1", 2, 3)]),
ScriptedTurn::Text("acknowledged"),
];
let replacement = json!({"x": 2, "y": 40});
let blocking_model =
MockCompletionModel::from_turns(turns.iter().map(ScriptedTurn::as_blocking_turn));
let blocking_hook = RecordingHook::default();
let blocking = AgentBuilder::new(blocking_model)
.tool(MockAddTool)
.build()
.prompt("add 2 and 3")
.max_turns(3)
.add_hook(blocking_hook.clone())
.add_hook(RewriteToolArgsHook(replacement.clone()))
.run()
.await
.expect("blocking run should succeed with rewritten tool arguments");
let streaming_model = MockCompletionModel::from_stream_turns(
turns
.iter()
.map(|turn| turn.as_stream_events(StreamShape::Complete)),
);
let streaming_hook = RecordingHook::default();
let mut stream = AgentBuilder::new(streaming_model)
.tool(MockAddTool)
.build()
.prompt("add 2 and 3")
.max_turns(3)
.add_hook(streaming_hook.clone())
.add_hook(RewriteToolArgsHook(replacement))
.stream();
let mut final_response = None;
while let Some(item) = stream.next().await {
if let Ok(MultiTurnStreamItem::FinalResponse(resp)) =
item.map_err(|err| panic!("stream item errored: {err}"))
{
final_response = Some(resp);
}
}
let final_response = final_response.expect("stream should yield a final response");
assert_eq!(blocking_hook.tool_results(), vec!["42".to_string()]);
assert_eq!(blocking.output(), "acknowledged");
assert_eq!(final_response.output(), blocking.output());
assert_eq!(
blocking_hook.shared_events(),
streaming_hook.shared_events()
);
assert_eq!(blocking_hook.tool_results(), streaming_hook.tool_results());
}
struct RewriteToolResultHook(&'static str);
impl AgentHook for RewriteToolResultHook {
async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
OutcomeAction::rewrite_tool_result(&event, self.0)
}
}
#[tokio::test]
async fn valid_tool_result_rewrite_parity_across_run_and_stream() {
let turns = [
ScriptedTurn::ToolCalls(vec![add_call("tc1", 2, 3)]),
ScriptedTurn::Text("acknowledged"),
];
let blocking_model =
MockCompletionModel::from_turns(turns.iter().map(ScriptedTurn::as_blocking_turn));
let blocking_hook = RecordingHook::default();
let blocking = AgentBuilder::new(blocking_model)
.tool(MockAddTool)
.build()
.prompt("add 2 and 3")
.max_turns(3)
.add_hook(blocking_hook.clone())
.add_hook(RewriteToolResultHook("redacted-result"))
.run()
.await
.expect("blocking run should succeed with a rewritten tool result");
let streaming_model = MockCompletionModel::from_stream_turns(
turns
.iter()
.map(|turn| turn.as_stream_events(StreamShape::Complete)),
);
let streaming_hook = RecordingHook::default();
let mut stream = AgentBuilder::new(streaming_model)
.tool(MockAddTool)
.build()
.prompt("add 2 and 3")
.max_turns(3)
.add_hook(streaming_hook.clone())
.add_hook(RewriteToolResultHook("redacted-result"))
.stream();
let mut final_response = None;
while let Some(item) = stream.next().await {
if let Ok(MultiTurnStreamItem::FinalResponse(resp)) =
item.map_err(|err| panic!("stream item errored: {err}"))
{
final_response = Some(resp);
}
}
let final_response = final_response.expect("stream should yield a final response");
assert_eq!(blocking.output(), "acknowledged");
assert_eq!(final_response.output(), blocking.output());
assert_eq!(blocking_hook.tool_results(), vec!["5".to_string()]);
assert_eq!(blocking_hook.tool_results(), streaming_hook.tool_results());
let blocking_messages = blocking.messages;
let streaming_messages = final_response.messages().to_vec();
assert_eq!(
serde_json::to_value(&blocking_messages).expect("serialize blocking"),
serde_json::to_value(&streaming_messages).expect("serialize streaming"),
);
assert!(
tool_result_text_in_history(&blocking_messages, "redacted-result"),
"the model-visible tool result must be the hook's replacement"
);
assert!(
!tool_result_text_in_history(&blocking_messages, "5"),
"the tool's original output must not reach the model after a rewrite"
);
}
fn hook_doc(id: &str, text: &str) -> crate::completion::Document {
crate::completion::Document {
id: id.to_string(),
text: text.to_string(),
additional_props: Default::default(),
}
}
struct ExtraContextHook {
id: &'static str,
text: &'static str,
}
impl AgentHook for ExtraContextHook {
async fn on_completion_call(
&self,
_ctx: &HookContext,
event: CompletionCallEvent<'_>,
) -> CompletionCallAction {
if let CompletionCallEvent { .. } = event {
CompletionCallAction::patch(RequestPatch::new().context(hook_doc(self.id, self.text)))
} else {
CompletionCallAction::continue_run()
}
}
}
#[derive(Clone)]
struct RecordingContextIndex {
id: &'static str,
queries: Arc<Mutex<Vec<(String, u64)>>>,
}
impl VectorStoreIndex for RecordingContextIndex {
type Filter = Filter<serde_json::Value>;
async fn top_n<T: for<'a> Deserialize<'a> + WasmCompatSend>(
&self,
req: VectorSearchRequest,
) -> Result<Vec<VectorSearchResult<T>>, VectorStoreError> {
self.queries
.lock()
.expect("context query recorder lock")
.push((req.query().to_string(), req.samples()));
let value = serde_json::from_value(json!({ "source": self.id }))?;
Ok(vec![VectorSearchResult {
score: 1.0,
id: self.id.to_string(),
document: value,
}])
}
async fn top_n_ids(
&self,
_req: VectorSearchRequest,
) -> Result<Vec<VectorSearchIdResult>, VectorStoreError> {
Ok(vec![VectorSearchIdResult {
score: 1.0,
id: self.id.to_string(),
}])
}
}
struct FailingContextIndex;
impl VectorStoreIndex for FailingContextIndex {
type Filter = Filter<serde_json::Value>;
async fn top_n<T: for<'a> Deserialize<'a> + WasmCompatSend>(
&self,
_req: VectorSearchRequest,
) -> Result<Vec<VectorSearchResult<T>>, VectorStoreError> {
Err(VectorStoreError::datastore(std::io::Error::other(
"context index unavailable",
)))
}
async fn top_n_ids(
&self,
_req: VectorSearchRequest,
) -> Result<Vec<VectorSearchIdResult>, VectorStoreError> {
Err(VectorStoreError::datastore(std::io::Error::other(
"context index unavailable",
)))
}
}
fn one_text_stream_turn(text: &'static str) -> Vec<MockStreamEvent> {
vec![
MockStreamEvent::text(text),
MockStreamEvent::final_response_with_total_tokens(0),
]
}
use rig_core::test_utils::sent_documents;
#[tokio::test]
async fn extra_context_appears_after_static_context_on_both_surfaces() {
fn assert_docs(req: &crate::completion::CompletionRequest) {
let documents = sent_documents(req);
let ids: Vec<&str> = documents.iter().map(|(id, _)| id.as_str()).collect();
let static_pos = ids
.iter()
.position(|id| id.starts_with("static_doc"))
.expect("static context document present");
let extra_pos = ids
.iter()
.position(|id| *id == "hook-doc")
.expect("hook extra_context document present");
assert!(
static_pos < extra_pos,
"static context precedes hook extras: {ids:?}"
);
assert!(
documents.iter().any(|(_, text)| text == "injected"),
"the hook document's text is present"
);
}
let blocking_model = MockCompletionModel::from_turns([MockTurn::text("done")]);
let blocking_probe = blocking_model.clone();
AgentBuilder::new(blocking_model)
.context("static context text")
.add_hook(ExtraContextHook {
id: "hook-doc",
text: "injected",
})
.build()
.prompt("go")
.run()
.await
.expect("blocking run should succeed");
assert_docs(blocking_probe.requests().first().expect("one request"));
let streaming_model = MockCompletionModel::from_stream_turns([one_text_stream_turn("done")]);
let streaming_probe = streaming_model.clone();
let mut stream = AgentBuilder::new(streaming_model)
.context("static context text")
.add_hook(ExtraContextHook {
id: "hook-doc",
text: "injected",
})
.build()
.prompt("go")
.stream();
while let Some(item) = stream.next().await {
let _ = item.map_err(|err| panic!("stream item errored: {err}"));
}
assert_docs(streaming_probe.requests().first().expect("one request"));
}
#[tokio::test]
async fn dynamic_context_preserves_query_selection_formatting_and_order_on_both_surfaces() {
fn assert_documents(request: &crate::completion::CompletionRequest) {
let sent = sent_documents(request);
let documents = sent
.iter()
.map(|(id, text)| (id.as_str(), text.as_str()))
.collect::<Vec<_>>();
assert_eq!(
documents,
vec![
("static_doc_0", "static context"),
("blocking", "{\n \"source\": \"blocking\"\n}"),
]
);
}
let blocking_queries = Arc::new(Mutex::new(Vec::new()));
let blocking_model = MockCompletionModel::from_turns([MockTurn::text("done")]);
let blocking_probe = blocking_model.clone();
AgentBuilder::new(blocking_model)
.context("static context")
.dynamic_context(
2,
RecordingContextIndex {
id: "blocking",
queries: blocking_queries.clone(),
},
)
.build()
.prompt("current blocking query")
.history(vec![Message::user("ignored history query")])
.run()
.await
.expect("blocking dynamic-context run should succeed");
assert_eq!(
*blocking_queries.lock().expect("blocking queries"),
vec![("current blocking query".to_string(), 2)]
);
assert_documents(blocking_probe.requests().first().expect("one request"));
let streaming_queries = Arc::new(Mutex::new(Vec::new()));
let streaming_model = MockCompletionModel::from_stream_turns([one_text_stream_turn("done")]);
let streaming_probe = streaming_model.clone();
let mut stream = AgentBuilder::new(streaming_model)
.dynamic_context(
3,
RecordingContextIndex {
id: "streaming",
queries: streaming_queries.clone(),
},
)
.build()
.prompt(Message::User {
content: vec![UserContent::image_url(
"https://example.com/prompt.png",
None,
None,
)],
})
.history(vec![
Message::user("older history query"),
Message::user("latest history query"),
])
.stream();
while let Some(item) = stream.next().await {
item.expect("streaming dynamic-context run should succeed");
}
assert_eq!(
*streaming_queries.lock().expect("streaming queries"),
vec![("latest history query".to_string(), 3)]
);
let streaming_requests = streaming_probe.requests();
let request = streaming_requests.first().expect("one request");
assert_eq!(
sent_documents(request),
[(
"streaming".to_owned(),
"{\n \"source\": \"streaming\"\n}".to_owned()
)]
);
}
#[tokio::test]
async fn dynamic_context_and_application_hooks_follow_registration_order() {
let queries = Arc::new(Mutex::new(Vec::new()));
let model = MockCompletionModel::from_turns([MockTurn::text("done")]);
let probe = model.clone();
AgentBuilder::new(model)
.context("static")
.add_hook(ExtraContextHook {
id: "before",
text: "before dynamic context",
})
.dynamic_context(
1,
RecordingContextIndex {
id: "first",
queries: queries.clone(),
},
)
.add_hook(ExtraContextHook {
id: "between",
text: "between dynamic contexts",
})
.dynamic_context(
2,
RecordingContextIndex {
id: "second",
queries: queries.clone(),
},
)
.add_hook(ExtraContextHook {
id: "after",
text: "after dynamic context",
})
.build()
.prompt("query")
.run()
.await
.expect("run should succeed");
assert_eq!(
sent_documents(&probe.requests()[0])
.into_iter()
.map(|(id, _)| id)
.collect::<Vec<_>>(),
vec![
"static_doc_0",
"before",
"first",
"between",
"second",
"after",
]
);
assert_eq!(
*queries.lock().expect("context queries"),
vec![("query".to_string(), 1), ("query".to_string(), 2)]
);
let skipped_queries = Arc::new(Mutex::new(Vec::new()));
let error = AgentBuilder::new(MockCompletionModel::from_turns([MockTurn::text("unused")]))
.add_hook(TerminateOn(TerminatePoint::CompletionCall))
.dynamic_context(
1,
RecordingContextIndex {
id: "skipped",
queries: skipped_queries.clone(),
},
)
.build()
.prompt("query")
.run()
.await
.expect_err("an earlier stop hook should terminate before retrieval");
assert!(matches!(error, PromptError::Cancelled { .. }));
assert!(skipped_queries.lock().expect("skipped queries").is_empty());
}
#[tokio::test]
async fn dynamic_context_retrieval_failure_stops_before_provider_io_on_both_surfaces() {
let blocking_model = MockCompletionModel::from_turns([MockTurn::text("unused")]);
let blocking_probe = blocking_model.clone();
let error = AgentBuilder::new(blocking_model)
.dynamic_context(1, FailingContextIndex)
.build()
.prompt("retrieve this")
.run()
.await
.expect_err("failed retrieval should stop the run");
assert!(matches!(
error,
PromptError::Cancelled { reason, .. }
if reason.contains("context index unavailable")
));
assert_eq!(blocking_probe.request_count(), 0);
let streaming_model = MockCompletionModel::from_stream_turns([one_text_stream_turn("unused")]);
let streaming_probe = streaming_model.clone();
let mut stream = AgentBuilder::new(streaming_model)
.dynamic_context(1, FailingContextIndex)
.build()
.prompt("retrieve this")
.stream();
let error = stream
.next()
.await
.expect("stream should report retrieval failure")
.expect_err("failed retrieval should stop the stream");
assert!(matches!(
error,
PromptError::Cancelled { reason, .. } if reason.contains("context index unavailable")
));
assert_eq!(streaming_probe.request_count(), 0);
}
#[tokio::test]
async fn history_patch_changes_sent_messages_not_transcript_on_both_surfaces() {
const SENTINEL: &str = "COMPACTED-HISTORY-SENTINEL";
struct HistoryOverrideHook;
impl AgentHook for HistoryOverrideHook {
async fn on_completion_call(
&self,
_ctx: &HookContext,
event: CompletionCallEvent<'_>,
) -> CompletionCallAction {
if let CompletionCallEvent { .. } = event {
CompletionCallAction::patch(RequestPatch::new().history([Message::user(SENTINEL)]))
} else {
CompletionCallAction::continue_run()
}
}
}
fn request_has_sentinel(req: &crate::completion::CompletionRequest) -> bool {
req.chat_history.iter().any(|m| match m {
Message::User { content } => content
.iter()
.any(|c| matches!(c, UserContent::Text(text) if text.text.contains(SENTINEL))),
_ => false,
})
}
fn messages_have_sentinel(messages: &[Message]) -> bool {
messages.iter().any(|m| match m {
Message::User { content } => content
.iter()
.any(|c| matches!(c, UserContent::Text(text) if text.text.contains(SENTINEL))),
_ => false,
})
}
let blocking_model = MockCompletionModel::from_turns([MockTurn::text("done")]);
let blocking_probe = blocking_model.clone();
let blocking = AgentBuilder::new(blocking_model)
.add_hook(HistoryOverrideHook)
.build()
.prompt("real prompt")
.run()
.await
.expect("blocking run should succeed");
assert!(
request_has_sentinel(blocking_probe.requests().first().expect("one request")),
"the overridden history reaches the provider"
);
assert!(
!messages_have_sentinel(&blocking.messages),
"the persisted transcript is untouched by the per-turn history override"
);
let streaming_model = MockCompletionModel::from_stream_turns([one_text_stream_turn("done")]);
let streaming_probe = streaming_model.clone();
let stream = AgentBuilder::new(streaming_model)
.add_hook(HistoryOverrideHook)
.build()
.prompt("real prompt")
.stream();
let final_response = drive_to_final_response(stream).await;
assert!(
request_has_sentinel(streaming_probe.requests().first().expect("one request")),
"the overridden history reaches the provider on the streaming surface too"
);
assert!(
!messages_have_sentinel(final_response.messages()),
"the persisted transcript is untouched by the per-turn history override on \
the streaming surface too"
);
}
#[tokio::test]
async fn chained_rewrites_compose_across_hooks() {
struct SetArg {
key: &'static str,
value: i64,
}
impl AgentHook for SetArg {
async fn on_dispatch(
&self,
_ctx: &HookContext,
event: DispatchEvent<'_>,
) -> DispatchAction {
if let Some(args) = event.tool_args() {
let mut parsed: serde_json::Value =
serde_json::from_str(args).unwrap_or_else(|_| json!({}));
parsed[self.key] = json!(self.value);
DispatchAction::rewrite_tool_args(event.kind, parsed)
} else {
DispatchAction::proceed()
}
}
}
struct WrapResult(&'static str);
impl AgentHook for WrapResult {
async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
if let Some(result) = event.tool_result() {
OutcomeAction::rewrite_tool_result(
&event,
format!("{}({})", self.0, result.output().render()),
)
} else {
OutcomeAction::proceed()
}
}
}
let recorder = RecordingHook::default();
let blocking = AgentBuilder::new(blocking_model())
.tool(MockAddTool)
.add_hook(SetArg {
key: "y",
value: 40,
})
.add_hook(SetArg {
key: "x",
value: 100,
})
.add_hook(WrapResult("A"))
.add_hook(WrapResult("B"))
.add_hook(recorder.clone())
.build()
.prompt("add 2 and 3")
.max_turns(3)
.run()
.await
.expect("blocking run should succeed");
assert_eq!(blocking.output(), "the answer is 5");
assert_eq!(
recorder.tool_results(),
vec!["B(A(140))".to_string()],
"arg rewrites compose (100+40=140) and result rewrites nest B(A(...))"
);
let stream_recorder = RecordingHook::default();
let mut stream = AgentBuilder::new(streaming_model())
.tool(MockAddTool)
.add_hook(SetArg {
key: "y",
value: 40,
})
.add_hook(SetArg {
key: "x",
value: 100,
})
.add_hook(WrapResult("A"))
.add_hook(WrapResult("B"))
.add_hook(stream_recorder.clone())
.build()
.prompt("add 2 and 3")
.max_turns(3)
.stream();
while let Some(item) = stream.next().await {
let _ = item.map_err(|err| panic!("stream item errored: {err}"));
}
assert_eq!(
stream_recorder.tool_results(),
vec!["B(A(140))".to_string()],
"chained rewrites compose identically on the streaming surface"
);
}
#[derive(serde::Deserialize, schemars::JsonSchema)]
#[allow(dead_code)]
struct Answer {
answer: String,
}
struct FinalResultTool;
impl Tool for FinalResultTool {
const NAME: &'static str = "final_result";
type Error = MockToolError;
type Args = serde_json::Value;
type Output = String;
fn description(&self) -> String {
"A real tool sharing the default output-tool name".to_string()
}
fn parameters(&self) -> serde_json::Value {
json!({ "type": "object", "properties": {} })
}
async fn call(
&self,
_context: &mut ToolContext,
_args: Self::Args,
) -> Result<Self::Output, Self::Error> {
Ok("real final_result output".to_string())
}
}
#[derive(Clone)]
struct RegisterLateFinalResultTool {
handle: ToolServerHandle,
second_turn_patch: Option<RequestPatch>,
}
impl AgentHook for RegisterLateFinalResultTool {
async fn on_model_turn_finished(
&self,
ctx: &HookContext,
_event: ModelTurnFinished<'_>,
) -> ModelTurnAction {
if ctx.turn() == 1 {
self.handle.add_tool(FinalResultTool);
}
ModelTurnAction::continue_run()
}
async fn on_completion_call(
&self,
ctx: &HookContext,
_event: CompletionCallEvent<'_>,
) -> CompletionCallAction {
if ctx.turn() == 2
&& let Some(patch) = &self.second_turn_patch
{
return CompletionCallAction::patch(patch.clone());
}
CompletionCallAction::continue_run()
}
}
fn assert_structured_output_collision_error(message: &str) {
assert!(
message.contains("final_result"),
"error should name the conflicting tool: {message}"
);
assert!(
message.contains("structured-output") && message.contains("reserved"),
"error should explain the structured-output reservation: {message}"
);
assert!(
message.contains("rename or remove"),
"error should provide an actionable resolution: {message}"
);
}
#[tokio::test]
async fn late_output_tool_collision_is_checked_after_active_tools_filtering() {
let handle = ToolServer::new().tool(MockAddTool).run();
let model = MockCompletionModel::from_turns([
MockTurn::tool_call("add-1", "add", json!({ "x": 1, "y": 2 })),
MockTurn::tool_call("add-2", "add", json!({ "x": 3, "y": 4 })),
MockTurn::tool_call(
"shadowed",
"final_result",
json!({ "answer": "wrongly finalized" }),
),
]);
let probe = model.clone();
let err = AgentBuilder::new(model)
.tool_server_handle(handle.clone())
.output_schema::<Answer>()
.output_mode(OutputMode::Tool)
.add_hook(RegisterLateFinalResultTool {
handle,
second_turn_patch: Some(RequestPatch::new().active_tools(["add"])),
})
.build()
.prompt("go")
.max_turns(4)
.run()
.await
.expect_err("the exposed third-turn collision should fail locally");
assert_eq!(
probe.request_count(),
2,
"the filtered second turn may run, but the exposed third turn may not"
);
let requests = probe.requests();
let second_turn_names = requests[1]
.tools
.iter()
.map(|tool| tool.name.as_str())
.collect::<Vec<_>>();
assert_eq!(second_turn_names.len(), 2);
for expected in ["add", "final_result"] {
assert_eq!(
second_turn_names
.iter()
.filter(|name| **name == expected)
.count(),
1,
"the second request should advertise `{expected}` exactly once: \
{second_turn_names:?}"
);
}
assert_structured_output_collision_error(&err.to_string());
}
struct ActiveToolsAddOnly;
impl AgentHook for ActiveToolsAddOnly {
async fn on_completion_call(
&self,
_ctx: &HookContext,
event: CompletionCallEvent<'_>,
) -> CompletionCallAction {
if let CompletionCallEvent { .. } = event {
CompletionCallAction::patch(RequestPatch::new().active_tools(["add"]))
} else {
CompletionCallAction::continue_run()
}
}
}
#[tokio::test]
async fn active_tools_filter_does_not_let_output_tool_collide_with_a_filtered_real_tool() {
let model = MockCompletionModel::from_turns([MockTurn::tool_call(
"out1",
"final_result_1",
json!({ "answer": "done" }),
)]);
let probe = model.clone();
let response = AgentBuilder::new(model)
.tool(MockAddTool)
.tool(FinalResultTool)
.output_schema::<Answer>()
.output_mode(OutputMode::Tool)
.add_hook(ActiveToolsAddOnly)
.build()
.prompt("go")
.max_turns(2)
.run()
.await
.expect("run should finalize via the picked output tool `final_result_1`");
assert!(
response.output().contains("done"),
"the intercepted output-tool call should produce the structured result, \
got {:?}",
response.output()
);
let requests = probe.requests();
assert!(
!requests.is_empty(),
"the first model request should be captured"
);
let tool_names: Vec<&str> = requests[0].tools.iter().map(|t| t.name.as_str()).collect();
assert!(
tool_names.contains(&"add"),
"active_tools keeps `add` advertised, saw {tool_names:?}"
);
assert!(
tool_names.contains(&"final_result_1"),
"the synthetic output tool must avoid the filtered real `final_result` name, \
saw {tool_names:?}"
);
assert!(
!tool_names.contains(&"final_result"),
"the real `final_result` is filtered out and the output tool must not reuse \
its name, saw {tool_names:?}"
);
}
#[tokio::test]
async fn output_tool_finalization_emits_no_complete_tool_call_stream_item() {
let mut stream = AgentBuilder::new(MockCompletionModel::from_stream_turns([vec![
MockStreamEvent::tool_call("out1", "final_result", json!({ "answer": "done" })),
MockStreamEvent::final_response_with_total_tokens(0),
]]))
.output_schema::<Answer>()
.output_mode(OutputMode::Tool)
.build()
.prompt("go")
.max_turns(2)
.stream();
let mut saw_complete_output_tool_call = false;
let mut final_has_output = false;
while let Some(item) = stream.next().await {
match item.expect("stream item") {
MultiTurnStreamItem::ToolCall { tool_call, .. }
if tool_call.function.name == "final_result" =>
{
saw_complete_output_tool_call = true;
}
MultiTurnStreamItem::FinalResponse(res) => {
final_has_output = res.output().contains("done");
}
_ => {}
}
}
assert!(
!saw_complete_output_tool_call,
"the output-tool call finalizes the run, so no complete \
StreamAssistantItem::ToolCall item must be emitted for it"
);
assert!(
final_has_output,
"the structured output must be surfaced via the FinalResponse"
);
}
enum Decision {
Approve,
Deny(&'static str),
Edit(serde_json::Value),
Abort(&'static str),
}
#[derive(Clone)]
struct HumanApprovalHook {
decisions: Arc<Mutex<std::collections::VecDeque<Decision>>>,
reviewed: Arc<Mutex<Vec<String>>>,
}
impl HumanApprovalHook {
fn new(decisions: impl IntoIterator<Item = Decision>) -> Self {
Self {
decisions: Arc::new(Mutex::new(decisions.into_iter().collect())),
reviewed: Arc::new(Mutex::new(Vec::new())),
}
}
fn reviewed(&self) -> Vec<String> {
self.reviewed.lock().unwrap().clone()
}
}
impl AgentHook for HumanApprovalHook {
async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
let (Some(tool_name), Some(args)) = (event.tool_name(), event.tool_args()) else {
return DispatchAction::proceed();
};
self.reviewed
.lock()
.unwrap()
.push(format!("{tool_name}({args})"));
let decision = self.decisions.lock().unwrap().pop_front();
match decision {
Some(Decision::Approve) => DispatchAction::proceed(),
Some(Decision::Deny(reason)) => DispatchAction::skip(reason),
Some(Decision::Edit(args)) => DispatchAction::rewrite_tool_args(event.kind, args),
Some(Decision::Abort(reason)) => DispatchAction::stop(reason),
None => DispatchAction::skip("denied: no scripted decision (fail-closed)"),
}
}
}
#[tokio::test]
async fn human_in_the_loop_approve_deny_edit_parity_across_run_and_stream() {
let turns = [
ScriptedTurn::ToolCalls(vec![
add_call("tc1", 2, 3), add_call("tc2", 10, 20), add_call("tc3", 1, 1), ]),
ScriptedTurn::Text("done"),
];
let denial = "denied by reviewer: amount too large";
let decisions = || {
vec![
Decision::Approve,
Decision::Deny(denial),
Decision::Edit(json!({"x": 1, "y": 100})),
]
};
let blocking_model =
MockCompletionModel::from_turns(turns.iter().map(ScriptedTurn::as_blocking_turn));
let blocking_recorder = RecordingHook::default();
let blocking_approver = HumanApprovalHook::new(decisions());
let blocking = AgentBuilder::new(blocking_model)
.tool(MockAddTool)
.build()
.prompt("carry out the plan")
.max_turns(3)
.add_hook(blocking_recorder.clone())
.add_hook(blocking_approver.clone())
.run()
.await
.expect("blocking HITL run should succeed");
let streaming_model = MockCompletionModel::from_stream_turns(
turns
.iter()
.map(|turn| turn.as_stream_events(StreamShape::Complete)),
);
let streaming_recorder = RecordingHook::default();
let streaming_approver = HumanApprovalHook::new(decisions());
let mut stream = AgentBuilder::new(streaming_model)
.tool(MockAddTool)
.build()
.prompt("carry out the plan")
.max_turns(3)
.add_hook(streaming_recorder.clone())
.add_hook(streaming_approver.clone())
.stream();
let mut final_response = None;
while let Some(item) = stream.next().await {
if let Ok(MultiTurnStreamItem::FinalResponse(resp)) =
item.map_err(|err| panic!("stream item errored: {err}"))
{
final_response = Some(resp);
}
}
let final_response = final_response.expect("stream should yield a final response");
assert_eq!(
blocking_recorder.tool_results(),
vec![
"5".to_string(),
"denied by reviewer: amount too large".to_string(),
"101".to_string()
]
);
assert_eq!(
blocking_recorder.tool_results(),
streaming_recorder.tool_results()
);
assert!(
!blocking_recorder.tool_results().contains(&"30".to_string()),
"the denied call must not have executed"
);
let reviewed = blocking_approver.reviewed();
assert_eq!(reviewed.len(), 3);
assert_eq!(reviewed, streaming_approver.reviewed());
assert!(
reviewed[0].contains('2') && reviewed[0].contains('3'),
"first reviewed call should be add(2, 3): {reviewed:?}"
);
assert!(
reviewed[1].contains("10") && reviewed[1].contains("20"),
"the denied (second) call should be add(10, 20): {reviewed:?}"
);
assert_eq!(blocking.output(), "done");
assert_eq!(final_response.output(), blocking.output());
assert_eq!(
blocking_recorder.shared_events(),
streaming_recorder.shared_events()
);
let blocking_messages = blocking.messages;
let streaming_messages = final_response.messages().to_vec();
assert_eq!(
serde_json::to_value(&blocking_messages).expect("serialize blocking"),
serde_json::to_value(&streaming_messages).expect("serialize streaming"),
);
assert!(
tool_result_text_in_history(&blocking_messages, denial),
"the denial reason must be the denied call's tool result in the history"
);
assert!(
tool_result_json_in_history(&blocking_messages, &json!(101)),
"the edited call must have executed with the rewritten arguments"
);
}
#[tokio::test]
async fn human_in_the_loop_abort_terminates_the_run() {
let turns = [
ScriptedTurn::ToolCalls(vec![add_call("tc1", 2, 3)]),
ScriptedTurn::Text("unreachable"),
];
const ABORT_REASON: &str = "aborted by the human reviewer";
let blocking_model =
MockCompletionModel::from_turns(turns.iter().map(ScriptedTurn::as_blocking_turn));
let err = AgentBuilder::new(blocking_model)
.tool(MockAddTool)
.build()
.prompt("do the sensitive thing")
.max_turns(3)
.add_hook(HumanApprovalHook::new([Decision::Abort(ABORT_REASON)]))
.run()
.await
.expect_err("an aborted tool call should terminate the blocking run");
assert!(
format!("{err}").contains(ABORT_REASON),
"the abort reason should surface in the blocking error, got: {err}"
);
let streaming_model = MockCompletionModel::from_stream_turns(
turns
.iter()
.map(|turn| turn.as_stream_events(StreamShape::Complete)),
);
let mut stream = AgentBuilder::new(streaming_model)
.tool(MockAddTool)
.build()
.prompt("do the sensitive thing")
.max_turns(3)
.add_hook(HumanApprovalHook::new([Decision::Abort(ABORT_REASON)]))
.stream();
let mut stream_error = None;
while let Some(item) = stream.next().await {
match item {
Err(err) => stream_error = Some(format!("{err}")),
Ok(MultiTurnStreamItem::FinalResponse(resp)) => {
panic!("aborted stream must not finalize, got: {}", resp.output())
}
Ok(_) => {}
}
}
let stream_error = stream_error.expect("an aborted tool call should error the stream");
assert!(
stream_error.contains(ABORT_REASON),
"the abort reason should surface in the streaming error, got: {stream_error}"
);
}
#[derive(Clone)]
struct PolicyHook {
auto_approve: std::collections::HashSet<&'static str>,
evaluated: Arc<Mutex<Vec<String>>>,
cache: Arc<Mutex<std::collections::HashMap<String, bool>>>,
}
impl PolicyHook {
fn new(auto_approve: impl IntoIterator<Item = &'static str>) -> Self {
Self {
auto_approve: auto_approve.into_iter().collect(),
evaluated: Arc::new(Mutex::new(Vec::new())),
cache: Arc::new(Mutex::new(std::collections::HashMap::new())),
}
}
fn evaluated(&self) -> Vec<String> {
self.evaluated.lock().unwrap().clone()
}
}
impl AgentHook for PolicyHook {
async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
let Some(tool_name) = event.tool_name() else {
return DispatchAction::proceed();
};
let cached = self.cache.lock().unwrap().get(tool_name).copied();
let approved = match cached {
Some(decision) => decision, None => {
self.evaluated.lock().unwrap().push(tool_name.to_string());
let decision = self.auto_approve.contains(tool_name);
self.cache
.lock()
.unwrap()
.insert(tool_name.to_string(), decision);
decision
}
};
if approved {
DispatchAction::proceed()
} else {
DispatchAction::skip(format!("denied by policy: `{tool_name}` not allowed"))
}
}
}
#[tokio::test]
async fn approval_policy_allow_list_with_sticky_decisions() {
let turns = [
ScriptedTurn::ToolCalls(vec![
add_call("c1", 2, 3),
ScriptedToolCall {
id: "c2",
name: "subtract",
args: json!({ "x": 10, "y": 4 }),
},
add_call("c3", 2, 3),
]),
ScriptedTurn::Text("done"),
];
let model = MockCompletionModel::from_turns(turns.iter().map(ScriptedTurn::as_blocking_turn));
let recorder = RecordingHook::default();
let policy = PolicyHook::new(["add"]);
let out = AgentBuilder::new(model)
.tool(MockAddTool)
.tool(MockSubtractTool)
.build()
.prompt("go")
.max_turns(3)
.add_hook(recorder.clone())
.add_hook(policy.clone())
.run()
.await
.expect("policy run should succeed");
assert_eq!(out.output(), "done");
assert_eq!(
recorder.tool_results(),
vec![
"5".to_string(),
"denied by policy: `subtract` not allowed".to_string(),
"5".to_string()
]
);
assert_eq!(
policy.evaluated(),
vec!["add".to_string(), "subtract".to_string()]
);
let messages = out.messages;
assert!(
tool_result_text_in_history(&messages, "denied by policy: `subtract` not allowed"),
"the policy denial reason must reach the model as the subtract tool result"
);
}
static NEXT_RESPONSE_RETRY_HOOK_ID: AtomicU64 = AtomicU64::new(1);
#[derive(Clone, Default)]
struct ResponseRetryAttempts(HashMap<u64, usize>);
#[derive(Clone)]
enum TestRetryMode {
Repeat,
Feedback(&'static str),
}
#[derive(Clone)]
struct BoundedResponseRetry {
id: u64,
rejected_text: &'static str,
max_retries: usize,
mode: TestRetryMode,
}
impl BoundedResponseRetry {
fn new(rejected_text: &'static str, max_retries: usize, mode: TestRetryMode) -> Self {
Self {
id: NEXT_RESPONSE_RETRY_HOOK_ID.fetch_add(1, SeqCst),
rejected_text,
max_retries,
mode,
}
}
}
impl AgentHook for BoundedResponseRetry {
async fn on_model_turn_finished(
&self,
ctx: &HookContext,
event: ModelTurnFinished<'_>,
) -> ModelTurnAction {
let has_rejected_text = event.content.iter().any(
|content| matches!(content, AssistantContent::Text(text) if text.text == self.rejected_text),
);
let rejected =
has_rejected_text || (self.rejected_text.is_empty() && event.content.is_empty());
if !rejected {
return ModelTurnAction::continue_run();
}
let attempt = ctx
.scratchpad()
.update::<ResponseRetryAttempts, _>(|attempts| {
let attempt = attempts.0.entry(self.id).or_default();
*attempt += 1;
*attempt
});
if attempt > self.max_retries {
return ModelTurnAction::stop(format!(
"response retry limit ({}) exceeded",
self.max_retries
));
}
match self.mode {
TestRetryMode::Repeat => ModelTurnAction::repeat(),
TestRetryMode::Feedback(feedback) => ModelTurnAction::retry_with_feedback(feedback),
}
}
}
fn retry_usage(input_tokens: u64, output_tokens: u64) -> Usage {
Usage::new()
.input_tokens(input_tokens)
.output_tokens(output_tokens)
.total_tokens(input_tokens + output_tokens)
}
#[tokio::test]
async fn blocking_empty_feedback_retry_omits_empty_assistant_history() {
let first_usage = retry_usage(5, 1);
let second_usage = retry_usage(7, 2);
let model = MockCompletionModel::from_turns([
MockTurn::text("").with_usage(first_usage),
MockTurn::text("accepted").with_usage(second_usage),
]);
let response = AgentBuilder::new(model.clone())
.add_hook(BoundedResponseRetry::new(
"",
1,
TestRetryMode::Feedback("provide an answer"),
))
.build()
.prompt("question")
.max_turns(2)
.run()
.await
.expect("feedback retry should recover from an empty turn");
assert_eq!(response.output(), "accepted");
assert_eq!(response.usage, first_usage + second_usage);
assert_eq!(response.completion_calls.len(), 2);
assert_eq!(
response.messages,
vec![
Message::user("question"),
Message::user("provide an answer"),
mock_reply("accepted"),
]
);
assert_eq!(
model.requests()[1].chat_history.clone(),
vec![Message::User {
content: vec![
UserContent::text("question"),
UserContent::text("provide an answer")
],
}],
"the retry request holds no empty assistant message, and the user messages it separated become one"
);
}
struct AlwaysRepeatModelTurn;
impl AgentHook for AlwaysRepeatModelTurn {
async fn on_model_turn_finished(
&self,
_ctx: &HookContext,
_event: ModelTurnFinished<'_>,
) -> ModelTurnAction {
ModelTurnAction::repeat()
}
}
#[tokio::test]
async fn model_turn_retry_rejects_tool_turn_before_tool_hooks_or_execution() {
let recorder = RecordingHook::default();
let executions = Arc::new(AtomicU32::new(0));
let err = AgentBuilder::new(MockCompletionModel::from_turns([MockTurn::tool_call(
"tc1",
"add",
json!({"x": 1, "y": 2}),
)]))
.tool(CountingAddTool {
calls: executions.clone(),
})
.add_hook(recorder.clone())
.add_hook(AlwaysRepeatModelTurn)
.build()
.prompt("add")
.max_turns(2)
.run()
.await
.expect_err("tool-bearing retry must fail closed");
let PromptError::Cancelled {
chat_history,
reason,
} = err
else {
panic!("tool-bearing retry should return Cancelled");
};
assert!(reason.contains("tool-bearing model turns"));
assert!(reason.contains("tool-call hooks"));
assert_eq!(chat_history, vec![Message::user("add")]);
assert_eq!(recorder.count(StepEventKind::ToolDispatch), 0);
assert_eq!(executions.load(SeqCst), 0);
}
#[tokio::test]
async fn streaming_model_turn_retry_rejects_tool_turn_without_committed_execution() {
let recorder = RecordingHook::default();
let executions = Arc::new(AtomicU32::new(0));
let mut stream = AgentBuilder::new(MockCompletionModel::from_stream_turns([[
MockStreamEvent::tool_call_name_delta("tc1", "add"),
MockStreamEvent::tool_call_arguments_delta("tc1", r#"{"x":1,"y":2}"#),
MockStreamEvent::tool_call("tc1", "add", json!({"x": 1, "y": 2})),
MockStreamEvent::final_response_with_default_usage(),
]]))
.tool(CountingAddTool {
calls: executions.clone(),
})
.add_hook(recorder.clone())
.add_hook(AlwaysRepeatModelTurn)
.build()
.prompt("add")
.max_turns(2)
.stream();
let mut execution_commits = 0;
let mut tool_results = 0;
let mut completion_calls = 0;
let mut agent_finals = 0;
let mut retry_markers = 0;
let mut error = None;
while let Some(item) = stream.next().await {
match item {
Ok(MultiTurnStreamItem::ToolExecutionCommitted { .. }) => execution_commits += 1,
Ok(MultiTurnStreamItem::ToolResult { .. }) => tool_results += 1,
Ok(MultiTurnStreamItem::CompletionCall(_)) => completion_calls += 1,
Ok(MultiTurnStreamItem::FinalResponse(_)) => agent_finals += 1,
Ok(MultiTurnStreamItem::ModelTurnRetried { .. }) => retry_markers += 1,
Ok(_) => {}
Err(err) => error = Some(err),
}
}
let Some(error) = error else {
panic!("tool-bearing streaming retry should return Cancelled");
};
let PromptError::Cancelled {
chat_history,
reason,
} = error
else {
panic!("tool-bearing streaming retry should return Cancelled");
};
assert!(reason.contains("tool-bearing model turns"));
assert!(reason.contains("tool-call hooks"));
assert_eq!(chat_history, &[Message::user("add")]);
assert_eq!(execution_commits, 0);
assert_eq!(tool_results, 0);
assert_eq!(completion_calls, 1);
assert_eq!(agent_finals, 0);
assert_eq!(retry_markers, 0);
assert_eq!(recorder.count(StepEventKind::ToolDispatch), 0);
assert_eq!(executions.load(SeqCst), 0);
}
#[derive(Clone)]
struct BarrierResponseRetry {
inner: BoundedResponseRetry,
barrier: Arc<Barrier>,
}
impl AgentHook for BarrierResponseRetry {
async fn on_model_turn_finished(
&self,
ctx: &HookContext,
event: ModelTurnFinished<'_>,
) -> ModelTurnAction {
let rejected = event.content.iter().any(
|content| matches!(content, AssistantContent::Text(text) if text.text == "rejected"),
);
if rejected {
self.barrier.wait().await;
}
self.inner.on_model_turn_finished(ctx, event).await
}
}
#[tokio::test]
async fn concurrent_runs_of_same_agent_have_independent_retry_budgets() {
let hook = BarrierResponseRetry {
inner: BoundedResponseRetry::new("rejected", 1, TestRetryMode::Repeat),
barrier: Arc::new(Barrier::new(2)),
};
let agent = AgentBuilder::new(MockCompletionModel::from_turns([
MockTurn::text("rejected"),
MockTurn::text("rejected"),
MockTurn::text("accepted one"),
MockTurn::text("accepted two"),
]))
.add_hook(hook)
.build();
let first = agent.prompt("first").max_turns(2).run();
let second = agent.prompt("second").max_turns(2).run();
let (first, second) = tokio::join!(first, second);
let first = first.expect("first run");
let second = second.expect("second run");
let outputs = std::collections::HashSet::from([first.output(), second.output()]);
assert_eq!(
outputs,
std::collections::HashSet::from(["accepted one".to_string(), "accepted two".to_string(),])
);
assert_eq!(first.completion_calls.len(), 2);
assert_eq!(second.completion_calls.len(), 2);
}
mod run_lifecycle {
use super::*;
use crate::agent::hook::{RunSettled, RunStart, RunStartAction, SettledOutcome};
#[derive(Clone, Default)]
struct LifecycleProbe {
starts: Arc<AtomicU32>,
seen_prompts: Arc<Mutex<Vec<String>>>,
settles: Arc<Mutex<Vec<String>>>,
rewrite_on_start: bool,
stop_on_start: bool,
}
impl AgentHook for LifecycleProbe {
async fn on_run_start(&self, _ctx: &HookContext, event: RunStart<'_>) -> RunStartAction {
self.starts.fetch_add(1, SeqCst);
if self.stop_on_start {
return RunStartAction::stop("vetoed at start");
}
if self.rewrite_on_start {
let current = event.prompt.rag_text().unwrap_or_default();
return RunStartAction::rewrite(Message::user(format!("{current} (rewritten)")));
}
RunStartAction::Continue
}
async fn on_completion_call(
&self,
_ctx: &HookContext,
event: CompletionCallEvent<'_>,
) -> CompletionCallAction {
self.seen_prompts
.lock()
.expect("prompts")
.push(event.prompt.rag_text().unwrap_or_default());
CompletionCallAction::Continue
}
async fn on_run_settled(&self, _ctx: &HookContext, event: RunSettled<'_>) {
let outcome = match event.outcome {
SettledOutcome::Response(_) => "ok".to_string(),
SettledOutcome::Error(reason) => format!("err:{reason}"),
};
self.settles.lock().expect("settles").push(outcome);
}
}
fn probe(rewrite_on_start: bool, stop_on_start: bool) -> LifecycleProbe {
LifecycleProbe {
rewrite_on_start,
stop_on_start,
..LifecycleProbe::default()
}
}
#[tokio::test]
async fn blocking_run_fires_start_once_rewrites_prompt_and_settles_ok() {
let hook = probe(true, false);
let model = MockCompletionModel::from_turns([MockTurn::text("done")]);
let response = AgentBuilder::new(model)
.add_hook(hook.clone())
.build()
.prompt("hi")
.await
.expect("prompt succeeds");
assert_eq!(response.output(), "done");
assert_eq!(hook.starts.load(SeqCst), 1);
assert_eq!(
hook.seen_prompts.lock().expect("prompts").as_slice(),
["hi (rewritten)".to_string()]
);
assert_eq!(hook.settles.lock().expect("settles").as_slice(), ["ok"]);
}
#[tokio::test]
async fn run_start_stop_terminates_before_any_provider_call_and_settles_err() {
let hook = probe(false, true);
let model = MockCompletionModel::from_stream_turns([vec![
MockStreamEvent::text("never sent"),
MockStreamEvent::final_response_with_total_tokens(0),
]]);
let mut stream = AgentBuilder::new(model)
.add_hook(hook.clone())
.build()
.prompt(Message::user("hi"))
.stream();
let first = stream.next().await.expect("terminal item");
assert!(matches!(
first,
Err(ref err)
if matches!(err, PromptError::Cancelled { .. })
));
assert!(stream.next().await.is_none());
assert_eq!(hook.starts.load(SeqCst), 1);
assert!(hook.seen_prompts.lock().expect("prompts").is_empty());
let settles = hook.settles.lock().expect("settles").clone();
assert_eq!(settles.len(), 1, "settled exactly once");
assert!(
settles[0].starts_with("err:"),
"stop settles as an error outcome: {settles:?}"
);
}
#[tokio::test]
async fn error_termination_settles_exactly_once() {
let hook = probe(false, false);
let model = MockCompletionModel::from_stream_turns([vec![
MockStreamEvent::tool_call_name_delta("tc1", "add"),
MockStreamEvent::tool_call_arguments_delta("tc1", "{\"x\":2,\"y\":3}"),
MockStreamEvent::tool_call("tc1", "add", json!({"x": 2, "y": 3})),
MockStreamEvent::final_response_with_total_tokens(0),
]]);
let mut stream = AgentBuilder::new(model)
.tool(crate::test_utils::MockAddTool)
.add_hook(hook.clone())
.build()
.prompt(Message::user("add 2 and 3"))
.max_turns(1)
.stream();
let mut saw_error = false;
while let Some(item) = stream.next().await {
if item.is_err() {
saw_error = true;
}
}
assert!(saw_error, "the exhausted budget surfaces as a stream error");
assert_eq!(hook.starts.load(SeqCst), 1);
let settles = hook.settles.lock().expect("settles").clone();
assert_eq!(settles.len(), 1, "settled exactly once: {settles:?}");
assert!(settles[0].starts_with("err:"), "error outcome: {settles:?}");
}
}
#[tokio::test]
async fn outcome_stop_is_terminal_through_nested_hooks_on_both_surfaces() {
struct StopTool;
impl AgentHook for StopTool {
async fn on_outcome(&self, _: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
if event.tool_name().is_some() {
OutcomeAction::stop("terminal policy")
} else {
OutcomeAction::Proceed
}
}
}
struct Revive(Arc<AtomicU32>);
impl AgentHook for Revive {
async fn on_outcome(&self, _: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
if event.tool_name().is_none() {
return OutcomeAction::Proceed;
}
self.0.fetch_add(1, SeqCst);
OutcomeAction::Replace(Ok(rig_core::effect::Outcome::ToolResult {
result: crate::tool::ToolResult::success(crate::tool::ToolOutput::text("revived")),
}))
}
}
for streaming in [false, true] {
for nested in [false, true] {
let later = Arc::new(AtomicU32::new(0));
let mut inner = HookStack::with(StopTool);
inner.push(Revive(later.clone()));
let mut stack = if nested {
HookStack::with(inner)
} else {
inner
};
stack.push(Revive(later.clone()));
let model = if streaming {
MockCompletionModel::from_stream_turns([
vec![
MockStreamEvent::tool_call("tc1", "add", json!({"x": 1, "y": 2})),
MockStreamEvent::final_response_with_total_tokens(0),
],
vec![
MockStreamEvent::text("done"),
MockStreamEvent::final_response_with_total_tokens(0),
],
])
} else {
MockCompletionModel::from_turns([
MockTurn::tool_call("tc1", "add", json!({"x": 1, "y": 2})),
MockTurn::text("done"),
])
};
let agent = AgentBuilder::new(model)
.tool(MockAddTool)
.add_hook(stack)
.build();
let error = if streaming {
let mut stream = agent.prompt("go").max_turns(3).stream();
let mut error = None;
while let Some(item) = stream.next().await {
match item {
Err(err) => error = Some(err.to_string()),
Ok(
MultiTurnStreamItem::ToolExecutionCommitted { .. }
| MultiTurnStreamItem::FinalResponse(_),
) => panic!("stopped outcome must not commit"),
_ => {}
}
}
error.expect("stream must stop")
} else {
agent
.prompt("go")
.max_turns(3)
.run()
.await
.expect_err("run must stop")
.to_string()
};
assert!(error.contains("terminal policy"), "{error}");
assert_eq!(later.load(SeqCst), 0, "later hooks must not undo stop");
}
}
}
fn mock_reply(text: &str) -> Message {
Message::Assistant(
rig_core::message::AssistantMessage::new(vec![AssistantContent::text(text)])
.with_origin(rig_core::message::Origin::new(
rig_core::test_utils::MOCK_API,
rig_core::test_utils::MOCK_PROVIDER,
rig_core::test_utils::MOCK_MODEL,
))
.with_stop(rig_core::message::StopReason::Stop),
)
}
fn canonical_history(messages: &[Message]) -> Vec<Message> {
messages
.iter()
.map(|message| match message {
Message::Assistant(turn) => {
Message::Assistant(rig_core::message::AssistantMessage::new(
turn.content
.iter()
.map(AssistantContent::canonical)
.collect(),
))
}
other => other.clone(),
})
.collect()
}