use std::collections::{BTreeMap, VecDeque};
use std::sync::{
Arc,
atomic::{AtomicU64, Ordering},
};
use futures::{Stream, StreamExt, stream};
use tracing::{Instrument, span::Id};
use crate::bus::{DispatchOptions, MemoryHandle};
use rig_core::error::ProviderError;
use rig_core::{
completion::ModelRef,
effect::{EffectId, EffectKind, Outcome},
error::{ErrorKind, ErrorReport},
message::{AssistantContent, Message, ToolCall, ToolFunction, ToolName, UserContent},
telemetry::SpanCombinator,
wasm_compat::{WasmBoxedStream, WasmCompatSend, WasmCompatSync},
};
use super::{
ModelHandle,
completion::{PreparedCompletionRequest, build_prepared_completion_request},
hook::{
AgentHook, CompletionCallAction, CompletionCallEvent, DispatchAction, DispatchEvent,
HookContext, HookStack, InvalidToolCallAction, InvalidToolCallContext, ModelSelection,
ModelSelectionAction, ModelTurnAction, ModelTurnFinished, ObservationAction, OutcomeAction,
OutcomeEvent, ReasoningDelta, RequestPatch, RunSettled, RunStart, RunStartAction,
SettledOutcome, StepEventKind, TextDelta, ToolCallDelta,
},
run::{
AgentRun, AgentRunStep, ModelTurn, ModelTurnOutcome, PendingToolCall,
streamed::{StreamedResolution, StreamedTurnAssembler, StreamedTurnEvent},
},
run::{
response::{MemoryAppend, PromptResponse, finalize_output_tool_choice},
transcript::{
assistant_text_from_choice, is_empty_assistant_turn, tool_result_message,
tool_result_output,
},
},
runner::AgentRunner,
streaming::MultiTurnStreamItem,
telemetry::{build_chat_span, new_execute_tool_span},
};
use crate::run::UnhandledInvalidToolCall;
use crate::{
completion::PromptError,
streaming::{Item, Part, StreamEvent},
tool::{ToolCatalog, ToolResult},
};
pub(crate) type DriveStream<'a> = WasmBoxedStream<'a, Result<MultiTurnStreamItem, PromptError>>;
pub(crate) enum DriveItem {
Item(MultiTurnStreamItem),
Done(PromptResponse),
}
pub(crate) trait TurnSource: WasmCompatSend + WasmCompatSync {
const FORWARDS_ITEMS: bool;
fn open_chat_span(
&self,
runner: &AgentRunner,
effective_preamble: Option<&str>,
) -> tracing::Span;
fn run_model_turn<'a>(
&'a mut self,
runner: &'a AgentRunner,
hook_ctx: &'a HookContext,
run: &'a mut AgentRun,
prepared: PreparedCompletionRequest,
chat_span: tracing::Span,
agent_span: &'a tracing::Span,
prompt: Message,
) -> DriveStream<'a>;
fn chain_span(&self, span: tracing::Span) -> tracing::Span;
fn record_run_level_telemetry(
&self,
agent_span: &tracing::Span,
response: &PromptResponse,
created_agent_span: bool,
);
fn final_item(&self, response: &PromptResponse) -> Option<MultiTurnStreamItem>;
}
pub(crate) fn store_error_usage(runner: &AgentRunner, run: &AgentRun) {
if let Some(usage) = &runner.error_usage {
*usage
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = run.usage();
}
}
pub(crate) fn drive_agent<S>(
runner: AgentRunner,
mut source: S,
mut run: AgentRun,
agent_span: tracing::Span,
created_agent_span: bool,
memory_handle: Option<(MemoryHandle, rig_core::id::ConversationId)>,
hook_ctx: HookContext,
) -> impl Stream<Item = Result<DriveItem, PromptError>>
where
S: TurnSource,
{
async_stream::stream! {
hook_ctx.seed_entries(run.entries());
macro_rules! flush_entries {
() => {
for entry in hook_ctx.drain_pending_entries() {
run.append_entry(entry);
}
};
}
macro_rules! settle_error {
($err:expr) => {{
if runner.config.hooks.observes(StepEventKind::RunSettled) {
let reason = $err.to_string();
runner
.config
.hooks
.on_run_settled(
&hook_ctx,
RunSettled {
outcome: SettledOutcome::Error(&reason),
messages: Some(run.messages()),
},
)
.await;
}
}};
}
let mut pending_tool_snapshot: Option<Arc<ToolCatalog>> = None;
let mut previous_model: Option<ModelRef> = run.previous_model().cloned();
if runner.config.hooks.observes(StepEventKind::RunStart) {
let action = match run.initial_prompt() {
Some(prompt) => {
runner
.config
.hooks
.on_run_start(
&hook_ctx,
RunStart {
prompt,
history: run.input_chat_history(),
},
)
.await
}
None => RunStartAction::Continue,
};
let early_stop = match action {
RunStartAction::Continue => None,
RunStartAction::Rewrite(prompt) => run
.rewrite_initial_prompt(prompt)
.err(),
RunStartAction::Stop(reason) => {
Some(run.cancel_error(reason))
}
};
if let Some(err) = early_stop {
store_error_usage(&runner, &run);
settle_error!(err);
yield Err(err);
return;
}
}
macro_rules! drive_step {
($label:lifetime, $step_stream:expr) => {{
let mut step_stream = $step_stream;
let mut step_error = None;
while let Some(item) = step_stream.next().await {
match item {
Ok(item) => yield Ok(DriveItem::Item(item)),
Err(err) => {
step_error = Some(err);
break;
}
}
}
drop(step_stream);
if let Some(err) = step_error {
store_error_usage(&runner, &run);
settle_error!(err);
yield Err(err);
break $label;
}
}};
}
'outer: loop {
flush_entries!();
let step = match run.next_step() {
Ok(step) => step,
Err(err) => {
store_error_usage(&runner, &run);
settle_error!(err);
yield Err(err);
break 'outer;
}
};
match step {
AgentRunStep::CallModel { prompt, history, turn } => {
drop(pending_tool_snapshot.take());
if runner.config.max_turns > 1 {
tracing::info!("Current conversation Turns: {}/{}", turn, runner.config.max_turns);
}
hook_ctx.set_turn(turn);
let request_patch =
match resolve_completion_call(&runner.config.hooks, &hook_ctx, &prompt, &history, turn).await {
CompletionCallOutcome::Terminate(reason) => {
store_error_usage(&runner, &run);
let err = run.cancel_error(reason);
settle_error!(err);
yield Err(err);
break 'outer;
}
CompletionCallOutcome::Proceed(request_patch) => request_patch,
};
let default_label = runner.config.model_label();
let selected_label = match runner.config.hooks.on_model_select(
&hook_ctx,
ModelSelection {
prompt: &prompt,
history: &history,
request_patch: request_patch.as_ref(),
previous_model: previous_model.as_ref(),
default_model: &default_label,
selected_model: &default_label,
},
) {
ModelSelectionAction::Continue => default_label.clone(),
ModelSelectionAction::Select(model) => model,
ModelSelectionAction::Stop(reason) => {
store_error_usage(&runner, &run);
let err = run.cancel_error(reason);
settle_error!(err);
yield Err(err);
break 'outer;
}
};
let selected_model = if selected_label == default_label {
runner.config.model_handle()
} else {
runner.config.model_by_ref(&selected_label)
};
let selected_model: ModelHandle = match selected_model {
Ok(model) => model,
Err(report) => {
store_error_usage(&runner, &run);
let err = PromptError::Report(report);
settle_error!(err);
yield Err(err);
break 'outer;
}
};
let effective_preamble = request_patch
.as_ref()
.and_then(|o| o.preamble.as_deref())
.or(runner.config.preamble.as_deref());
let chat_span = source.open_chat_span(&runner, effective_preamble);
let committed_output_tool = run.output_tool_name().map(str::to_owned);
let mut prepared = match build_prepared_completion_request(
&runner,
&hook_ctx,
&selected_model,
prompt.clone(),
&history,
committed_output_tool.as_deref(),
request_patch.as_ref(),
)
.await
{
Ok(prepared) => prepared,
Err(err) => {
store_error_usage(&runner, &run);
let err = PromptError::from(err);
settle_error!(err);
yield Err(err);
break 'outer;
}
};
if let Err(err) = runner
.config
.check_call_options(&selected_label, &mut prepared.request.options)
{
store_error_usage(&runner, &run);
let err = PromptError::from(err);
settle_error!(err);
yield Err(err);
break 'outer;
}
if let Some(name) = &prepared.output_tool_name {
let _ = run.commit_output_tool_name(name.clone());
}
let turn_tool_snapshot = prepared.tool_snapshot.clone();
run.advertise_tools(turn, std::mem::take(&mut prepared.advertised_tools));
if runner.config.record_telemetry_content {
let input_messages = std::mem::take(&mut prepared.telemetry_messages);
rig_core::telemetry::record_model_input(&chat_span, &input_messages, true);
}
run.set_previous_model(selected_label.clone());
previous_model = Some(selected_label);
drive_step!('outer, source.run_model_turn(
&runner,
&hook_ctx,
&mut run,
prepared,
chat_span,
&agent_span,
prompt,
));
pending_tool_snapshot = Some(turn_tool_snapshot);
}
AgentRunStep::CallTools { calls } => {
if pending_tool_snapshot.is_none()
&& let Some(advertised) = run.advertised_tools()
{
let names: Vec<String> = advertised
.definitions
.iter()
.map(|definition| definition.name.to_string())
.collect();
pending_tool_snapshot = Some(Arc::new(
runner.tool_server_handle.snapshot_with_dynamic(&names),
));
}
let Some(tool_snapshot) = pending_tool_snapshot.take() else {
store_error_usage(&runner, &run);
let err = PromptError::Provider(ProviderError::Response(
"agent requested tool execution without a prepared registry snapshot"
.to_string(),
));
settle_error!(err);
yield Err(err);
break 'outer;
};
drive_step!('outer, drive_tool_calls(
&runner,
&hook_ctx,
&mut run,
calls,
tool_snapshot,
|span| source.chain_span(span),
S::FORWARDS_ITEMS,
));
}
AgentRunStep::Done(response) => {
flush_entries!();
tracing::info!(
turn = run.turn(),
max_turns = runner.config.max_turns,
"Agent run finished"
);
source.record_run_level_telemetry(&agent_span, &response, created_agent_span);
let memory_append = append_run_messages(
&runner,
&hook_ctx,
memory_handle.as_ref(),
&response.messages,
)
.await;
let response = response.with_memory_append(memory_append);
if runner.config.hooks.observes(StepEventKind::RunSettled) {
runner
.config
.hooks
.on_run_settled(
&hook_ctx,
RunSettled {
outcome: SettledOutcome::Response(&response),
messages: Some(run.messages()),
},
)
.await;
}
if let Some(final_item) = source.final_item(&response) {
yield Ok(DriveItem::Item(final_item));
}
yield Ok(DriveItem::Done(response));
break 'outer;
}
}
}
}
}
pub(crate) fn drive_tool_calls<'a, F>(
runner: &'a AgentRunner,
hook_ctx: &'a HookContext,
run: &'a mut AgentRun,
calls: Vec<PendingToolCall>,
tool_snapshot: Arc<ToolCatalog>,
chain_tool_span: F,
forward_items: bool,
) -> DriveStream<'a>
where
F: Fn(tracing::Span) -> tracing::Span + WasmCompatSend + 'a,
{
struct PreparedToolCall {
tool_call: rig_core::message::ToolCall,
preresolved_result: Option<UserContent>,
malformed: Option<InvalidToolCallContext>,
span: tracing::Span,
}
enum ToolSurface {
Executed(rig_core::message::ToolCall),
Skipped,
Preresolved,
}
struct CollectedToolResult {
content: UserContent,
surface: ToolSurface,
}
Box::pin(async_stream::stream! {
let full_history_for_errors = run.full_history();
let call_count = calls.len();
let mut prepared: Vec<PreparedToolCall> = Vec::with_capacity(call_count);
for pending in calls {
let (span, preresolved_result) = match pending.preresolved_result {
Some(result) => (tracing::Span::none(), Some(result)),
None => {
if forward_items {
yield Ok(MultiTurnStreamItem::ToolCall {
tool_call: pending.tool_call.clone(),
});
}
(chain_tool_span(new_execute_tool_span()), None)
}
};
let malformed = match preresolved_result {
Some(_) => None,
None => run.malformed_tool_call_context(&pending.tool_call, forward_items),
};
prepared.push(PreparedToolCall {
tool_call: pending.tool_call,
preresolved_result,
malformed,
span,
});
}
let mut collected: Vec<Option<CollectedToolResult>> =
(0..call_count).map(|_| None).collect();
let mut first_error: Option<(usize, PromptError)> = None;
{
let terminating = Arc::new(std::sync::atomic::AtomicBool::new(false));
let unordered = stream::iter(prepared.into_iter().enumerate())
.map(|(index, call)| {
let PreparedToolCall { tool_call, preresolved_result, malformed, span } = call;
let tool_snapshot = &tool_snapshot;
let full_history_for_errors = &full_history_for_errors;
let terminating = terminating.clone();
async move {
if let Some(result) = preresolved_result {
return (
index,
Some(Ok(CollectedToolResult {
content: result,
surface: ToolSurface::Preresolved,
})),
);
}
if terminating.load(std::sync::atomic::Ordering::SeqCst) {
return (index, None);
}
let outcome = run_single_tool(
runner,
hook_ctx,
tool_snapshot,
&tool_call,
malformed.as_ref(),
full_history_for_errors,
)
.await;
let mapped = outcome.map(|o| {
let surface = match o.execution {
ToolExecution::Executed(effective) => {
ToolSurface::Executed(effective)
}
ToolExecution::Skipped => ToolSurface::Skipped,
};
CollectedToolResult {
content: o.content,
surface,
}
});
(index, Some(mapped))
}
.instrument(span)
})
.buffer_unordered(runner.concurrency.max(1));
futures::pin_mut!(unordered);
while let Some((index, outcome)) = unordered.next().await {
let Some(result) = outcome else { continue };
match result {
Ok(collected_result) => {
if let Some(slot) = collected.get_mut(index) {
*slot = Some(collected_result);
}
}
Err(err) => {
terminating.store(true, std::sync::atomic::Ordering::SeqCst);
if first_error.as_ref().is_none_or(|(i, _)| index < *i) {
first_error = Some((index, err));
}
}
}
}
}
if let Some((_, err)) = first_error {
yield Err(err);
return;
}
let mut committed: Vec<UserContent> = Vec::with_capacity(call_count);
let mut surface_items: Vec<MultiTurnStreamItem> =
Vec::with_capacity(call_count.saturating_mul(2));
for slot in collected {
let Some(CollectedToolResult { content, surface }) = slot else {
yield Err(PromptError::Provider(ProviderError::Response(
"tool execution finished without producing every result".to_string(),
)));
return;
};
if forward_items {
let surface_result = match surface {
ToolSurface::Executed(tool_call) => {
surface_items.push(MultiTurnStreamItem::ToolExecutionCommitted {
tool_call,
});
true
}
ToolSurface::Skipped => true,
ToolSurface::Preresolved => false,
};
if surface_result
&& let UserContent::ToolResult(tool_result) = &content
{
surface_items.push(MultiTurnStreamItem::ToolResult {
tool_result: tool_result.clone(),
});
}
}
committed.push(content);
}
if let Err(err) = run.tool_results(committed) {
yield Err(err);
return;
}
for item in surface_items {
yield Ok(item);
}
})
}
pub(crate) struct StreamingTurnSource {
last_final_choice: Vec<AssistantContent>,
last_response_id: Option<String>,
agent_name: String,
created_agent_span: bool,
record_telemetry_content: bool,
observes_text_delta: bool,
observes_reasoning_delta: bool,
observes_tool_call_delta: bool,
has_hooks: bool,
}
impl StreamingTurnSource {
pub(crate) fn new(
hooks: &HookStack,
agent_name: String,
created_agent_span: bool,
record_telemetry_content: bool,
) -> Self {
Self {
last_final_choice: Vec::new(),
last_response_id: None,
agent_name,
created_agent_span,
record_telemetry_content,
observes_text_delta: hooks.observes(StepEventKind::TextDelta),
observes_reasoning_delta: hooks.observes(StepEventKind::ReasoningDelta),
observes_tool_call_delta: hooks.observes(StepEventKind::ToolCallDelta),
has_hooks: !hooks.is_empty(),
}
}
fn record_turn_telemetry(
&self,
agent_span: &tracing::Span,
chat_span: &tracing::Span,
choice: &[AssistantContent],
record_content: bool,
) {
if self.created_agent_span && self.record_telemetry_content {
agent_span.record("gen_ai.completion", assistant_text_from_choice(choice));
}
rig_core::telemetry::record_model_output(chat_span, choice, record_content);
}
}
impl TurnSource for StreamingTurnSource {
const FORWARDS_ITEMS: bool = true;
fn open_chat_span(
&self,
runner: &AgentRunner,
effective_preamble: Option<&str>,
) -> tracing::Span {
build_chat_span!(runner, effective_preamble, "chat", "chat")
}
fn run_model_turn<'a>(
&'a mut self,
runner: &'a AgentRunner,
hook_ctx: &'a HookContext,
run: &'a mut AgentRun,
prepared: PreparedCompletionRequest,
chat_span: tracing::Span,
agent_span: &'a tracing::Span,
_current_prompt: Message,
) -> DriveStream<'a> {
Box::pin(async_stream::stream! {
let attempt_max_tokens = prepared.max_tokens;
let dispatched =
dispatch_completion_stream(runner, hook_ctx, run, &prepared.model, prepared.request)
.instrument(chat_span.clone())
.await;
let CompletionDispatch { id: dispatch_id, kind: dispatched_kind, answer: mut stream } =
match dispatched {
Ok(dispatched) => dispatched,
Err(err) => {
yield Err(err);
return;
}
};
let mut assembler = StreamedTurnAssembler::new(
prepared.executable_tool_names.clone(),
prepared.allowed_tool_names.clone(),
);
let mut turn_recovered = false;
let mut held: BTreeMap<Part, Vec<StreamEvent>> = BTreeMap::new();
'turn: while let Some(item) = stream.next().await {
let (mut item_slot, mut events, ended): (
Option<Item<StreamEvent>>,
VecDeque<StreamedTurnEvent>,
bool,
) = match item {
Ok(item) => match assembler.ingest(&item) {
Ok(events) => (Some(item), events.into(), false),
Err(err) => {
yield Err(err.into());
return;
}
},
Err(err) => {
yield Err(err.into());
return;
}
};
while let Some(event) = events.pop_front() {
match event {
StreamedTurnEvent::EmitIngested => {
if self.observes_text_delta
&& let Some(Item::Event(StreamEvent::Text { text, .. })) =
item_slot.as_ref()
&& let Some(reason) = observe_action(
runner
.config.hooks
.on_text_delta(
hook_ctx,
TextDelta {
delta: text,
aggregated: assembler.aggregated_text(),
},
)
.await,
)
{
drop(stream);
yield Err(run.cancel_error(reason));
return;
}
if self.observes_reasoning_delta
&& let Some(Item::Event(StreamEvent::Reasoning {
part,
text: reasoning,
})) = item_slot.as_ref()
&& let Some(reason) = observe_action(
runner
.config.hooks
.on_reasoning_delta(
hook_ctx,
ReasoningDelta {
part: *part,
delta: reasoning,
aggregated: assembler
.aggregated_reasoning(part.index())
.unwrap_or_default(),
},
)
.await,
)
{
drop(stream);
yield Err(run.cancel_error(reason));
return;
}
if let Some(item) = item_slot.take() {
yield Ok(MultiTurnStreamItem::stream_item(item));
}
}
StreamedTurnEvent::EmitToolCallDelta => {
if self.observes_tool_call_delta
&& let Some(Item::Event(StreamEvent::Arguments { part, json })) =
item_slot.as_ref()
&& let Some(reason) = observe_action(
runner
.config.hooks
.on_tool_call_delta(
hook_ctx,
ToolCallDelta {
part: *part,
tool_name: assembler
.streaming_tool_name(part.index())
.map(ToolName::as_str)
.unwrap_or_default(),
delta: json,
aggregated: assembler
.aggregated_arguments(part.index())
.unwrap_or_default(),
},
)
.await,
)
{
drop(stream);
yield Err(run.cancel_error(reason));
return;
}
if let Some(item) = item_slot.take() {
yield Ok(MultiTurnStreamItem::stream_item(item));
}
}
StreamedTurnEvent::HoldToolCall => {
if let Some(Item::Event(event)) = item_slot.take() {
held.entry(event.part()).or_default().push(event);
}
}
StreamedTurnEvent::EmitToolCall { call } => {
let end = match item_slot.take() {
Some(Item::Event(StreamEvent::End { part, .. })) => Some(part),
_ => None,
};
let replay = end
.and_then(|part| held.remove(&part))
.unwrap_or_default();
let mut aggregated = String::new();
for mut event in replay {
if let StreamEvent::Start { name, .. } = &mut event {
*name = Some(call.function.name.clone());
}
if let StreamEvent::Arguments { json, .. } = &event {
aggregated.push_str(json);
}
if self.observes_tool_call_delta
&& let StreamEvent::Arguments { part, json } = &event
&& let Some(reason) = observe_action(
runner
.config.hooks
.on_tool_call_delta(
hook_ctx,
ToolCallDelta {
part: *part,
tool_name: call.function.name.as_str(),
delta: json,
aggregated: &aggregated,
},
)
.await,
)
{
drop(stream);
yield Err(run.cancel_error(reason));
return;
}
yield Ok(MultiTurnStreamItem::stream_item(Item::Event(event)));
}
if let Some(part) = end {
yield Ok(MultiTurnStreamItem::stream_item(Item::Event(
StreamEvent::End {
part,
content: AssistantContent::ToolCall(call),
},
)));
}
}
StreamedTurnEvent::InvalidToolCall(invalid) => {
let rejected = match item_slot.as_ref() {
Some(Item::Event(event)) => Some(event.part()),
_ => None,
};
let partial = assembler.partial_turn(&stream.partial());
let hook_action = if self.has_hooks {
let context =
run.streamed_invalid_tool_call_context(&partial, &invalid);
runner
.config.hooks
.on_invalid_tool_call(hook_ctx, &context)
.await
} else {
None
};
let resolved = match hook_action {
Some(action) => {
run.resolve_streamed_invalid_tool_call(&partial, &invalid, action)
}
None => match run.unhandled_invalid_tool_call() {
UnhandledInvalidToolCall::Fail => run
.resolve_streamed_invalid_tool_call(
&partial,
&invalid,
InvalidToolCallAction::fail(),
),
UnhandledInvalidToolCall::Ignore => {
run.ignore_streamed_invalid_tool_call()
}
},
};
let resolution = match resolved {
Ok(resolution) => resolution,
Err(err) => {
yield Err(err);
return;
}
};
match resolution {
StreamedResolution::Ignored => {
assembler.resolve_pending_invalid(&resolution);
if let Some(part) = rejected {
held.remove(&part);
}
item_slot = None;
}
StreamedResolution::Repaired { .. } => {
turn_recovered = true;
events.extend(assembler.resolve_pending_invalid(&resolution));
}
StreamedResolution::TurnAbandoned {
ref skipped_tool_result,
} => {
let skipped_tool_result = skipped_tool_result.clone();
assembler.resolve_pending_invalid(&resolution);
held.clear();
if !ended {
let response = match stream.finish().await {
Ok(response) => response,
Err(err) => {
yield Err(err.into());
return;
}
};
chat_span.record_token_usage(&response.usage);
match run.record_streamed_completion_call(
response.usage,
response.identity(),
response.finish_reason(),
response.raw.clone(),
) {
Ok(call) => {
yield Ok(MultiTurnStreamItem::CompletionCall(call));
}
Err(err) => {
yield Err(err);
return;
}
}
}
if let Some(tool_result) = skipped_tool_result {
yield Ok(MultiTurnStreamItem::ToolResult { tool_result });
}
return;
}
}
}
}
}
if ended {
break 'turn;
}
}
let mut response = match stream.finish().await {
Ok(response) => response,
Err(err) => {
yield Err(err.into());
return;
}
};
chat_span.record_token_usage(&response.usage);
match run.record_streamed_completion_call(
response.usage,
response.identity(),
response.finish_reason(),
response.raw.clone(),
) {
Ok(call) => yield Ok(MultiTurnStreamItem::CompletionCall(call)),
Err(err) => {
yield Err(err);
return;
}
}
let streamed_turn = assembler.finish(&response);
self.last_response_id = response.response_id().map(str::to_owned);
let mut final_turn_content =
std::mem::replace(&mut response.choice, streamed_turn.choice.clone());
if let Err(err) = run.streamed_turn(streamed_turn) {
yield Err(err);
return;
}
let turn = CompletionDispatch {
id: dispatch_id,
kind: dispatched_kind,
answer: response,
};
let settlement = settle_model_turn(
&runner.config.hooks,
hook_ctx,
run,
&turn,
attempt_max_tokens,
turn_recovered,
)
.await;
match settlement {
Ok(ModelTurnDecision::Advance { replaced }) => {
if let Some(choice) = replaced {
final_turn_content = choice;
}
}
Ok(ModelTurnDecision::Retried) => {
yield Ok(MultiTurnStreamItem::ModelTurnRetried {
turn: hook_ctx.turn(),
});
return;
}
Ok(ModelTurnDecision::Terminate(reason)) => {
self.record_turn_telemetry(
agent_span,
&chat_span,
&turn.answer.choice,
runner.config.record_telemetry_content,
);
yield Err(run.cancel_error(reason));
return;
}
Err(err) => {
yield Err(err);
return;
}
}
self.record_turn_telemetry(
agent_span,
&chat_span,
&turn.answer.choice,
runner.config.record_telemetry_content,
);
self.last_final_choice = final_turn_content;
})
}
fn chain_span(&self, span: tracing::Span) -> tracing::Span {
span
}
fn record_run_level_telemetry(
&self,
agent_span: &tracing::Span,
response: &PromptResponse,
created_agent_span: bool,
) {
if created_agent_span {
agent_span.record_token_usage(&response.usage);
}
}
fn final_item(&self, response: &PromptResponse) -> Option<MultiTurnStreamItem> {
let final_choice = finalize_output_tool_choice(&self.last_final_choice, &response.output())
.unwrap_or_else(|| {
if is_empty_assistant_turn(&self.last_final_choice) {
tracing::warn!(
agent_name = self.agent_name.as_str(),
response_id = ?self.last_response_id,
"Streaming turn completed without assistant text; final response will be empty"
);
}
self.last_final_choice.clone()
});
Some(
MultiTurnStreamItem::final_response_with_completion_calls(
final_choice,
response.usage,
response.completion_calls.clone(),
response.messages.clone(),
)
.with_memory_append(response.memory_append.clone()),
)
}
}
pub(crate) fn observe_action(action: ObservationAction) -> Option<String> {
match action {
ObservationAction::Continue => None,
ObservationAction::Stop(reason) => Some(reason),
}
}
pub(crate) enum ModelTurnDecision {
Advance {
replaced: Option<Vec<AssistantContent>>,
},
Retried,
Terminate(String),
}
pub(crate) async fn settle_model_turn(
hooks: &HookStack,
hook_ctx: &HookContext,
run: &mut AgentRun,
turn: &CompletionDispatch<rig_core::completion::CompletionResponse>,
max_tokens: Option<u64>,
recovered: bool,
) -> Result<ModelTurnDecision, PromptError> {
if recovered {
return Ok(ModelTurnDecision::Advance { replaced: None });
}
let response = &turn.answer;
let identity = response.identity();
let finish_reason = response.finish_reason();
let mut folded = rig_core::completion::CompletionResponse::new(
response.choice.clone(),
response.usage,
response.origin.clone(),
response.raw.clone(),
)
.with_optional_finish_reason(finish_reason.clone())
.accept_unknown_finish_reasons(response.accepts_unknown_finish_reasons());
folded.error = response.error.clone();
folded.provider_request_id = identity.provider_request_id.clone();
let outcome: Result<Outcome, ErrorReport> = Ok(Outcome::Completion(folded));
let mut replaced: Option<Vec<AssistantContent>> = None;
match hooks
.on_outcome(
hook_ctx,
OutcomeEvent {
id: turn.id,
kind: &turn.kind,
outcome: &outcome,
turn: hook_ctx.turn(),
call_id: None,
context: None,
},
)
.await
{
OutcomeAction::Proceed => {}
OutcomeAction::Replace(Ok(Outcome::Completion(replacement))) => {
run.replace_accepted_turn_choice(replacement.choice.clone())?;
replaced = Some(replacement.choice);
}
OutcomeAction::Replace(Ok(other)) => {
return Err(PromptError::Report(wrong_outcome("a completion", &other)));
}
OutcomeAction::Replace(Err(report)) => {
if report.kind == ErrorKind::Cancelled {
return Ok(ModelTurnDecision::Terminate(report.message));
}
return Err(PromptError::Report(report));
}
}
let content = replaced.as_ref().unwrap_or(&response.choice);
let action = hooks
.on_model_turn_finished(
hook_ctx,
ModelTurnFinished {
turn: hook_ctx.turn(),
content,
usage: response.usage,
identity: &identity,
finish_reason: finish_reason.as_ref(),
max_tokens,
raw: &response.raw,
},
)
.await;
match action {
ModelTurnAction::Continue => Ok(ModelTurnDecision::Advance { replaced }),
ModelTurnAction::Retry(request) => {
run.retry_model_turn(request)?;
Ok(ModelTurnDecision::Retried)
}
ModelTurnAction::Stop(reason) => Ok(ModelTurnDecision::Terminate(reason)),
}
}
pub(crate) enum CompletionCallOutcome {
Proceed(Option<RequestPatch>),
Terminate(String),
}
pub(crate) async fn resolve_completion_call(
hooks: &HookStack,
ctx: &HookContext,
prompt: &Message,
history: &[Message],
turn: usize,
) -> CompletionCallOutcome {
match hooks
.on_completion_call(
ctx,
CompletionCallEvent {
prompt,
history,
turn,
},
)
.await
{
CompletionCallAction::Stop(reason) => CompletionCallOutcome::Terminate(reason),
CompletionCallAction::Patch(patch) => CompletionCallOutcome::Proceed(Some(patch)),
CompletionCallAction::Continue => CompletionCallOutcome::Proceed(None),
}
}
pub(crate) async fn append_run_messages(
runner: &AgentRunner,
ctx: &HookContext,
memory_handle: Option<&(MemoryHandle, rig_core::id::ConversationId)>,
messages: &[Message],
) -> Option<MemoryAppend> {
let (memory, id) = memory_handle?;
let appended = dispatch_effect(
&runner.config.hooks,
ctx,
runner.config.bus.dispatcher(),
memory.key(),
EffectKind::Memory {
op: rig_core::effect::MemoryOp::Append {
conversation: id.clone(),
messages: messages.to_vec(),
},
},
)
.await
.and_then(|outcome| match outcome {
Outcome::Memory(rig_core::effect::MemoryOutcome::Appended) => Ok(()),
other => Err(wrong_outcome("appended memory", &other)),
});
Some(match appended {
Ok(()) => MemoryAppend::Acknowledged,
Err(report) => {
tracing::warn!(
error = %report,
conversation_id = %id,
"conversation memory append failed; surfacing final response anyway"
);
MemoryAppend::Failed { report }
}
})
}
pub(crate) async fn dispatch_effect(
hooks: &HookStack,
ctx: &HookContext,
dispatcher: &crate::bus::Dispatcher,
key: &rig_core::effect::HandlerKey,
kind: EffectKind,
) -> Result<Outcome, ErrorReport> {
let id = dispatcher.mint_id();
let family = kind.family();
let kind = match hooks
.on_dispatch(
ctx,
DispatchEvent {
id,
kind: &kind,
turn: ctx.turn(),
call_id: None,
context: None,
},
)
.await
{
DispatchAction::Proceed => kind,
DispatchAction::Patch(patched) if patched.family() == family => patched,
DispatchAction::Patch(other) => return Err(wrong_family_patch(kind.name(), &other)),
DispatchAction::Deny(report) => return Err(report),
};
let outcome = dispatcher
.dispatch_with(key, kind.clone(), DispatchOptions::default().with_id(id))
.await;
match hooks
.on_outcome(
ctx,
OutcomeEvent {
id,
kind: &kind,
outcome: &outcome,
turn: ctx.turn(),
call_id: None,
context: None,
},
)
.await
{
OutcomeAction::Proceed => outcome,
OutcomeAction::Replace(replaced) => replaced,
}
}
pub(crate) fn wrong_outcome(expected: &str, outcome: &Outcome) -> ErrorReport {
ErrorReport::new(
ErrorKind::Internal,
format!(
"expected {expected}, the handler answered with a {} outcome",
outcome.family()
),
)
}
pub(crate) enum ToolExecution {
Executed(ToolCall),
Skipped,
}
pub(crate) struct ToolCallOutcome {
pub content: UserContent,
pub execution: ToolExecution,
}
pub(crate) async fn run_single_tool(
runner: &AgentRunner,
ctx: &HookContext,
tool_snapshot: &ToolCatalog,
tool_call: &ToolCall,
malformed: Option<&InvalidToolCallContext>,
error_history: &[Message],
) -> Result<ToolCallOutcome, PromptError> {
let tool_context = &runner.tool_context;
let record_content = runner.config.record_telemetry_content;
let tool_name = tool_call.function.name.as_str();
if let Some(raw) = &tool_call.function.invalid_arguments {
return answer_malformed_tool_call(runner, ctx, tool_call, raw, malformed, error_history)
.await;
}
let args = tool_call.function.arguments_value().to_string();
let tool_span = tracing::Span::current();
tool_span.record("gen_ai.tool.name", tool_name);
tool_span.record(
"gen_ai.tool.call.id",
tracing::field::display(&tool_call.id),
);
if record_content {
tool_span.record("gen_ai.tool.call.arguments", &args);
}
let ToolCallDispatch {
result: exec,
executed,
context: _dispatch_context,
args: effective_args,
} = match dispatch_tool_call(
runner,
ctx,
tool_snapshot,
tool_name,
args.clone(),
&tool_call.id,
tool_context,
)
.await
{
Ok(dispatch) => dispatch,
Err(ToolDispatchAbort::Cancelled(reason)) => {
return Err(PromptError::cancelled(error_history.to_vec(), reason));
}
Err(ToolDispatchAbort::Failed(report)) => return Err(PromptError::Report(report)),
};
if effective_args != args {
if record_content {
tool_span.record("gen_ai.tool.call.arguments", &effective_args);
}
tracing::debug!(
tool_name = tool_name,
"tool-call arguments rewritten by a hook"
);
}
let execution = if !executed {
ToolExecution::Skipped
} else {
let mut effective_tool_call = tool_call.clone();
effective_tool_call.function =
ToolFunction::parse(tool_call.function.name.clone(), &effective_args);
ToolExecution::Executed(effective_tool_call)
};
record_tool_result(&tool_span, &exec);
if record_content {
tool_span.record("gen_ai.tool.call.result", exec.output().render());
}
let content = tool_result_output(tool_call.id.clone(), tool_call.function.name.clone(), &exec);
Ok(ToolCallOutcome { content, execution })
}
async fn answer_malformed_tool_call(
runner: &AgentRunner,
ctx: &HookContext,
tool_call: &ToolCall,
raw: &str,
context: Option<&InvalidToolCallContext>,
error_history: &[Message],
) -> Result<ToolCallOutcome, PromptError> {
let action = futures::future::OptionFuture::from(
context.map(|context| runner.config.hooks.on_invalid_tool_call(ctx, context)),
)
.await
.flatten();
let id = tool_call.id.clone();
let name = tool_call.function.name.clone();
let content = match action {
None => tool_result_message(
id,
name,
rig_core::transcript::invalid_arguments_feedback(tool_call.function.name.as_str(), raw),
),
Some(InvalidToolCallAction::Retry { feedback }) => tool_result_message(id, name, feedback),
Some(InvalidToolCallAction::Skip { reason }) => {
tool_result_output(id, name, &ToolResult::skipped(reason))
}
Some(InvalidToolCallAction::Stop { reason }) => {
return Err(PromptError::cancelled(error_history.to_vec(), reason));
}
Some(InvalidToolCallAction::Fail | InvalidToolCallAction::Repair { .. }) => {
return Err(ProviderError::Response(format!(
"tool `{}` was called with arguments that are not a JSON object: {}",
tool_call.function.name,
crate::run::policy::arguments_parse_error(raw)
))
.into());
}
};
Ok(ToolCallOutcome {
content,
execution: ToolExecution::Skipped,
})
}
fn record_tool_result(span: &tracing::Span, result: &ToolResult) {
span.record("gen_ai.tool.call.outcome", result.status_name());
if let Some(error) = result.error() {
span.record("gen_ai.tool.error.type", error.kind().as_str());
}
}
pub(crate) struct UnaryTurnSource {
current_span_id: AtomicU64,
record_telemetry_content: bool,
}
impl UnaryTurnSource {
pub(crate) fn new(record_telemetry_content: bool) -> Self {
Self {
current_span_id: AtomicU64::new(0),
record_telemetry_content,
}
}
}
impl TurnSource for UnaryTurnSource {
const FORWARDS_ITEMS: bool = false;
fn chain_span(&self, span: tracing::Span) -> tracing::Span {
let span = match self.current_span_id.load(Ordering::Relaxed) {
0 => span,
id => {
span.follows_from(Id::from_u64(id));
span
}
};
if let Some(id) = span.id() {
self.current_span_id.store(id.into_u64(), Ordering::Relaxed);
}
span
}
fn open_chat_span(
&self,
runner: &AgentRunner,
effective_preamble: Option<&str>,
) -> tracing::Span {
let chat_span = build_chat_span!(runner, effective_preamble, "chat", "chat");
self.chain_span(chat_span)
}
fn run_model_turn<'a>(
&'a mut self,
runner: &'a AgentRunner,
hook_ctx: &'a HookContext,
run: &'a mut AgentRun,
prepared: PreparedCompletionRequest,
chat_span: tracing::Span,
_agent_span: &'a tracing::Span,
_current_prompt: Message,
) -> DriveStream<'a> {
Box::pin(async_stream::stream! {
let record_accepted_turn = |run: &AgentRun| {
if runner.config.record_telemetry_content
&& let Some(choice) = run.accepted_turn_choice()
{
rig_core::telemetry::record_model_output(&chat_span, &choice, true);
}
};
let attempt_max_tokens = prepared.max_tokens;
let dispatched =
dispatch_completion_response(runner, hook_ctx, run, &prepared.model, prepared.request)
.instrument(chat_span.clone())
.await;
let dispatched = match dispatched {
Ok(dispatched) => dispatched,
Err(err) => {
yield Err(err);
return;
}
};
let mut outcome = match run.model_response(ModelTurn::from_response_parts(
&dispatched.answer,
prepared.executable_tool_names,
prepared.allowed_tool_names,
)) {
Ok(outcome) => outcome,
Err(err) => {
yield Err(err);
return;
}
};
loop {
match outcome {
ModelTurnOutcome::NeedsResolution(context) => {
let action = runner
.config.hooks
.on_invalid_tool_call(hook_ctx, &context)
.await;
let resolution = match action {
Some(action) => run.resolve_invalid_tool_call(action),
None => run.resolve_unhandled_invalid_tool_call(),
};
outcome = match resolution {
Ok(outcome) => outcome,
Err(err) => {
yield Err(err);
return;
}
};
}
ModelTurnOutcome::TurnRetried => break,
ModelTurnOutcome::Continue {
response_hook_suppressed,
} => {
let settlement = settle_model_turn(
&runner.config.hooks,
hook_ctx,
run,
&dispatched,
attempt_max_tokens,
response_hook_suppressed,
)
.await;
match settlement {
Ok(ModelTurnDecision::Advance { .. }) => {}
Ok(ModelTurnDecision::Retried) => break,
Ok(ModelTurnDecision::Terminate(reason)) => {
record_accepted_turn(run);
yield Err(run.cancel_error(reason));
return;
}
Err(err) => {
yield Err(err);
return;
}
}
record_accepted_turn(run);
break;
}
}
}
})
}
fn record_run_level_telemetry(
&self,
agent_span: &tracing::Span,
response: &PromptResponse,
created_agent_span: bool,
) {
if created_agent_span {
if self.record_telemetry_content {
agent_span.record("gen_ai.completion", response.output());
}
agent_span.record_token_usage(&response.usage);
}
}
fn final_item(&self, _response: &PromptResponse) -> Option<MultiTurnStreamItem> {
None
}
}
#[cfg(test)]
#[allow(irrefutable_let_patterns, unreachable_patterns)]
mod tests;
pub(crate) struct CompletionDispatch<A> {
pub(crate) id: EffectId,
pub(crate) kind: EffectKind,
pub(crate) answer: A,
}
fn wrong_family_patch(expected: &str, kind: &EffectKind) -> ErrorReport {
ErrorReport::new(
ErrorKind::Internal,
format!(
"a hook patched a {expected} dispatch into a `{}` effect",
kind.name()
),
)
}
async fn completion_effect(
runner: &AgentRunner,
ctx: &HookContext,
run: &AgentRun,
request: rig_core::completion::CompletionRequest,
stream: bool,
) -> Result<(EffectId, EffectKind), PromptError> {
let id = runner.config.bus.dispatcher().mint_id();
let kind = EffectKind::Completion { request, stream };
let kind = match runner
.config
.hooks
.on_dispatch(
ctx,
DispatchEvent {
id,
kind: &kind,
turn: ctx.turn(),
call_id: None,
context: None,
},
)
.await
{
DispatchAction::Proceed => kind,
DispatchAction::Patch(EffectKind::Completion { request, .. }) => {
EffectKind::Completion { request, stream }
}
DispatchAction::Patch(other) => {
return Err(PromptError::Report(wrong_family_patch(
"completion",
&other,
)));
}
DispatchAction::Deny(report) if report.kind == ErrorKind::Cancelled => {
return Err(run.cancel_error(report.message));
}
DispatchAction::Deny(report) => return Err(PromptError::Report(report)),
};
Ok((id, kind))
}
pub(crate) async fn dispatch_completion_stream(
runner: &AgentRunner,
ctx: &HookContext,
run: &AgentRun,
model: &ModelHandle,
request: rig_core::completion::CompletionRequest,
) -> Result<CompletionDispatch<rig_core::streaming::CompletionStream>, PromptError> {
let (id, kind) = completion_effect(runner, ctx, run, request, true).await?;
let events = runner.config.bus.dispatcher().dispatch_stream_with(
model.key(),
kind.clone(),
DispatchOptions::default().with_id(id),
);
Ok(CompletionDispatch {
id,
kind,
answer: crate::bus::wrap_stream(model.label().to_string(), events),
})
}
pub(crate) async fn dispatch_completion_response(
runner: &AgentRunner,
ctx: &HookContext,
run: &AgentRun,
model: &ModelHandle,
request: rig_core::completion::CompletionRequest,
) -> Result<CompletionDispatch<rig_core::completion::CompletionResponse>, PromptError> {
let (id, kind) = completion_effect(runner, ctx, run, request, false).await?;
let outcome = runner
.config
.bus
.dispatcher()
.dispatch_with(
model.key(),
kind.clone(),
DispatchOptions::default().with_id(id),
)
.await;
match outcome {
Ok(Outcome::Completion(answer)) => Ok(CompletionDispatch { id, kind, answer }),
Ok(other) => Err(PromptError::Report(ErrorReport::new(
ErrorKind::Internal,
format!(
"the completion handler answered with a {} outcome",
other.family()
),
))),
Err(report) => Err(PromptError::Report(report)),
}
}
pub(crate) struct ToolCallDispatch {
pub(crate) result: ToolResult,
pub(crate) executed: bool,
pub(crate) context: crate::tool::ToolContext,
pub(crate) args: String,
}
pub(crate) enum ToolDispatchAbort {
Cancelled(String),
Failed(ErrorReport),
}
pub(crate) async fn dispatch_tool_call(
runner: &AgentRunner,
ctx: &HookContext,
tool_snapshot: &ToolCatalog,
tool_name: &str,
args: String,
call_id: &rig_core::message::CallId,
tool_context: &crate::tool::ToolContext,
) -> Result<ToolCallDispatch, ToolDispatchAbort> {
let hooks = &runner.config.hooks;
let dispatcher = runner.config.bus.dispatcher();
let id = dispatcher.mint_id();
let kind = EffectKind::ToolCall {
name: tool_name.to_owned(),
args,
};
let inbound = tool_context.for_dispatch();
let (kind, denied) = match hooks
.on_dispatch(
ctx,
DispatchEvent {
id,
kind: &kind,
turn: ctx.turn(),
call_id: Some(call_id),
context: Some(&inbound),
},
)
.await
{
DispatchAction::Proceed => (kind, None),
DispatchAction::Patch(patched) => {
match super::hook::validate_dispatch_patch(&kind, &patched) {
Ok(()) => (patched, None),
Err(report) => (kind, Some(report)),
}
}
DispatchAction::Deny(report) => {
if report.kind == ErrorKind::Cancelled {
return Err(ToolDispatchAbort::Cancelled(report.message));
}
tracing::info!(tool_name = tool_name, reason = %report.message, "Tool call rejected");
let salvaged = ctx.take_salvaged_patch(id);
let kind = salvaged
.filter(|patched| super::hook::validate_dispatch_patch(&kind, patched).is_ok())
.unwrap_or(kind);
(kind, Some(report))
}
};
let effective_args = match &kind {
EffectKind::ToolCall { args, .. } => args.clone(),
_ => String::new(),
};
let mut published: Option<crate::tool::ToolContext> = None;
let mut executed = false;
let outcome: Result<Outcome, ErrorReport> = match denied {
Some(report) => Ok(Outcome::ToolResult {
result: ToolResult::skipped(report.message),
}),
None => match tool_snapshot.key(tool_name) {
Some(key) => {
let pending = dispatcher.dispatch_with(
key.raw(),
kind.clone(),
DispatchOptions::default()
.with_id(id)
.with_tool_context(inbound.clone()),
);
let published_at = pending.published_context();
let outcome = pending.await;
executed =
matches!(&outcome, Ok(Outcome::ToolResult { result }) if !result.is_skipped());
published = published_at.and_then(|published| published.take());
outcome
}
None => Ok(Outcome::ToolResult {
result: ToolResult::failed(
crate::tool::ToolExecutionError::not_found(format!(
"no tool named `{tool_name}` is registered"
))
.with_model_feedback(format!("tool `{tool_name}` not found")),
),
}),
},
};
let context = published.unwrap_or_else(|| tool_context.for_dispatch());
let outcome = match hooks
.on_outcome(
ctx,
OutcomeEvent {
id,
kind: &kind,
outcome: &outcome,
turn: ctx.turn(),
call_id: Some(call_id),
context: Some(&context),
},
)
.await
{
OutcomeAction::Proceed => outcome,
OutcomeAction::Replace(replaced) => replaced,
};
Ok(match outcome {
Ok(Outcome::ToolResult { result }) => ToolCallDispatch {
executed,
result,
context,
args: effective_args,
},
Ok(other) => ToolCallDispatch {
executed,
result: ToolResult::failed(crate::tool::ToolExecutionError::other(format!(
"the tool handler answered with a {} outcome",
other.family()
))),
context: tool_context.for_dispatch(),
args: effective_args,
},
Err(report) if report.kind == ErrorKind::Cancelled => {
return Err(ToolDispatchAbort::Cancelled(report.message));
}
Err(report) if report.kind == ErrorKind::Denied => {
tracing::info!(tool_name = tool_name, reason = %report.message, "Tool call denied");
ToolCallDispatch {
executed,
result: ToolResult::skipped(report.message),
context: tool_context.for_dispatch(),
args: effective_args,
}
}
Err(report)
if matches!(
report.kind,
ErrorKind::BusClosed | ErrorKind::HandlerUnavailable | ErrorKind::Divergence
) =>
{
return Err(ToolDispatchAbort::Failed(report));
}
Err(report) => ToolCallDispatch {
executed,
result: ToolResult::failed(report.into()),
context: tool_context.for_dispatch(),
args: effective_args,
},
})
}