use std::collections::BTreeMap;
use std::sync::{
Arc,
atomic::{AtomicU64, Ordering},
};
use futures::{Stream, StreamExt, stream};
use tracing::{Instrument, span::Id};
use rig_core::error::ProviderError;
use rig_core::{
completion::ModelRef,
effect::{Answered, EffectKind},
error::{ErrorKind, ErrorReport},
message::{AssistantContent, Message, ToolCall, ToolFunction, ToolName},
telemetry::SpanCombinator,
wasm_compat::{WasmBoxedFuture, WasmBoxedStream, WasmCompatSend, WasmCompatSync},
};
use super::{
ModelHandle,
completion::{PreparedCompletionRequest, build_prepared_completion_request},
hook::{
AgentHook, CompletionCallAction, CompletionCallEvent, HookContext, HookStack,
InvalidToolCallAction, ModelSelection, ModelSelectionAction, ModelTurnAction,
ModelTurnFinished, ObservationAction, OutcomeAction, ReasoningDelta, RequestPatch,
RunSettled, RunStart, RunStartAction, SettledOutcome, StepEventKind, TextDelta,
ToolCallDelta,
},
runner::AgentRunner,
telemetry::{build_chat_span, new_execute_tool_span},
};
use crate::run::{
AgentRun, AgentRunStep, ModelTurn, ModelTurnOutcome, PendingToolCall, ToolAnswer,
UnhandledInvalidToolCall,
response::{CompletionCall, PromptResponse, finalize_output_tool_choice},
streamed::{StreamedResolution, StreamedTurnAssembler, StreamedTurnEvent},
transcript::choice_text,
};
use crate::{
completion::PromptError,
streaming::{Item, Part, StreamEvent},
tool::{ToolCatalog, ToolResult},
};
use dispatch::{CompletionScope, DispatchScope, model_step};
use projection::{ProjectedItems, committed_stream_items};
mod dispatch;
mod projection;
pub(crate) type DriveStream<'a> = WasmBoxedStream<'a, Result<Surfaced, PromptError>>;
pub(crate) enum Surfaced {
Provider(Item<StreamEvent>),
CompletionCall(CompletionCall),
ModelTurnRetried { turn: usize },
Final(PromptResponse),
}
pub(crate) enum DriveItem {
Surfaced(Surfaced),
Projected(ProjectedItems),
Done(PromptResponse),
}
pub(crate) trait TurnSource: Sized + WasmCompatSend + WasmCompatSync {
const STREAMS: bool;
fn open_chat_span(
&self,
runner: &AgentRunner,
effective_preamble: Option<&str>,
) -> tracing::Span;
fn run_model_turn<'a>(
&'a mut self,
runner: &'a AgentRunner,
hook_ctx: &'a HookContext,
run: &'a mut AgentRun,
prepared: PreparedCompletionRequest,
chat_span: tracing::Span,
agent_span: &'a tracing::Span,
scope: &'a mut CompletionScope<Self>,
) -> DriveStream<'a>;
fn chain_span(&self, span: tracing::Span) -> tracing::Span;
fn record_run_level_telemetry(
&self,
agent_span: &tracing::Span,
response: &PromptResponse,
created_agent_span: bool,
);
fn finish(&mut self, response: PromptResponse) -> DriveItem;
}
pub(crate) fn store_error_usage(runner: &AgentRunner, run: &AgentRun) {
if let Some(usage) = &runner.error_usage {
*usage
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = run.usage();
}
}
pub(crate) fn drive_agent<S>(
runner: AgentRunner,
mut source: S,
mut run: AgentRun,
agent_span: tracing::Span,
created_agent_span: bool,
hook_ctx: HookContext,
) -> impl Stream<Item = Result<DriveItem, PromptError>>
where
S: TurnSource,
{
async_stream::stream! {
hook_ctx.seed_entries(run.entries());
macro_rules! flush_entries {
() => {
for entry in hook_ctx.drain_pending_entries() {
run.append_entry(entry);
}
};
}
macro_rules! fail {
($err:expr, $($exit:tt)+) => {{
let err: PromptError = $err;
store_error_usage(&runner, &run);
if runner.config.hooks.observes(StepEventKind::RunSettled) {
let reason = err.to_string();
runner
.config
.hooks
.on_run_settled(
&hook_ctx,
RunSettled {
outcome: SettledOutcome::Error(&reason),
messages: Some(run.messages()),
},
)
.await;
}
yield Err(err);
$($exit)+;
}};
}
let mut pending_tool_snapshot: Option<Arc<ToolCatalog>> = None;
let mut previous_model: Option<ModelRef> = run.previous_model().cloned();
let mut cursor = run.projection_start();
let mut executed: Vec<Option<ToolCall>> = Vec::new();
macro_rules! project {
($label:lifetime) => {{
let committed = run.messages().get(cursor..).unwrap_or_default();
match committed_stream_items(committed, &std::mem::take(&mut executed)) {
Ok(items) => yield Ok(DriveItem::Projected(items)),
Err(at) => fail!(run.cancel_error(format!(
"agent run driver protocol violation: tool batch \
does not line up with its committed results at {at}"
)), break $label),
}
cursor = run.messages().len();
}};
}
let mut memory = None;
if let Some((store, log)) = runner.memory_log() {
match store.load(&log).await {
Ok(conversation) => {
if matches!(runner.origin, super::runner::RunOrigin::Prompt(_)) {
run = run.with_history(conversation.messages().iter().cloned());
}
memory = Some((store, log, conversation));
}
Err(error) => fail!(PromptError::Memory { error, response: None }, return),
}
}
if runner.config.hooks.observes(StepEventKind::RunStart) {
let action = match run.initial_prompt() {
Some(prompt) => {
runner
.config
.hooks
.on_run_start(
&hook_ctx,
RunStart {
prompt,
history: run.input_chat_history(),
},
)
.await
}
None => RunStartAction::Continue,
};
let early_stop = match action {
RunStartAction::Continue => None,
RunStartAction::Rewrite(prompt) => run
.rewrite_initial_prompt(prompt)
.err(),
RunStartAction::Stop(reason) => {
Some(run.cancel_error(reason))
}
};
if let Some(err) = early_stop {
fail!(err, return);
}
}
'outer: loop {
flush_entries!();
let step = match run.next_step() {
Ok(step) => step,
Err(err) => fail!(err, break 'outer),
};
project!('outer);
match step {
AgentRunStep::CallModel { prompt, history, turn } => {
drop(pending_tool_snapshot.take());
if runner.config.max_turns() > 1 {
tracing::info!("Current conversation Turns: {}/{}", turn, runner.config.max_turns());
}
hook_ctx.set_turn(turn);
let request_patch =
match resolve_completion_call(&runner.config.hooks, &hook_ctx, &prompt, &history, turn).await {
CompletionCallOutcome::Terminate(reason) => {
fail!(run.cancel_error(reason), break 'outer)
}
CompletionCallOutcome::Proceed(request_patch) => request_patch,
};
let default_label = runner.config.model_label();
let selected_label = match runner.config.hooks.on_model_select(
&hook_ctx,
ModelSelection {
prompt: &prompt,
history: &history,
request_patch: request_patch.as_ref(),
previous_model: previous_model.as_ref(),
default_model: &default_label,
selected_model: &default_label,
},
) {
ModelSelectionAction::Continue => default_label.clone(),
ModelSelectionAction::Select(model) => model,
ModelSelectionAction::Stop(reason) => {
fail!(run.cancel_error(reason), break 'outer)
}
};
let selected_model = if selected_label == default_label {
runner.config.model_handle()
} else {
runner.config.model_by_ref(&selected_label)
};
let selected_model: ModelHandle = match selected_model {
Ok(model) => model,
Err(report) => fail!(PromptError::Report(report), break 'outer),
};
let effective_preamble = request_patch
.as_ref()
.and_then(|o| o.preamble.as_deref())
.or(runner.config.spec.preamble.as_deref());
let chat_span = source.open_chat_span(&runner, effective_preamble);
let committed_output_tool = run.output_tool_name().map(str::to_owned);
let (request, mut prepared) = match build_prepared_completion_request(
&runner,
&hook_ctx,
&selected_model,
prompt,
&history,
committed_output_tool.as_deref(),
request_patch.as_ref(),
)
.await
{
Ok(prepared) => prepared,
Err(err) => fail!(PromptError::from(err), break 'outer),
};
let turn_tool_snapshot = prepared.tool_snapshot.clone();
run.advertise_tools(turn, std::mem::take(&mut prepared.advertised_tools));
if runner.config.record_telemetry_content {
let input_messages = std::mem::take(&mut prepared.telemetry_messages);
rig_core::telemetry::record_model_input(&chat_span, &input_messages, true);
}
run.set_previous_model(selected_label.clone());
previous_model = Some(selected_label);
let mut turn_stream = model_step(
&mut source,
&runner,
&hook_ctx,
&mut run,
request,
prepared,
chat_span,
&agent_span,
);
let turn_error = loop {
match turn_stream.next().await {
Some(Ok(item)) => yield Ok(DriveItem::Surfaced(item)),
Some(Err(err)) => break Some(err),
None => break None,
}
};
drop(turn_stream);
if let Some(err) = turn_error {
fail!(err, break 'outer);
}
project!('outer);
pending_tool_snapshot = Some(turn_tool_snapshot);
}
AgentRunStep::CallTools { calls } => {
if pending_tool_snapshot.is_none()
&& let Some(advertised) = run.advertised_tools()
{
let names: Vec<String> = advertised
.definitions
.iter()
.map(|definition| definition.name.to_string())
.collect();
pending_tool_snapshot = Some(Arc::new(
runner.tool_server_handle.snapshot_with_dynamic(&names),
));
}
let Some(tool_snapshot) = pending_tool_snapshot.take() else {
fail!(PromptError::Provider(ProviderError::Response(
"agent requested tool execution without a prepared registry snapshot"
.to_string(),
)), break 'outer);
};
let settled = drive_tool_calls(
&runner,
&hook_ctx,
&mut run,
calls,
tool_snapshot,
|span| source.chain_span(span),
S::STREAMS,
)
.await;
match settled {
Ok(ran) => executed = ran,
Err(err) => fail!(err, break 'outer),
}
project!('outer);
}
AgentRunStep::Done(response) => {
flush_entries!();
if let Some((store, log, conversation)) = &mut memory {
conversation.extend(run.messages().iter().cloned());
if let Err(error) = store.save(log, conversation).await {
let response = Some(Box::new(response));
fail!(PromptError::Memory { error, response }, break 'outer);
}
}
tracing::info!(
turn = run.turn(),
max_turns = runner.config.max_turns(),
"Agent run finished"
);
source.record_run_level_telemetry(&agent_span, &response, created_agent_span);
if runner.config.hooks.observes(StepEventKind::RunSettled) {
runner
.config
.hooks
.on_run_settled(
&hook_ctx,
RunSettled {
outcome: SettledOutcome::Response(&response),
messages: Some(run.messages()),
},
)
.await;
}
yield Ok(source.finish(response));
break 'outer;
}
}
}
}
}
pub(crate) fn drive_tool_calls<'a, F>(
runner: &'a AgentRunner,
hook_ctx: &'a HookContext,
run: &'a mut AgentRun,
calls: Vec<PendingToolCall>,
tool_snapshot: Arc<ToolCatalog>,
chain_tool_span: F,
is_streaming: bool,
) -> WasmBoxedFuture<'a, Result<Vec<Option<ToolCall>>, PromptError>>
where
F: Fn(tracing::Span) -> tracing::Span + WasmCompatSend + 'a,
{
Box::pin(async move {
let call_count = calls.len();
let prepared: Vec<_> = calls
.into_iter()
.map(|call| {
let call = match call {
PendingToolCall::Execute(call) => Ok(call),
PendingToolCall::Malformed(call) => {
let context = run.malformed_context(&call, is_streaming);
Err((call, context))
}
};
(call, chain_tool_span(new_execute_tool_span()))
})
.collect();
let mut collected: Vec<Option<(ToolAnswer, Option<ToolCall>)>> =
(0..call_count).map(|_| None).collect();
let mut first_error: Option<(usize, ToolExit)> = None;
{
let terminating = Arc::new(std::sync::atomic::AtomicBool::new(false));
let unordered = stream::iter(prepared.into_iter().enumerate())
.map(|(index, (call, span))| {
let tool_snapshot = &tool_snapshot;
let terminating = terminating.clone();
async move {
if terminating.load(std::sync::atomic::Ordering::SeqCst) {
return (index, None);
}
let outcome = match call {
Ok(call) => {
run_single_tool(runner, hook_ctx, tool_snapshot, call.tool_call())
.await
.map(|outcome| (call.answer(outcome.result), outcome.executed))
}
Err((call, context)) => {
let action = runner
.config
.hooks
.on_invalid_tool_call(hook_ctx, &context)
.await;
Ok((call.answer(action), None))
}
};
(index, Some(outcome))
}
.instrument(span)
})
.buffer_unordered(runner.concurrency.max(1));
futures::pin_mut!(unordered);
while let Some((index, outcome)) = unordered.next().await {
let Some(result) = outcome else { continue };
match result {
Ok(answered) => {
if answered.0.ends_run() {
terminating.store(true, std::sync::atomic::Ordering::SeqCst);
}
if let Some(slot) = collected.get_mut(index) {
*slot = Some(answered);
}
}
Err(err) => {
terminating.store(true, std::sync::atomic::Ordering::SeqCst);
if first_error.as_ref().is_none_or(|(i, _)| index < *i) {
first_error = Some((index, err));
}
}
}
}
}
let mut answers = Vec::with_capacity(call_count);
let mut ran = Vec::with_capacity(call_count);
for (index, slot) in collected.into_iter().enumerate() {
let Some((answer, executed)) = slot else {
continue;
};
if answer.ends_run() && first_error.as_ref().is_some_and(|(i, _)| *i < index) {
continue;
}
ran.push((answer.index, executed));
answers.push(answer);
}
if let Err(err) = run.answer_all(answers) {
first_error = Some((0, ToolExit::Fail(err)));
}
if let Some((_, exit)) = first_error {
return Err(match exit {
ToolExit::Stop(reason) => run.cancel_error(reason),
ToolExit::Fail(err) => err,
});
}
let committed = match run.messages().last().map(|message| &**message) {
Some(Message::User { content }) => content.len(),
_ => 0,
};
let mut executed: Vec<Option<ToolCall>> = vec![None; committed];
for (index, call) in ran {
if let Some(slot) = executed.get_mut(index) {
*slot = call;
}
}
Ok(executed)
})
}
pub(crate) struct StreamingTurnSource {
last_final_choice: Vec<AssistantContent>,
last_response_id: Option<String>,
agent_name: String,
created_agent_span: bool,
record_telemetry_content: bool,
observes_text_delta: bool,
observes_reasoning_delta: bool,
observes_tool_call_delta: bool,
has_hooks: bool,
}
impl StreamingTurnSource {
pub(crate) fn new(
hooks: &HookStack,
agent_name: String,
created_agent_span: bool,
record_telemetry_content: bool,
) -> Self {
Self {
last_final_choice: Vec::new(),
last_response_id: None,
agent_name,
created_agent_span,
record_telemetry_content,
observes_text_delta: hooks.observes(StepEventKind::TextDelta),
observes_reasoning_delta: hooks.observes(StepEventKind::ReasoningDelta),
observes_tool_call_delta: hooks.observes(StepEventKind::ToolCallDelta),
has_hooks: !hooks.is_empty(),
}
}
fn record_turn_telemetry(
&self,
agent_span: &tracing::Span,
chat_span: &tracing::Span,
choice: &[AssistantContent],
record_content: bool,
) {
if self.created_agent_span && self.record_telemetry_content {
agent_span.record("gen_ai.completion", choice_text(choice));
}
rig_core::telemetry::record_model_output(chat_span, choice, record_content);
}
}
impl TurnSource for StreamingTurnSource {
const STREAMS: bool = true;
fn open_chat_span(
&self,
runner: &AgentRunner,
effective_preamble: Option<&str>,
) -> tracing::Span {
build_chat_span!(runner, effective_preamble, "chat", "chat")
}
fn run_model_turn<'a>(
&'a mut self,
runner: &'a AgentRunner,
hook_ctx: &'a HookContext,
run: &'a mut AgentRun,
prepared: PreparedCompletionRequest,
chat_span: tracing::Span,
agent_span: &'a tracing::Span,
scope: &'a mut CompletionScope<Self>,
) -> DriveStream<'a> {
Box::pin(async_stream::stream! {
let attempt_max_tokens = prepared.max_tokens;
let stream = chat_span.in_scope(|| scope.dispatch_stream(runner, &prepared.model));
let mut stream = match stream {
Ok(stream) => stream,
Err(err) => {
yield Err(err);
return;
}
};
let mut assembler = StreamedTurnAssembler::new(prepared.policy);
let mut text = String::new();
let mut reasoning: BTreeMap<usize, String> = BTreeMap::new();
let mut arguments: BTreeMap<usize, String> = BTreeMap::new();
let mut turn_recovered = false;
let mut held: BTreeMap<Part, Vec<StreamEvent>> = BTreeMap::new();
'turn: while let Some(item) = stream.next().await {
let (mut item_slot, mut next, ended): (
Option<Item<StreamEvent>>,
Option<StreamedTurnEvent>,
bool,
) = match item {
Ok(item) => match assembler.ingest(&item) {
Ok(event) => (Some(item), Some(event), false),
Err(err) => {
yield Err(err.into());
return;
}
},
Err(err) => {
yield Err(err.into());
return;
}
};
while let Some(event) = next.take() {
match event {
StreamedTurnEvent::EmitIngested => {
if self.observes_text_delta
&& let Some(Item::Event(StreamEvent::Text { text: delta, .. })) =
item_slot.as_ref()
&& let Some(reason) = observe_action(
runner
.config.hooks
.on_text_delta(
hook_ctx,
TextDelta {
delta,
aggregated: {
text.push_str(delta);
&text
},
},
)
.await,
)
{
drop(stream);
yield Err(run.cancel_error(reason));
return;
}
if self.observes_reasoning_delta
&& let Some(Item::Event(StreamEvent::Reasoning { part, text: delta })) =
item_slot.as_ref()
&& let Some(reason) = observe_action(
runner
.config.hooks
.on_reasoning_delta(
hook_ctx,
ReasoningDelta {
part: *part,
delta,
aggregated: {
let aggregated =
reasoning.entry(part.index()).or_default();
aggregated.push_str(delta);
aggregated
},
},
)
.await,
)
{
drop(stream);
yield Err(run.cancel_error(reason));
return;
}
if let Some(item) = item_slot.take() {
yield Ok(Surfaced::Provider(item));
}
}
StreamedTurnEvent::EmitToolCallDelta => {
if self.observes_tool_call_delta
&& let Some(Item::Event(StreamEvent::Arguments { part, json })) =
item_slot.as_ref()
&& let Some(reason) = observe_action(
runner
.config.hooks
.on_tool_call_delta(
hook_ctx,
ToolCallDelta {
part: *part,
tool_name: assembler
.streaming_tool_name(part.index())
.map(ToolName::as_str)
.unwrap_or_default(),
delta: json,
aggregated: {
let aggregated =
arguments.entry(part.index()).or_default();
aggregated.push_str(json);
aggregated
},
},
)
.await,
)
{
drop(stream);
yield Err(run.cancel_error(reason));
return;
}
if let Some(item) = item_slot.take() {
yield Ok(Surfaced::Provider(item));
}
}
StreamedTurnEvent::HoldToolCall => {
if let Some(Item::Event(event)) = item_slot.take() {
held.entry(event.part()).or_default().push(event);
}
}
StreamedTurnEvent::EmitToolCall { call } => {
let end = match item_slot.take() {
Some(Item::Event(StreamEvent::End { part, .. })) => Some(part),
_ => None,
};
let replay = end
.and_then(|part| held.remove(&part))
.unwrap_or_default();
let mut aggregated = String::new();
for mut event in replay {
if let StreamEvent::Start { name, .. } = &mut event {
*name = Some(call.function.name.clone());
}
if let StreamEvent::Arguments { json, .. } = &event {
aggregated.push_str(json);
}
if self.observes_tool_call_delta
&& let StreamEvent::Arguments { part, json } = &event
&& let Some(reason) = observe_action(
runner
.config.hooks
.on_tool_call_delta(
hook_ctx,
ToolCallDelta {
part: *part,
tool_name: call.function.name.as_str(),
delta: json,
aggregated: &aggregated,
},
)
.await,
)
{
drop(stream);
yield Err(run.cancel_error(reason));
return;
}
yield Ok(Surfaced::Provider(Item::Event(event)));
}
if let Some(part) = end {
yield Ok(Surfaced::Provider(Item::Event(
StreamEvent::End {
part,
content: AssistantContent::ToolCall(call),
},
)));
}
}
StreamedTurnEvent::InvalidToolCall(invalid) => {
let rejected = match item_slot.as_ref() {
Some(Item::Event(event)) => Some(event.part()),
_ => None,
};
let partial = assembler.partial_turn(&stream.partial());
let hook_action = if self.has_hooks {
let context =
run.streamed_invalid_tool_call_context(&partial, &invalid);
runner
.config.hooks
.on_invalid_tool_call(hook_ctx, &context)
.await
} else {
None
};
let resolved = match hook_action {
Some(action) => {
run.resolve_streamed_invalid_tool_call(&partial, &invalid, action)
}
None => match run.unhandled_invalid_tool_call() {
UnhandledInvalidToolCall::Fail => run
.resolve_streamed_invalid_tool_call(
&partial,
&invalid,
InvalidToolCallAction::fail(),
),
UnhandledInvalidToolCall::Ignore => {
run.ignore_streamed_invalid_tool_call()
}
},
};
let resolution = match resolved {
Ok(resolution) => resolution,
Err(err) => {
yield Err(err);
return;
}
};
match resolution {
StreamedResolution::Ignored => {
assembler.resolve_pending_invalid(&resolution);
if let Some(part) = rejected {
held.remove(&part);
}
item_slot = None;
}
StreamedResolution::Repaired { .. } => {
turn_recovered = true;
next = assembler.resolve_pending_invalid(&resolution);
}
StreamedResolution::TurnAbandoned => {
assembler.resolve_pending_invalid(&resolution);
held.clear();
if !ended {
match scope.finish_stream(stream, run, &chat_span).await {
Ok((_, call)) => {
yield Ok(Surfaced::CompletionCall(call));
}
Err(err) => {
yield Err(err);
return;
}
}
}
return;
}
}
}
}
}
if ended {
break 'turn;
}
}
let response = match scope.finish_stream(stream, run, &chat_span).await {
Ok((response, call)) => {
yield Ok(Surfaced::CompletionCall(call));
response
}
Err(err) => {
yield Err(err);
return;
}
};
let streamed_turn = assembler.finish(response);
self.last_response_id = response.response_id().map(str::to_owned);
let mut final_turn_content =
std::mem::replace(&mut response.choice, streamed_turn.choice.clone());
if let Err(err) = run.streamed_turn(streamed_turn) {
yield Err(err);
return;
}
let settlement = settle_model_turn(
runner,
hook_ctx,
run,
scope,
attempt_max_tokens,
turn_recovered,
)
.await;
match settlement {
Ok(ModelTurnDecision::Advance { replaced }) => {
if let Some(choice) = replaced {
final_turn_content = choice;
}
}
Ok(ModelTurnDecision::Retried) => {
yield Ok(Surfaced::ModelTurnRetried {
turn: hook_ctx.turn(),
});
return;
}
Ok(ModelTurnDecision::Terminate(reason)) => {
self.record_turn_telemetry(
agent_span,
&chat_span,
scope.choice(),
runner.config.record_telemetry_content,
);
yield Err(run.cancel_error(reason));
return;
}
Err(err) => {
yield Err(err);
return;
}
}
self.record_turn_telemetry(
agent_span,
&chat_span,
scope.choice(),
runner.config.record_telemetry_content,
);
self.last_final_choice = final_turn_content;
})
}
fn chain_span(&self, span: tracing::Span) -> tracing::Span {
span
}
fn record_run_level_telemetry(
&self,
agent_span: &tracing::Span,
response: &PromptResponse,
created_agent_span: bool,
) {
if created_agent_span {
agent_span.record_token_usage(&response.usage);
}
}
fn finish(&mut self, mut response: PromptResponse) -> DriveItem {
let final_choice = finalize_output_tool_choice(&self.last_final_choice, &response.output())
.unwrap_or_else(|| {
if self.last_final_choice.iter().all(AssistantContent::is_blank) {
tracing::warn!(
agent_name = self.agent_name.as_str(),
response_id = ?self.last_response_id,
"Streaming turn completed without assistant text; final response will be empty"
);
}
std::mem::take(&mut self.last_final_choice)
});
response.content = final_choice;
DriveItem::Surfaced(Surfaced::Final(response.with_output_tool_calls(0)))
}
}
pub(crate) fn observe_action(action: ObservationAction) -> Option<String> {
match action {
ObservationAction::Continue => None,
ObservationAction::Stop(reason) => Some(reason),
}
}
pub(crate) enum ModelTurnDecision {
Advance {
replaced: Option<Vec<AssistantContent>>,
},
Retried,
Terminate(String),
}
pub(crate) async fn settle_model_turn<M>(
runner: &AgentRunner,
hook_ctx: &HookContext,
run: &mut AgentRun,
scope: &mut CompletionScope<M>,
max_tokens: Option<u64>,
recovered: bool,
) -> Result<ModelTurnDecision, PromptError> {
if recovered {
return Ok(ModelTurnDecision::Advance { replaced: None });
}
let (action, response) = scope.close(runner, hook_ctx).await?;
let hooks = &runner.config.hooks;
let mut replaced: Option<Vec<AssistantContent>> = None;
match action {
OutcomeAction::Proceed => {}
OutcomeAction::Replace(Ok(Answered::Completion(replacement))) => {
let replacement = Arc::unwrap_or_clone(replacement);
run.replace_accepted_turn_choice(replacement.choice.clone())?;
replaced = Some(replacement.choice);
}
OutcomeAction::Replace(Ok(other)) => {
return Err(PromptError::Report(wrong_outcome("a completion", &other)));
}
OutcomeAction::Replace(Err(report)) => {
if report.kind == ErrorKind::Cancelled {
return Ok(ModelTurnDecision::Terminate(report.message));
}
return Err(PromptError::Report(report));
}
}
let identity = response.identity();
let finish_reason = response.finish_reason();
let content = replaced.as_ref().unwrap_or(&response.choice);
let action = hooks
.on_model_turn_finished(
hook_ctx,
ModelTurnFinished {
turn: hook_ctx.turn(),
content,
usage: response.usage,
identity: &identity,
finish_reason: finish_reason.as_ref(),
max_tokens,
raw: &response.raw,
},
)
.await;
match action {
ModelTurnAction::Continue => Ok(ModelTurnDecision::Advance { replaced }),
ModelTurnAction::Retry(request) => {
run.retry_model_turn(request)?;
Ok(ModelTurnDecision::Retried)
}
ModelTurnAction::Stop(reason) => Ok(ModelTurnDecision::Terminate(reason)),
}
}
pub(crate) enum CompletionCallOutcome {
Proceed(Option<RequestPatch>),
Terminate(String),
}
pub(crate) async fn resolve_completion_call(
hooks: &HookStack,
ctx: &HookContext,
prompt: &Message,
history: &[Arc<Message>],
turn: usize,
) -> CompletionCallOutcome {
match hooks
.on_completion_call(
ctx,
CompletionCallEvent {
prompt,
history,
turn,
},
)
.await
{
CompletionCallAction::Stop(reason) => CompletionCallOutcome::Terminate(reason),
CompletionCallAction::Patch(patch) => CompletionCallOutcome::Proceed(Some(patch)),
CompletionCallAction::Continue => CompletionCallOutcome::Proceed(None),
}
}
pub(crate) async fn dispatch_effect(
runner: &AgentRunner,
ctx: &HookContext,
key: &rig_core::effect::HandlerKey,
kind: EffectKind,
) -> Result<Answered, ErrorReport> {
let scope = DispatchScope::open(runner, ctx, kind, None)
.await
.map_err(|(_, report)| report)?;
let kind = scope.kind().clone();
let beside = crate::handlers::Beside::for_kind(&kind);
let outcome = runner
.config
.handlers
.serve(scope.id(), key.clone(), kind, beside)
.await;
match scope.close(runner, ctx, &outcome, None).await {
OutcomeAction::Proceed => outcome,
OutcomeAction::Replace(replaced) => replaced,
}
}
pub(crate) fn wrong_outcome(expected: &str, outcome: &Answered) -> ErrorReport {
ErrorReport::new(
ErrorKind::Internal,
format!(
"expected {expected}, the handler answered with a {} outcome",
outcome.family()
),
)
}
pub(crate) enum ToolExit {
Stop(String),
Fail(PromptError),
}
pub(crate) struct ToolCallOutcome {
pub result: ToolResult,
pub executed: Option<ToolCall>,
}
pub(crate) async fn run_single_tool(
runner: &AgentRunner,
ctx: &HookContext,
tool_snapshot: &ToolCatalog,
tool_call: &ToolCall,
) -> Result<ToolCallOutcome, ToolExit> {
let record_content = runner.config.record_telemetry_content;
let tool_name = tool_call.function.name.as_str();
let args = tool_call.function.arguments_json();
let tool_span = tracing::Span::current();
tool_span.record("gen_ai.tool.name", tool_name);
tool_span.record(
"gen_ai.tool.call.id",
tracing::field::display(&tool_call.id),
);
if record_content {
tool_span.record("gen_ai.tool.call.arguments", &args);
}
let ToolCallDispatch {
result: exec,
executed,
args: effective_args,
} = dispatch_tool_call(runner, ctx, tool_snapshot, tool_call, args.clone()).await?;
if effective_args != args {
if record_content {
tool_span.record("gen_ai.tool.call.arguments", &effective_args);
}
tracing::debug!(
tool_name = tool_name,
"tool-call arguments rewritten by a hook"
);
}
let executed = executed.then(|| {
let mut effective_tool_call = tool_call.clone();
effective_tool_call.function =
ToolFunction::parse(tool_call.function.name.clone(), &effective_args);
effective_tool_call
});
record_tool_result(&tool_span, &exec);
if record_content {
tool_span.record("gen_ai.tool.call.result", exec.output().render());
}
Ok(ToolCallOutcome {
result: exec,
executed,
})
}
fn record_tool_result(span: &tracing::Span, result: &ToolResult) {
span.record("gen_ai.tool.call.outcome", result.status_name());
if let Some(error) = result.error() {
span.record("gen_ai.tool.error.type", error.kind().as_str());
}
}
pub(crate) struct UnaryTurnSource {
current_span_id: AtomicU64,
record_telemetry_content: bool,
}
impl UnaryTurnSource {
pub(crate) fn new(record_telemetry_content: bool) -> Self {
Self {
current_span_id: AtomicU64::new(0),
record_telemetry_content,
}
}
}
impl TurnSource for UnaryTurnSource {
const STREAMS: bool = false;
fn chain_span(&self, span: tracing::Span) -> tracing::Span {
let span = match self.current_span_id.load(Ordering::Relaxed) {
0 => span,
id => {
span.follows_from(Id::from_u64(id));
span
}
};
if let Some(id) = span.id() {
self.current_span_id.store(id.into_u64(), Ordering::Relaxed);
}
span
}
fn open_chat_span(
&self,
runner: &AgentRunner,
effective_preamble: Option<&str>,
) -> tracing::Span {
let chat_span = build_chat_span!(runner, effective_preamble, "chat", "chat");
self.chain_span(chat_span)
}
fn run_model_turn<'a>(
&'a mut self,
runner: &'a AgentRunner,
hook_ctx: &'a HookContext,
run: &'a mut AgentRun,
prepared: PreparedCompletionRequest,
chat_span: tracing::Span,
_agent_span: &'a tracing::Span,
scope: &'a mut CompletionScope<Self>,
) -> DriveStream<'a> {
Box::pin(async_stream::stream! {
let record_accepted_turn = |run: &AgentRun| {
if runner.config.record_telemetry_content
&& let Some(choice) = run.accepted_turn_choice()
{
rig_core::telemetry::record_model_output(&chat_span, &choice, true);
}
};
let attempt_max_tokens = prepared.max_tokens;
let dispatched = scope
.dispatch_response(runner, &prepared.model)
.instrument(chat_span.clone())
.await;
let response = match dispatched {
Ok(response) => response,
Err(err) => {
yield Err(err);
return;
}
};
let mut outcome = match run.model_response(ModelTurn::from_policy(
response,
prepared.policy,
)) {
Ok(outcome) => outcome,
Err(err) => {
yield Err(err);
return;
}
};
loop {
match outcome {
ModelTurnOutcome::NeedsResolution(context) => {
let action = runner
.config.hooks
.on_invalid_tool_call(hook_ctx, &context)
.await;
let resolution = match action {
Some(action) => run.resolve_invalid_tool_call(action),
None => run.resolve_unhandled_invalid_tool_call(),
};
outcome = match resolution {
Ok(outcome) => outcome,
Err(err) => {
yield Err(err);
return;
}
};
}
ModelTurnOutcome::TurnRetried => break,
ModelTurnOutcome::Continue {
response_hook_suppressed,
} => {
let settlement = settle_model_turn(
runner,
hook_ctx,
run,
scope,
attempt_max_tokens,
response_hook_suppressed,
)
.await;
match settlement {
Ok(ModelTurnDecision::Advance { .. }) => {}
Ok(ModelTurnDecision::Retried) => break,
Ok(ModelTurnDecision::Terminate(reason)) => {
record_accepted_turn(run);
yield Err(run.cancel_error(reason));
return;
}
Err(err) => {
yield Err(err);
return;
}
}
record_accepted_turn(run);
break;
}
}
}
})
}
fn record_run_level_telemetry(
&self,
agent_span: &tracing::Span,
response: &PromptResponse,
created_agent_span: bool,
) {
if created_agent_span {
if self.record_telemetry_content {
agent_span.record("gen_ai.completion", response.output());
}
agent_span.record_token_usage(&response.usage);
}
}
fn finish(&mut self, response: PromptResponse) -> DriveItem {
DriveItem::Done(response)
}
}
#[cfg(test)]
#[allow(irrefutable_let_patterns, unreachable_patterns)]
mod tests;
pub(crate) struct ToolCallDispatch {
pub(crate) result: ToolResult,
pub(crate) executed: bool,
pub(crate) args: String,
}
pub(crate) async fn dispatch_tool_call(
runner: &AgentRunner,
ctx: &HookContext,
tool_snapshot: &ToolCatalog,
tool_call: &ToolCall,
args: String,
) -> Result<ToolCallDispatch, ToolExit> {
let (tool_name, call_id) = (tool_call.function.name.as_str(), &tool_call.id);
let kind = EffectKind::ToolCall {
name: tool_name.to_owned(),
args,
};
let inbound = runner.tool_context.for_dispatch();
let (scope, denied) = match DispatchScope::open(runner, ctx, kind, Some((call_id, &inbound)))
.await
{
Ok(scope) => (scope, None),
Err((_, report)) if report.kind == ErrorKind::Cancelled => {
return Err(ToolExit::Stop(report.message));
}
Err((scope, report)) => {
tracing::info!(tool_name = tool_name, reason = %report.message, "Tool call rejected");
(scope, Some(report))
}
};
let args = match scope.kind() {
EffectKind::ToolCall { args, .. } => args.clone(),
_ => String::new(),
};
let mut published: Option<crate::tool::ToolContext> = None;
let mut executed = false;
let outcome: Result<Answered, ErrorReport> = match denied {
Some(report) => Ok(Answered::ToolResult {
result: ToolResult::skipped(report.message),
}),
None => match tool_snapshot.get(tool_name) {
Some(tool) => {
let slot = rig_core::tool::PublishedContext::new();
let beside = crate::handlers::Beside {
handler: Some(tool.handler().clone()),
context: Some(inbound),
published: Some(std::sync::Arc::clone(&slot)),
};
let key = tool.key().raw().clone();
let served =
runner
.config
.handlers
.serve(scope.id(), key, scope.kind().clone(), beside);
let outcome = served.await;
executed =
matches!(&outcome, Ok(Answered::ToolResult { result }) if !result.is_skipped());
published = slot.take();
outcome
}
None => Ok(Answered::ToolResult {
result: ToolResult::failed(
crate::tool::ToolExecutionError::not_found(format!(
"no tool named `{tool_name}` is registered"
))
.with_model_feedback(format!("tool `{tool_name}` not found")),
),
}),
},
};
let context = published.unwrap_or_else(|| runner.tool_context.for_dispatch());
let outcome = match scope
.close(runner, ctx, &outcome, Some((call_id, &context)))
.await
{
OutcomeAction::Proceed => outcome,
OutcomeAction::Replace(replaced) => replaced,
};
let result = match outcome {
Ok(Answered::ToolResult { result }) => result,
Ok(other) => ToolResult::failed(crate::tool::ToolExecutionError::other(format!(
"the tool handler answered with a {} outcome",
other.family()
))),
Err(report) if report.kind == ErrorKind::Cancelled => {
return Err(ToolExit::Stop(report.message));
}
Err(report) if report.kind == ErrorKind::Denied => {
tracing::info!(tool_name = tool_name, reason = %report.message, "Tool call denied");
ToolResult::skipped(report.message)
}
Err(report)
if matches!(
report.kind,
ErrorKind::HandlerUnavailable | ErrorKind::Divergence
) =>
{
return Err(ToolExit::Fail(PromptError::Report(report)));
}
Err(report) => ToolResult::failed(report.into()),
};
Ok(ToolCallDispatch {
result,
executed,
args,
})
}