use std::collections::{BTreeMap, VecDeque};
use std::sync::{
Arc,
atomic::{AtomicU64, Ordering},
};
use futures::{
Stream, StreamExt,
channel::{mpsc, oneshot},
future::{self, Either},
stream,
};
use tracing::{Instrument, span::Id};
use rig_core::error::ProviderError;
use rig_core::{
completion::ModelRef,
effect::{Answered, EffectKind, Family, family},
error::{ErrorKind, ErrorReport},
message::{
self, AssistantContent, Message, ToolCall, ToolFunction, ToolName, ToolResultContent,
},
telemetry::SpanCombinator,
tool::Touches,
wasm_compat::{WasmBoxedFuture, WasmBoxedStream, WasmCompatSend, WasmCompatSync},
};
use super::{
ModelHandle,
completion::{PreparedCompletionRequest, build_prepared_completion_request},
hook::{
AgentHook, CompletionCallAction, CompletionCallEvent, HookContext, HookStack,
InvalidToolCallAction, InvalidToolCallContext, ModelSelection, ModelSelectionAction,
ModelTurnAction, ModelTurnFinished, ObservationAction, OutcomeAction, ReasoningDelta,
RunSettled, RunStart, RunStartAction, SettledOutcome, StepEventKind, TextDelta,
ToolCallDelta,
},
runner::AgentRunner,
telemetry::{build_chat_span, new_execute_tool_span},
};
use crate::run::{
AgentRun, AgentRunStep, ExecCall, MalformedCall, ModelTurn, ModelTurnOutcome, PendingToolCall,
ToolAnswer, UnhandledInvalidToolCall,
response::{CompletionCall, PromptResponse, finalize_output_tool_choice},
streamed::{StreamedResolution, StreamedTurnAssembler, StreamedTurnEvent},
};
use crate::{
completion::PromptError,
streaming::{Item, Part, StreamEvent},
tool::{ToolCatalog, ToolResult},
};
use dispatch::{CompletionScope, DispatchScope, model_step};
use projection::{ProjectedItems, committed_stream_items};
mod dispatch;
mod projection;
pub(crate) type DriveStream<'a> = WasmBoxedStream<'a, Result<Surfaced, PromptError>>;
pub(crate) enum Surfaced {
Provider(Item<StreamEvent>),
CompletionCall(CompletionCall),
ModelTurnRetried { turn: usize },
Final(PromptResponse),
}
pub(crate) enum DriveItem {
Surfaced(Surfaced),
Live(LiveTool),
Projected(ProjectedItems),
Done(PromptResponse),
}
pub(crate) trait TurnSource: Sized + WasmCompatSend + WasmCompatSync {
const STREAMS: bool;
fn run_model_turn<'a>(
&'a mut self,
runner: &'a AgentRunner,
hook_ctx: &'a HookContext,
run: &'a mut AgentRun,
prepared: PreparedCompletionRequest,
chat_span: tracing::Span,
scope: &'a mut CompletionScope<Self>,
) -> DriveStream<'a>;
fn chain_span(&self, span: tracing::Span) -> tracing::Span;
fn finish(&mut self, response: PromptResponse) -> DriveItem;
}
pub(crate) fn store_error_usage(runner: &AgentRunner, run: &AgentRun) {
if let Some(usage) = &runner.error_usage {
*crate::lock(usage) = run.usage();
}
}
pub(crate) fn drive_agent<S>(
runner: AgentRunner,
mut source: S,
ambient: tracing::Span,
) -> impl Stream<Item = Result<DriveItem, PromptError>>
where
S: TurnSource,
{
let (agent_span, created_agent_span) = runner.open_agent_span(ambient);
let hook_ctx = runner.hook_context(S::STREAMS);
let mut run = runner.build_run();
let driver_span = agent_span.clone();
let driver = async_stream::stream! {
hook_ctx.seed_entries(run.entries());
macro_rules! flush_entries {
() => {
for entry in hook_ctx.drain_pending_entries() {
run.append_entry(entry);
}
};
}
macro_rules! fail {
($err:expr, $($exit:tt)+) => {{
let err: PromptError = $err;
store_error_usage(&runner, &run);
let reason = err.to_string();
settle_run(&runner, &hook_ctx, &run, SettledOutcome::Error(&reason)).await;
yield Err(err);
$($exit)+;
}};
}
let mut pending_tool_snapshot: Option<Arc<ToolCatalog>> = None;
let mut previous_model: Option<ModelRef> = run.previous_model().cloned();
let mut cursor = run.projection_start();
let mut executed: Vec<Option<ToolCall>> = Vec::new();
macro_rules! project {
($label:lifetime) => {{
let committed = run.messages().get(cursor..).unwrap_or_default();
match committed_stream_items(committed, &std::mem::take(&mut executed)) {
Ok(items) => yield Ok(DriveItem::Projected(items)),
Err(at) => fail!(run.cancel_error(format!(
"agent run driver protocol violation: tool batch \
does not line up with its committed results at {at}"
)), break $label),
}
cursor = run.messages().len();
}};
}
let mut memory = None;
if let Some((store, log)) = runner.memory_log() {
match store.load(&log).await {
Ok(conversation) => {
if matches!(runner.origin, super::runner::RunOrigin::Prompt(_)) {
run = run.with_history(conversation.messages().iter().cloned());
}
memory = Some((store, log, conversation));
}
Err(error) => fail!(PromptError::Memory { error, response: None }, return),
}
}
if runner.config.hooks.observes(StepEventKind::RunStart) {
let action = match run.initial_prompt() {
Some(prompt) => {
runner
.config
.hooks
.on_run_start(
&hook_ctx,
RunStart {
prompt,
history: run.input_chat_history(),
},
)
.await
}
None => RunStartAction::Continue,
};
let early_stop = match action {
RunStartAction::Continue => None,
RunStartAction::Rewrite(prompt) => run
.rewrite_initial_prompt(prompt)
.err(),
RunStartAction::Stop(reason) => {
Some(run.cancel_error(reason))
}
};
if let Some(err) = early_stop {
fail!(err, return);
}
}
'outer: loop {
flush_entries!();
let step = match run.next_step() {
Ok(step) => step,
Err(err) => fail!(err, break 'outer),
};
project!('outer);
match step {
AgentRunStep::CallModel { prompt, history, turn } => {
drop(pending_tool_snapshot.take());
if runner.config.max_turns() > 1 {
tracing::info!("Current conversation Turns: {}/{}", turn, runner.config.max_turns());
}
hook_ctx.set_turn(turn);
let completion_call = CompletionCallEvent { prompt: &prompt, history: &history, turn };
let request_patch = match runner.config.hooks.on_completion_call(&hook_ctx, completion_call).await {
CompletionCallAction::Stop(reason) => {
fail!(run.cancel_error(reason), break 'outer)
}
CompletionCallAction::Patch(patch) => Some(patch),
CompletionCallAction::Continue => None,
};
let default_label = runner.config.model_label();
let selected_label = match runner.config.hooks.on_model_select(
&hook_ctx,
ModelSelection {
prompt: &prompt,
history: &history,
request_patch: request_patch.as_ref(),
previous_model: previous_model.as_ref(),
default_model: &default_label,
selected_model: &default_label,
},
) {
ModelSelectionAction::Continue => default_label.clone(),
ModelSelectionAction::Select(model) => model,
ModelSelectionAction::Stop(reason) => {
fail!(run.cancel_error(reason), break 'outer)
}
};
let selected_model = if selected_label == default_label {
runner.config.model_handle()
} else {
runner.config.model_by_ref(&selected_label)
};
let selected_model: ModelHandle = match selected_model {
Ok(model) => model,
Err(report) => fail!(PromptError::Report(report), break 'outer),
};
let effective_preamble = request_patch
.as_ref()
.and_then(|o| o.preamble.as_deref())
.or(runner.config.spec.preamble.as_deref());
let chat_span =
source.chain_span(build_chat_span!(runner, effective_preamble));
let committed_output_tool = run.output_tool_name().map(str::to_owned);
let (request, prepared) = match build_prepared_completion_request(
&runner,
&hook_ctx,
&selected_model,
prompt,
&history,
committed_output_tool.as_deref(),
request_patch.as_ref(),
)
.await
{
Ok(prepared) => prepared,
Err(err) => fail!(PromptError::from(err), break 'outer),
};
let turn_tool_snapshot = prepared.tool_snapshot.clone();
run.advertise_tools(turn, request.tools.clone());
if runner.config.record_telemetry_content {
let input_messages = request.messages_for_telemetry();
rig_core::telemetry::record_model_input(&chat_span, &input_messages, true);
}
run.set_previous_model(selected_label.clone());
previous_model = Some(selected_label);
let mut turn_stream = model_step(
&mut source,
&runner,
&hook_ctx,
&mut run,
request,
prepared,
chat_span,
);
let turn_error = loop {
match turn_stream.next().await {
Some(Ok(item)) => yield Ok(DriveItem::Surfaced(item)),
Some(Err(err)) => break Some(err),
None => break None,
}
};
drop(turn_stream);
if let Some(err) = turn_error {
fail!(err, break 'outer);
}
project!('outer);
pending_tool_snapshot = Some(turn_tool_snapshot);
}
AgentRunStep::CallTools { calls } => {
if pending_tool_snapshot.is_none()
&& let Some(advertised) = run.advertised_tools()
{
let names: Vec<String> = advertised
.definitions
.iter()
.map(|definition| definition.name.to_string())
.collect();
pending_tool_snapshot = Some(Arc::new(
runner.tool_server_handle.snapshot_with_dynamic(&names),
));
}
let Some(tool_snapshot) = pending_tool_snapshot.take() else {
fail!(PromptError::Provider(ProviderError::Response(
"agent requested tool execution without a prepared registry snapshot"
.to_string(),
)), break 'outer);
};
let (report, mut reports) = mpsc::unbounded();
let settled = {
let mut settle = drive_tool_calls(
&runner,
&hook_ctx,
&mut run,
calls,
tool_snapshot,
|span| source.chain_span(span),
S::STREAMS.then_some(report),
);
loop {
match future::select(&mut settle, reports.next()).await {
Either::Left((settled, _)) => break settled,
Either::Right((Some((live, taken)), _)) => {
yield Ok(DriveItem::Live(live));
let _ = taken.send(());
}
Either::Right((None, _)) => break settle.await,
}
}
};
match settled {
Ok(ran) => executed = ran,
Err(err) => fail!(err, break 'outer),
}
project!('outer);
}
AgentRunStep::Done(response) => {
flush_entries!();
if let Some((store, log, conversation)) = &mut memory {
conversation.extend(run.messages().iter().cloned());
if let Err(error) = store.save(log, conversation).await {
let response = Some(Box::new(response));
fail!(PromptError::Memory { error, response }, break 'outer);
}
}
tracing::info!(
turn = run.turn(),
max_turns = runner.config.max_turns(),
"Agent run finished"
);
if created_agent_span {
if runner.config.record_telemetry_content {
agent_span.record("gen_ai.completion", response.output());
}
agent_span.record_token_usage(&response.usage);
}
settle_run(&runner, &hook_ctx, &run, SettledOutcome::Response(&response)).await;
yield Ok(source.finish(response));
break 'outer;
}
}
}
};
tracing_futures::Instrument::instrument(driver, driver_span)
}
async fn settle_run(
runner: &AgentRunner,
hook_ctx: &HookContext,
run: &AgentRun,
outcome: SettledOutcome<'_>,
) {
let hooks = &runner.config.hooks;
if hooks.observes(StepEventKind::RunSettled) {
let messages = Some(run.messages());
hooks
.on_run_settled(hook_ctx, RunSettled { outcome, messages })
.await;
}
}
pub(crate) fn drive_tool_calls<'a, F>(
runner: &'a AgentRunner,
hook_ctx: &'a HookContext,
run: &'a mut AgentRun,
calls: Vec<PendingToolCall>,
tool_snapshot: Arc<ToolCatalog>,
chain_tool_span: F,
live: Option<LiveSender>,
) -> WasmBoxedFuture<'a, Result<Vec<Option<ToolCall>>, PromptError>>
where
F: Fn(tracing::Span) -> tracing::Span + WasmCompatSend + 'a,
{
Box::pin(async move {
let call_count = calls.len();
let prepared: Vec<_> = calls
.into_iter()
.map(|call| {
let call = match call {
PendingToolCall::Execute(call) => Ok(call),
PendingToolCall::Malformed(call) => {
let context = run.malformed_context(&call, hook_ctx.is_streaming());
Err((call, context))
}
};
(call, chain_tool_span(new_execute_tool_span()))
})
.collect();
let mut collected: Vec<Option<(ToolAnswer, Option<ToolCall>)>> =
(0..call_count).map(|_| None).collect();
let mut first_error: Option<(usize, ToolExit)> = None;
{
let terminating = Arc::new(std::sync::atomic::AtomicBool::new(false));
let footprints: Vec<Option<Touches>> = prepared
.iter()
.map(|(call, _)| match call {
Ok(call) if runner.ordered => Some(footprint(&tool_snapshot, call.tool_call())),
_ => None,
})
.collect();
let mut queued: Vec<_> = prepared.into_iter().map(Some).collect();
let mut settled = vec![false; call_count];
let start = |index: usize,
(call, span): (
Result<ExecCall, (MalformedCall, InvalidToolCallContext)>,
tracing::Span,
)| {
let tool_snapshot = &tool_snapshot;
let terminating = terminating.clone();
let live = live.clone();
async move {
if terminating.load(std::sync::atomic::Ordering::SeqCst) {
return (index, None);
}
let outcome = match call {
Ok(call) => {
let tool_call = call.tool_call();
if let Some(live) = &live {
report(live, LiveTool::Started(tool_call.clone())).await;
}
let outcome =
run_single_tool(runner, hook_ctx, tool_snapshot, tool_call).await;
if let (Some(live), Ok(outcome)) = (&live, &outcome) {
for item in output_items(tool_call.answer(&outcome.result)) {
report(live, item).await;
}
}
outcome.map(|outcome| (call.answer(outcome.result), outcome.executed))
}
Err((call, context)) => {
let action = runner
.config
.hooks
.on_invalid_tool_call(hook_ctx, &context)
.await;
Ok((call.answer(action), None))
}
};
(index, Some(outcome))
}
.instrument(span)
};
let mut running = stream::FuturesUnordered::new();
loop {
for index in 0..call_count {
if running.len() >= runner.concurrency.max(1) {
break;
}
if waits(&footprints, &settled, index) {
continue;
}
if let Some(call) = queued.get_mut(index).and_then(Option::take) {
running.push(start(index, call));
}
}
let Some((index, outcome)) = running.next().await else {
break;
};
if let Some(slot) = settled.get_mut(index) {
*slot = true;
}
let Some(result) = outcome else { continue };
match result {
Ok(answered) => {
if answered.0.ends_run() {
terminating.store(true, std::sync::atomic::Ordering::SeqCst);
}
if let Some(slot) = collected.get_mut(index) {
*slot = Some(answered);
}
}
Err(err) => {
terminating.store(true, std::sync::atomic::Ordering::SeqCst);
if first_error.as_ref().is_none_or(|(i, _)| index < *i) {
first_error = Some((index, err));
}
}
}
}
}
let mut answers = Vec::with_capacity(call_count);
let mut ran = Vec::with_capacity(call_count);
for (index, slot) in collected.into_iter().enumerate() {
let Some((answer, executed)) = slot else {
continue;
};
if answer.ends_run() && first_error.as_ref().is_some_and(|(i, _)| *i < index) {
continue;
}
ran.push((answer.index, executed));
answers.push(answer);
}
if let Err(err) = run.answer_all(answers) {
first_error = Some((0, ToolExit::Fail(err)));
}
if let Some((_, exit)) = first_error {
return Err(match exit {
ToolExit::Stop(reason) => run.cancel_error(reason),
ToolExit::Fail(err) => err,
});
}
let committed = match run.messages().last().map(|message| &**message) {
Some(Message::User { content }) => content.len(),
_ => 0,
};
let mut executed: Vec<Option<ToolCall>> = vec![None; committed];
for (index, call) in ran {
if let Some(slot) = executed.get_mut(index) {
*slot = call;
}
}
Ok(executed)
})
}
pub(crate) enum LiveTool {
Started(ToolCall),
Output {
tool_result: message::ToolResult,
index: usize,
count: usize,
},
}
pub(crate) type LiveSender = mpsc::UnboundedSender<(LiveTool, oneshot::Sender<()>)>;
async fn report(sender: &LiveSender, live: LiveTool) {
let (taken, wait) = oneshot::channel();
if sender.unbounded_send((live, taken)).is_ok() {
let _ = wait.await;
}
}
fn output_items(result: message::ToolResult) -> Vec<LiveTool> {
let message::ToolResult {
call,
name,
content,
is_error,
} = result;
let items: Vec<Vec<ToolResultContent>> = match <[ToolResultContent; 1]>::try_from(content) {
Ok(
[
ToolResultContent::Json {
value: serde_json::Value::Array(elements),
},
],
) => elements
.into_iter()
.map(|value| vec![ToolResultContent::Json { value }])
.collect(),
Ok(one) => vec![Vec::from(one)],
Err(content) => vec![content],
};
let count = items.len();
items
.into_iter()
.enumerate()
.map(|(index, content)| LiveTool::Output {
tool_result: message::ToolResult {
call: call.clone(),
name: name.clone(),
content,
is_error,
},
index,
count,
})
.collect()
}
fn waits(footprints: &[Option<Touches>], settled: &[bool], index: usize) -> bool {
let earlier = footprints
.iter()
.zip(settled)
.take(index)
.filter(|(_, settled)| !**settled)
.filter_map(|(footprint, _)| footprint.as_ref());
footprints
.get(index)
.and_then(Option::as_ref)
.is_some_and(|footprint| footprint.must_wait(earlier))
}
fn footprint(catalog: &ToolCatalog, call: &ToolCall) -> Touches {
catalog
.get(call.function.name.as_str())
.and_then(|tool| tool.handler().footprint(&EffectKind::from(call)))
.unwrap_or_default()
.normalized()
}
pub(crate) struct StreamingTurnSource {
last_final_choice: Vec<AssistantContent>,
last_response_id: Option<String>,
agent_name: String,
record_telemetry_content: bool,
observes_text_delta: bool,
observes_reasoning_delta: bool,
observes_tool_call_delta: bool,
has_hooks: bool,
}
impl StreamingTurnSource {
pub(crate) fn new(
hooks: &HookStack,
agent_name: String,
record_telemetry_content: bool,
) -> Self {
Self {
last_final_choice: Vec::new(),
last_response_id: None,
agent_name,
record_telemetry_content,
observes_text_delta: hooks.observes(StepEventKind::TextDelta),
observes_reasoning_delta: hooks.observes(StepEventKind::ReasoningDelta),
observes_tool_call_delta: hooks.observes(StepEventKind::ToolCallDelta),
has_hooks: !hooks.is_empty(),
}
}
}
impl TurnSource for StreamingTurnSource {
const STREAMS: bool = true;
fn run_model_turn<'a>(
&'a mut self,
runner: &'a AgentRunner,
hook_ctx: &'a HookContext,
run: &'a mut AgentRun,
prepared: PreparedCompletionRequest,
chat_span: tracing::Span,
scope: &'a mut CompletionScope<Self>,
) -> DriveStream<'a> {
Box::pin(async_stream::try_stream! {
let attempt_max_tokens = prepared.max_tokens;
let mut stream =
chat_span.in_scope(|| scope.dispatch_stream(runner, &prepared.model))?;
let mut assembler = StreamedTurnAssembler::new(prepared.policy);
let mut text = String::new();
let mut reasoning: BTreeMap<usize, String> = BTreeMap::new();
let mut arguments: BTreeMap<usize, String> = BTreeMap::new();
let mut turn_recovered = false;
let mut held: BTreeMap<Part, Vec<StreamEvent>> = BTreeMap::new();
let mut ahead = VecDeque::new();
while let Some(item) = match ahead.pop_front() {
Some(item) => item,
None => stream.next().await,
} {
let item = item?;
let mut next = Some(assembler.ingest(&item)?);
let mut item_slot = Some(item);
while let Some(event) = next.take() {
match event {
StreamedTurnEvent::EmitIngested => {
if self.observes_text_delta
&& let Some(Item::Event(StreamEvent::Text { text: delta, .. })) =
item_slot.as_ref()
&& let Some(reason) = observe_action(
runner
.config.hooks
.on_text_delta(
hook_ctx,
TextDelta {
delta,
aggregated: {
text.push_str(delta);
&text
},
},
)
.await,
)
{
drop(stream);
return Err(run.cancel_error(reason))?;
}
if self.observes_reasoning_delta
&& let Some(Item::Event(StreamEvent::Reasoning { part, text: delta })) =
item_slot.as_ref()
&& let Some(reason) = observe_action(
runner
.config.hooks
.on_reasoning_delta(
hook_ctx,
ReasoningDelta {
part: *part,
delta,
aggregated: {
let aggregated =
reasoning.entry(part.index()).or_default();
aggregated.push_str(delta);
aggregated
},
},
)
.await,
)
{
drop(stream);
return Err(run.cancel_error(reason))?;
}
if let Some(item) = item_slot.take() {
yield Surfaced::Provider(item);
}
}
StreamedTurnEvent::EmitToolCallDelta => {
if self.observes_tool_call_delta
&& let Some(Item::Event(StreamEvent::Arguments { part, json })) =
item_slot.as_ref()
&& let Some(reason) = observe_action(
runner
.config.hooks
.on_tool_call_delta(
hook_ctx,
ToolCallDelta {
part: *part,
tool_name: assembler
.streaming_tool_name(part.index())
.map(ToolName::as_str)
.unwrap_or_default(),
delta: json,
aggregated: {
let aggregated =
arguments.entry(part.index()).or_default();
aggregated.push_str(json);
aggregated
},
},
)
.await,
)
{
drop(stream);
return Err(run.cancel_error(reason))?;
}
if let Some(item) = item_slot.take() {
yield Surfaced::Provider(item);
}
}
StreamedTurnEvent::HoldToolCall => {
if let Some(Item::Event(event)) = item_slot.take() {
held.entry(event.part()).or_default().push(event);
}
}
StreamedTurnEvent::EmitToolCall { call } => {
let end = match item_slot.take() {
Some(Item::Event(StreamEvent::End { part, .. })) => Some(part),
_ => None,
};
let replay = end
.and_then(|part| held.remove(&part))
.unwrap_or_default();
let mut aggregated = String::new();
for mut event in replay {
if let StreamEvent::Start { name, .. } = &mut event {
*name = Some(call.function.name.clone());
}
if let StreamEvent::Arguments { json, .. } = &event {
aggregated.push_str(json);
}
if self.observes_tool_call_delta
&& let StreamEvent::Arguments { part, json } = &event
&& let Some(reason) = observe_action(
runner
.config.hooks
.on_tool_call_delta(
hook_ctx,
ToolCallDelta {
part: *part,
tool_name: call.function.name.as_str(),
delta: json,
aggregated: &aggregated,
},
)
.await,
)
{
drop(stream);
return Err(run.cancel_error(reason))?;
}
yield Surfaced::Provider(Item::Event(event));
}
if let Some(part) = end {
yield Surfaced::Provider(Item::Event(StreamEvent::End {
part,
content: AssistantContent::ToolCall(call),
}));
}
}
StreamedTurnEvent::InvalidToolCall(invalid) => {
while !matches!(ahead.back(), Some(None)) {
match futures::FutureExt::now_or_never(stream.next()) {
Some(item) => ahead.push_back(item),
None => break,
}
}
let cut = matches!(ahead.back(), Some(None))
&& stream
.partial()
.cut_off_calls()
.any(|cut| cut.id == invalid.tool_call.id);
if cut {
next = assembler.keep_pending_cut_off();
continue;
}
let rejected = match item_slot.as_ref() {
Some(Item::Event(event)) => Some(event.part()),
_ => None,
};
let partial = assembler.partial_turn(&stream.partial());
let hook_action = if self.has_hooks {
let context =
run.streamed_invalid_tool_call_context(&partial, &invalid);
runner
.config.hooks
.on_invalid_tool_call(hook_ctx, &context)
.await
} else {
None
};
let resolved = match hook_action {
Some(action) => {
run.resolve_streamed_invalid_tool_call(&partial, &invalid, action)
}
None => match run.unhandled_invalid_tool_call() {
UnhandledInvalidToolCall::Fail => run
.resolve_streamed_invalid_tool_call(
&partial,
&invalid,
InvalidToolCallAction::fail(),
),
UnhandledInvalidToolCall::Ignore => {
run.ignore_streamed_invalid_tool_call()
}
},
};
let resolution = resolved?;
match resolution {
StreamedResolution::Ignored => {
assembler.resolve_pending_invalid(&resolution);
if let Some(part) = rejected {
held.remove(&part);
}
item_slot = None;
}
StreamedResolution::Repaired { .. } => {
turn_recovered = true;
next = assembler.resolve_pending_invalid(&resolution);
}
StreamedResolution::TurnAbandoned => {
assembler.resolve_pending_invalid(&resolution);
held.clear();
let (_, call) =
scope.finish_stream(stream, run, &chat_span).await?;
yield Surfaced::CompletionCall(call);
return;
}
}
}
}
}
}
let (response, call) = scope.finish_stream(stream, run, &chat_span).await?;
yield Surfaced::CompletionCall(call);
let streamed_turn = assembler.finish(response);
self.last_response_id = response.response_id().map(str::to_owned);
let mut final_turn_content =
std::mem::replace(&mut response.choice, streamed_turn.choice.clone());
run.streamed_turn(streamed_turn)?;
let settlement = settle_model_turn(
runner,
hook_ctx,
run,
scope,
attempt_max_tokens,
turn_recovered,
)
.await?;
match settlement {
ModelTurnDecision::Advance { replaced } => {
if let Some(choice) = replaced {
final_turn_content = choice;
}
}
ModelTurnDecision::Retried => {
yield Surfaced::ModelTurnRetried {
turn: hook_ctx.turn(),
};
return;
}
ModelTurnDecision::Terminate(reason) => {
rig_core::telemetry::record_model_output(
&chat_span,
scope.choice(),
self.record_telemetry_content,
);
return Err(run.cancel_error(reason))?;
}
}
rig_core::telemetry::record_model_output(
&chat_span,
scope.choice(),
self.record_telemetry_content,
);
self.last_final_choice = final_turn_content;
})
}
fn chain_span(&self, span: tracing::Span) -> tracing::Span {
span
}
fn finish(&mut self, mut response: PromptResponse) -> DriveItem {
let final_choice = finalize_output_tool_choice(&self.last_final_choice, &response.output())
.unwrap_or_else(|| {
if self.last_final_choice.iter().all(AssistantContent::is_blank) {
tracing::warn!(
agent_name = self.agent_name.as_str(),
response_id = ?self.last_response_id,
"Streaming turn completed without assistant text; final response will be empty"
);
}
std::mem::take(&mut self.last_final_choice)
});
response.content = final_choice;
DriveItem::Surfaced(Surfaced::Final(response.with_output_tool_calls(0)))
}
}
pub(crate) fn observe_action(action: ObservationAction) -> Option<String> {
match action {
ObservationAction::Continue => None,
ObservationAction::Stop(reason) => Some(reason),
}
}
pub(crate) enum ModelTurnDecision {
Advance {
replaced: Option<Vec<AssistantContent>>,
},
Retried,
Terminate(String),
}
pub(crate) async fn settle_model_turn<M>(
runner: &AgentRunner,
hook_ctx: &HookContext,
run: &mut AgentRun,
scope: &mut CompletionScope<M>,
max_tokens: Option<u64>,
recovered: bool,
) -> Result<ModelTurnDecision, PromptError> {
if recovered {
return Ok(ModelTurnDecision::Advance { replaced: None });
}
let (action, response) = scope.close(runner, hook_ctx).await?;
let hooks = &runner.config.hooks;
let mut replaced: Option<Vec<AssistantContent>> = None;
match action {
OutcomeAction::Proceed => {}
OutcomeAction::Replace(Ok(Answered::Completion(replacement))) => {
let replacement = Arc::unwrap_or_clone(replacement);
run.replace_accepted_turn_choice(replacement.choice.clone())?;
replaced = Some(replacement.choice);
}
OutcomeAction::Replace(Ok(other)) => {
return Err(PromptError::Report(family::Completion::mismatch(&other)));
}
OutcomeAction::Replace(Err(report)) => {
if report.kind == ErrorKind::Cancelled {
return Ok(ModelTurnDecision::Terminate(report.message));
}
return Err(PromptError::Report(report));
}
}
let identity = response.identity();
let finish_reason = response.finish_reason();
let content = replaced.as_ref().unwrap_or(&response.choice);
let action = hooks
.on_model_turn_finished(
hook_ctx,
ModelTurnFinished {
turn: hook_ctx.turn(),
content,
usage: response.usage,
identity: &identity,
finish_reason: finish_reason.as_ref(),
max_tokens,
raw: &response.raw,
},
)
.await;
match action {
ModelTurnAction::Continue => Ok(ModelTurnDecision::Advance { replaced }),
ModelTurnAction::Retry(request) => {
run.retry_model_turn(request)?;
Ok(ModelTurnDecision::Retried)
}
ModelTurnAction::Stop(reason) => Ok(ModelTurnDecision::Terminate(reason)),
}
}
pub(crate) async fn dispatch_effect(
runner: &AgentRunner,
ctx: &HookContext,
key: &rig_core::effect::HandlerKey,
kind: EffectKind,
) -> Result<Answered, ErrorReport> {
let scope = DispatchScope::open(runner, ctx, kind, None)
.await
.map_err(|(_, report)| report)?;
let kind = scope.kind().clone();
let beside = crate::handlers::Beside::for_kind(&kind);
let outcome = runner
.config
.handlers
.serve(scope.id(), key.clone(), kind, beside)
.await;
match scope.close(runner, ctx, &outcome, None).await {
OutcomeAction::Proceed => outcome,
OutcomeAction::Replace(replaced) => replaced,
}
}
pub(crate) enum ToolExit {
Stop(String),
Fail(PromptError),
}
pub(crate) struct ToolCallOutcome {
pub result: ToolResult,
pub executed: Option<ToolCall>,
}
fn record_tool_result(span: &tracing::Span, result: &ToolResult) {
span.record("gen_ai.tool.call.outcome", result.status_name());
if let Some(error) = result.error() {
span.record("gen_ai.tool.error.type", error.kind().as_str());
}
}
#[derive(Default)]
pub(crate) struct UnaryTurnSource {
current_span_id: AtomicU64,
}
impl TurnSource for UnaryTurnSource {
const STREAMS: bool = false;
fn chain_span(&self, span: tracing::Span) -> tracing::Span {
let span = match self.current_span_id.load(Ordering::Relaxed) {
0 => span,
id => {
span.follows_from(Id::from_u64(id));
span
}
};
if let Some(id) = span.id() {
self.current_span_id.store(id.into_u64(), Ordering::Relaxed);
}
span
}
fn run_model_turn<'a>(
&'a mut self,
runner: &'a AgentRunner,
hook_ctx: &'a HookContext,
run: &'a mut AgentRun,
prepared: PreparedCompletionRequest,
chat_span: tracing::Span,
scope: &'a mut CompletionScope<Self>,
) -> DriveStream<'a> {
let turn = async move {
let record_accepted_turn = |run: &AgentRun| {
if runner.config.record_telemetry_content
&& let Some(choice) = run.accepted_turn_choice()
{
rig_core::telemetry::record_model_output(&chat_span, &choice, true);
}
};
let attempt_max_tokens = prepared.max_tokens;
let response = scope
.dispatch_response(runner, &prepared.model)
.instrument(chat_span.clone())
.await?;
let mut outcome =
run.model_response(ModelTurn::from_policy(response, prepared.policy))?;
loop {
match outcome {
ModelTurnOutcome::NeedsResolution(context) => {
let action = runner
.config
.hooks
.on_invalid_tool_call(hook_ctx, &context)
.await;
let resolution = match action {
Some(action) => run.resolve_invalid_tool_call(action),
None => run.resolve_unhandled_invalid_tool_call(),
};
outcome = resolution?;
}
ModelTurnOutcome::TurnRetried => break,
ModelTurnOutcome::Continue {
response_hook_suppressed,
} => {
let settlement = settle_model_turn(
runner,
hook_ctx,
run,
scope,
attempt_max_tokens,
response_hook_suppressed,
)
.await?;
match settlement {
ModelTurnDecision::Advance { .. } => {}
ModelTurnDecision::Retried => break,
ModelTurnDecision::Terminate(reason) => {
record_accepted_turn(run);
return Err(run.cancel_error(reason));
}
}
record_accepted_turn(run);
break;
}
}
}
Ok(())
};
Box::pin(
futures::stream::once(turn)
.filter_map(|done| std::future::ready(done.err().map(Err::<Surfaced, _>))),
)
}
fn finish(&mut self, response: PromptResponse) -> DriveItem {
DriveItem::Done(response)
}
}
#[cfg(test)]
mod tests;
pub(crate) async fn run_single_tool(
runner: &AgentRunner,
ctx: &HookContext,
tool_snapshot: &ToolCatalog,
tool_call: &ToolCall,
) -> Result<ToolCallOutcome, ToolExit> {
let record_content = runner.config.record_telemetry_content;
let (tool_name, call_id) = (tool_call.function.name.as_str(), &tool_call.id);
let model_args = tool_call.function.arguments_json();
let tool_span = tracing::Span::current();
tool_span.record("gen_ai.tool.name", tool_name);
tool_span.record("gen_ai.tool.call.id", tracing::field::display(call_id));
if record_content {
tool_span.record("gen_ai.tool.call.arguments", &model_args);
}
let kind = EffectKind::ToolCall {
name: tool_name.to_owned(),
args: model_args.clone(),
};
let inbound = runner.tool_context.for_dispatch();
let (scope, denied) = match DispatchScope::open(runner, ctx, kind, Some((call_id, &inbound)))
.await
{
Ok(scope) => (scope, None),
Err((_, report)) if report.kind == ErrorKind::Cancelled => {
return Err(ToolExit::Stop(report.message));
}
Err((scope, report)) => {
tracing::info!(tool_name = tool_name, reason = %report.message, "Tool call rejected");
(scope, Some(report))
}
};
let args = match scope.kind() {
EffectKind::ToolCall { args, .. } => args.clone(),
_ => String::new(),
};
let mut published: Option<crate::tool::ToolContext> = None;
let mut executed = false;
let outcome: Result<Answered, ErrorReport> = match denied {
Some(report) if report.kind == ErrorKind::Denied => Ok(Answered::ToolResult {
result: ToolResult::failed(report.refused().into()),
}),
Some(report) => Ok(Answered::ToolResult {
result: ToolResult::skipped(report.message),
}),
None => match tool_snapshot.get(tool_name) {
Some(tool) => {
let slot = rig_core::tool::PublishedContext::new();
let beside = crate::handlers::Beside {
handler: Some(tool.handler().clone()),
context: Some(inbound),
published: Some(std::sync::Arc::clone(&slot)),
};
let key = tool.key().raw().clone();
let served =
runner
.config
.handlers
.serve(scope.id(), key, scope.kind().clone(), beside);
let outcome = served.await;
executed =
matches!(&outcome, Ok(Answered::ToolResult { result }) if !result.is_skipped());
published = slot.take();
outcome
}
None => Ok(Answered::ToolResult {
result: ToolResult::failed(
crate::tool::ToolExecutionError::not_found(format!(
"no tool named `{tool_name}` is registered"
))
.with_model_feedback(
rig_core::tool::feedback::unknown_tool_feedback(
tool_name,
tool_snapshot.names(),
),
),
),
}),
},
};
let context = published.unwrap_or_else(|| runner.tool_context.for_dispatch());
let outcome = match scope
.close(runner, ctx, &outcome, Some((call_id, &context)))
.await
{
OutcomeAction::Proceed => outcome,
OutcomeAction::Replace(replaced) => replaced,
};
let result = match outcome.and_then(family::Tool::unwrap) {
Ok(result) => result,
Err(report) if report.kind == ErrorKind::Cancelled => {
return Err(ToolExit::Stop(report.message));
}
Err(report) if report.kind == ErrorKind::Denied => {
tracing::info!(tool_name = tool_name, reason = %report.message, "Tool call denied");
ToolResult::failed(report.refused().into())
}
Err(report)
if matches!(
report.kind,
ErrorKind::HandlerUnavailable | ErrorKind::Divergence
) =>
{
return Err(ToolExit::Fail(PromptError::Report(report)));
}
Err(report) => ToolResult::failed(report.into()),
};
if args != model_args {
if record_content {
tool_span.record("gen_ai.tool.call.arguments", &args);
}
tracing::debug!(
tool_name = tool_name,
"tool-call arguments rewritten by a hook"
);
}
let executed = executed.then(|| {
let mut effective_tool_call = tool_call.clone();
effective_tool_call.function = ToolFunction::parse(tool_call.function.name.clone(), &args);
effective_tool_call
});
record_tool_result(&tool_span, &result);
if record_content {
tool_span.record("gen_ai.tool.call.result", result.output().render());
}
Ok(ToolCallOutcome { result, executed })
}