use std::sync::mpsc::channel;
use crate::message::{Message, ToolCall, ToolResult};
use crate::request::{Cancel, ToolSchema};
use crate::{Chunk, Error, Provider, Request, Result, Usage};
pub trait Tools {
fn schemas(&self) -> Vec<ToolSchema>;
fn run(&mut self, call: &ToolCall) -> std::result::Result<String, String>;
}
impl<T: Tools + ?Sized> Tools for &mut T {
fn schemas(&self) -> Vec<ToolSchema> {
(**self).schemas()
}
fn run(&mut self, call: &ToolCall) -> std::result::Result<String, String> {
(**self).run(call)
}
}
pub struct NoTools;
impl Tools for NoTools {
fn schemas(&self) -> Vec<ToolSchema> {
Vec::new()
}
fn run(&mut self, call: &ToolCall) -> std::result::Result<String, String> {
Err(format!("no tool named `{}`", call.name))
}
}
#[derive(Clone, Copy, Debug)]
pub enum Step<'a> {
Turn(usize),
Token(&'a str),
Planning(&'a str),
Calling(&'a ToolCall),
Called(&'a ToolResult),
}
#[derive(Clone, Debug, Default)]
pub struct Answer {
pub text: String,
pub usage: Usage,
pub turns: usize,
}
pub struct Agent<'a, T: Tools> {
provider: &'a dyn Provider,
tools: T,
instructions: String,
max_turns: usize,
max_output_tokens: Option<u32>,
model: Option<String>,
cancel: Option<Cancel>,
history: Vec<Message>,
}
impl<'a, T: Tools> Agent<'a, T> {
pub fn new(provider: &'a dyn Provider, tools: T) -> Self {
Self {
provider,
tools,
instructions: String::new(),
max_turns: 8,
max_output_tokens: None,
model: None,
cancel: None,
history: Vec::new(),
}
}
pub fn instructions(mut self, instructions: impl Into<String>) -> Self {
self.instructions = instructions.into();
self
}
pub fn max_turns(mut self, turns: usize) -> Self {
self.max_turns = turns.max(1);
self
}
pub fn max_output_tokens(mut self, tokens: u32) -> Self {
self.max_output_tokens = Some(tokens);
self
}
pub fn model(mut self, model: impl Into<String>) -> Self {
self.model = Some(model.into());
self
}
pub fn cancel(mut self, cancel: Cancel) -> Self {
self.cancel = Some(cancel);
self
}
pub fn history(mut self, history: Vec<Message>) -> Self {
self.history = history;
self
}
pub fn set_instructions(&mut self, instructions: impl Into<String>) {
self.instructions = instructions.into();
}
pub fn transcript(&self) -> &[Message] {
&self.history
}
pub fn into_history(self) -> Vec<Message> {
self.history
}
pub fn clear(&mut self) {
self.history.clear();
}
pub fn tools(&self) -> &T {
&self.tools
}
pub fn tools_mut(&mut self) -> &mut T {
&mut self.tools
}
fn cancelled(&self) -> bool {
self.cancel
.as_ref()
.is_some_and(|flag| flag.load(std::sync::atomic::Ordering::Relaxed))
}
fn request(&self, messages: Vec<Message>, tools: Vec<ToolSchema>) -> Request {
let mut request = Request::new(self.instructions.clone())
.messages(messages)
.tools(tools);
if let Some(tokens) = self.max_output_tokens {
request = request.max_output_tokens(tokens);
}
if let Some(model) = &self.model {
request = request.model(model.clone());
}
if let Some(cancel) = &self.cancel {
request = request.cancel(cancel.clone());
}
request
}
pub fn ask(&mut self, question: &str, observe: &mut dyn FnMut(Step)) -> Result<Answer> {
let mut messages = self.history.clone();
messages.push(Message::user(question));
let mut answer = Answer::default();
let mut already_run: Vec<String> = Vec::new();
for turn in 0..self.max_turns {
if self.cancelled() {
return Err(Error::Cancelled);
}
observe(Step::Turn(turn));
answer.turns = turn + 1;
let request = self.request(messages.clone(), self.tools.schemas());
let (text, calls, usage) = self.turn(&request, observe)?;
answer.usage = answer.usage + usage;
if calls.is_empty() {
answer.text = text;
messages.push(Message::assistant(&answer.text));
self.history = messages;
return Ok(answer);
}
observe(Step::Planning(&text));
let called = Message::tool_calls(text, calls);
let mut results = Vec::new();
for (index, call) in called.calls.iter().enumerate() {
if self.cancelled() {
return Err(Error::Cancelled);
}
observe(Step::Calling(call));
let signature = format!("{}{}", call.name, call.args);
let result = if already_run.contains(&signature) {
ToolResult::failure(
call,
index,
"You already ran this exact call. Use what it returned, try different \
arguments, or answer now.",
)
} else {
already_run.push(signature);
match self.tools.run(call) {
Ok(output) => ToolResult::new(call, index, output),
Err(message) => ToolResult::failure(call, index, message),
}
};
observe(Step::Called(&result));
results.push(result);
}
messages.push(called);
messages.push(Message::tool_results(results));
}
observe(Step::Turn(self.max_turns));
let mut closing = messages.clone();
closing.push(Message::user(
"Answer now with what you have gathered. Do not call any more tools.",
));
let request = self.request(closing, Vec::new());
let (text, _, usage) = self.turn(&request, observe)?;
answer.usage = answer.usage + usage;
answer.text = text;
messages.push(Message::assistant(&answer.text));
self.history = messages;
Ok(answer)
}
fn turn(
&self,
request: &Request,
observe: &mut dyn FnMut(Step),
) -> Result<(String, Vec<ToolCall>, Usage)> {
let (tx, rx) = channel::<Chunk>();
let provider = self.provider;
std::thread::scope(|scope| {
let sending = scope.spawn(move || {
let result = provider.stream(request, &tx);
drop(tx);
result
});
let mut text = String::new();
let mut calls = Vec::new();
let mut usage = Usage::default();
let mut failure = None;
for chunk in rx {
match chunk {
Chunk::Text(piece) => {
observe(Step::Token(&piece));
text.push_str(&piece);
}
Chunk::Tool(call) => calls.push(call),
Chunk::Done {
input_tokens,
output_tokens,
} => {
usage.input = input_tokens as u64;
usage.output = output_tokens as u64;
usage.calls = 1;
}
Chunk::Failed(error) => failure = Some(error),
}
}
sending
.join()
.map_err(|_| Error::Stream("the model stream panicked".into()))??;
match failure {
Some(error) if text.trim().is_empty() && calls.is_empty() => Err(error),
_ => Ok((text, calls, usage)),
}
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::request::ToolSchema;
use serde_json::json;
use std::sync::mpsc::Sender;
use std::sync::Mutex;
struct Script {
turns: Mutex<Vec<(String, Vec<ToolCall>)>>,
seen: Mutex<Vec<Request>>,
}
impl Script {
fn new(turns: Vec<(&str, Vec<ToolCall>)>) -> Self {
Self {
turns: Mutex::new(
turns
.into_iter()
.map(|(text, calls)| (text.to_string(), calls))
.collect(),
),
seen: Mutex::new(Vec::new()),
}
}
}
impl Provider for Script {
fn stream(&self, request: &Request, sender: &Sender<Chunk>) -> Result<()> {
self.seen.lock().unwrap().push(request.clone());
let mut turns = self.turns.lock().unwrap();
let (text, calls) = if turns.is_empty() {
("out of script".to_string(), Vec::new())
} else {
turns.remove(0)
};
if !text.is_empty() {
sender.send(Chunk::Text(text)).unwrap();
}
for call in calls {
sender.send(Chunk::Tool(call)).unwrap();
}
sender
.send(Chunk::Done {
input_tokens: 5,
output_tokens: 2,
})
.unwrap();
Ok(())
}
fn label(&self) -> String {
"script".into()
}
}
#[derive(Default)]
struct Counter {
runs: Vec<String>,
fail: bool,
}
impl Tools for Counter {
fn schemas(&self) -> Vec<ToolSchema> {
vec![ToolSchema::new(
"search",
"Search",
json!({"type": "object"}),
)]
}
fn run(&mut self, call: &ToolCall) -> std::result::Result<String, String> {
self.runs.push(call.args.to_string());
if self.fail {
Err("the page would not load".into())
} else {
Ok(format!("results for {}", call.args))
}
}
}
fn search(query: &str) -> ToolCall {
ToolCall::new("search", json!({ "q": query }))
}
#[test]
fn a_tool_call_is_run_and_its_output_is_fed_back() {
let script = Script::new(vec![
("Looking it up.", vec![search("rust gats")]),
("Generic associated types.", Vec::new()),
]);
let mut agent = Agent::new(&script, Counter::default()).instructions("Be brief");
let answer = agent.ask("what are gats?", &mut |_| {}).unwrap();
assert_eq!(answer.text, "Generic associated types.");
assert_eq!(answer.turns, 2);
assert_eq!(answer.usage.calls, 2);
assert_eq!(agent.tools().runs, [r#"{"q":"rust gats"}"#]);
let second = &script.seen.lock().unwrap()[1];
let called = &second.messages[1];
let answered = &second.messages[2];
assert_eq!(called.calls[0].name, "search");
assert_eq!(answered.results[0].id, called.calls[0].id);
assert!(answered.results[0].output.contains("results for"));
}
#[test]
fn a_repeated_call_is_refused_rather_than_run_twice() {
let script = Script::new(vec![
("", vec![search("same")]),
("", vec![search("same")]),
("Answering.", Vec::new()),
]);
let mut agent = Agent::new(&script, Counter::default());
agent.ask("q", &mut |_| {}).unwrap();
assert_eq!(agent.tools().runs.len(), 1, "the tool ran once");
let third = &script.seen.lock().unwrap()[2];
let refusal = third
.messages
.iter()
.flat_map(|message| &message.results)
.find(|result| result.failed)
.expect("the repeat was reported as a failed call");
assert!(refusal.output.contains("already ran"));
}
#[test]
fn a_failing_tool_is_reported_to_the_model_as_a_failure() {
let script = Script::new(vec![
("", vec![search("x")]),
("Could not check.", Vec::new()),
]);
let tools = Counter {
fail: true,
..Counter::default()
};
let mut agent = Agent::new(&script, tools);
let mut failures = 0;
agent
.ask("q", &mut |step| {
if matches!(step, Step::Called(result) if result.failed) {
failures += 1;
}
})
.unwrap();
assert_eq!(failures, 1);
}
#[test]
fn running_out_of_turns_forces_an_answer_without_tools() {
let script = Script::new(vec![
("", vec![search("one")]),
("", vec![search("two")]),
("Here is what I found.", Vec::new()),
]);
let mut agent = Agent::new(&script, Counter::default()).max_turns(2);
let answer = agent.ask("q", &mut |_| {}).unwrap();
assert_eq!(answer.text, "Here is what I found.");
let closing = script.seen.lock().unwrap().last().unwrap().clone();
assert!(closing.tools.is_empty(), "no tools were offered");
assert!(closing.messages.last().unwrap().text.contains("Answer now"));
}
#[test]
fn tokens_and_planning_are_reported_separately() {
let script = Script::new(vec![
("Let me check.", vec![search("x")]),
("The answer.", Vec::new()),
]);
let mut agent = Agent::new(&script, Counter::default());
let (mut tokens, mut planning) = (String::new(), String::new());
agent
.ask("q", &mut |step| match step {
Step::Token(text) => tokens.push_str(text),
Step::Planning(text) => planning.push_str(text),
_ => {}
})
.unwrap();
assert_eq!(tokens, "Let me check.The answer.");
assert_eq!(planning, "Let me check.");
}
#[test]
fn history_carries_the_conversation_into_the_next_question() {
let script = Script::new(vec![("First.", Vec::new()), ("Second.", Vec::new())]);
let mut agent = Agent::new(&script, NoTools);
agent.ask("one", &mut |_| {}).unwrap();
agent.ask("two", &mut |_| {}).unwrap();
assert_eq!(agent.transcript().len(), 4);
assert_eq!(script.seen.lock().unwrap()[1].messages.len(), 3);
agent.clear();
assert!(agent.transcript().is_empty());
}
#[test]
fn a_raised_cancel_flag_stops_before_the_first_request() {
let script = Script::new(vec![("never", Vec::new())]);
let cancel = Cancel::default();
cancel.store(true, std::sync::atomic::Ordering::Relaxed);
let mut agent = Agent::new(&script, NoTools).cancel(cancel);
assert!(matches!(agent.ask("q", &mut |_| {}), Err(Error::Cancelled)));
assert!(script.seen.lock().unwrap().is_empty());
}
}