1use std::{
8 collections::BTreeSet,
9 sync::{
10 Arc, Mutex, MutexGuard,
11 atomic::{AtomicUsize, Ordering},
12 },
13 time::{Duration, Instant},
14};
15
16use futures::StreamExt;
17use schemars::JsonSchema;
18use serde::{Deserialize, Serialize};
19
20use crate::{
21 agent::{
22 AgentBuilder, AgentHook, CompletionCallAction, CompletionCallEvent, DispatchAction,
23 DispatchEvent, HookContext, InvalidToolCallAction, MultiTurnStreamItem, OutcomeAction,
24 OutcomeEvent, OutputMode, RequestPatch,
25 },
26 completion::{AssistantContent, Message, PromptError, ToolDefinition},
27 run::{AgentRun, AgentRunStep, ModelTurn, ModelTurnOutcome, PendingToolCall, TurnPolicy},
28 tool::{Tool, ToolContext},
29};
30use rig_core::completion::CompletionRequest;
31use rig_core::error::ProviderError;
32use rig_core::message::{ToolChoice, UserContent};
33
34#[derive(Debug, thiserror::Error)]
36pub enum ScenarioError {
37 #[error(transparent)]
39 Prompt(#[from] PromptError),
40 #[error(transparent)]
42 Completion(#[from] ProviderError),
43 #[error(transparent)]
45 Report(#[from] rig_core::error::ErrorReport),
46 #[error(transparent)]
48 Json(#[from] serde_json::Error),
49 #[error(transparent)]
51 Extraction(#[from] crate::completion::StructuredOutputError),
52 #[error("{scenario} conformance failed: {details}")]
54 Contract {
55 scenario: &'static str,
57 details: String,
59 },
60}
61
62impl ScenarioError {
63 fn contract(scenario: &'static str, details: impl Into<String>) -> Self {
64 Self::Contract {
65 scenario,
66 details: details.into(),
67 }
68 }
69}
70
71pub fn validate_unknown_tool_failure(
74 error: &PromptError,
75 expected_tool: &str,
76 expected_allowed_tools: &[&str],
77) -> Result<(), ScenarioError> {
78 const SCENARIO: &str = "unknown_tool_failure";
79 let PromptError::UnknownToolCall {
80 tool_name,
81 allowed_tools,
82 chat_history,
83 ..
84 } = error
85 else {
86 return Err(ScenarioError::contract(
87 SCENARIO,
88 format!("expected UnknownToolCall, observed {error:?}"),
89 ));
90 };
91 let expected_allowed = expected_allowed_tools
92 .iter()
93 .map(|name| (*name).to_string())
94 .collect::<Vec<_>>();
95 let history_has_call = chat_history.iter().any(|message| {
99 matches!(
100 message,
101 Message::Assistant(rig_core::message::AssistantMessage { content, .. })
102 if content.iter().any(|item| matches!(item, AssistantContent::ToolCall(_)))
103 )
104 });
105 if tool_name != expected_tool || allowed_tools != &expected_allowed || !history_has_call {
106 return Err(ScenarioError::contract(
107 SCENARIO,
108 format!(
109 "tool={tool_name:?}, allowed={allowed_tools:?}, expected_tool={expected_tool:?}, expected_allowed={expected_allowed:?}, history_has_call={history_has_call}, history={chat_history:?}"
110 ),
111 ));
112 }
113 Ok(())
114}
115
116pub fn validate_cancelled_failure(
119 error: &PromptError,
120 expected_reason: &str,
121 expected_tool: &str,
122) -> Result<(), ScenarioError> {
123 const SCENARIO: &str = "cancelled_failure";
124 let PromptError::Cancelled {
125 chat_history,
126 reason,
127 } = error
128 else {
129 return Err(ScenarioError::contract(
130 SCENARIO,
131 format!("expected Cancelled, observed {error:?}"),
132 ));
133 };
134 let history_has_call = chat_history.iter().any(|message| {
135 matches!(
136 message,
137 Message::Assistant(rig_core::message::AssistantMessage { content, .. })
138 if content.iter().any(|item| matches!(
139 item,
140 AssistantContent::ToolCall(call) if call.function.name == expected_tool
141 ))
142 )
143 });
144 if reason != expected_reason || !history_has_call {
145 return Err(ScenarioError::contract(
146 SCENARIO,
147 format!(
148 "reason={reason:?}, expected={expected_reason:?}, history_has_call={history_has_call}, history={chat_history:?}"
149 ),
150 ));
151 }
152 Ok(())
153}
154
155pub fn validate_max_turns_failure(
158 error: &PromptError,
159 expected_max_turns: usize,
160) -> Result<(), ScenarioError> {
161 const SCENARIO: &str = "max_turns_failure";
162 let PromptError::MaxTurns {
163 max_turns,
164 chat_history,
165 prompt,
166 } = error
167 else {
168 return Err(ScenarioError::contract(
169 SCENARIO,
170 format!("expected MaxTurns, observed {error:?}"),
171 ));
172 };
173 let pending_prompt_retained = matches!(
174 prompt,
175 Message::User { content } if content.iter().next().is_some()
176 );
177 if *max_turns != expected_max_turns || chat_history.is_empty() || !pending_prompt_retained {
178 return Err(ScenarioError::contract(
179 SCENARIO,
180 format!(
181 "max_turns={max_turns}, expected={expected_max_turns}, history={chat_history:?}, pending_prompt_retained={pending_prompt_retained}, pending_prompt={prompt:?}"
182 ),
183 ));
184 }
185 Ok(())
186}
187
188pub fn decode_structured_output<T>(
191 scenario: &'static str,
192 response: &str,
193) -> Result<T, ScenarioError>
194where
195 T: serde::de::DeserializeOwned,
196{
197 serde_json::from_str(response).map_err(|error| {
198 ScenarioError::contract(
199 scenario,
200 format!("structured output did not decode: {error}; response={response:?}"),
201 )
202 })
203}
204
205pub fn validate_protocol_hygiene(
208 scenario: &'static str,
209 visible_output: &str,
210 messages: &[Message],
211 forbidden_markers: &[&str],
212) -> Result<(), ScenarioError> {
213 let serialized = serde_json::to_string(messages)?;
214 let leaked = forbidden_markers
215 .iter()
216 .filter(|marker| visible_output.contains(**marker) || serialized.contains(**marker))
217 .copied()
218 .collect::<Vec<_>>();
219 if !leaked.is_empty() {
220 return Err(ScenarioError::contract(
221 scenario,
222 format!(
223 "protocol markers leaked: {leaked:?}; output={visible_output:?}, history={messages:?}"
224 ),
225 ));
226 }
227 Ok(())
228}
229
230pub fn validate_rewritten_arguments(
233 scenario: &'static str,
234 observations: &[serde_json::Value],
235 expected_fields: &serde_json::Value,
236) -> Result<(), ScenarioError> {
237 let Some(expected) = expected_fields.as_object() else {
238 return Err(ScenarioError::contract(
239 scenario,
240 "expected rewritten fields must be a JSON object",
241 ));
242 };
243 if observations.is_empty() {
244 return Err(ScenarioError::contract(
245 scenario,
246 "the rewritten tool was never invoked",
247 ));
248 }
249 for observation in observations {
250 let Some(actual) = observation.as_object() else {
251 return Err(ScenarioError::contract(
252 scenario,
253 format!("observed rewritten arguments were not an object: {observation:?}"),
254 ));
255 };
256 for (key, expected_value) in expected {
257 if actual.get(key) != Some(expected_value) {
258 return Err(ScenarioError::contract(
259 scenario,
260 format!(
261 "rewritten field {key:?} expected {expected_value:?}, observed {observation:?}"
262 ),
263 ));
264 }
265 }
266 }
267 Ok(())
268}
269
270pub fn validate_extraction_fields(
272 scenario: &'static str,
273 first_name: Option<&str>,
274 last_name: Option<&str>,
275 job: Option<&str>,
276 usage: crate::completion::Usage,
277) -> Result<(), ScenarioError> {
278 let fields_match = first_name.is_some_and(|value| value.eq_ignore_ascii_case("Ada"))
279 && last_name.is_some_and(|value| value.eq_ignore_ascii_case("Lovelace"))
280 && job.is_some_and(|value| value.to_ascii_lowercase().contains("mathematician"));
281 if !fields_match || !usage.is_reported() {
282 return Err(ScenarioError::contract(
283 scenario,
284 format!(
285 "first_name={first_name:?}, last_name={last_name:?}, job={job:?}, usage={usage:?}"
286 ),
287 ));
288 }
289 Ok(())
290}
291
292#[derive(Debug, Clone)]
294pub struct ScenarioReport {
295 pub name: &'static str,
297 pub tool_calls: usize,
299 pub prompt_tokens: u64,
302 pub generated_tokens: u64,
305 pub history_messages: usize,
307 pub duration: Duration,
309 pub response: String,
311}
312
313#[derive(Debug, thiserror::Error)]
315#[error("model-conformance tool failed")]
316pub struct ConformanceToolError;
317
318const FORCE_TOOLS_PREAMBLE: &str = "You are a calculator assistant. You MUST use the provided tools for every arithmetic operation instead of computing results yourself. Once you have all the tool results you need, reply with the final numeric answer in plain text.";
319const PARALLEL_PROMPT: &str = "Compute 3 + 4 and 10 - 2. You MUST call the add tool and the subtract tool together in your first response, as two parallel function calls, then report both results.";
320const PING_OUTPUT: &str = "pong-crimson-7423";
321const MOTTO_OUTPUT: &str = "steady hands\ncalm waters";
322
323fn lock_recover<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
324 mutex
325 .lock()
326 .unwrap_or_else(std::sync::PoisonError::into_inner)
327}
328
329fn tool_result_values(message: &Message) -> Vec<serde_json::Value> {
330 let Message::User { content } = message else {
331 return Vec::new();
332 };
333 content
334 .iter()
335 .filter_map(|item| match item {
336 UserContent::ToolResult(result) => Some(result),
337 _ => None,
338 })
339 .flat_map(|result| result.content.iter())
340 .filter_map(|content| match content {
341 rig_core::message::ToolResultContent::Text(text) => {
342 Some(serde_json::Value::String(text.text.clone()))
343 }
344 rig_core::message::ToolResultContent::Json { value } => Some(value.clone()),
345 rig_core::message::ToolResultContent::Image(_) => None,
346 })
347 .collect()
348}
349
350fn validate_tool_correlation(
351 scenario: &'static str,
352 messages: &[Message],
353) -> Result<(), ScenarioError> {
354 let mut calls = Vec::new();
355 let mut results = Vec::new();
356 let mut turn = 0usize;
357 for message in messages {
358 match message {
359 Message::Assistant(rig_core::message::AssistantMessage { content, .. }) => {
360 turn += 1;
361 calls.extend(content.iter().filter_map(|item| match item {
362 AssistantContent::ToolCall(call) => Some((
363 turn,
364 &call.id,
365 call.id.provider().map(|provider| provider.as_str()),
366 )),
367 _ => None,
368 }));
369 }
370 Message::User { content } => {
371 results.extend(content.iter().filter_map(|item| match item {
372 UserContent::ToolResult(result) => Some((
373 turn,
374 &result.call,
375 result.call.provider().map(|provider| provider.as_str()),
376 )),
377 _ => None,
378 }));
379 }
380 Message::System { .. } => {}
381 }
382 }
383 if calls.is_empty() {
384 return Err(ScenarioError::contract(
385 scenario,
386 format!("history has no assistant tool calls: {messages:?}"),
387 ));
388 }
389 for (turn, id, call_id) in &calls {
390 let matches = results
391 .iter()
392 .filter(|(result_turn, result_id, result_call_id)| {
393 result_turn == turn && result_id == id && call_id == result_call_id
394 })
395 .count();
396 if matches != 1 {
397 return Err(ScenarioError::contract(
398 scenario,
399 format!(
400 "tool call id={id:?} call_id={call_id:?} has {matches} correlated results; calls={calls:?}, results={results:?}"
401 ),
402 ));
403 }
404 }
405 if results.len() != calls.len() {
406 return Err(ScenarioError::contract(
407 scenario,
408 format!(
409 "history contains dangling calls or results: calls={calls:?}, results={results:?}"
410 ),
411 ));
412 }
413 Ok(())
414}
415
416#[derive(Debug, Deserialize, JsonSchema)]
417struct OperationArgs {
418 x: i64,
419 y: i64,
420}
421
422#[derive(Clone)]
423struct CountingAdd(Arc<AtomicUsize>);
424
425impl Tool for CountingAdd {
426 const NAME: &'static str = "add";
427 type Error = ConformanceToolError;
428 type Args = OperationArgs;
429 type Output = i64;
430
431 fn description(&self) -> String {
432 "Add x and y together".to_string()
433 }
434
435 fn parameters(&self) -> serde_json::Value {
436 serde_json::json!({
437 "type": "object",
438 "properties": {
439 "x": { "type": "number", "description": "The first operand" },
440 "y": { "type": "number", "description": "The second operand" }
441 },
442 "required": ["x", "y"]
443 })
444 }
445
446 async fn call(
447 &self,
448 _context: &mut ToolContext,
449 args: Self::Args,
450 ) -> Result<Self::Output, Self::Error> {
451 self.0.fetch_add(1, Ordering::SeqCst);
452 Ok(args.x + args.y)
453 }
454}
455
456#[derive(Clone)]
457struct CountingSum(Arc<AtomicUsize>);
458
459impl Tool for CountingSum {
460 const NAME: &'static str = "sum";
461 type Error = ConformanceToolError;
462 type Args = OperationArgs;
463 type Output = i64;
464
465 fn description(&self) -> String {
466 "Add x and y together (alias of add)".to_string()
467 }
468
469 fn parameters(&self) -> serde_json::Value {
470 CountingAdd(Arc::new(AtomicUsize::new(0))).parameters()
471 }
472
473 async fn call(
474 &self,
475 _context: &mut ToolContext,
476 args: Self::Args,
477 ) -> Result<Self::Output, Self::Error> {
478 self.0.fetch_add(1, Ordering::SeqCst);
479 Ok(args.x + args.y)
480 }
481}
482
483#[derive(Clone)]
484struct CountingSubtract(Arc<AtomicUsize>);
485
486impl Tool for CountingSubtract {
487 const NAME: &'static str = "subtract";
488 type Error = ConformanceToolError;
489 type Args = OperationArgs;
490 type Output = i64;
491
492 fn description(&self) -> String {
493 "Subtract y from x (i.e. x - y)".to_string()
494 }
495
496 fn parameters(&self) -> serde_json::Value {
497 serde_json::json!({
498 "type": "object",
499 "properties": {
500 "x": { "type": "number", "description": "The first operand" },
501 "y": { "type": "number", "description": "The second operand" }
502 },
503 "required": ["x", "y"]
504 })
505 }
506
507 async fn call(
508 &self,
509 _context: &mut ToolContext,
510 args: Self::Args,
511 ) -> Result<Self::Output, Self::Error> {
512 self.0.fetch_add(1, Ordering::SeqCst);
513 Ok(args.x - args.y)
514 }
515}
516
517#[derive(Clone)]
518struct RewriteArgument {
519 key: &'static str,
520 value: serde_json::Value,
521}
522
523impl AgentHook for RewriteArgument {
524 async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
525 let (Some(name), Some(args)) = (event.tool_name(), event.tool_args()) else {
526 return DispatchAction::proceed();
527 };
528 if name != CountingAdd::NAME {
529 return DispatchAction::proceed();
530 }
531 let Ok(mut arguments) = serde_json::from_str::<serde_json::Value>(args) else {
532 return DispatchAction::proceed();
533 };
534 let Some(object) = arguments.as_object_mut() else {
535 return DispatchAction::proceed();
536 };
537 object.insert(self.key.to_string(), self.value.clone());
538 DispatchAction::rewrite_tool_args(event.kind, arguments)
539 }
540}
541
542#[derive(Clone, Default)]
543struct ObserveArguments(Arc<Mutex<Vec<serde_json::Value>>>);
544
545impl AgentHook for ObserveArguments {
546 async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
547 if let Some(args) = event.tool_args() {
548 let value = serde_json::from_str(args)
549 .unwrap_or_else(|_| serde_json::Value::String(args.to_string()));
550 lock_recover(&self.0).push(value);
551 }
552 DispatchAction::proceed()
553 }
554}
555
556#[derive(Clone)]
557struct ReplaceResult(&'static str);
558
559impl AgentHook for ReplaceResult {
560 async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
561 if event.tool_name() == Some(CountingAdd::NAME) {
562 OutcomeAction::rewrite_tool_result(&event, self.0)
563 } else {
564 OutcomeAction::proceed()
565 }
566 }
567}
568
569#[derive(Clone)]
570struct WrapResult;
571
572impl AgentHook for WrapResult {
573 async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
574 match (event.tool_name(), event.tool_result()) {
575 (Some(name), Some(result)) if name == CountingAdd::NAME => {
576 OutcomeAction::rewrite_tool_result(
577 &event,
578 format!("[{}]", result.output().render()),
579 )
580 }
581 _ => OutcomeAction::proceed(),
582 }
583 }
584}
585
586#[derive(Clone)]
587struct FirstTurnPatch(RequestPatch);
588
589impl AgentHook for FirstTurnPatch {
590 async fn on_completion_call(
591 &self,
592 ctx: &HookContext,
593 _event: CompletionCallEvent<'_>,
594 ) -> CompletionCallAction {
595 if ctx.turn() == 1 {
596 CompletionCallAction::patch(self.0.clone())
597 } else {
598 CompletionCallAction::continue_run()
599 }
600 }
601}
602
603#[derive(Clone)]
604struct StopAfterResult(&'static str);
605
606impl AgentHook for StopAfterResult {
607 async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
608 if event.tool_name() == Some(CountingAdd::NAME) {
609 OutcomeAction::stop(self.0)
610 } else {
611 OutcomeAction::proceed()
612 }
613 }
614}
615
616#[derive(Debug, Deserialize, JsonSchema)]
617struct EmptyArgs {}
618
619#[derive(Clone)]
620struct PingTool(Arc<AtomicUsize>);
621
622impl Tool for PingTool {
623 const NAME: &'static str = "ping";
624 type Error = ConformanceToolError;
625 type Args = EmptyArgs;
626 type Output = String;
627
628 fn description(&self) -> String {
629 "Return the current ping marker. Takes no arguments.".to_string()
630 }
631
632 fn parameters(&self) -> serde_json::Value {
633 serde_json::json!({ "type": "object", "properties": {}, "required": [] })
634 }
635
636 async fn call(
637 &self,
638 _context: &mut ToolContext,
639 _args: Self::Args,
640 ) -> Result<Self::Output, Self::Error> {
641 self.0.fetch_add(1, Ordering::SeqCst);
642 Ok(PING_OUTPUT.to_string())
643 }
644}
645
646#[derive(Clone)]
647struct MottoTool(Arc<AtomicUsize>);
648
649impl Tool for MottoTool {
650 const NAME: &'static str = "fetch_motto";
651 type Error = ConformanceToolError;
652 type Args = EmptyArgs;
653 type Output = String;
654
655 fn description(&self) -> String {
656 "Fetch the two-line workshop motto.".to_string()
657 }
658
659 fn parameters(&self) -> serde_json::Value {
660 serde_json::json!({ "type": "object", "properties": {}, "required": [] })
661 }
662
663 async fn call(
664 &self,
665 _context: &mut ToolContext,
666 _args: Self::Args,
667 ) -> Result<Self::Output, Self::Error> {
668 self.0.fetch_add(1, Ordering::SeqCst);
669 Ok(MOTTO_OUTPUT.to_string())
670 }
671}
672
673#[derive(Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
674struct ConfigOutput {
675 service: String,
676 max_retries: u64,
677}
678
679#[derive(Clone)]
680struct ConfigTool(Arc<AtomicUsize>);
681
682impl Tool for ConfigTool {
683 const NAME: &'static str = "fetch_config";
684 type Error = ConformanceToolError;
685 type Args = EmptyArgs;
686 type Output = ConfigOutput;
687
688 fn description(&self) -> String {
689 "Fetch the service configuration object.".to_string()
690 }
691
692 fn parameters(&self) -> serde_json::Value {
693 serde_json::json!({ "type": "object", "properties": {}, "required": [] })
694 }
695
696 async fn call(
697 &self,
698 _context: &mut ToolContext,
699 _args: Self::Args,
700 ) -> Result<Self::Output, Self::Error> {
701 self.0.fetch_add(1, Ordering::SeqCst);
702 Ok(ConfigOutput {
703 service: "cassette-lab".to_string(),
704 max_retries: 3,
705 })
706 }
707}
708
709#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
710#[serde(rename_all = "snake_case")]
711enum ComplexMode {
712 Careful,
713 Fast,
714}
715
716#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
717struct ComplexProfile {
718 name: String,
719 tags: Vec<String>,
720}
721
722#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
723struct ComplexArgs {
724 profile: ComplexProfile,
725 mode: ComplexMode,
726 note: Option<String>,
727 quote: String,
728}
729
730#[derive(Debug, Deserialize, Serialize, JsonSchema)]
731struct ExtractedPerson {
732 #[schemars(required)]
733 first_name: Option<String>,
734 #[schemars(required)]
735 last_name: Option<String>,
736 #[schemars(required)]
737 job: Option<String>,
738}
739
740#[derive(Clone)]
741struct CaptureComplexTool {
742 calls: Arc<AtomicUsize>,
743 captured: Arc<Mutex<Option<ComplexArgs>>>,
744}
745
746impl Tool for CaptureComplexTool {
747 const NAME: &'static str = "store_profile";
748 type Error = ConformanceToolError;
749 type Args = ComplexArgs;
750 type Output = ComplexArgs;
751
752 fn description(&self) -> String {
753 "Store one profile with its nested tags, mode, optional note, and exact quoted text."
754 .to_string()
755 }
756
757 fn parameters(&self) -> serde_json::Value {
758 serde_json::to_value(schemars::schema_for!(ComplexArgs)).unwrap_or_default()
759 }
760
761 async fn call(
762 &self,
763 _context: &mut ToolContext,
764 args: Self::Args,
765 ) -> Result<Self::Output, Self::Error> {
766 self.calls.fetch_add(1, Ordering::SeqCst);
767 *lock_recover(&self.captured) = Some(args.clone());
768 Ok(args)
769 }
770}
771
772fn has_tool_roundtrip(messages: &[Message]) -> bool {
773 let saw_call = messages.iter().any(|message| {
774 matches!(
775 message,
776 Message::Assistant(rig_core::message::AssistantMessage { content, .. })
777 if content.iter().any(|item| matches!(item, AssistantContent::ToolCall(_)))
778 )
779 });
780 let saw_result = messages.iter().any(|message| {
781 matches!(
782 message,
783 Message::User { content }
784 if content.iter().any(|item| matches!(item, UserContent::ToolResult(_)))
785 )
786 });
787 saw_call && saw_result
788}
789
790#[derive(Debug, Deserialize, JsonSchema)]
791struct RepeatArgs {
792 text: String,
794 times: Option<u32>,
796}
797
798#[derive(Clone)]
799struct RepeatTool {
800 calls: Arc<AtomicUsize>,
801}
802
803impl Tool for RepeatTool {
804 const NAME: &'static str = "repeat_text";
805 type Error = ConformanceToolError;
806 type Args = RepeatArgs;
807 type Output = String;
808
809 fn description(&self) -> String {
810 "Repeat `text`. `times` is optional and defaults to 2.".to_string()
811 }
812
813 fn parameters(&self) -> serde_json::Value {
814 serde_json::to_value(schemars::schema_for!(RepeatArgs)).unwrap_or_default()
815 }
816
817 async fn call(
818 &self,
819 _context: &mut ToolContext,
820 args: Self::Args,
821 ) -> Result<Self::Output, Self::Error> {
822 self.calls.fetch_add(1, Ordering::SeqCst);
823 Ok(vec![args.text.as_str(); args.times.unwrap_or(2) as usize].join(" "))
824 }
825}
826
827#[derive(Debug, Deserialize, JsonSchema)]
828struct BinOpArgs {
829 a: i64,
830 b: i64,
831}
832
833#[derive(Debug, Deserialize, JsonSchema)]
834struct ArithmeticResult {
835 answer: i64,
836 explanation: Option<String>,
837}
838
839#[derive(Clone)]
840struct AddTool(Arc<AtomicUsize>);
841
842impl Tool for AddTool {
843 const NAME: &'static str = "add";
844 type Error = ConformanceToolError;
845 type Args = BinOpArgs;
846 type Output = i64;
847
848 fn description(&self) -> String {
849 "Add two integers a and b.".to_string()
850 }
851
852 fn parameters(&self) -> serde_json::Value {
853 serde_json::to_value(schemars::schema_for!(BinOpArgs)).unwrap_or_default()
854 }
855
856 async fn call(
857 &self,
858 _context: &mut ToolContext,
859 args: Self::Args,
860 ) -> Result<Self::Output, Self::Error> {
861 self.0.fetch_add(1, Ordering::SeqCst);
862 Ok(args.a + args.b)
863 }
864}
865
866#[derive(Clone)]
867struct MultiplyTool(Arc<AtomicUsize>);
868
869impl Tool for MultiplyTool {
870 const NAME: &'static str = "multiply";
871 type Error = ConformanceToolError;
872 type Args = BinOpArgs;
873 type Output = i64;
874
875 fn description(&self) -> String {
876 "Multiply two integers a and b.".to_string()
877 }
878
879 fn parameters(&self) -> serde_json::Value {
880 serde_json::to_value(schemars::schema_for!(BinOpArgs)).unwrap_or_default()
881 }
882
883 async fn call(
884 &self,
885 _context: &mut ToolContext,
886 args: Self::Args,
887 ) -> Result<Self::Output, Self::Error> {
888 self.0.fetch_add(1, Ordering::SeqCst);
889 Ok(args.a * args.b)
890 }
891}
892
893fn report_from_response(
894 name: &'static str,
895 started: Instant,
896 tool_calls: usize,
897 response: crate::agent::PromptResponse,
898) -> Result<ScenarioReport, ScenarioError> {
899 validate_protocol_hygiene(
900 name,
901 &response.output(),
902 &response.messages,
903 &[
904 "<tool_call>",
905 "</tool_call>",
906 "<tool_response>",
907 "</tool_response>",
908 "<|im_start|>",
909 "<|im_end|>",
910 "<think>",
911 "</think>",
912 ],
913 )?;
914 Ok(ScenarioReport {
915 name,
916 tool_calls,
917 prompt_tokens: response.usage.input_tokens.unwrap_or(0),
918 generated_tokens: response.usage.output_tokens.unwrap_or(0),
919 history_messages: response.messages.len(),
920 duration: started.elapsed(),
921 response: response.output(),
922 })
923}
924
925fn correlated_messages<'a>(
928 scenario: &'static str,
929 response: &'a crate::agent::PromptResponse,
930) -> Result<&'a [Message], ScenarioError> {
931 let messages = response.messages.as_slice();
932 if messages.is_empty() {
933 return Err(ScenarioError::contract(
934 scenario,
935 "extended run omitted accumulated message history",
936 ));
937 }
938 validate_tool_correlation(scenario, messages)?;
939 Ok(messages)
940}
941
942fn correlated_result_values(
944 scenario: &'static str,
945 response: &crate::agent::PromptResponse,
946) -> Result<Vec<serde_json::Value>, ScenarioError> {
947 Ok(correlated_messages(scenario, response)?
948 .iter()
949 .flat_map(tool_result_values)
950 .collect())
951}
952
953fn value_matches_integer(value: &serde_json::Value, expected: i64) -> bool {
954 value.as_i64() == Some(expected)
955 || value
956 .as_str()
957 .and_then(|text| text.trim().parse::<i64>().ok())
958 == Some(expected)
959}
960
961pub async fn parallel_tools<W, T, F>(
967 model: rig_core::driver::Model<W, T>,
968 configure: F,
969 tool_concurrency: Option<usize>,
970) -> Result<ScenarioReport, ScenarioError>
971where
972 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
973 T: rig_core::driver::Transport<W>,
974 F: FnOnce(AgentBuilder) -> AgentBuilder,
975{
976 let add_calls = Arc::new(AtomicUsize::new(0));
977 let subtract_calls = Arc::new(AtomicUsize::new(0));
978 let started = Instant::now();
979 let agent = configure(AgentBuilder::new(model).temperature(0.0))
980 .preamble(FORCE_TOOLS_PREAMBLE)
981 .tool(CountingAdd(add_calls.clone()))
982 .tool(CountingSubtract(subtract_calls.clone()))
983 .default_max_turns(3)
984 .build();
985 let request = agent.prompt(PARALLEL_PROMPT).max_turns(3);
986 let response = match tool_concurrency {
987 Some(concurrency) => request.tool_concurrency(concurrency).await?,
988 None => request.await?,
989 };
990 let scenario = if tool_concurrency == Some(1) {
991 "parallel_tools_serial_execution"
992 } else {
993 "parallel_tools"
994 };
995 let messages = correlated_messages(scenario, &response)?;
996
997 let Some((call_index, calls)) = messages.iter().enumerate().find_map(|(index, message)| {
998 let Message::Assistant(rig_core::message::AssistantMessage { content, .. }) = message
999 else {
1000 return None;
1001 };
1002 let calls = content
1003 .iter()
1004 .filter_map(|item| match item {
1005 AssistantContent::ToolCall(call) => Some(call),
1006 _ => None,
1007 })
1008 .collect::<Vec<_>>();
1009 (calls.len() == 2).then_some((index, calls))
1010 }) else {
1011 return Err(ScenarioError::contract(
1012 scenario,
1013 format!("no assistant turn contained exactly two tool calls: {messages:?}"),
1014 ));
1015 };
1016 let mut names = calls
1017 .iter()
1018 .map(|call| call.function.name.as_str())
1019 .collect::<Vec<_>>();
1020 names.sort_unstable();
1021 if names != ["add", "subtract"] {
1022 return Err(ScenarioError::contract(
1023 scenario,
1024 format!("parallel turn called {names:?}, expected add and subtract"),
1025 ));
1026 }
1027 let results_message = messages.get(call_index + 1).ok_or_else(|| {
1028 ScenarioError::contract(
1029 scenario,
1030 "parallel call turn has no following result message",
1031 )
1032 })?;
1033 let values = tool_result_values(results_message);
1034 if !values.iter().any(|value| value_matches_integer(value, 7))
1035 || !values.iter().any(|value| value_matches_integer(value, 8))
1036 || values.len() != 2
1037 {
1038 return Err(ScenarioError::contract(
1039 scenario,
1040 format!("parallel result message did not contain exactly 7 and 8: {values:?}"),
1041 ));
1042 }
1043 let add = add_calls.load(Ordering::SeqCst);
1044 let subtract = subtract_calls.load(Ordering::SeqCst);
1045 if add != 1 || subtract != 1 {
1046 return Err(ScenarioError::contract(
1047 scenario,
1048 format!("execution counts were add={add}, subtract={subtract}, expected one each"),
1049 ));
1050 }
1051 report_from_response(scenario, started, add + subtract, response)
1052}
1053
1054pub async fn zero_argument_tool<W, T, F>(
1056 model: rig_core::driver::Model<W, T>,
1057 configure: F,
1058) -> Result<ScenarioReport, ScenarioError>
1059where
1060 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1061 T: rig_core::driver::Transport<W>,
1062 F: FnOnce(AgentBuilder) -> AgentBuilder,
1063{
1064 const SCENARIO: &str = "zero_argument_tool";
1065 let calls = Arc::new(AtomicUsize::new(0));
1066 let started = Instant::now();
1067 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1068 .preamble("You must use the provided tools. Report tool outputs exactly as returned.")
1069 .tool(PingTool(calls.clone()))
1070 .default_max_turns(2)
1071 .build();
1072 let response = agent
1073 .prompt("Call the ping tool, then report the exact marker it returns.")
1074 .max_turns(2)
1075 .await?;
1076 let values = correlated_result_values(SCENARIO, &response)?;
1077 if calls.load(Ordering::SeqCst) != 1
1078 || !values
1079 .iter()
1080 .any(|value| value.as_str() == Some(PING_OUTPUT))
1081 || !response.output().contains(PING_OUTPUT)
1082 {
1083 return Err(ScenarioError::contract(
1084 SCENARIO,
1085 format!(
1086 "calls={}, results={values:?}, response={:?}",
1087 calls.load(Ordering::SeqCst),
1088 response.output()
1089 ),
1090 ));
1091 }
1092 report_from_response(SCENARIO, started, 1, response)
1093}
1094
1095pub async fn tool_output_serialization<W, T, F>(
1098 model: rig_core::driver::Model<W, T>,
1099 configure: F,
1100) -> Result<ScenarioReport, ScenarioError>
1101where
1102 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1103 T: rig_core::driver::Transport<W>,
1104 F: FnOnce(AgentBuilder) -> AgentBuilder,
1105{
1106 const SCENARIO: &str = "tool_output_serialization";
1107 let started = Instant::now();
1108 let motto_calls = Arc::new(AtomicUsize::new(0));
1109 let config_calls = Arc::new(AtomicUsize::new(0));
1110 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1111 .preamble("You must use the provided tools before answering.")
1112 .tool(MottoTool(motto_calls.clone()))
1113 .tool(ConfigTool(config_calls.clone()))
1114 .default_max_turns(3)
1115 .build();
1116 let response = agent
1117 .prompt("Call fetch_motto and fetch_config, then summarize both outputs in one sentence.")
1118 .max_turns(3)
1119 .await?;
1120 let values = correlated_result_values(SCENARIO, &response)?;
1121 let expected_config = serde_json::to_value(ConfigOutput {
1122 service: "cassette-lab".to_string(),
1123 max_retries: 3,
1124 })?;
1125 let motto_ok = values
1126 .iter()
1127 .any(|value| value.as_str() == Some(MOTTO_OUTPUT));
1128 let config_ok = values.iter().any(|value| {
1129 value == &expected_config
1130 || value
1131 .as_str()
1132 .and_then(|text| serde_json::from_str::<serde_json::Value>(text).ok())
1133 .as_ref()
1134 == Some(&expected_config)
1135 });
1136 let motto_count = motto_calls.load(Ordering::SeqCst);
1137 let config_count = config_calls.load(Ordering::SeqCst);
1138 if !motto_ok || !config_ok || motto_count != 1 || config_count != 1 {
1139 return Err(ScenarioError::contract(
1140 SCENARIO,
1141 format!(
1142 "expected one verbatim motto and one semantic config JSON; motto_calls={motto_count}, config_calls={config_count}, values={values:?}"
1143 ),
1144 ));
1145 }
1146 report_from_response(SCENARIO, started, motto_count + config_count, response)
1147}
1148
1149pub async fn complex_tool_arguments<W, T, F>(
1152 model: rig_core::driver::Model<W, T>,
1153 configure: F,
1154) -> Result<ScenarioReport, ScenarioError>
1155where
1156 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1157 T: rig_core::driver::Transport<W>,
1158 F: FnOnce(AgentBuilder) -> AgentBuilder,
1159{
1160 complex_tool_arguments_with_prompt(model, configure, COMPLEX_ARGUMENTS_PROMPT).await
1161}
1162
1163const COMPLEX_ARGUMENTS_PROMPT: &str = "Call store_profile with profile.name exactly `Zoë \\\"Z\\\"`, profile.tags exactly [`rust`, `東京`], mode `careful`, note containing the two lines `line one` and `line two` separated by a newline, and quote exactly `path C:\\\\tmp and \\\"quoted\\\"`. Then confirm it was stored.";
1166
1167pub const COMPLEX_ARGUMENTS_RAW_PROMPT: &str = "Call store_profile once. Each value below is raw text between <<< and >>>, with no escaping: profile.name <<<Zoë \"Z\">>>, profile.tags <<<rust>>> and <<<東京>>>, mode <<<careful>>>, note the two lines <<<line one>>> and <<<line two>>> joined by one newline, and quote <<<path C:\\tmp and \"quoted\">>>. Then confirm it was stored.";
1170
1171pub async fn complex_tool_arguments_with_prompt<W, T, F>(
1174 model: rig_core::driver::Model<W, T>,
1175 configure: F,
1176 prompt: &str,
1177) -> Result<ScenarioReport, ScenarioError>
1178where
1179 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1180 T: rig_core::driver::Transport<W>,
1181 F: FnOnce(AgentBuilder) -> AgentBuilder,
1182{
1183 const SCENARIO: &str = "complex_tool_arguments";
1184 let expected = ComplexArgs {
1185 profile: ComplexProfile {
1186 name: "Zoë \"Z\"".to_string(),
1187 tags: vec!["rust".to_string(), "東京".to_string()],
1188 },
1189 mode: ComplexMode::Careful,
1190 note: Some("line one\nline two".to_string()),
1191 quote: "path C:\\tmp and \"quoted\"".to_string(),
1192 };
1193 let calls = Arc::new(AtomicUsize::new(0));
1194 let captured = Arc::new(Mutex::new(None));
1195 let started = Instant::now();
1196 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1197 .preamble("Use store_profile exactly once with every value supplied by the user.")
1198 .tool(CaptureComplexTool {
1199 calls: calls.clone(),
1200 captured: captured.clone(),
1201 })
1202 .default_max_turns(3)
1203 .build();
1204 let response = agent.prompt(prompt).max_turns(3).await?;
1205 let observed = lock_recover(&captured).clone();
1206 if calls.load(Ordering::SeqCst) != 1 || observed.as_ref() != Some(&expected) {
1207 return Err(ScenarioError::contract(
1208 SCENARIO,
1209 format!(
1210 "calls={}, expected={expected:?}, observed={observed:?}, response={:?}",
1211 calls.load(Ordering::SeqCst),
1212 response.output()
1213 ),
1214 ));
1215 }
1216 correlated_messages(SCENARIO, &response)?;
1217 report_from_response(SCENARIO, started, 1, response)
1218}
1219
1220pub async fn buffered_streaming_text_parity<W, T, A>(
1223 model: rig_core::driver::Model<W, T>,
1224 adjust: A,
1225) -> Result<ScenarioReport, ScenarioError>
1226where
1227 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1228 T: rig_core::driver::Transport<W>,
1229 A: Fn(CompletionRequest) -> CompletionRequest,
1230{
1231 const SCENARIO: &str = "buffered_streaming_text_parity";
1232 const PROMPT: &str = "Answer with exactly the single word Paris.";
1233 let started = Instant::now();
1234 let request = || {
1235 adjust(
1236 CompletionRequest::new(PROMPT)
1237 .temperature(0.0)
1238 .max_tokens(32),
1239 )
1240 };
1241 let buffered = model.call(request()).await?;
1242 let buffered_text = buffered
1243 .choice
1244 .iter()
1245 .filter_map(|item| match item {
1246 AssistantContent::Text(text) => Some(text.text.as_str()),
1247 _ => None,
1248 })
1249 .collect::<String>();
1250
1251 let mut stream = model.stream(request())?;
1252 let mut streamed_text = String::new();
1253 while let Some(item) = stream.next().await {
1254 if let crate::streaming::Item::Event(crate::streaming::StreamEvent::Text { text, .. }) =
1255 item?
1256 {
1257 streamed_text.push_str(&text);
1258 }
1259 }
1260 let streamed_usage = Some(stream.finish().await?.usage);
1261 let usage = streamed_usage.ok_or_else(|| {
1262 ScenarioError::contract(SCENARIO, "raw stream omitted its final response metadata")
1263 })?;
1264 let normalize = |text: &str| {
1265 text.trim()
1266 .trim_matches(|character: char| !character.is_alphanumeric())
1267 .to_string()
1268 };
1269 let buffered_answer = normalize(&buffered_text);
1270 let streamed_answer = normalize(&streamed_text);
1271 if !buffered_answer.eq_ignore_ascii_case("Paris")
1272 || !streamed_answer.eq_ignore_ascii_case("Paris")
1273 || !buffered.usage.is_reported()
1274 || !usage.is_reported()
1275 {
1276 return Err(ScenarioError::contract(
1277 SCENARIO,
1278 format!(
1279 "buffered={buffered_text:?}, streamed={streamed_text:?}, buffered_usage={:?}, streamed_usage={usage:?}",
1280 buffered.usage
1281 ),
1282 ));
1283 }
1284 Ok(ScenarioReport {
1285 name: SCENARIO,
1286 tool_calls: 0,
1287 prompt_tokens: usage.input_tokens.unwrap_or(0),
1288 generated_tokens: usage.output_tokens.unwrap_or(0),
1289 history_messages: 0,
1290 duration: started.elapsed(),
1291 response: streamed_text,
1292 })
1293}
1294
1295pub async fn structured_extraction<W, T>(
1298 model: rig_core::driver::Model<W, T>,
1299 additional_params: Option<serde_json::Value>,
1300) -> Result<ScenarioReport, ScenarioError>
1301where
1302 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1303 T: rig_core::driver::Transport<W>,
1304{
1305 const SCENARIO: &str = "structured_extraction";
1306 const INPUT: &str = "Hello, my name is Ada Lovelace and I work as a mathematician.";
1307 let started = Instant::now();
1308 let mut builder = crate::extractor::ExtractorBuilder::<ExtractedPerson>::new(model)
1309 .max_tokens(384)
1310 .retries(0);
1311 if let Some(params) = additional_params {
1312 builder = builder.additional_params(params);
1313 }
1314 let response = builder.build().extract(INPUT).await?;
1315 validate_extraction_fields(
1316 SCENARIO,
1317 response.output.first_name.as_deref(),
1318 response.output.last_name.as_deref(),
1319 response.output.job.as_deref(),
1320 response.usage,
1321 )?;
1322 Ok(ScenarioReport {
1323 name: SCENARIO,
1324 tool_calls: 1,
1325 prompt_tokens: response.usage.input_tokens.unwrap_or(0),
1326 generated_tokens: response.usage.output_tokens.unwrap_or(0),
1327 history_messages: 0,
1328 duration: started.elapsed(),
1329 response: format!(
1330 "{} {} — {}",
1331 response.output.first_name.as_deref().unwrap_or_default(),
1332 response.output.last_name.as_deref().unwrap_or_default(),
1333 response.output.job.as_deref().unwrap_or_default()
1334 ),
1335 })
1336}
1337
1338fn restricted_recovery_run(
1339 prompt: &str,
1340 turn: ModelTurn,
1341 retries: usize,
1342) -> Result<AgentRun, ScenarioError> {
1343 const SCENARIO: &str = "invalid_tool_recovery";
1344 let mut run = AgentRun::new(prompt)
1345 .max_turns(2)
1346 .max_invalid_tool_call_retries(retries);
1347 if !matches!(run.next_step()?, AgentRunStep::CallModel { .. }) {
1348 return Err(ScenarioError::contract(
1349 SCENARIO,
1350 "fresh AgentRun did not request a model turn",
1351 ));
1352 }
1353 let outcome = run.model_response(turn)?;
1354 let ModelTurnOutcome::NeedsResolution(context) = outcome else {
1355 return Err(ScenarioError::contract(
1356 SCENARIO,
1357 format!("disallowed tool call did not require resolution: {outcome:?}"),
1358 ));
1359 };
1360 if context.tool_name != CountingAdd::NAME {
1361 return Err(ScenarioError::contract(
1362 SCENARIO,
1363 format!("expected rejected add call, observed {context:?}"),
1364 ));
1365 }
1366 Ok(run)
1367}
1368
1369pub async fn invalid_tool_recovery<W, T, F>(
1372 model: rig_core::driver::Model<W, T>,
1373 configure: F,
1374) -> Result<ScenarioReport, ScenarioError>
1375where
1376 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1377 T: rig_core::driver::Transport<W>,
1378 F: FnOnce(AgentBuilder) -> AgentBuilder,
1379{
1380 invalid_tool_recovery_with_choice(model, configure, ToolChoice::Required).await
1381}
1382
1383pub async fn invalid_tool_recovery_with_choice<W, T, F>(
1387 model: rig_core::driver::Model<W, T>,
1388 configure: F,
1389 tool_choice: ToolChoice,
1390) -> Result<ScenarioReport, ScenarioError>
1391where
1392 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1393 T: rig_core::driver::Transport<W>,
1394 F: FnOnce(AgentBuilder) -> AgentBuilder,
1395{
1396 const SCENARIO: &str = "invalid_tool_recovery";
1397 const PROMPT: &str = "Call the add tool exactly once with x=2 and y=3. Do not call sum.";
1398 let started = Instant::now();
1399 let add_calls = Arc::new(AtomicUsize::new(0));
1400 let sum_calls = Arc::new(AtomicUsize::new(0));
1401 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1402 .preamble(FORCE_TOOLS_PREAMBLE)
1403 .tool(CountingAdd(add_calls.clone()))
1404 .tool(CountingSum(sum_calls.clone()))
1405 .tool_choice(tool_choice)
1406 .build();
1407 #[derive(Clone)]
1408 struct CaptureTurn(Arc<Mutex<Option<ModelTurn>>>);
1409
1410 impl AgentHook for CaptureTurn {
1411 async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
1412 let Some(response) = event.completion() else {
1413 return OutcomeAction::proceed();
1414 };
1415 *lock_recover(&self.0) = Some(ModelTurn::new(
1416 response.head(),
1417 response.choice.clone(),
1418 response.usage,
1419 TurnPolicy::default(),
1420 response.raw.clone(),
1421 ));
1422 OutcomeAction::stop("captured conformance model turn")
1423 }
1424 }
1425
1426 let captured = Arc::new(Mutex::new(None));
1427 let stopped = agent
1428 .prompt(PROMPT)
1429 .add_hook(CaptureTurn(captured.clone()))
1430 .run()
1431 .await;
1432 if !matches!(stopped, Err(PromptError::Cancelled { .. })) {
1433 return Err(ScenarioError::contract(
1434 SCENARIO,
1435 format!("capture hook did not stop after the model response: {stopped:?}"),
1436 ));
1437 }
1438 let response = lock_recover(&captured).take().ok_or_else(|| {
1439 ScenarioError::contract(SCENARIO, "capture hook observed no model response")
1440 })?;
1441 let emitted = response
1442 .choice
1443 .iter()
1444 .filter(|item| {
1445 matches!(item, AssistantContent::ToolCall(call) if call.function.name == CountingAdd::NAME)
1446 })
1447 .count();
1448 if emitted != 1 {
1449 return Err(ScenarioError::contract(
1450 SCENARIO,
1451 format!(
1452 "model emitted {emitted} add calls, response={:?}",
1453 response.choice
1454 ),
1455 ));
1456 }
1457 let executable = BTreeSet::from([CountingAdd::NAME.to_string(), CountingSum::NAME.to_string()]);
1458 let only_sum = ToolChoice::Specific {
1459 function_names: vec![
1460 rig_core::message::ToolName::new(CountingSum::NAME)
1461 .map_err(|error| ScenarioError::contract(SCENARIO, error.to_string()))?,
1462 ],
1463 };
1464 let policy = TurnPolicy::new(executable, Some(only_sum), None)
1465 .map_err(|error| ScenarioError::contract(SCENARIO, error.to_string()))?;
1466 let turn = ModelTurn::new(
1467 response.head,
1468 response.choice,
1469 response.usage,
1470 policy,
1471 response.raw,
1472 );
1473
1474 let mut fail = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1475 let error = match fail.resolve_invalid_tool_call(InvalidToolCallAction::fail()) {
1476 Err(error) => error,
1477 Ok(outcome) => {
1478 return Err(ScenarioError::contract(
1479 SCENARIO,
1480 format!("fail action unexpectedly returned {outcome:?}"),
1481 ));
1482 }
1483 };
1484 validate_unknown_tool_failure(&error, CountingAdd::NAME, &[CountingSum::NAME])?;
1485
1486 let mut retry = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1487 let error = match retry
1488 .resolve_invalid_tool_call(InvalidToolCallAction::retry("choose an allowed tool"))
1489 {
1490 Err(error) => error,
1491 Ok(outcome) => {
1492 return Err(ScenarioError::contract(
1493 SCENARIO,
1494 format!("exhausted retry unexpectedly returned {outcome:?}"),
1495 ));
1496 }
1497 };
1498 validate_unknown_tool_failure(&error, CountingAdd::NAME, &[CountingSum::NAME])?;
1499
1500 let mut rejected_repair = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1501 let error =
1502 match rejected_repair.resolve_invalid_tool_call(InvalidToolCallAction::repair("missing")) {
1503 Err(error) => error,
1504 Ok(outcome) => {
1505 return Err(ScenarioError::contract(
1506 SCENARIO,
1507 format!("disallowed repair unexpectedly returned {outcome:?}"),
1508 ));
1509 }
1510 };
1511 validate_unknown_tool_failure(&error, "missing", &[CountingSum::NAME])?;
1512
1513 let mut repaired = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1514 if !matches!(
1515 repaired.resolve_invalid_tool_call(InvalidToolCallAction::repair(CountingSum::NAME))?,
1516 ModelTurnOutcome::Continue { .. }
1517 ) {
1518 return Err(ScenarioError::contract(
1519 SCENARIO,
1520 "valid repair did not continue",
1521 ));
1522 }
1523 let AgentRunStep::CallTools { calls } = repaired.next_step()? else {
1524 return Err(ScenarioError::contract(
1525 SCENARIO,
1526 "valid repair did not produce pending tool execution",
1527 ));
1528 };
1529 let repaired_call = calls.first();
1530 if calls.len() != 1
1531 || !repaired_call.is_some_and(|call| {
1532 matches!(call, PendingToolCall::Execute(call) if call.name() == CountingSum::NAME)
1533 })
1534 {
1535 return Err(ScenarioError::contract(
1536 SCENARIO,
1537 format!("repaired pending calls were incorrect: {calls:?}"),
1538 ));
1539 }
1540
1541 let mut skipped = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1542 if !matches!(
1543 skipped.resolve_invalid_tool_call(InvalidToolCallAction::skip("disabled for this turn"))?,
1544 ModelTurnOutcome::Continue { .. }
1545 ) {
1546 return Err(ScenarioError::contract(SCENARIO, "skip did not continue"));
1547 }
1548 let step = skipped.next_step()?;
1550 let skipped_answered = matches!(
1551 skipped.messages().last().map(|message| &**message),
1552 Some(Message::User { content }) if content.len() == 1
1553 );
1554 if !matches!(step, AgentRunStep::CallModel { .. }) || !skipped_answered {
1555 return Err(ScenarioError::contract(
1556 SCENARIO,
1557 format!("skip did not answer the turn and continue: {step:?}"),
1558 ));
1559 }
1560 if add_calls.load(Ordering::SeqCst) != 0 || sum_calls.load(Ordering::SeqCst) != 0 {
1561 return Err(ScenarioError::contract(
1562 SCENARIO,
1563 "recovery scenario executed a tool body",
1564 ));
1565 }
1566
1567 Ok(ScenarioReport {
1568 name: SCENARIO,
1569 tool_calls: emitted,
1570 prompt_tokens: turn.usage.input_tokens.unwrap_or(0),
1571 generated_tokens: turn.usage.output_tokens.unwrap_or(0),
1572 history_messages: 2,
1573 duration: started.elapsed(),
1574 response: "fail, retry, repair, rejected repair, and skip passed".to_string(),
1575 })
1576}
1577
1578pub async fn hook_rewrites_and_request_patch<W, T, F>(
1581 model: rig_core::driver::Model<W, T>,
1582 configure: F,
1583) -> Result<ScenarioReport, ScenarioError>
1584where
1585 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1586 T: rig_core::driver::Transport<W>,
1587 F: FnOnce(AgentBuilder) -> AgentBuilder,
1588{
1589 hook_rewrites_and_request_patch_with_choice(model, configure, ToolChoice::Required).await
1590}
1591
1592pub async fn hook_rewrites_and_request_patch_with_choice<W, T, F>(
1596 model: rig_core::driver::Model<W, T>,
1597 configure: F,
1598 first_turn_choice: ToolChoice,
1599) -> Result<ScenarioReport, ScenarioError>
1600where
1601 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1602 T: rig_core::driver::Transport<W>,
1603 F: FnOnce(AgentBuilder) -> AgentBuilder,
1604{
1605 const SCENARIO: &str = "hook_rewrites_and_request_patch";
1606 let started = Instant::now();
1607 let calls = Arc::new(AtomicUsize::new(0));
1608 let observed = ObserveArguments::default();
1609 let observed_probe = observed.clone();
1610 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1611 .preamble("Use add for arithmetic and report only the tool result.")
1612 .tool(CountingAdd(calls.clone()))
1613 .default_max_turns(3)
1614 .build();
1615 let response = agent
1616 .prompt("Use add once for x=1 and y=1, then report what the tool returns.")
1617 .max_turns(3)
1618 .add_hook(FirstTurnPatch(
1619 RequestPatch::new()
1620 .active_tools([CountingAdd::NAME])
1621 .tool_choice(first_turn_choice),
1622 ))
1623 .add_hook(RewriteArgument {
1624 key: "x",
1625 value: serde_json::json!(7),
1626 })
1627 .add_hook(RewriteArgument {
1628 key: "y",
1629 value: serde_json::json!(8),
1630 })
1631 .add_hook(observed)
1632 .add_hook(ReplaceResult("portable-redacted"))
1633 .add_hook(WrapResult)
1634 .await?;
1635 let observations = lock_recover(&observed_probe.0).clone();
1636 validate_rewritten_arguments(
1637 SCENARIO,
1638 &observations,
1639 &serde_json::json!({ "x": 7, "y": 8 }),
1640 )?;
1641 let messages = response.messages.as_slice();
1642 if messages.is_empty() {
1643 return Err(ScenarioError::contract(
1644 SCENARIO,
1645 "extended hook run omitted message history",
1646 ));
1647 }
1648 let results = messages
1649 .iter()
1650 .flat_map(tool_result_values)
1651 .collect::<Vec<_>>();
1652 if calls.load(Ordering::SeqCst) != 1
1653 || !results
1654 .iter()
1655 .any(|value| value == &serde_json::json!("[portable-redacted]"))
1656 || response.completion_calls.len() != 2
1657 {
1658 return Err(ScenarioError::contract(
1659 SCENARIO,
1660 format!(
1661 "calls={}, completion_calls={}, results={results:?}, output={:?}",
1662 calls.load(Ordering::SeqCst),
1663 response.completion_calls.len(),
1664 response.output()
1665 ),
1666 ));
1667 }
1668 report_from_response(SCENARIO, started, 1, response)
1669}
1670
1671pub async fn cancellation_and_max_turns<W, T, F>(
1674 model: rig_core::driver::Model<W, T>,
1675 configure: F,
1676) -> Result<ScenarioReport, ScenarioError>
1677where
1678 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1679 T: rig_core::driver::Transport<W>,
1680 F: Fn(AgentBuilder) -> AgentBuilder,
1681{
1682 const SCENARIO: &str = "cancellation_and_max_turns";
1683 const REASON: &str = "portable result veto";
1684 let started = Instant::now();
1685 let cancelled_calls = Arc::new(AtomicUsize::new(0));
1686 let cancelled_agent = configure(AgentBuilder::new(model.clone()).temperature(0.0))
1687 .preamble("Use add for arithmetic; never calculate by hand.")
1688 .tool(CountingAdd(cancelled_calls.clone()))
1689 .build();
1690 let cancelled = match cancelled_agent
1691 .prompt("Use add once to compute x=20 plus y=22.")
1692 .max_turns(2)
1693 .add_hook(StopAfterResult(REASON))
1694 .await
1695 {
1696 Err(error) => error,
1697 Ok(output) => {
1698 return Err(ScenarioError::contract(
1699 SCENARIO,
1700 format!("result cancellation unexpectedly completed: {output:?}"),
1701 ));
1702 }
1703 };
1704 validate_cancelled_failure(&cancelled, REASON, CountingAdd::NAME)?;
1705
1706 let max_turn_calls = Arc::new(AtomicUsize::new(0));
1707 let max_turn_agent = configure(AgentBuilder::new(model).temperature(0.0))
1708 .preamble("Use add for arithmetic; never calculate by hand.")
1709 .tool(CountingAdd(max_turn_calls.clone()))
1710 .build();
1711 let max_turn = match max_turn_agent
1712 .prompt("Use add once to compute x=20 plus y=22, then report the result.")
1713 .max_turns(1)
1714 .await
1715 {
1716 Err(error) => error,
1717 Ok(output) => {
1718 return Err(ScenarioError::contract(
1719 SCENARIO,
1720 format!("one-turn budget unexpectedly completed: {output:?}"),
1721 ));
1722 }
1723 };
1724 validate_max_turns_failure(&max_turn, 1)?;
1725 let cancelled_count = cancelled_calls.load(Ordering::SeqCst);
1726 let max_turn_count = max_turn_calls.load(Ordering::SeqCst);
1727 if cancelled_count != 1 || max_turn_count != 1 {
1728 return Err(ScenarioError::contract(
1729 SCENARIO,
1730 format!("cancelled executions={cancelled_count}, max-turn executions={max_turn_count}"),
1731 ));
1732 }
1733 Ok(ScenarioReport {
1734 name: SCENARIO,
1735 tool_calls: cancelled_count + max_turn_count,
1736 prompt_tokens: 0,
1737 generated_tokens: 0,
1738 history_messages: 4,
1739 duration: started.elapsed(),
1740 response: "post-result cancellation and max-turn diagnostics passed".to_string(),
1741 })
1742}
1743
1744pub async fn optional_argument<W, T, F>(
1750 model: rig_core::driver::Model<W, T>,
1751 configure: F,
1752) -> Result<ScenarioReport, ScenarioError>
1753where
1754 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1755 T: rig_core::driver::Transport<W>,
1756 F: FnOnce(AgentBuilder) -> AgentBuilder,
1757{
1758 let calls = Arc::new(AtomicUsize::new(0));
1759 let started = Instant::now();
1760 let agent = configure(AgentBuilder::new(model))
1761 .preamble("Use the repeat_text tool whenever asked to repeat text.")
1762 .tool(RepeatTool {
1763 calls: calls.clone(),
1764 })
1765 .default_max_turns(4)
1766 .build();
1767 let result = agent
1768 .prompt(
1769 "Use the repeat_text tool to repeat the word \"banana\" 3 times, then show me the exact result.",
1770 )
1771 .await?;
1772 let response = result.output();
1773 let tool_calls = calls.load(Ordering::SeqCst);
1774 if tool_calls == 0
1775 || response.matches("banana").count() < 1
1776 || !has_tool_roundtrip(&result.messages)
1777 {
1778 return Err(ScenarioError::contract(
1779 "optional_argument",
1780 format!("calls={tool_calls}, response={response:?}"),
1781 ));
1782 }
1783 report_from_response("optional_argument", started, tool_calls, result)
1784}
1785
1786pub async fn sequential_tools<W, T, F>(
1788 model: rig_core::driver::Model<W, T>,
1789 configure: F,
1790) -> Result<ScenarioReport, ScenarioError>
1791where
1792 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1793 T: rig_core::driver::Transport<W>,
1794 F: FnOnce(AgentBuilder) -> AgentBuilder,
1795{
1796 let add_calls = Arc::new(AtomicUsize::new(0));
1797 let multiply_calls = Arc::new(AtomicUsize::new(0));
1798 let started = Instant::now();
1799 let agent = configure(AgentBuilder::new(model))
1800 .preamble(
1801 "You are a calculator. Use the add and multiply tools for arithmetic; never compute by hand.",
1802 )
1803 .tool(AddTool(add_calls.clone()))
1804 .tool(MultiplyTool(multiply_calls.clone()))
1805 .default_max_turns(6)
1806 .build();
1807 let result = agent
1808 .prompt(
1809 "Compute (4 + 6) * 2. First call the add tool, then call the multiply tool on the result. Tell me the final number.",
1810 )
1811 .await?;
1812 let response = result.output();
1813 let add = add_calls.load(Ordering::SeqCst);
1814 let multiply = multiply_calls.load(Ordering::SeqCst);
1815 if add == 0
1816 || multiply == 0
1817 || !response.contains("20")
1818 || !has_tool_roundtrip(&result.messages)
1819 {
1820 return Err(ScenarioError::contract(
1821 "sequential_tools",
1822 format!("add={add}, multiply={multiply}, response={response:?}"),
1823 ));
1824 }
1825 report_from_response("sequential_tools", started, add + multiply, result)
1826}
1827
1828pub async fn streaming_tool<W, T, F>(
1830 model: rig_core::driver::Model<W, T>,
1831 configure: F,
1832) -> Result<ScenarioReport, ScenarioError>
1833where
1834 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1835 T: rig_core::driver::Transport<W>,
1836 F: FnOnce(AgentBuilder) -> AgentBuilder,
1837{
1838 let calls = Arc::new(AtomicUsize::new(0));
1839 let started = Instant::now();
1840 let agent = configure(AgentBuilder::new(model))
1841 .preamble("Use the add tool for arithmetic; do not calculate by hand.")
1842 .tool(AddTool(calls.clone()))
1843 .default_max_turns(4)
1844 .build();
1845 let mut stream = agent
1846 .prompt("Use add to calculate 17 + 25, then state the final number.")
1847 .max_turns(4)
1848 .stream();
1849 let mut final_response = None;
1850 let mut final_count = 0_usize;
1851 let mut completion_usage = crate::completion::Usage::default();
1852 let mut streamed_call_ids = Vec::new();
1853 let mut streamed_result_ids = Vec::new();
1854 while let Some(item) = stream.next().await {
1855 match item? {
1856 MultiTurnStreamItem::ToolCall { tool_call } => streamed_call_ids.push(tool_call.id),
1857 MultiTurnStreamItem::ToolResult { tool_result } => {
1858 streamed_result_ids.push(tool_result.call)
1859 }
1860 MultiTurnStreamItem::CompletionCall(call) => completion_usage += call.usage,
1861 MultiTurnStreamItem::FinalResponse(response) => {
1862 final_count += 1;
1863 final_response = Some(response);
1864 }
1865 MultiTurnStreamItem::StreamAssistantItem(_)
1866 | MultiTurnStreamItem::ToolExecutionCommitted { .. }
1867 | MultiTurnStreamItem::ModelTurnRetried { .. } => {}
1868 }
1869 }
1870 let result = final_response.ok_or_else(|| {
1871 ScenarioError::contract("streaming_tool", "stream produced no final response")
1872 })?;
1873 let response = result.output();
1874 let history_messages = result.messages.len();
1875 let tool_calls = calls.load(Ordering::SeqCst);
1876 streamed_call_ids.sort();
1877 streamed_result_ids.sort();
1878 let correlated_stream =
1879 !streamed_call_ids.is_empty() && streamed_call_ids == streamed_result_ids;
1880 let correlated_history = validate_tool_correlation("streaming_tool", &result.messages).is_ok();
1881 if tool_calls == 0
1882 || !response.contains("42")
1883 || history_messages < 4
1884 || final_count != 1
1885 || !correlated_stream
1886 || !correlated_history
1887 || completion_usage != result.usage
1888 || result.completion_calls.is_empty()
1889 {
1890 return Err(ScenarioError::contract(
1891 "streaming_tool",
1892 format!(
1893 "calls={tool_calls}, final_count={final_count}, streamed_call_ids={streamed_call_ids:?}, streamed_result_ids={streamed_result_ids:?}, completion_usage={completion_usage:?}, final_usage={:?}, history_messages={history_messages}, response={response:?}",
1894 result.usage
1895 ),
1896 ));
1897 }
1898 validate_protocol_hygiene(
1899 "streaming_tool",
1900 &response,
1901 &result.messages,
1902 &["<tool_call>", "</tool_call>", "<think>", "</think>"],
1903 )?;
1904 Ok(ScenarioReport {
1905 name: "streaming_tool",
1906 tool_calls,
1907 prompt_tokens: result.usage.input_tokens.unwrap_or(0),
1908 generated_tokens: result.usage.output_tokens.unwrap_or(0),
1909 history_messages,
1910 duration: started.elapsed(),
1911 response,
1912 })
1913}
1914
1915pub async fn structured_after_tool<W, T, F>(
1917 model: rig_core::driver::Model<W, T>,
1918 configure: F,
1919) -> Result<ScenarioReport, ScenarioError>
1920where
1921 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1922 T: rig_core::driver::Transport<W>,
1923 F: FnOnce(AgentBuilder) -> AgentBuilder,
1924{
1925 let calls = Arc::new(AtomicUsize::new(0));
1926 let started = Instant::now();
1927 let agent = configure(AgentBuilder::new(model))
1928 .preamble(
1929 "Use add for arithmetic, then finish by calling the structured output tool exactly once.",
1930 )
1931 .output_schema::<ArithmeticResult>()
1932 .output_mode(OutputMode::Tool)
1933 .tool(AddTool(calls.clone()))
1934 .default_max_turns(5)
1935 .build();
1936 let result = agent
1937 .prompt("Use add to calculate 19 + 23. Return answer=42 and a short optional explanation.")
1938 .await?;
1939 let response = result.output();
1940 let parsed: ArithmeticResult = serde_json::from_str(&response)?;
1941 let tool_calls = calls.load(Ordering::SeqCst);
1942 if tool_calls == 0 || parsed.answer != 42 || !has_tool_roundtrip(&result.messages) {
1943 return Err(ScenarioError::contract(
1944 "structured_after_tool",
1945 format!("calls={tool_calls}, response={response:?}"),
1946 ));
1947 }
1948 let _ = parsed.explanation;
1949 report_from_response("structured_after_tool", started, tool_calls + 1, result)
1950}
1951
1952pub async fn tool_choice_modes<W, T, A>(
1954 model: rig_core::driver::Model<W, T>,
1955 adjust: A,
1956) -> Result<ScenarioReport, ScenarioError>
1957where
1958 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1959 T: rig_core::driver::Transport<W>,
1960 A: Fn(CompletionRequest) -> CompletionRequest,
1961{
1962 let tool_name = |name: &str| {
1963 rig_core::message::ToolName::new(name)
1964 .map_err(|error| ScenarioError::contract("tool_choice_modes", error.to_string()))
1965 };
1966 let definition = |name: rig_core::message::ToolName| ToolDefinition {
1967 description: format!("Return the supplied integer using {name}."),
1968 name,
1969 parameters: serde_json::json!({
1970 "type": "object",
1971 "properties": {"value": {"type": "integer"}},
1972 "required": ["value"]
1973 }),
1974 annotations: Default::default(),
1975 };
1976 let tools = vec![
1977 definition(tool_name("alpha")?),
1978 definition(tool_name("beta")?),
1979 ];
1980 let started = Instant::now();
1981 let none = model
1982 .call(adjust(
1983 CompletionRequest::new("Answer with only the number 4. Do not call a function.")
1984 .tools(tools.clone())
1985 .tool_choice(ToolChoice::None)
1986 .temperature(0.0)
1987 .max_tokens(64),
1988 ))
1989 .await?;
1990 if none
1991 .choice
1992 .iter()
1993 .any(|item| matches!(item, AssistantContent::ToolCall(_)))
1994 {
1995 return Err(ScenarioError::contract(
1996 "tool_choice_modes",
1997 "tool_choice none emitted a tool call",
1998 ));
1999 }
2000
2001 let required = model
2002 .call(adjust(
2003 CompletionRequest::new("Call alpha with value 7.")
2004 .tools(tools.clone())
2005 .tool_choice(ToolChoice::Required)
2006 .temperature(0.0)
2007 .max_tokens(96),
2008 ))
2009 .await?;
2010 let required_calls = required
2011 .choice
2012 .iter()
2013 .filter(|item| matches!(item, AssistantContent::ToolCall(_)))
2014 .count();
2015 if required_calls == 0 {
2016 return Err(ScenarioError::contract(
2017 "tool_choice_modes",
2018 "tool_choice required emitted no tool call",
2019 ));
2020 }
2021
2022 let specific = model
2023 .call(adjust(
2024 CompletionRequest::new("Call beta with value 9.")
2025 .tools(tools)
2026 .tool_choice(ToolChoice::Specific {
2027 function_names: vec![tool_name("beta")?],
2028 })
2029 .temperature(0.0)
2030 .max_tokens(96),
2031 ))
2032 .await?;
2033 let specific_calls = specific
2034 .choice
2035 .iter()
2036 .filter_map(|item| match item {
2037 AssistantContent::ToolCall(call) => Some(call),
2038 _ => None,
2039 })
2040 .collect::<Vec<_>>();
2041 if specific_calls.is_empty()
2042 || specific_calls
2043 .iter()
2044 .any(|call| call.function.name != "beta")
2045 {
2046 return Err(ScenarioError::contract(
2047 "tool_choice_modes",
2048 "specific tool choice did not select only beta",
2049 ));
2050 }
2051
2052 let usage = none.usage + required.usage + specific.usage;
2053 Ok(ScenarioReport {
2054 name: "tool_choice_modes",
2055 tool_calls: required_calls + specific_calls.len(),
2056 prompt_tokens: usage.input_tokens.unwrap_or(0),
2057 generated_tokens: usage.output_tokens.unwrap_or(0),
2058 history_messages: 0,
2059 duration: started.elapsed(),
2060 response: "none, required, and specific modes passed".to_string(),
2061 })
2062}
2063
2064pub async fn streaming_structured_after_tool<W, T, F>(
2066 model: rig_core::driver::Model<W, T>,
2067 configure: F,
2068) -> Result<ScenarioReport, ScenarioError>
2069where
2070 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
2071 T: rig_core::driver::Transport<W>,
2072 F: FnOnce(AgentBuilder) -> AgentBuilder,
2073{
2074 let calls = Arc::new(AtomicUsize::new(0));
2075 let started = Instant::now();
2076 let agent = configure(AgentBuilder::new(model))
2077 .preamble(
2078 "Use add for arithmetic, then finish by calling the structured output tool exactly once.",
2079 )
2080 .output_schema::<ArithmeticResult>()
2081 .output_mode(OutputMode::Tool)
2082 .tool(AddTool(calls.clone()))
2083 .default_max_turns(5)
2084 .build();
2085 let mut stream = agent
2086 .prompt("Use add to calculate 19 + 23. Return answer=42 and a short optional explanation.")
2087 .max_turns(5)
2088 .stream();
2089 let mut final_response = None;
2090 let mut final_count = 0_usize;
2091 while let Some(item) = stream.next().await {
2092 if let MultiTurnStreamItem::FinalResponse(response) = item? {
2093 final_count += 1;
2094 final_response = Some(response);
2095 }
2096 }
2097 let result = final_response.ok_or_else(|| {
2098 ScenarioError::contract(
2099 "streaming_structured_after_tool",
2100 "stream produced no final response",
2101 )
2102 })?;
2103 let parsed: ArithmeticResult = serde_json::from_str(&result.output())?;
2104 let calls = calls.load(Ordering::SeqCst);
2105 if calls == 0
2106 || final_count != 1
2107 || parsed.answer != 42
2108 || !has_tool_roundtrip(&result.messages)
2109 {
2110 return Err(ScenarioError::contract(
2111 "streaming_structured_after_tool",
2112 format!(
2113 "calls={calls}, final_count={final_count}, response={:?}",
2114 result.output()
2115 ),
2116 ));
2117 }
2118 report_from_response(
2119 "streaming_structured_after_tool",
2120 started,
2121 calls + 1,
2122 result,
2123 )
2124}
2125
2126#[cfg(test)]
2127mod tests;