1use std::{
8 collections::BTreeSet,
9 sync::{
10 Arc, Mutex,
11 atomic::{AtomicUsize, Ordering},
12 },
13 time::{Duration, Instant},
14};
15
16use futures::StreamExt;
17use schemars::JsonSchema;
18use serde::{Deserialize, Serialize};
19
20use crate::{
21 agent::{
22 AgentBuilder, AgentHook, CompletionCallAction, CompletionCallEvent, DispatchAction,
23 DispatchEvent, HookContext, InvalidToolCallAction, MultiTurnStreamItem, OutcomeAction,
24 OutcomeEvent, OutputMode, RequestPatch,
25 },
26 completion::{AssistantContent, Message, PromptError, ToolDefinition},
27 run::{AgentRun, AgentRunStep, ModelTurn, ModelTurnOutcome, PendingToolCall, TurnPolicy},
28 tool::{Tool, ToolContext},
29};
30use rig_core::completion::CompletionRequest;
31use rig_core::error::ProviderError;
32use rig_core::message::{ToolChoice, UserContent};
33
34#[derive(Debug, thiserror::Error)]
36pub enum ScenarioError {
37 #[error(transparent)]
39 Prompt(#[from] PromptError),
40 #[error(transparent)]
42 Completion(#[from] ProviderError),
43 #[error(transparent)]
45 Report(#[from] rig_core::error::ErrorReport),
46 #[error(transparent)]
48 Json(#[from] serde_json::Error),
49 #[error(transparent)]
51 Extraction(#[from] crate::completion::StructuredOutputError),
52 #[error("{scenario} conformance failed: {details}")]
54 Contract {
55 scenario: &'static str,
57 details: String,
59 },
60}
61
62impl ScenarioError {
63 fn contract(scenario: &'static str, details: impl Into<String>) -> Self {
64 Self::Contract {
65 scenario,
66 details: details.into(),
67 }
68 }
69}
70
71pub fn validate_unknown_tool_failure(
74 error: &PromptError,
75 expected_tool: &str,
76 expected_allowed_tools: &[&str],
77) -> Result<(), ScenarioError> {
78 const SCENARIO: &str = "unknown_tool_failure";
79 let PromptError::UnknownToolCall {
80 tool_name,
81 allowed_tools,
82 chat_history,
83 ..
84 } = error
85 else {
86 return Err(ScenarioError::contract(
87 SCENARIO,
88 format!("expected UnknownToolCall, observed {error:?}"),
89 ));
90 };
91 let expected_allowed = expected_allowed_tools
92 .iter()
93 .map(|name| (*name).to_string())
94 .collect::<Vec<_>>();
95 let history_has_call = chat_history.iter().any(|message| {
99 matches!(
100 message,
101 Message::Assistant(rig_core::message::AssistantMessage { content, .. })
102 if content.iter().any(|item| matches!(item, AssistantContent::ToolCall(_)))
103 )
104 });
105 if tool_name != expected_tool || allowed_tools != &expected_allowed || !history_has_call {
106 return Err(ScenarioError::contract(
107 SCENARIO,
108 format!(
109 "tool={tool_name:?}, allowed={allowed_tools:?}, expected_tool={expected_tool:?}, expected_allowed={expected_allowed:?}, history_has_call={history_has_call}, history={chat_history:?}"
110 ),
111 ));
112 }
113 Ok(())
114}
115
116pub fn validate_cancelled_failure(
119 error: &PromptError,
120 expected_reason: &str,
121 expected_tool: &str,
122) -> Result<(), ScenarioError> {
123 const SCENARIO: &str = "cancelled_failure";
124 let PromptError::Cancelled {
125 chat_history,
126 reason,
127 } = error
128 else {
129 return Err(ScenarioError::contract(
130 SCENARIO,
131 format!("expected Cancelled, observed {error:?}"),
132 ));
133 };
134 let history_has_call = chat_history.iter().any(|message| {
135 matches!(
136 message,
137 Message::Assistant(rig_core::message::AssistantMessage { content, .. })
138 if content.iter().any(|item| matches!(
139 item,
140 AssistantContent::ToolCall(call) if call.function.name == expected_tool
141 ))
142 )
143 });
144 if reason != expected_reason || !history_has_call {
145 return Err(ScenarioError::contract(
146 SCENARIO,
147 format!(
148 "reason={reason:?}, expected={expected_reason:?}, history_has_call={history_has_call}, history={chat_history:?}"
149 ),
150 ));
151 }
152 Ok(())
153}
154
155pub fn validate_max_turns_failure(
158 error: &PromptError,
159 expected_max_turns: usize,
160) -> Result<(), ScenarioError> {
161 const SCENARIO: &str = "max_turns_failure";
162 let PromptError::MaxTurns {
163 max_turns,
164 chat_history,
165 prompt,
166 } = error
167 else {
168 return Err(ScenarioError::contract(
169 SCENARIO,
170 format!("expected MaxTurns, observed {error:?}"),
171 ));
172 };
173 let pending_prompt_retained = matches!(
174 prompt,
175 Message::User { content } if content.iter().next().is_some()
176 );
177 if *max_turns != expected_max_turns || chat_history.is_empty() || !pending_prompt_retained {
178 return Err(ScenarioError::contract(
179 SCENARIO,
180 format!(
181 "max_turns={max_turns}, expected={expected_max_turns}, history={chat_history:?}, pending_prompt_retained={pending_prompt_retained}, pending_prompt={prompt:?}"
182 ),
183 ));
184 }
185 Ok(())
186}
187
188pub fn decode_structured_output<T>(
191 scenario: &'static str,
192 response: &str,
193) -> Result<T, ScenarioError>
194where
195 T: serde::de::DeserializeOwned,
196{
197 serde_json::from_str(response).map_err(|error| {
198 ScenarioError::contract(
199 scenario,
200 format!("structured output did not decode: {error}; response={response:?}"),
201 )
202 })
203}
204
205pub fn validate_protocol_hygiene(
208 scenario: &'static str,
209 visible_output: &str,
210 messages: &[Message],
211 forbidden_markers: &[&str],
212) -> Result<(), ScenarioError> {
213 let serialized = serde_json::to_string(messages)?;
214 let leaked = forbidden_markers
215 .iter()
216 .filter(|marker| visible_output.contains(**marker) || serialized.contains(**marker))
217 .copied()
218 .collect::<Vec<_>>();
219 if !leaked.is_empty() {
220 return Err(ScenarioError::contract(
221 scenario,
222 format!(
223 "protocol markers leaked: {leaked:?}; output={visible_output:?}, history={messages:?}"
224 ),
225 ));
226 }
227 Ok(())
228}
229
230pub fn validate_rewritten_arguments(
233 scenario: &'static str,
234 observations: &[serde_json::Value],
235 expected_fields: &serde_json::Value,
236) -> Result<(), ScenarioError> {
237 let Some(expected) = expected_fields.as_object() else {
238 return Err(ScenarioError::contract(
239 scenario,
240 "expected rewritten fields must be a JSON object",
241 ));
242 };
243 if observations.is_empty() {
244 return Err(ScenarioError::contract(
245 scenario,
246 "the rewritten tool was never invoked",
247 ));
248 }
249 for observation in observations {
250 let Some(actual) = observation.as_object() else {
251 return Err(ScenarioError::contract(
252 scenario,
253 format!("observed rewritten arguments were not an object: {observation:?}"),
254 ));
255 };
256 for (key, expected_value) in expected {
257 if actual.get(key) != Some(expected_value) {
258 return Err(ScenarioError::contract(
259 scenario,
260 format!(
261 "rewritten field {key:?} expected {expected_value:?}, observed {observation:?}"
262 ),
263 ));
264 }
265 }
266 }
267 Ok(())
268}
269
270pub fn validate_extraction_fields(
272 scenario: &'static str,
273 first_name: Option<&str>,
274 last_name: Option<&str>,
275 job: Option<&str>,
276 usage: crate::completion::Usage,
277) -> Result<(), ScenarioError> {
278 let fields_match = first_name.is_some_and(|value| value.eq_ignore_ascii_case("Ada"))
279 && last_name.is_some_and(|value| value.eq_ignore_ascii_case("Lovelace"))
280 && job.is_some_and(|value| value.to_ascii_lowercase().contains("mathematician"));
281 if !fields_match || !usage.is_reported() {
282 return Err(ScenarioError::contract(
283 scenario,
284 format!(
285 "first_name={first_name:?}, last_name={last_name:?}, job={job:?}, usage={usage:?}"
286 ),
287 ));
288 }
289 Ok(())
290}
291
292#[derive(Debug, Clone)]
294pub struct ScenarioReport {
295 pub name: &'static str,
297 pub tool_calls: usize,
299 pub prompt_tokens: u64,
302 pub generated_tokens: u64,
305 pub history_messages: usize,
307 pub duration: Duration,
309 pub response: String,
311}
312
313#[derive(Debug, thiserror::Error)]
315#[error("model-conformance tool failed")]
316pub struct ConformanceToolError;
317
318const FORCE_TOOLS_PREAMBLE: &str = "You are a calculator assistant. You MUST use the provided tools for every arithmetic operation instead of computing results yourself. Once you have all the tool results you need, reply with the final numeric answer in plain text.";
319const PARALLEL_PROMPT: &str = "Compute 3 + 4 and 10 - 2. You MUST call the add tool and the subtract tool together in your first response, as two parallel function calls, then report both results.";
320const PING_OUTPUT: &str = "pong-crimson-7423";
321const MOTTO_OUTPUT: &str = "steady hands\ncalm waters";
322
323fn tool_result_values(message: &Message) -> Vec<serde_json::Value> {
324 message
325 .results()
326 .flat_map(|result| result.content.iter())
327 .filter_map(|content| match content {
328 rig_core::message::ToolResultContent::Text(text) => {
329 Some(serde_json::Value::String(text.text.clone()))
330 }
331 rig_core::message::ToolResultContent::Json { value } => Some(value.clone()),
332 rig_core::message::ToolResultContent::Image(_) => None,
333 })
334 .collect()
335}
336
337fn validate_tool_correlation(
338 scenario: &'static str,
339 messages: &[Message],
340) -> Result<(), ScenarioError> {
341 let mut calls = Vec::new();
342 let mut results = Vec::new();
343 let mut turn = 0usize;
344 for message in messages {
345 match message {
346 Message::Assistant(rig_core::message::AssistantMessage { content, .. }) => {
347 turn += 1;
348 calls.extend(content.iter().filter_map(|item| match item {
349 AssistantContent::ToolCall(call) => Some((
350 turn,
351 &call.id,
352 call.id.provider().map(|provider| provider.as_str()),
353 )),
354 _ => None,
355 }));
356 }
357 Message::User { .. } => {
358 results.extend(message.results().map(|result| {
359 let provider = result.call.provider().map(|provider| provider.as_str());
360 (turn, &result.call, provider)
361 }));
362 }
363 Message::System { .. } => {}
364 }
365 }
366 if calls.is_empty() {
367 return Err(ScenarioError::contract(
368 scenario,
369 format!("history has no assistant tool calls: {messages:?}"),
370 ));
371 }
372 for (turn, id, call_id) in &calls {
373 let matches = results
374 .iter()
375 .filter(|(result_turn, result_id, result_call_id)| {
376 result_turn == turn && result_id == id && call_id == result_call_id
377 })
378 .count();
379 if matches != 1 {
380 return Err(ScenarioError::contract(
381 scenario,
382 format!(
383 "tool call id={id:?} call_id={call_id:?} has {matches} correlated results; calls={calls:?}, results={results:?}"
384 ),
385 ));
386 }
387 }
388 if results.len() != calls.len() {
389 return Err(ScenarioError::contract(
390 scenario,
391 format!(
392 "history contains dangling calls or results: calls={calls:?}, results={results:?}"
393 ),
394 ));
395 }
396 Ok(())
397}
398
399#[derive(Debug, Deserialize, JsonSchema)]
400struct OperationArgs {
401 x: i64,
402 y: i64,
403}
404
405#[derive(Clone)]
406struct CountingAdd(Arc<AtomicUsize>);
407
408impl Tool for CountingAdd {
409 const NAME: &'static str = "add";
410 type Error = ConformanceToolError;
411 type Args = OperationArgs;
412 type Output = i64;
413
414 fn description(&self) -> String {
415 "Add x and y together".to_string()
416 }
417
418 fn parameters(&self) -> serde_json::Value {
419 serde_json::json!({
420 "type": "object",
421 "properties": {
422 "x": { "type": "number", "description": "The first operand" },
423 "y": { "type": "number", "description": "The second operand" }
424 },
425 "required": ["x", "y"]
426 })
427 }
428
429 async fn call(
430 &self,
431 _context: &mut ToolContext,
432 args: Self::Args,
433 ) -> Result<Self::Output, Self::Error> {
434 self.0.fetch_add(1, Ordering::SeqCst);
435 Ok(args.x + args.y)
436 }
437}
438
439#[derive(Clone)]
440struct CountingSum(Arc<AtomicUsize>);
441
442impl Tool for CountingSum {
443 const NAME: &'static str = "sum";
444 type Error = ConformanceToolError;
445 type Args = OperationArgs;
446 type Output = i64;
447
448 fn description(&self) -> String {
449 "Add x and y together (alias of add)".to_string()
450 }
451
452 fn parameters(&self) -> serde_json::Value {
453 CountingAdd(Arc::new(AtomicUsize::new(0))).parameters()
454 }
455
456 async fn call(
457 &self,
458 _context: &mut ToolContext,
459 args: Self::Args,
460 ) -> Result<Self::Output, Self::Error> {
461 self.0.fetch_add(1, Ordering::SeqCst);
462 Ok(args.x + args.y)
463 }
464}
465
466#[derive(Clone)]
467struct CountingSubtract(Arc<AtomicUsize>);
468
469impl Tool for CountingSubtract {
470 const NAME: &'static str = "subtract";
471 type Error = ConformanceToolError;
472 type Args = OperationArgs;
473 type Output = i64;
474
475 fn description(&self) -> String {
476 "Subtract y from x (i.e. x - y)".to_string()
477 }
478
479 fn parameters(&self) -> serde_json::Value {
480 serde_json::json!({
481 "type": "object",
482 "properties": {
483 "x": { "type": "number", "description": "The first operand" },
484 "y": { "type": "number", "description": "The second operand" }
485 },
486 "required": ["x", "y"]
487 })
488 }
489
490 async fn call(
491 &self,
492 _context: &mut ToolContext,
493 args: Self::Args,
494 ) -> Result<Self::Output, Self::Error> {
495 self.0.fetch_add(1, Ordering::SeqCst);
496 Ok(args.x - args.y)
497 }
498}
499
500#[derive(Clone)]
501struct RewriteArgument {
502 key: &'static str,
503 value: serde_json::Value,
504}
505
506impl AgentHook for RewriteArgument {
507 async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
508 let (Some(name), Some(args)) = (event.tool_name(), event.tool_args()) else {
509 return DispatchAction::proceed();
510 };
511 if name != CountingAdd::NAME {
512 return DispatchAction::proceed();
513 }
514 let Ok(mut arguments) = serde_json::from_str::<serde_json::Value>(args) else {
515 return DispatchAction::proceed();
516 };
517 let Some(object) = arguments.as_object_mut() else {
518 return DispatchAction::proceed();
519 };
520 object.insert(self.key.to_string(), self.value.clone());
521 DispatchAction::rewrite_tool_args(event.kind, arguments)
522 }
523}
524
525#[derive(Clone, Default)]
526struct ObserveArguments(Arc<Mutex<Vec<serde_json::Value>>>);
527
528impl AgentHook for ObserveArguments {
529 async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
530 if let Some(args) = event.tool_args() {
531 let value = serde_json::from_str(args)
532 .unwrap_or_else(|_| serde_json::Value::String(args.to_string()));
533 crate::lock(&self.0).push(value);
534 }
535 DispatchAction::proceed()
536 }
537}
538
539#[derive(Clone)]
540struct ReplaceResult(&'static str);
541
542impl AgentHook for ReplaceResult {
543 async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
544 if event.tool_name() == Some(CountingAdd::NAME) {
545 OutcomeAction::rewrite_tool_result(&event, self.0)
546 } else {
547 OutcomeAction::proceed()
548 }
549 }
550}
551
552#[derive(Clone)]
553struct WrapResult;
554
555impl AgentHook for WrapResult {
556 async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
557 match (event.tool_name(), event.tool_result()) {
558 (Some(name), Some(result)) if name == CountingAdd::NAME => {
559 OutcomeAction::rewrite_tool_result(
560 &event,
561 format!("[{}]", result.output().render()),
562 )
563 }
564 _ => OutcomeAction::proceed(),
565 }
566 }
567}
568
569#[derive(Clone)]
570struct FirstTurnPatch(RequestPatch);
571
572impl AgentHook for FirstTurnPatch {
573 async fn on_completion_call(
574 &self,
575 ctx: &HookContext,
576 _event: CompletionCallEvent<'_>,
577 ) -> CompletionCallAction {
578 if ctx.turn() == 1 {
579 CompletionCallAction::patch(self.0.clone())
580 } else {
581 CompletionCallAction::continue_run()
582 }
583 }
584}
585
586#[derive(Clone)]
587struct StopAfterResult(&'static str);
588
589impl AgentHook for StopAfterResult {
590 async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
591 if event.tool_name() == Some(CountingAdd::NAME) {
592 OutcomeAction::stop(self.0)
593 } else {
594 OutcomeAction::proceed()
595 }
596 }
597}
598
599#[derive(Debug, Deserialize, JsonSchema)]
600struct EmptyArgs {}
601
602#[derive(Clone)]
603struct PingTool(Arc<AtomicUsize>);
604
605impl Tool for PingTool {
606 const NAME: &'static str = "ping";
607 type Error = ConformanceToolError;
608 type Args = EmptyArgs;
609 type Output = String;
610
611 fn description(&self) -> String {
612 "Return the current ping marker. Takes no arguments.".to_string()
613 }
614
615 fn parameters(&self) -> serde_json::Value {
616 serde_json::json!({ "type": "object", "properties": {}, "required": [] })
617 }
618
619 async fn call(
620 &self,
621 _context: &mut ToolContext,
622 _args: Self::Args,
623 ) -> Result<Self::Output, Self::Error> {
624 self.0.fetch_add(1, Ordering::SeqCst);
625 Ok(PING_OUTPUT.to_string())
626 }
627}
628
629#[derive(Clone)]
630struct MottoTool(Arc<AtomicUsize>);
631
632impl Tool for MottoTool {
633 const NAME: &'static str = "fetch_motto";
634 type Error = ConformanceToolError;
635 type Args = EmptyArgs;
636 type Output = String;
637
638 fn description(&self) -> String {
639 "Fetch the two-line workshop motto.".to_string()
640 }
641
642 fn parameters(&self) -> serde_json::Value {
643 serde_json::json!({ "type": "object", "properties": {}, "required": [] })
644 }
645
646 async fn call(
647 &self,
648 _context: &mut ToolContext,
649 _args: Self::Args,
650 ) -> Result<Self::Output, Self::Error> {
651 self.0.fetch_add(1, Ordering::SeqCst);
652 Ok(MOTTO_OUTPUT.to_string())
653 }
654}
655
656#[derive(Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
657struct ConfigOutput {
658 service: String,
659 max_retries: u64,
660}
661
662#[derive(Clone)]
663struct ConfigTool(Arc<AtomicUsize>);
664
665impl Tool for ConfigTool {
666 const NAME: &'static str = "fetch_config";
667 type Error = ConformanceToolError;
668 type Args = EmptyArgs;
669 type Output = ConfigOutput;
670
671 fn description(&self) -> String {
672 "Fetch the service configuration object.".to_string()
673 }
674
675 fn parameters(&self) -> serde_json::Value {
676 serde_json::json!({ "type": "object", "properties": {}, "required": [] })
677 }
678
679 async fn call(
680 &self,
681 _context: &mut ToolContext,
682 _args: Self::Args,
683 ) -> Result<Self::Output, Self::Error> {
684 self.0.fetch_add(1, Ordering::SeqCst);
685 Ok(ConfigOutput {
686 service: "cassette-lab".to_string(),
687 max_retries: 3,
688 })
689 }
690}
691
692#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
693#[serde(rename_all = "snake_case")]
694enum ComplexMode {
695 Careful,
696 Fast,
697}
698
699#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
700struct ComplexProfile {
701 name: String,
702 tags: Vec<String>,
703}
704
705#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
706struct ComplexArgs {
707 profile: ComplexProfile,
708 mode: ComplexMode,
709 note: Option<String>,
710 quote: String,
711}
712
713#[derive(Debug, Deserialize, Serialize, JsonSchema)]
714struct ExtractedPerson {
715 #[schemars(required)]
716 first_name: Option<String>,
717 #[schemars(required)]
718 last_name: Option<String>,
719 #[schemars(required)]
720 job: Option<String>,
721}
722
723#[derive(Clone)]
724struct CaptureComplexTool {
725 calls: Arc<AtomicUsize>,
726 captured: Arc<Mutex<Option<ComplexArgs>>>,
727}
728
729impl Tool for CaptureComplexTool {
730 const NAME: &'static str = "store_profile";
731 type Error = ConformanceToolError;
732 type Args = ComplexArgs;
733 type Output = ComplexArgs;
734
735 fn description(&self) -> String {
736 "Store one profile with its nested tags, mode, optional note, and exact quoted text."
737 .to_string()
738 }
739
740 fn parameters(&self) -> serde_json::Value {
741 serde_json::to_value(schemars::schema_for!(ComplexArgs)).unwrap_or_default()
742 }
743
744 async fn call(
745 &self,
746 _context: &mut ToolContext,
747 args: Self::Args,
748 ) -> Result<Self::Output, Self::Error> {
749 self.calls.fetch_add(1, Ordering::SeqCst);
750 *crate::lock(&self.captured) = Some(args.clone());
751 Ok(args)
752 }
753}
754
755fn has_tool_roundtrip(messages: &[Message]) -> bool {
756 let saw_call = messages.iter().any(|message| {
757 matches!(
758 message,
759 Message::Assistant(rig_core::message::AssistantMessage { content, .. })
760 if content.iter().any(|item| matches!(item, AssistantContent::ToolCall(_)))
761 )
762 });
763 let saw_result = messages.iter().any(|message| {
764 matches!(
765 message,
766 Message::User { content }
767 if content.iter().any(|item| matches!(item, UserContent::ToolResult(_)))
768 )
769 });
770 saw_call && saw_result
771}
772
773#[derive(Debug, Deserialize, JsonSchema)]
774struct RepeatArgs {
775 text: String,
777 times: Option<u32>,
779}
780
781#[derive(Clone)]
782struct RepeatTool {
783 calls: Arc<AtomicUsize>,
784}
785
786impl Tool for RepeatTool {
787 const NAME: &'static str = "repeat_text";
788 type Error = ConformanceToolError;
789 type Args = RepeatArgs;
790 type Output = String;
791
792 fn description(&self) -> String {
793 "Repeat `text`. `times` is optional and defaults to 2.".to_string()
794 }
795
796 fn parameters(&self) -> serde_json::Value {
797 serde_json::to_value(schemars::schema_for!(RepeatArgs)).unwrap_or_default()
798 }
799
800 async fn call(
801 &self,
802 _context: &mut ToolContext,
803 args: Self::Args,
804 ) -> Result<Self::Output, Self::Error> {
805 self.calls.fetch_add(1, Ordering::SeqCst);
806 Ok(vec![args.text.as_str(); args.times.unwrap_or(2) as usize].join(" "))
807 }
808}
809
810#[derive(Debug, Deserialize, JsonSchema)]
811struct BinOpArgs {
812 a: i64,
813 b: i64,
814}
815
816#[derive(Debug, Deserialize, JsonSchema)]
817struct ArithmeticResult {
818 answer: i64,
819 explanation: Option<String>,
820}
821
822#[derive(Clone)]
823struct AddTool(Arc<AtomicUsize>);
824
825impl Tool for AddTool {
826 const NAME: &'static str = "add";
827 type Error = ConformanceToolError;
828 type Args = BinOpArgs;
829 type Output = i64;
830
831 fn description(&self) -> String {
832 "Add two integers a and b.".to_string()
833 }
834
835 fn parameters(&self) -> serde_json::Value {
836 serde_json::to_value(schemars::schema_for!(BinOpArgs)).unwrap_or_default()
837 }
838
839 async fn call(
840 &self,
841 _context: &mut ToolContext,
842 args: Self::Args,
843 ) -> Result<Self::Output, Self::Error> {
844 self.0.fetch_add(1, Ordering::SeqCst);
845 Ok(args.a + args.b)
846 }
847}
848
849#[derive(Clone)]
850struct MultiplyTool(Arc<AtomicUsize>);
851
852impl Tool for MultiplyTool {
853 const NAME: &'static str = "multiply";
854 type Error = ConformanceToolError;
855 type Args = BinOpArgs;
856 type Output = i64;
857
858 fn description(&self) -> String {
859 "Multiply two integers a and b.".to_string()
860 }
861
862 fn parameters(&self) -> serde_json::Value {
863 serde_json::to_value(schemars::schema_for!(BinOpArgs)).unwrap_or_default()
864 }
865
866 async fn call(
867 &self,
868 _context: &mut ToolContext,
869 args: Self::Args,
870 ) -> Result<Self::Output, Self::Error> {
871 self.0.fetch_add(1, Ordering::SeqCst);
872 Ok(args.a * args.b)
873 }
874}
875
876fn report_from_response(
877 name: &'static str,
878 started: Instant,
879 tool_calls: usize,
880 response: crate::agent::PromptResponse,
881) -> Result<ScenarioReport, ScenarioError> {
882 validate_protocol_hygiene(
883 name,
884 &response.output(),
885 &response.messages,
886 &[
887 "<tool_call>",
888 "</tool_call>",
889 "<tool_response>",
890 "</tool_response>",
891 "<|im_start|>",
892 "<|im_end|>",
893 "<think>",
894 "</think>",
895 ],
896 )?;
897 Ok(ScenarioReport {
898 name,
899 tool_calls,
900 prompt_tokens: response.usage.input_tokens.unwrap_or(0),
901 generated_tokens: response.usage.output_tokens.unwrap_or(0),
902 history_messages: response.messages.len(),
903 duration: started.elapsed(),
904 response: response.output(),
905 })
906}
907
908fn correlated_messages<'a>(
911 scenario: &'static str,
912 response: &'a crate::agent::PromptResponse,
913) -> Result<&'a [Message], ScenarioError> {
914 let messages = response.messages.as_slice();
915 if messages.is_empty() {
916 return Err(ScenarioError::contract(
917 scenario,
918 "extended run omitted accumulated message history",
919 ));
920 }
921 validate_tool_correlation(scenario, messages)?;
922 Ok(messages)
923}
924
925fn correlated_result_values(
927 scenario: &'static str,
928 response: &crate::agent::PromptResponse,
929) -> Result<Vec<serde_json::Value>, ScenarioError> {
930 Ok(correlated_messages(scenario, response)?
931 .iter()
932 .flat_map(tool_result_values)
933 .collect())
934}
935
936fn value_matches_integer(value: &serde_json::Value, expected: i64) -> bool {
937 value.as_i64() == Some(expected)
938 || value
939 .as_str()
940 .and_then(|text| text.trim().parse::<i64>().ok())
941 == Some(expected)
942}
943
944pub async fn parallel_tools<W, T, F>(
950 model: rig_core::driver::Model<W, T>,
951 configure: F,
952 tool_concurrency: Option<usize>,
953) -> Result<ScenarioReport, ScenarioError>
954where
955 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
956 T: rig_core::driver::Transport<W>,
957 F: FnOnce(AgentBuilder) -> AgentBuilder,
958{
959 let add_calls = Arc::new(AtomicUsize::new(0));
960 let subtract_calls = Arc::new(AtomicUsize::new(0));
961 let started = Instant::now();
962 let agent = configure(AgentBuilder::new(model).temperature(0.0))
963 .preamble(FORCE_TOOLS_PREAMBLE)
964 .tool(CountingAdd(add_calls.clone()))
965 .tool(CountingSubtract(subtract_calls.clone()))
966 .default_max_turns(3)
967 .build();
968 let request = agent.prompt(PARALLEL_PROMPT).max_turns(3);
969 let response = match tool_concurrency {
970 Some(concurrency) => request.tool_concurrency(concurrency).await?,
971 None => request.await?,
972 };
973 let scenario = if tool_concurrency == Some(1) {
974 "parallel_tools_serial_execution"
975 } else {
976 "parallel_tools"
977 };
978 let messages = correlated_messages(scenario, &response)?;
979
980 let Some((call_index, calls)) = messages.iter().enumerate().find_map(|(index, message)| {
981 let Message::Assistant(rig_core::message::AssistantMessage { content, .. }) = message
982 else {
983 return None;
984 };
985 let calls = content
986 .iter()
987 .filter_map(|item| match item {
988 AssistantContent::ToolCall(call) => Some(call),
989 _ => None,
990 })
991 .collect::<Vec<_>>();
992 (calls.len() == 2).then_some((index, calls))
993 }) else {
994 return Err(ScenarioError::contract(
995 scenario,
996 format!("no assistant turn contained exactly two tool calls: {messages:?}"),
997 ));
998 };
999 let mut names = calls
1000 .iter()
1001 .map(|call| call.function.name.as_str())
1002 .collect::<Vec<_>>();
1003 names.sort_unstable();
1004 if names != ["add", "subtract"] {
1005 return Err(ScenarioError::contract(
1006 scenario,
1007 format!("parallel turn called {names:?}, expected add and subtract"),
1008 ));
1009 }
1010 let results_message = messages.get(call_index + 1).ok_or_else(|| {
1011 ScenarioError::contract(
1012 scenario,
1013 "parallel call turn has no following result message",
1014 )
1015 })?;
1016 let values = tool_result_values(results_message);
1017 if !values.iter().any(|value| value_matches_integer(value, 7))
1018 || !values.iter().any(|value| value_matches_integer(value, 8))
1019 || values.len() != 2
1020 {
1021 return Err(ScenarioError::contract(
1022 scenario,
1023 format!("parallel result message did not contain exactly 7 and 8: {values:?}"),
1024 ));
1025 }
1026 let add = add_calls.load(Ordering::SeqCst);
1027 let subtract = subtract_calls.load(Ordering::SeqCst);
1028 if add != 1 || subtract != 1 {
1029 return Err(ScenarioError::contract(
1030 scenario,
1031 format!("execution counts were add={add}, subtract={subtract}, expected one each"),
1032 ));
1033 }
1034 report_from_response(scenario, started, add + subtract, response)
1035}
1036
1037pub async fn zero_argument_tool<W, T, F>(
1039 model: rig_core::driver::Model<W, T>,
1040 configure: F,
1041) -> Result<ScenarioReport, ScenarioError>
1042where
1043 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1044 T: rig_core::driver::Transport<W>,
1045 F: FnOnce(AgentBuilder) -> AgentBuilder,
1046{
1047 const SCENARIO: &str = "zero_argument_tool";
1048 let calls = Arc::new(AtomicUsize::new(0));
1049 let started = Instant::now();
1050 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1051 .preamble("You must use the provided tools. Report tool outputs exactly as returned.")
1052 .tool(PingTool(calls.clone()))
1053 .default_max_turns(2)
1054 .build();
1055 let response = agent
1056 .prompt("Call the ping tool, then report the exact marker it returns.")
1057 .max_turns(2)
1058 .await?;
1059 let values = correlated_result_values(SCENARIO, &response)?;
1060 if calls.load(Ordering::SeqCst) != 1
1061 || !values
1062 .iter()
1063 .any(|value| value.as_str() == Some(PING_OUTPUT))
1064 || !response.output().contains(PING_OUTPUT)
1065 {
1066 return Err(ScenarioError::contract(
1067 SCENARIO,
1068 format!(
1069 "calls={}, results={values:?}, response={:?}",
1070 calls.load(Ordering::SeqCst),
1071 response.output()
1072 ),
1073 ));
1074 }
1075 report_from_response(SCENARIO, started, 1, response)
1076}
1077
1078pub async fn tool_output_serialization<W, T, F>(
1081 model: rig_core::driver::Model<W, T>,
1082 configure: F,
1083) -> Result<ScenarioReport, ScenarioError>
1084where
1085 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1086 T: rig_core::driver::Transport<W>,
1087 F: FnOnce(AgentBuilder) -> AgentBuilder,
1088{
1089 const SCENARIO: &str = "tool_output_serialization";
1090 let started = Instant::now();
1091 let motto_calls = Arc::new(AtomicUsize::new(0));
1092 let config_calls = Arc::new(AtomicUsize::new(0));
1093 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1094 .preamble("You must use the provided tools before answering.")
1095 .tool(MottoTool(motto_calls.clone()))
1096 .tool(ConfigTool(config_calls.clone()))
1097 .default_max_turns(3)
1098 .build();
1099 let response = agent
1100 .prompt("Call fetch_motto and fetch_config, then summarize both outputs in one sentence.")
1101 .max_turns(3)
1102 .await?;
1103 let values = correlated_result_values(SCENARIO, &response)?;
1104 let expected_config = serde_json::to_value(ConfigOutput {
1105 service: "cassette-lab".to_string(),
1106 max_retries: 3,
1107 })?;
1108 let motto_ok = values
1109 .iter()
1110 .any(|value| value.as_str() == Some(MOTTO_OUTPUT));
1111 let config_ok = values.iter().any(|value| {
1112 value == &expected_config
1113 || value
1114 .as_str()
1115 .and_then(|text| serde_json::from_str::<serde_json::Value>(text).ok())
1116 .as_ref()
1117 == Some(&expected_config)
1118 });
1119 let motto_count = motto_calls.load(Ordering::SeqCst);
1120 let config_count = config_calls.load(Ordering::SeqCst);
1121 if !motto_ok || !config_ok || motto_count != 1 || config_count != 1 {
1122 return Err(ScenarioError::contract(
1123 SCENARIO,
1124 format!(
1125 "expected one verbatim motto and one semantic config JSON; motto_calls={motto_count}, config_calls={config_count}, values={values:?}"
1126 ),
1127 ));
1128 }
1129 report_from_response(SCENARIO, started, motto_count + config_count, response)
1130}
1131
1132pub async fn complex_tool_arguments<W, T, F>(
1135 model: rig_core::driver::Model<W, T>,
1136 configure: F,
1137) -> Result<ScenarioReport, ScenarioError>
1138where
1139 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1140 T: rig_core::driver::Transport<W>,
1141 F: FnOnce(AgentBuilder) -> AgentBuilder,
1142{
1143 complex_tool_arguments_with_prompt(model, configure, COMPLEX_ARGUMENTS_PROMPT).await
1144}
1145
1146const COMPLEX_ARGUMENTS_PROMPT: &str = "Call store_profile with profile.name exactly `Zoë \\\"Z\\\"`, profile.tags exactly [`rust`, `東京`], mode `careful`, note containing the two lines `line one` and `line two` separated by a newline, and quote exactly `path C:\\\\tmp and \\\"quoted\\\"`. Then confirm it was stored.";
1149
1150pub const COMPLEX_ARGUMENTS_RAW_PROMPT: &str = "Call store_profile once. Each value below is raw text between <<< and >>>, with no escaping: profile.name <<<Zoë \"Z\">>>, profile.tags <<<rust>>> and <<<東京>>>, mode <<<careful>>>, note the two lines <<<line one>>> and <<<line two>>> joined by one newline, and quote <<<path C:\\tmp and \"quoted\">>>. Then confirm it was stored.";
1153
1154pub async fn complex_tool_arguments_with_prompt<W, T, F>(
1157 model: rig_core::driver::Model<W, T>,
1158 configure: F,
1159 prompt: &str,
1160) -> Result<ScenarioReport, ScenarioError>
1161where
1162 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1163 T: rig_core::driver::Transport<W>,
1164 F: FnOnce(AgentBuilder) -> AgentBuilder,
1165{
1166 const SCENARIO: &str = "complex_tool_arguments";
1167 let expected = ComplexArgs {
1168 profile: ComplexProfile {
1169 name: "Zoë \"Z\"".to_string(),
1170 tags: vec!["rust".to_string(), "東京".to_string()],
1171 },
1172 mode: ComplexMode::Careful,
1173 note: Some("line one\nline two".to_string()),
1174 quote: "path C:\\tmp and \"quoted\"".to_string(),
1175 };
1176 let calls = Arc::new(AtomicUsize::new(0));
1177 let captured = Arc::new(Mutex::new(None));
1178 let started = Instant::now();
1179 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1180 .preamble("Use store_profile exactly once with every value supplied by the user.")
1181 .tool(CaptureComplexTool {
1182 calls: calls.clone(),
1183 captured: captured.clone(),
1184 })
1185 .default_max_turns(3)
1186 .build();
1187 let response = agent.prompt(prompt).max_turns(3).await?;
1188 let observed = crate::lock(&captured).clone();
1189 if calls.load(Ordering::SeqCst) != 1 || observed.as_ref() != Some(&expected) {
1190 return Err(ScenarioError::contract(
1191 SCENARIO,
1192 format!(
1193 "calls={}, expected={expected:?}, observed={observed:?}, response={:?}",
1194 calls.load(Ordering::SeqCst),
1195 response.output()
1196 ),
1197 ));
1198 }
1199 correlated_messages(SCENARIO, &response)?;
1200 report_from_response(SCENARIO, started, 1, response)
1201}
1202
1203pub async fn buffered_streaming_text_parity<W, T, A>(
1206 model: rig_core::driver::Model<W, T>,
1207 adjust: A,
1208) -> Result<ScenarioReport, ScenarioError>
1209where
1210 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1211 T: rig_core::driver::Transport<W>,
1212 A: Fn(CompletionRequest) -> CompletionRequest,
1213{
1214 const SCENARIO: &str = "buffered_streaming_text_parity";
1215 const PROMPT: &str = "Answer with exactly the single word Paris.";
1216 let started = Instant::now();
1217 let request = || {
1218 adjust(
1219 CompletionRequest::new(PROMPT)
1220 .temperature(0.0)
1221 .max_tokens(32),
1222 )
1223 };
1224 let buffered = model.call(request()).await?;
1225 let buffered_text = buffered
1226 .choice
1227 .iter()
1228 .filter_map(|item| match item {
1229 AssistantContent::Text(text) => Some(text.text.as_str()),
1230 _ => None,
1231 })
1232 .collect::<String>();
1233
1234 let mut stream = model.stream(request())?;
1235 let mut streamed_text = String::new();
1236 while let Some(item) = stream.next().await {
1237 if let crate::streaming::Item::Event(crate::streaming::StreamEvent::Text { text, .. }) =
1238 item?
1239 {
1240 streamed_text.push_str(&text);
1241 }
1242 }
1243 let streamed_usage = Some(stream.finish().await?.usage);
1244 let usage = streamed_usage.ok_or_else(|| {
1245 ScenarioError::contract(SCENARIO, "raw stream omitted its final response metadata")
1246 })?;
1247 let normalize = |text: &str| {
1248 text.trim()
1249 .trim_matches(|character: char| !character.is_alphanumeric())
1250 .to_string()
1251 };
1252 let buffered_answer = normalize(&buffered_text);
1253 let streamed_answer = normalize(&streamed_text);
1254 if !buffered_answer.eq_ignore_ascii_case("Paris")
1255 || !streamed_answer.eq_ignore_ascii_case("Paris")
1256 || !buffered.usage.is_reported()
1257 || !usage.is_reported()
1258 {
1259 return Err(ScenarioError::contract(
1260 SCENARIO,
1261 format!(
1262 "buffered={buffered_text:?}, streamed={streamed_text:?}, buffered_usage={:?}, streamed_usage={usage:?}",
1263 buffered.usage
1264 ),
1265 ));
1266 }
1267 Ok(ScenarioReport {
1268 name: SCENARIO,
1269 tool_calls: 0,
1270 prompt_tokens: usage.input_tokens.unwrap_or(0),
1271 generated_tokens: usage.output_tokens.unwrap_or(0),
1272 history_messages: 0,
1273 duration: started.elapsed(),
1274 response: streamed_text,
1275 })
1276}
1277
1278pub async fn structured_extraction<W, T>(
1281 model: rig_core::driver::Model<W, T>,
1282 additional_params: Option<serde_json::Value>,
1283) -> Result<ScenarioReport, ScenarioError>
1284where
1285 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1286 T: rig_core::driver::Transport<W>,
1287{
1288 const SCENARIO: &str = "structured_extraction";
1289 const INPUT: &str = "Hello, my name is Ada Lovelace and I work as a mathematician.";
1290 let started = Instant::now();
1291 let mut builder = crate::extractor::ExtractorBuilder::<ExtractedPerson>::new(model)
1292 .max_tokens(384)
1293 .retries(0);
1294 if let Some(params) = additional_params {
1295 builder = builder.additional_params(params);
1296 }
1297 let response = builder.build().extract(INPUT).await?;
1298 validate_extraction_fields(
1299 SCENARIO,
1300 response.output.first_name.as_deref(),
1301 response.output.last_name.as_deref(),
1302 response.output.job.as_deref(),
1303 response.usage,
1304 )?;
1305 Ok(ScenarioReport {
1306 name: SCENARIO,
1307 tool_calls: 1,
1308 prompt_tokens: response.usage.input_tokens.unwrap_or(0),
1309 generated_tokens: response.usage.output_tokens.unwrap_or(0),
1310 history_messages: 0,
1311 duration: started.elapsed(),
1312 response: format!(
1313 "{} {} — {}",
1314 response.output.first_name.as_deref().unwrap_or_default(),
1315 response.output.last_name.as_deref().unwrap_or_default(),
1316 response.output.job.as_deref().unwrap_or_default()
1317 ),
1318 })
1319}
1320
1321fn restricted_recovery_run(
1322 prompt: &str,
1323 turn: ModelTurn,
1324 retries: usize,
1325) -> Result<AgentRun, ScenarioError> {
1326 const SCENARIO: &str = "invalid_tool_recovery";
1327 let mut run = AgentRun::new(prompt)
1328 .max_turns(2)
1329 .max_invalid_tool_call_retries(retries);
1330 if !matches!(run.next_step()?, AgentRunStep::CallModel { .. }) {
1331 return Err(ScenarioError::contract(
1332 SCENARIO,
1333 "fresh AgentRun did not request a model turn",
1334 ));
1335 }
1336 let outcome = run.model_response(turn)?;
1337 let ModelTurnOutcome::NeedsResolution(context) = outcome else {
1338 return Err(ScenarioError::contract(
1339 SCENARIO,
1340 format!("disallowed tool call did not require resolution: {outcome:?}"),
1341 ));
1342 };
1343 if context.tool_name != CountingAdd::NAME {
1344 return Err(ScenarioError::contract(
1345 SCENARIO,
1346 format!("expected rejected add call, observed {context:?}"),
1347 ));
1348 }
1349 Ok(run)
1350}
1351
1352pub async fn invalid_tool_recovery<W, T, F>(
1355 model: rig_core::driver::Model<W, T>,
1356 configure: F,
1357) -> Result<ScenarioReport, ScenarioError>
1358where
1359 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1360 T: rig_core::driver::Transport<W>,
1361 F: FnOnce(AgentBuilder) -> AgentBuilder,
1362{
1363 invalid_tool_recovery_with_choice(model, configure, ToolChoice::Required).await
1364}
1365
1366pub async fn invalid_tool_recovery_with_choice<W, T, F>(
1370 model: rig_core::driver::Model<W, T>,
1371 configure: F,
1372 tool_choice: ToolChoice,
1373) -> Result<ScenarioReport, ScenarioError>
1374where
1375 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1376 T: rig_core::driver::Transport<W>,
1377 F: FnOnce(AgentBuilder) -> AgentBuilder,
1378{
1379 const SCENARIO: &str = "invalid_tool_recovery";
1380 const PROMPT: &str = "Call the add tool exactly once with x=2 and y=3. Do not call sum.";
1381 let started = Instant::now();
1382 let add_calls = Arc::new(AtomicUsize::new(0));
1383 let sum_calls = Arc::new(AtomicUsize::new(0));
1384 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1385 .preamble(FORCE_TOOLS_PREAMBLE)
1386 .tool(CountingAdd(add_calls.clone()))
1387 .tool(CountingSum(sum_calls.clone()))
1388 .tool_choice(tool_choice)
1389 .build();
1390 #[derive(Clone)]
1391 struct CaptureTurn(Arc<Mutex<Option<ModelTurn>>>);
1392
1393 impl AgentHook for CaptureTurn {
1394 async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
1395 let Some(response) = event.completion() else {
1396 return OutcomeAction::proceed();
1397 };
1398 *crate::lock(&self.0) = Some(ModelTurn::new(
1399 response.head(),
1400 response.choice.clone(),
1401 response.usage,
1402 TurnPolicy::default(),
1403 response.raw.clone(),
1404 ));
1405 OutcomeAction::stop("captured conformance model turn")
1406 }
1407 }
1408
1409 let captured = Arc::new(Mutex::new(None));
1410 let stopped = agent
1411 .prompt(PROMPT)
1412 .add_hook(CaptureTurn(captured.clone()))
1413 .run()
1414 .await;
1415 if !matches!(stopped, Err(PromptError::Cancelled { .. })) {
1416 return Err(ScenarioError::contract(
1417 SCENARIO,
1418 format!("capture hook did not stop after the model response: {stopped:?}"),
1419 ));
1420 }
1421 let response = crate::lock(&captured).take().ok_or_else(|| {
1422 ScenarioError::contract(SCENARIO, "capture hook observed no model response")
1423 })?;
1424 let emitted = response
1425 .choice
1426 .iter()
1427 .filter(|item| {
1428 matches!(item, AssistantContent::ToolCall(call) if call.function.name == CountingAdd::NAME)
1429 })
1430 .count();
1431 if emitted != 1 {
1432 return Err(ScenarioError::contract(
1433 SCENARIO,
1434 format!(
1435 "model emitted {emitted} add calls, response={:?}",
1436 response.choice
1437 ),
1438 ));
1439 }
1440 let executable = BTreeSet::from([CountingAdd::NAME.to_string(), CountingSum::NAME.to_string()]);
1441 let only_sum = ToolChoice::Specific {
1442 function_names: vec![
1443 rig_core::message::ToolName::new(CountingSum::NAME)
1444 .map_err(|error| ScenarioError::contract(SCENARIO, error.to_string()))?,
1445 ],
1446 };
1447 let policy = TurnPolicy::new(executable, Some(only_sum), None)
1448 .map_err(|error| ScenarioError::contract(SCENARIO, error.to_string()))?;
1449 let turn = ModelTurn::new(
1450 response.head,
1451 response.choice,
1452 response.usage,
1453 policy,
1454 response.raw,
1455 );
1456
1457 let mut fail = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1458 let error = match fail.resolve_invalid_tool_call(InvalidToolCallAction::fail()) {
1459 Err(error) => error,
1460 Ok(outcome) => {
1461 return Err(ScenarioError::contract(
1462 SCENARIO,
1463 format!("fail action unexpectedly returned {outcome:?}"),
1464 ));
1465 }
1466 };
1467 validate_unknown_tool_failure(&error, CountingAdd::NAME, &[CountingSum::NAME])?;
1468
1469 let mut retry = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1470 let error = match retry
1471 .resolve_invalid_tool_call(InvalidToolCallAction::retry("choose an allowed tool"))
1472 {
1473 Err(error) => error,
1474 Ok(outcome) => {
1475 return Err(ScenarioError::contract(
1476 SCENARIO,
1477 format!("exhausted retry unexpectedly returned {outcome:?}"),
1478 ));
1479 }
1480 };
1481 validate_unknown_tool_failure(&error, CountingAdd::NAME, &[CountingSum::NAME])?;
1482
1483 let mut rejected_repair = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1484 let error =
1485 match rejected_repair.resolve_invalid_tool_call(InvalidToolCallAction::repair("missing")) {
1486 Err(error) => error,
1487 Ok(outcome) => {
1488 return Err(ScenarioError::contract(
1489 SCENARIO,
1490 format!("disallowed repair unexpectedly returned {outcome:?}"),
1491 ));
1492 }
1493 };
1494 validate_unknown_tool_failure(&error, "missing", &[CountingSum::NAME])?;
1495
1496 let mut repaired = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1497 if !matches!(
1498 repaired.resolve_invalid_tool_call(InvalidToolCallAction::repair(CountingSum::NAME))?,
1499 ModelTurnOutcome::Continue { .. }
1500 ) {
1501 return Err(ScenarioError::contract(
1502 SCENARIO,
1503 "valid repair did not continue",
1504 ));
1505 }
1506 let AgentRunStep::CallTools { calls } = repaired.next_step()? else {
1507 return Err(ScenarioError::contract(
1508 SCENARIO,
1509 "valid repair did not produce pending tool execution",
1510 ));
1511 };
1512 let repaired_call = calls.first();
1513 if calls.len() != 1
1514 || !repaired_call.is_some_and(|call| {
1515 matches!(call, PendingToolCall::Execute(call) if call.name() == CountingSum::NAME)
1516 })
1517 {
1518 return Err(ScenarioError::contract(
1519 SCENARIO,
1520 format!("repaired pending calls were incorrect: {calls:?}"),
1521 ));
1522 }
1523
1524 let mut skipped = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1525 if !matches!(
1526 skipped.resolve_invalid_tool_call(InvalidToolCallAction::skip("disabled for this turn"))?,
1527 ModelTurnOutcome::Continue { .. }
1528 ) {
1529 return Err(ScenarioError::contract(SCENARIO, "skip did not continue"));
1530 }
1531 let step = skipped.next_step()?;
1533 let skipped_answered = matches!(
1534 skipped.messages().last().map(|message| &**message),
1535 Some(Message::User { content }) if content.len() == 1
1536 );
1537 if !matches!(step, AgentRunStep::CallModel { .. }) || !skipped_answered {
1538 return Err(ScenarioError::contract(
1539 SCENARIO,
1540 format!("skip did not answer the turn and continue: {step:?}"),
1541 ));
1542 }
1543 if add_calls.load(Ordering::SeqCst) != 0 || sum_calls.load(Ordering::SeqCst) != 0 {
1544 return Err(ScenarioError::contract(
1545 SCENARIO,
1546 "recovery scenario executed a tool body",
1547 ));
1548 }
1549
1550 Ok(ScenarioReport {
1551 name: SCENARIO,
1552 tool_calls: emitted,
1553 prompt_tokens: turn.usage.input_tokens.unwrap_or(0),
1554 generated_tokens: turn.usage.output_tokens.unwrap_or(0),
1555 history_messages: 2,
1556 duration: started.elapsed(),
1557 response: "fail, retry, repair, rejected repair, and skip passed".to_string(),
1558 })
1559}
1560
1561pub async fn hook_rewrites_and_request_patch<W, T, F>(
1564 model: rig_core::driver::Model<W, T>,
1565 configure: F,
1566) -> Result<ScenarioReport, ScenarioError>
1567where
1568 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1569 T: rig_core::driver::Transport<W>,
1570 F: FnOnce(AgentBuilder) -> AgentBuilder,
1571{
1572 hook_rewrites_and_request_patch_with_choice(model, configure, ToolChoice::Required).await
1573}
1574
1575pub async fn hook_rewrites_and_request_patch_with_choice<W, T, F>(
1579 model: rig_core::driver::Model<W, T>,
1580 configure: F,
1581 first_turn_choice: ToolChoice,
1582) -> Result<ScenarioReport, ScenarioError>
1583where
1584 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1585 T: rig_core::driver::Transport<W>,
1586 F: FnOnce(AgentBuilder) -> AgentBuilder,
1587{
1588 const SCENARIO: &str = "hook_rewrites_and_request_patch";
1589 let started = Instant::now();
1590 let calls = Arc::new(AtomicUsize::new(0));
1591 let observed = ObserveArguments::default();
1592 let observed_probe = observed.clone();
1593 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1594 .preamble("Use add for arithmetic and report only the tool result.")
1595 .tool(CountingAdd(calls.clone()))
1596 .default_max_turns(3)
1597 .build();
1598 let response = agent
1599 .prompt("Use add once for x=1 and y=1, then report what the tool returns.")
1600 .max_turns(3)
1601 .add_hook(FirstTurnPatch(
1602 RequestPatch::new()
1603 .active_tools([CountingAdd::NAME])
1604 .tool_choice(first_turn_choice),
1605 ))
1606 .add_hook(RewriteArgument {
1607 key: "x",
1608 value: serde_json::json!(7),
1609 })
1610 .add_hook(RewriteArgument {
1611 key: "y",
1612 value: serde_json::json!(8),
1613 })
1614 .add_hook(observed)
1615 .add_hook(ReplaceResult("portable-redacted"))
1616 .add_hook(WrapResult)
1617 .await?;
1618 let observations = crate::lock(&observed_probe.0).clone();
1619 validate_rewritten_arguments(
1620 SCENARIO,
1621 &observations,
1622 &serde_json::json!({ "x": 7, "y": 8 }),
1623 )?;
1624 let messages = response.messages.as_slice();
1625 if messages.is_empty() {
1626 return Err(ScenarioError::contract(
1627 SCENARIO,
1628 "extended hook run omitted message history",
1629 ));
1630 }
1631 let results = messages
1632 .iter()
1633 .flat_map(tool_result_values)
1634 .collect::<Vec<_>>();
1635 if calls.load(Ordering::SeqCst) != 1
1636 || !results
1637 .iter()
1638 .any(|value| value == &serde_json::json!("[portable-redacted]"))
1639 || response.completion_calls.len() != 2
1640 {
1641 return Err(ScenarioError::contract(
1642 SCENARIO,
1643 format!(
1644 "calls={}, completion_calls={}, results={results:?}, output={:?}",
1645 calls.load(Ordering::SeqCst),
1646 response.completion_calls.len(),
1647 response.output()
1648 ),
1649 ));
1650 }
1651 report_from_response(SCENARIO, started, 1, response)
1652}
1653
1654pub async fn cancellation_and_max_turns<W, T, F>(
1657 model: rig_core::driver::Model<W, T>,
1658 configure: F,
1659) -> Result<ScenarioReport, ScenarioError>
1660where
1661 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1662 T: rig_core::driver::Transport<W>,
1663 F: Fn(AgentBuilder) -> AgentBuilder,
1664{
1665 const SCENARIO: &str = "cancellation_and_max_turns";
1666 const REASON: &str = "portable result veto";
1667 let started = Instant::now();
1668 let cancelled_calls = Arc::new(AtomicUsize::new(0));
1669 let cancelled_agent = configure(AgentBuilder::new(model.clone()).temperature(0.0))
1670 .preamble("Use add for arithmetic; never calculate by hand.")
1671 .tool(CountingAdd(cancelled_calls.clone()))
1672 .build();
1673 let cancelled = match cancelled_agent
1674 .prompt("Use add once to compute x=20 plus y=22.")
1675 .max_turns(2)
1676 .add_hook(StopAfterResult(REASON))
1677 .await
1678 {
1679 Err(error) => error,
1680 Ok(output) => {
1681 return Err(ScenarioError::contract(
1682 SCENARIO,
1683 format!("result cancellation unexpectedly completed: {output:?}"),
1684 ));
1685 }
1686 };
1687 validate_cancelled_failure(&cancelled, REASON, CountingAdd::NAME)?;
1688
1689 let max_turn_calls = Arc::new(AtomicUsize::new(0));
1690 let max_turn_agent = configure(AgentBuilder::new(model).temperature(0.0))
1691 .preamble("Use add for arithmetic; never calculate by hand.")
1692 .tool(CountingAdd(max_turn_calls.clone()))
1693 .build();
1694 let max_turn = match max_turn_agent
1695 .prompt("Use add once to compute x=20 plus y=22, then report the result.")
1696 .max_turns(1)
1697 .await
1698 {
1699 Err(error) => error,
1700 Ok(output) => {
1701 return Err(ScenarioError::contract(
1702 SCENARIO,
1703 format!("one-turn budget unexpectedly completed: {output:?}"),
1704 ));
1705 }
1706 };
1707 validate_max_turns_failure(&max_turn, 1)?;
1708 let cancelled_count = cancelled_calls.load(Ordering::SeqCst);
1709 let max_turn_count = max_turn_calls.load(Ordering::SeqCst);
1710 if cancelled_count != 1 || max_turn_count != 1 {
1711 return Err(ScenarioError::contract(
1712 SCENARIO,
1713 format!("cancelled executions={cancelled_count}, max-turn executions={max_turn_count}"),
1714 ));
1715 }
1716 Ok(ScenarioReport {
1717 name: SCENARIO,
1718 tool_calls: cancelled_count + max_turn_count,
1719 prompt_tokens: 0,
1720 generated_tokens: 0,
1721 history_messages: 4,
1722 duration: started.elapsed(),
1723 response: "post-result cancellation and max-turn diagnostics passed".to_string(),
1724 })
1725}
1726
1727pub async fn optional_argument<W, T, F>(
1733 model: rig_core::driver::Model<W, T>,
1734 configure: F,
1735) -> Result<ScenarioReport, ScenarioError>
1736where
1737 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1738 T: rig_core::driver::Transport<W>,
1739 F: FnOnce(AgentBuilder) -> AgentBuilder,
1740{
1741 let calls = Arc::new(AtomicUsize::new(0));
1742 let started = Instant::now();
1743 let agent = configure(AgentBuilder::new(model))
1744 .preamble("Use the repeat_text tool whenever asked to repeat text.")
1745 .tool(RepeatTool {
1746 calls: calls.clone(),
1747 })
1748 .default_max_turns(4)
1749 .build();
1750 let result = agent
1751 .prompt(
1752 "Use the repeat_text tool to repeat the word \"banana\" 3 times, then show me the exact result.",
1753 )
1754 .await?;
1755 let response = result.output();
1756 let tool_calls = calls.load(Ordering::SeqCst);
1757 if tool_calls == 0
1758 || response.matches("banana").count() < 1
1759 || !has_tool_roundtrip(&result.messages)
1760 {
1761 return Err(ScenarioError::contract(
1762 "optional_argument",
1763 format!("calls={tool_calls}, response={response:?}"),
1764 ));
1765 }
1766 report_from_response("optional_argument", started, tool_calls, result)
1767}
1768
1769pub async fn sequential_tools<W, T, F>(
1771 model: rig_core::driver::Model<W, T>,
1772 configure: F,
1773) -> Result<ScenarioReport, ScenarioError>
1774where
1775 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1776 T: rig_core::driver::Transport<W>,
1777 F: FnOnce(AgentBuilder) -> AgentBuilder,
1778{
1779 let add_calls = Arc::new(AtomicUsize::new(0));
1780 let multiply_calls = Arc::new(AtomicUsize::new(0));
1781 let started = Instant::now();
1782 let agent = configure(AgentBuilder::new(model))
1783 .preamble(
1784 "You are a calculator. Use the add and multiply tools for arithmetic; never compute by hand.",
1785 )
1786 .tool(AddTool(add_calls.clone()))
1787 .tool(MultiplyTool(multiply_calls.clone()))
1788 .default_max_turns(6)
1789 .build();
1790 let result = agent
1791 .prompt(
1792 "Compute (4 + 6) * 2. First call the add tool, then call the multiply tool on the result. Tell me the final number.",
1793 )
1794 .await?;
1795 let response = result.output();
1796 let add = add_calls.load(Ordering::SeqCst);
1797 let multiply = multiply_calls.load(Ordering::SeqCst);
1798 if add == 0
1799 || multiply == 0
1800 || !response.contains("20")
1801 || !has_tool_roundtrip(&result.messages)
1802 {
1803 return Err(ScenarioError::contract(
1804 "sequential_tools",
1805 format!("add={add}, multiply={multiply}, response={response:?}"),
1806 ));
1807 }
1808 report_from_response("sequential_tools", started, add + multiply, result)
1809}
1810
1811pub async fn streaming_tool<W, T, F>(
1813 model: rig_core::driver::Model<W, T>,
1814 configure: F,
1815) -> Result<ScenarioReport, ScenarioError>
1816where
1817 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1818 T: rig_core::driver::Transport<W>,
1819 F: FnOnce(AgentBuilder) -> AgentBuilder,
1820{
1821 let calls = Arc::new(AtomicUsize::new(0));
1822 let started = Instant::now();
1823 let agent = configure(AgentBuilder::new(model))
1824 .preamble("Use the add tool for arithmetic; do not calculate by hand.")
1825 .tool(AddTool(calls.clone()))
1826 .default_max_turns(4)
1827 .build();
1828 let mut stream = agent
1829 .prompt("Use add to calculate 17 + 25, then state the final number.")
1830 .max_turns(4)
1831 .stream();
1832 let mut final_response = None;
1833 let mut final_count = 0_usize;
1834 let mut completion_usage = crate::completion::Usage::default();
1835 let mut streamed_call_ids = Vec::new();
1836 let mut streamed_result_ids = Vec::new();
1837 while let Some(item) = stream.next().await {
1838 match item? {
1839 MultiTurnStreamItem::ToolCall { tool_call } => streamed_call_ids.push(tool_call.id),
1840 MultiTurnStreamItem::ToolResult { tool_result } => {
1841 streamed_result_ids.push(tool_result.call)
1842 }
1843 MultiTurnStreamItem::CompletionCall(call) => completion_usage += call.usage,
1844 MultiTurnStreamItem::FinalResponse(response) => {
1845 final_count += 1;
1846 final_response = Some(response);
1847 }
1848 MultiTurnStreamItem::StreamAssistantItem(_)
1849 | MultiTurnStreamItem::ToolExecutionStarted { .. }
1850 | MultiTurnStreamItem::ToolResultItem { .. }
1851 | MultiTurnStreamItem::ToolExecutionCommitted { .. }
1852 | MultiTurnStreamItem::ModelTurnRetried { .. } => {}
1853 }
1854 }
1855 let result = final_response.ok_or_else(|| {
1856 ScenarioError::contract("streaming_tool", "stream produced no final response")
1857 })?;
1858 let response = result.output();
1859 let history_messages = result.messages.len();
1860 let tool_calls = calls.load(Ordering::SeqCst);
1861 streamed_call_ids.sort();
1862 streamed_result_ids.sort();
1863 let correlated_stream =
1864 !streamed_call_ids.is_empty() && streamed_call_ids == streamed_result_ids;
1865 let correlated_history = validate_tool_correlation("streaming_tool", &result.messages).is_ok();
1866 if tool_calls == 0
1867 || !response.contains("42")
1868 || history_messages < 4
1869 || final_count != 1
1870 || !correlated_stream
1871 || !correlated_history
1872 || completion_usage != result.usage
1873 || result.completion_calls.is_empty()
1874 {
1875 return Err(ScenarioError::contract(
1876 "streaming_tool",
1877 format!(
1878 "calls={tool_calls}, final_count={final_count}, streamed_call_ids={streamed_call_ids:?}, streamed_result_ids={streamed_result_ids:?}, completion_usage={completion_usage:?}, final_usage={:?}, history_messages={history_messages}, response={response:?}",
1879 result.usage
1880 ),
1881 ));
1882 }
1883 validate_protocol_hygiene(
1884 "streaming_tool",
1885 &response,
1886 &result.messages,
1887 &["<tool_call>", "</tool_call>", "<think>", "</think>"],
1888 )?;
1889 Ok(ScenarioReport {
1890 name: "streaming_tool",
1891 tool_calls,
1892 prompt_tokens: result.usage.input_tokens.unwrap_or(0),
1893 generated_tokens: result.usage.output_tokens.unwrap_or(0),
1894 history_messages,
1895 duration: started.elapsed(),
1896 response,
1897 })
1898}
1899
1900pub async fn structured_after_tool<W, T, F>(
1902 model: rig_core::driver::Model<W, T>,
1903 configure: F,
1904) -> Result<ScenarioReport, ScenarioError>
1905where
1906 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1907 T: rig_core::driver::Transport<W>,
1908 F: FnOnce(AgentBuilder) -> AgentBuilder,
1909{
1910 let calls = Arc::new(AtomicUsize::new(0));
1911 let started = Instant::now();
1912 let agent = configure(AgentBuilder::new(model))
1913 .preamble(
1914 "Use add for arithmetic, then finish by calling the structured output tool exactly once.",
1915 )
1916 .output_schema::<ArithmeticResult>()
1917 .output_mode(OutputMode::Tool)
1918 .tool(AddTool(calls.clone()))
1919 .default_max_turns(5)
1920 .build();
1921 let result = agent
1922 .prompt("Use add to calculate 19 + 23. Return answer=42 and a short optional explanation.")
1923 .await?;
1924 let response = result.output();
1925 let parsed: ArithmeticResult = serde_json::from_str(&response)?;
1926 let tool_calls = calls.load(Ordering::SeqCst);
1927 if tool_calls == 0 || parsed.answer != 42 || !has_tool_roundtrip(&result.messages) {
1928 return Err(ScenarioError::contract(
1929 "structured_after_tool",
1930 format!("calls={tool_calls}, response={response:?}"),
1931 ));
1932 }
1933 let _ = parsed.explanation;
1934 report_from_response("structured_after_tool", started, tool_calls + 1, result)
1935}
1936
1937pub async fn tool_choice_modes<W, T, A>(
1939 model: rig_core::driver::Model<W, T>,
1940 adjust: A,
1941) -> Result<ScenarioReport, ScenarioError>
1942where
1943 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1944 T: rig_core::driver::Transport<W>,
1945 A: Fn(CompletionRequest) -> CompletionRequest,
1946{
1947 let tool_name = |name: &str| {
1948 rig_core::message::ToolName::new(name)
1949 .map_err(|error| ScenarioError::contract("tool_choice_modes", error.to_string()))
1950 };
1951 let definition = |name: rig_core::message::ToolName| ToolDefinition {
1952 description: format!("Return the supplied integer using {name}."),
1953 name,
1954 parameters: serde_json::json!({
1955 "type": "object",
1956 "properties": {"value": {"type": "integer"}},
1957 "required": ["value"]
1958 }),
1959 annotations: Default::default(),
1960 };
1961 let tools = vec![
1962 definition(tool_name("alpha")?),
1963 definition(tool_name("beta")?),
1964 ];
1965 let started = Instant::now();
1966 let none = model
1967 .call(adjust(
1968 CompletionRequest::new("Answer with only the number 4. Do not call a function.")
1969 .tools(tools.clone())
1970 .tool_choice(ToolChoice::None)
1971 .temperature(0.0)
1972 .max_tokens(64),
1973 ))
1974 .await?;
1975 if none
1976 .choice
1977 .iter()
1978 .any(|item| matches!(item, AssistantContent::ToolCall(_)))
1979 {
1980 return Err(ScenarioError::contract(
1981 "tool_choice_modes",
1982 "tool_choice none emitted a tool call",
1983 ));
1984 }
1985
1986 let required = model
1987 .call(adjust(
1988 CompletionRequest::new("Call alpha with value 7.")
1989 .tools(tools.clone())
1990 .tool_choice(ToolChoice::Required)
1991 .temperature(0.0)
1992 .max_tokens(96),
1993 ))
1994 .await?;
1995 let required_calls = required
1996 .choice
1997 .iter()
1998 .filter(|item| matches!(item, AssistantContent::ToolCall(_)))
1999 .count();
2000 if required_calls == 0 {
2001 return Err(ScenarioError::contract(
2002 "tool_choice_modes",
2003 "tool_choice required emitted no tool call",
2004 ));
2005 }
2006
2007 let specific = model
2008 .call(adjust(
2009 CompletionRequest::new("Call beta with value 9.")
2010 .tools(tools)
2011 .tool_choice(ToolChoice::Specific {
2012 function_names: vec![tool_name("beta")?],
2013 })
2014 .temperature(0.0)
2015 .max_tokens(96),
2016 ))
2017 .await?;
2018 let specific_calls = specific
2019 .choice
2020 .iter()
2021 .filter_map(|item| match item {
2022 AssistantContent::ToolCall(call) => Some(call),
2023 _ => None,
2024 })
2025 .collect::<Vec<_>>();
2026 if specific_calls.is_empty()
2027 || specific_calls
2028 .iter()
2029 .any(|call| call.function.name != "beta")
2030 {
2031 return Err(ScenarioError::contract(
2032 "tool_choice_modes",
2033 "specific tool choice did not select only beta",
2034 ));
2035 }
2036
2037 let usage = none.usage + required.usage + specific.usage;
2038 Ok(ScenarioReport {
2039 name: "tool_choice_modes",
2040 tool_calls: required_calls + specific_calls.len(),
2041 prompt_tokens: usage.input_tokens.unwrap_or(0),
2042 generated_tokens: usage.output_tokens.unwrap_or(0),
2043 history_messages: 0,
2044 duration: started.elapsed(),
2045 response: "none, required, and specific modes passed".to_string(),
2046 })
2047}
2048
2049pub async fn streaming_structured_after_tool<W, T, F>(
2051 model: rig_core::driver::Model<W, T>,
2052 configure: F,
2053) -> Result<ScenarioReport, ScenarioError>
2054where
2055 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
2056 T: rig_core::driver::Transport<W>,
2057 F: FnOnce(AgentBuilder) -> AgentBuilder,
2058{
2059 let calls = Arc::new(AtomicUsize::new(0));
2060 let started = Instant::now();
2061 let agent = configure(AgentBuilder::new(model))
2062 .preamble(
2063 "Use add for arithmetic, then finish by calling the structured output tool exactly once.",
2064 )
2065 .output_schema::<ArithmeticResult>()
2066 .output_mode(OutputMode::Tool)
2067 .tool(AddTool(calls.clone()))
2068 .default_max_turns(5)
2069 .build();
2070 let mut stream = agent
2071 .prompt("Use add to calculate 19 + 23. Return answer=42 and a short optional explanation.")
2072 .max_turns(5)
2073 .stream();
2074 let mut final_response = None;
2075 let mut final_count = 0_usize;
2076 while let Some(item) = stream.next().await {
2077 if let MultiTurnStreamItem::FinalResponse(response) = item? {
2078 final_count += 1;
2079 final_response = Some(response);
2080 }
2081 }
2082 let result = final_response.ok_or_else(|| {
2083 ScenarioError::contract(
2084 "streaming_structured_after_tool",
2085 "stream produced no final response",
2086 )
2087 })?;
2088 let parsed: ArithmeticResult = serde_json::from_str(&result.output())?;
2089 let calls = calls.load(Ordering::SeqCst);
2090 if calls == 0
2091 || final_count != 1
2092 || parsed.answer != 42
2093 || !has_tool_roundtrip(&result.messages)
2094 {
2095 return Err(ScenarioError::contract(
2096 "streaming_structured_after_tool",
2097 format!(
2098 "calls={calls}, final_count={final_count}, response={:?}",
2099 result.output()
2100 ),
2101 ));
2102 }
2103 report_from_response(
2104 "streaming_structured_after_tool",
2105 started,
2106 calls + 1,
2107 result,
2108 )
2109}
2110
2111#[cfg(test)]
2112mod tests;