1use std::{
8 collections::BTreeSet,
9 sync::{
10 Arc, Mutex, MutexGuard,
11 atomic::{AtomicUsize, Ordering},
12 },
13 time::{Duration, Instant},
14};
15
16use futures::StreamExt;
17use schemars::JsonSchema;
18use serde::{Deserialize, Serialize};
19
20use crate::{
21 agent::{
22 AgentBuilder, AgentHook, CompletionCallAction, CompletionCallEvent, DispatchAction,
23 DispatchEvent, HookContext, InvalidToolCallAction, MultiTurnStreamItem, NoToolConfig,
24 OutcomeAction, OutcomeEvent, OutputMode, RequestPatch,
25 run::{AgentRun, AgentRunStep, ModelTurn, ModelTurnOutcome},
26 },
27 completion::{AssistantContent, Message, PromptError, ToolDefinition},
28 tool::{Tool, ToolContext},
29};
30use rig_core::completion::CompletionRequest;
31use rig_core::error::ProviderError;
32use rig_core::message::{ToolChoice, UserContent};
33
34#[derive(Debug, thiserror::Error)]
36pub enum ScenarioError {
37 #[error(transparent)]
39 Prompt(#[from] PromptError),
40 #[error(transparent)]
42 Completion(#[from] ProviderError),
43 #[error(transparent)]
45 Report(#[from] rig_core::error::ErrorReport),
46 #[error(transparent)]
48 Json(#[from] serde_json::Error),
49 #[error(transparent)]
51 Extraction(#[from] crate::completion::StructuredOutputError),
52 #[error("{scenario} conformance failed: {details}")]
54 Contract {
55 scenario: &'static str,
57 details: String,
59 },
60}
61
62impl ScenarioError {
63 fn contract(scenario: &'static str, details: impl Into<String>) -> Self {
64 Self::Contract {
65 scenario,
66 details: details.into(),
67 }
68 }
69}
70
71pub fn validate_unknown_tool_failure(
74 error: &PromptError,
75 expected_tool: &str,
76 expected_allowed_tools: &[&str],
77) -> Result<(), ScenarioError> {
78 const SCENARIO: &str = "unknown_tool_failure";
79 let PromptError::UnknownToolCall {
80 tool_name,
81 allowed_tools,
82 chat_history,
83 ..
84 } = error
85 else {
86 return Err(ScenarioError::contract(
87 SCENARIO,
88 format!("expected UnknownToolCall, observed {error:?}"),
89 ));
90 };
91 let expected_allowed = expected_allowed_tools
92 .iter()
93 .map(|name| (*name).to_string())
94 .collect::<Vec<_>>();
95 let history_has_call = chat_history.iter().any(|message| {
99 matches!(
100 message,
101 Message::Assistant(rig_core::message::AssistantMessage { content, .. })
102 if content.iter().any(|item| matches!(item, AssistantContent::ToolCall(_)))
103 )
104 });
105 if tool_name != expected_tool || allowed_tools != &expected_allowed || !history_has_call {
106 return Err(ScenarioError::contract(
107 SCENARIO,
108 format!(
109 "tool={tool_name:?}, allowed={allowed_tools:?}, expected_tool={expected_tool:?}, expected_allowed={expected_allowed:?}, history_has_call={history_has_call}, history={chat_history:?}"
110 ),
111 ));
112 }
113 Ok(())
114}
115
116pub fn validate_cancelled_failure(
119 error: &PromptError,
120 expected_reason: &str,
121 expected_tool: &str,
122) -> Result<(), ScenarioError> {
123 const SCENARIO: &str = "cancelled_failure";
124 let PromptError::Cancelled {
125 chat_history,
126 reason,
127 } = error
128 else {
129 return Err(ScenarioError::contract(
130 SCENARIO,
131 format!("expected Cancelled, observed {error:?}"),
132 ));
133 };
134 let history_has_call = chat_history.iter().any(|message| {
135 matches!(
136 message,
137 Message::Assistant(rig_core::message::AssistantMessage { content, .. })
138 if content.iter().any(|item| matches!(
139 item,
140 AssistantContent::ToolCall(call) if call.function.name == expected_tool
141 ))
142 )
143 });
144 if reason != expected_reason || !history_has_call {
145 return Err(ScenarioError::contract(
146 SCENARIO,
147 format!(
148 "reason={reason:?}, expected={expected_reason:?}, history_has_call={history_has_call}, history={chat_history:?}"
149 ),
150 ));
151 }
152 Ok(())
153}
154
155pub fn validate_max_turns_failure(
158 error: &PromptError,
159 expected_max_turns: usize,
160) -> Result<(), ScenarioError> {
161 const SCENARIO: &str = "max_turns_failure";
162 let PromptError::MaxTurns {
163 max_turns,
164 chat_history,
165 prompt,
166 } = error
167 else {
168 return Err(ScenarioError::contract(
169 SCENARIO,
170 format!("expected MaxTurns, observed {error:?}"),
171 ));
172 };
173 let pending_prompt_retained = matches!(
174 prompt,
175 Message::User { content } if content.iter().next().is_some()
176 );
177 if *max_turns != expected_max_turns || chat_history.is_empty() || !pending_prompt_retained {
178 return Err(ScenarioError::contract(
179 SCENARIO,
180 format!(
181 "max_turns={max_turns}, expected={expected_max_turns}, history={chat_history:?}, pending_prompt_retained={pending_prompt_retained}, pending_prompt={prompt:?}"
182 ),
183 ));
184 }
185 Ok(())
186}
187
188pub fn decode_structured_output<T>(
191 scenario: &'static str,
192 response: &str,
193) -> Result<T, ScenarioError>
194where
195 T: serde::de::DeserializeOwned,
196{
197 serde_json::from_str(response).map_err(|error| {
198 ScenarioError::contract(
199 scenario,
200 format!("structured output did not decode: {error}; response={response:?}"),
201 )
202 })
203}
204
205pub fn validate_protocol_hygiene(
208 scenario: &'static str,
209 visible_output: &str,
210 messages: &[Message],
211 forbidden_markers: &[&str],
212) -> Result<(), ScenarioError> {
213 let serialized = serde_json::to_string(messages)?;
214 let leaked = forbidden_markers
215 .iter()
216 .filter(|marker| visible_output.contains(**marker) || serialized.contains(**marker))
217 .copied()
218 .collect::<Vec<_>>();
219 if !leaked.is_empty() {
220 return Err(ScenarioError::contract(
221 scenario,
222 format!(
223 "protocol markers leaked: {leaked:?}; output={visible_output:?}, history={messages:?}"
224 ),
225 ));
226 }
227 Ok(())
228}
229
230pub fn validate_rewritten_arguments(
233 scenario: &'static str,
234 observations: &[serde_json::Value],
235 expected_fields: &serde_json::Value,
236) -> Result<(), ScenarioError> {
237 let Some(expected) = expected_fields.as_object() else {
238 return Err(ScenarioError::contract(
239 scenario,
240 "expected rewritten fields must be a JSON object",
241 ));
242 };
243 if observations.is_empty() {
244 return Err(ScenarioError::contract(
245 scenario,
246 "the rewritten tool was never invoked",
247 ));
248 }
249 for observation in observations {
250 let Some(actual) = observation.as_object() else {
251 return Err(ScenarioError::contract(
252 scenario,
253 format!("observed rewritten arguments were not an object: {observation:?}"),
254 ));
255 };
256 for (key, expected_value) in expected {
257 if actual.get(key) != Some(expected_value) {
258 return Err(ScenarioError::contract(
259 scenario,
260 format!(
261 "rewritten field {key:?} expected {expected_value:?}, observed {observation:?}"
262 ),
263 ));
264 }
265 }
266 }
267 Ok(())
268}
269
270pub fn validate_result_redaction(
273 scenario: &'static str,
274 tool_produced_secret: bool,
275 visible_output: &str,
276 secret: &str,
277) -> Result<(), ScenarioError> {
278 if !tool_produced_secret || visible_output.is_empty() || visible_output.contains(secret) {
279 return Err(ScenarioError::contract(
280 scenario,
281 format!(
282 "produced_secret={tool_produced_secret}, secret_visible={}, output={visible_output:?}",
283 visible_output.contains(secret)
284 ),
285 ));
286 }
287 Ok(())
288}
289
290pub fn validate_extraction_fields(
292 scenario: &'static str,
293 first_name: Option<&str>,
294 last_name: Option<&str>,
295 job: Option<&str>,
296 usage: crate::completion::Usage,
297) -> Result<(), ScenarioError> {
298 let fields_match = first_name.is_some_and(|value| value.eq_ignore_ascii_case("Ada"))
299 && last_name.is_some_and(|value| value.eq_ignore_ascii_case("Lovelace"))
300 && job.is_some_and(|value| value.to_ascii_lowercase().contains("mathematician"));
301 if !fields_match || !usage.is_reported() {
302 return Err(ScenarioError::contract(
303 scenario,
304 format!(
305 "first_name={first_name:?}, last_name={last_name:?}, job={job:?}, usage={usage:?}"
306 ),
307 ));
308 }
309 Ok(())
310}
311
312#[derive(Debug, Clone)]
314pub struct ScenarioReport {
315 pub name: &'static str,
317 pub tool_calls: usize,
319 pub prompt_tokens: u64,
322 pub generated_tokens: u64,
325 pub history_messages: usize,
327 pub duration: Duration,
329 pub response: String,
331}
332
333#[derive(Debug, thiserror::Error)]
335#[error("model-conformance tool failed")]
336pub struct ConformanceToolError;
337
338const FORCE_TOOLS_PREAMBLE: &str = "You are a calculator assistant. You MUST use the provided tools for every arithmetic operation instead of computing results yourself. Once you have all the tool results you need, reply with the final numeric answer in plain text.";
339const PARALLEL_PROMPT: &str = "Compute 3 + 4 and 10 - 2. You MUST call the add tool and the subtract tool together in your first response, as two parallel function calls, then report both results.";
340const PING_OUTPUT: &str = "pong-crimson-7423";
341const MOTTO_OUTPUT: &str = "steady hands\ncalm waters";
342
343fn lock_recover<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
344 mutex
345 .lock()
346 .unwrap_or_else(std::sync::PoisonError::into_inner)
347}
348
349fn tool_result_values(message: &Message) -> Vec<serde_json::Value> {
350 let Message::User { content } = message else {
351 return Vec::new();
352 };
353 content
354 .iter()
355 .filter_map(|item| match item {
356 UserContent::ToolResult(result) => Some(result),
357 _ => None,
358 })
359 .flat_map(|result| result.content.iter())
360 .filter_map(|content| match content {
361 rig_core::message::ToolResultContent::Text(text) => {
362 Some(serde_json::Value::String(text.text.clone()))
363 }
364 rig_core::message::ToolResultContent::Json { value } => Some(value.clone()),
365 rig_core::message::ToolResultContent::Image(_) => None,
366 })
367 .collect()
368}
369
370fn validate_tool_correlation(
371 scenario: &'static str,
372 messages: &[Message],
373) -> Result<(), ScenarioError> {
374 let mut calls = Vec::new();
375 let mut results = Vec::new();
376 let mut turn = 0usize;
377 for message in messages {
378 match message {
379 Message::Assistant(rig_core::message::AssistantMessage { content, .. }) => {
380 turn += 1;
381 calls.extend(content.iter().filter_map(|item| match item {
382 AssistantContent::ToolCall(call) => Some((
383 turn,
384 &call.id,
385 call.id.provider().map(|provider| provider.as_str()),
386 )),
387 _ => None,
388 }));
389 }
390 Message::User { content } => {
391 results.extend(content.iter().filter_map(|item| match item {
392 UserContent::ToolResult(result) => Some((
393 turn,
394 &result.call,
395 result.call.provider().map(|provider| provider.as_str()),
396 )),
397 _ => None,
398 }));
399 }
400 Message::System { .. } => {}
401 }
402 }
403 if calls.is_empty() {
404 return Err(ScenarioError::contract(
405 scenario,
406 format!("history has no assistant tool calls: {messages:?}"),
407 ));
408 }
409 for (turn, id, call_id) in &calls {
410 let matches = results
411 .iter()
412 .filter(|(result_turn, result_id, result_call_id)| {
413 result_turn == turn && result_id == id && call_id == result_call_id
414 })
415 .count();
416 if matches != 1 {
417 return Err(ScenarioError::contract(
418 scenario,
419 format!(
420 "tool call id={id:?} call_id={call_id:?} has {matches} correlated results; calls={calls:?}, results={results:?}"
421 ),
422 ));
423 }
424 }
425 if results.len() != calls.len() {
426 return Err(ScenarioError::contract(
427 scenario,
428 format!(
429 "history contains dangling calls or results: calls={calls:?}, results={results:?}"
430 ),
431 ));
432 }
433 Ok(())
434}
435
436#[derive(Debug, Deserialize, JsonSchema)]
437struct OperationArgs {
438 x: i64,
439 y: i64,
440}
441
442#[derive(Clone)]
443struct CountingAdd(Arc<AtomicUsize>);
444
445impl Tool for CountingAdd {
446 const NAME: &'static str = "add";
447 type Error = ConformanceToolError;
448 type Args = OperationArgs;
449 type Output = i64;
450
451 fn description(&self) -> String {
452 "Add x and y together".to_string()
453 }
454
455 fn parameters(&self) -> serde_json::Value {
456 serde_json::json!({
457 "type": "object",
458 "properties": {
459 "x": { "type": "number", "description": "The first operand" },
460 "y": { "type": "number", "description": "The second operand" }
461 },
462 "required": ["x", "y"]
463 })
464 }
465
466 async fn call(
467 &self,
468 _context: &mut ToolContext,
469 args: Self::Args,
470 ) -> Result<Self::Output, Self::Error> {
471 self.0.fetch_add(1, Ordering::SeqCst);
472 Ok(args.x + args.y)
473 }
474}
475
476#[derive(Clone)]
477struct CountingSum(Arc<AtomicUsize>);
478
479impl Tool for CountingSum {
480 const NAME: &'static str = "sum";
481 type Error = ConformanceToolError;
482 type Args = OperationArgs;
483 type Output = i64;
484
485 fn description(&self) -> String {
486 "Add x and y together (alias of add)".to_string()
487 }
488
489 fn parameters(&self) -> serde_json::Value {
490 CountingAdd(Arc::new(AtomicUsize::new(0))).parameters()
491 }
492
493 async fn call(
494 &self,
495 _context: &mut ToolContext,
496 args: Self::Args,
497 ) -> Result<Self::Output, Self::Error> {
498 self.0.fetch_add(1, Ordering::SeqCst);
499 Ok(args.x + args.y)
500 }
501}
502
503#[derive(Clone)]
504struct CountingSubtract(Arc<AtomicUsize>);
505
506impl Tool for CountingSubtract {
507 const NAME: &'static str = "subtract";
508 type Error = ConformanceToolError;
509 type Args = OperationArgs;
510 type Output = i64;
511
512 fn description(&self) -> String {
513 "Subtract y from x (i.e. x - y)".to_string()
514 }
515
516 fn parameters(&self) -> serde_json::Value {
517 serde_json::json!({
518 "type": "object",
519 "properties": {
520 "x": { "type": "number", "description": "The first operand" },
521 "y": { "type": "number", "description": "The second operand" }
522 },
523 "required": ["x", "y"]
524 })
525 }
526
527 async fn call(
528 &self,
529 _context: &mut ToolContext,
530 args: Self::Args,
531 ) -> Result<Self::Output, Self::Error> {
532 self.0.fetch_add(1, Ordering::SeqCst);
533 Ok(args.x - args.y)
534 }
535}
536
537#[derive(Clone)]
538struct RewriteArgument {
539 key: &'static str,
540 value: serde_json::Value,
541}
542
543impl AgentHook for RewriteArgument {
544 async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
545 let (Some(name), Some(args)) = (event.tool_name(), event.tool_args()) else {
546 return DispatchAction::proceed();
547 };
548 if name != CountingAdd::NAME {
549 return DispatchAction::proceed();
550 }
551 let Ok(mut arguments) = serde_json::from_str::<serde_json::Value>(args) else {
552 return DispatchAction::proceed();
553 };
554 let Some(object) = arguments.as_object_mut() else {
555 return DispatchAction::proceed();
556 };
557 object.insert(self.key.to_string(), self.value.clone());
558 DispatchAction::rewrite_tool_args(event.kind, arguments)
559 }
560}
561
562#[derive(Clone, Default)]
563struct ObserveArguments(Arc<Mutex<Vec<serde_json::Value>>>);
564
565impl AgentHook for ObserveArguments {
566 async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
567 if let Some(args) = event.tool_args() {
568 let value = serde_json::from_str(args)
569 .unwrap_or_else(|_| serde_json::Value::String(args.to_string()));
570 lock_recover(&self.0).push(value);
571 }
572 DispatchAction::proceed()
573 }
574}
575
576#[derive(Clone)]
577struct ReplaceResult(&'static str);
578
579impl AgentHook for ReplaceResult {
580 async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
581 if event.tool_name() == Some(CountingAdd::NAME) {
582 OutcomeAction::rewrite_tool_result(&event, self.0)
583 } else {
584 OutcomeAction::proceed()
585 }
586 }
587}
588
589#[derive(Clone)]
590struct WrapResult;
591
592impl AgentHook for WrapResult {
593 async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
594 match (event.tool_name(), event.tool_result()) {
595 (Some(name), Some(result)) if name == CountingAdd::NAME => {
596 OutcomeAction::rewrite_tool_result(
597 &event,
598 format!("[{}]", result.output().render()),
599 )
600 }
601 _ => OutcomeAction::proceed(),
602 }
603 }
604}
605
606#[derive(Clone)]
607struct FirstTurnPatch(RequestPatch);
608
609impl AgentHook for FirstTurnPatch {
610 async fn on_completion_call(
611 &self,
612 ctx: &HookContext,
613 _event: CompletionCallEvent<'_>,
614 ) -> CompletionCallAction {
615 if ctx.turn() == 1 {
616 CompletionCallAction::patch(self.0.clone())
617 } else {
618 CompletionCallAction::continue_run()
619 }
620 }
621}
622
623#[derive(Clone)]
624struct StopAfterResult(&'static str);
625
626impl AgentHook for StopAfterResult {
627 async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
628 if event.tool_name() == Some(CountingAdd::NAME) {
629 OutcomeAction::stop(self.0)
630 } else {
631 OutcomeAction::proceed()
632 }
633 }
634}
635
636#[derive(Debug, Deserialize, JsonSchema)]
637struct EmptyArgs {}
638
639#[derive(Clone)]
640struct PingTool(Arc<AtomicUsize>);
641
642impl Tool for PingTool {
643 const NAME: &'static str = "ping";
644 type Error = ConformanceToolError;
645 type Args = EmptyArgs;
646 type Output = String;
647
648 fn description(&self) -> String {
649 "Return the current ping marker. Takes no arguments.".to_string()
650 }
651
652 fn parameters(&self) -> serde_json::Value {
653 serde_json::json!({ "type": "object", "properties": {}, "required": [] })
654 }
655
656 async fn call(
657 &self,
658 _context: &mut ToolContext,
659 _args: Self::Args,
660 ) -> Result<Self::Output, Self::Error> {
661 self.0.fetch_add(1, Ordering::SeqCst);
662 Ok(PING_OUTPUT.to_string())
663 }
664}
665
666#[derive(Clone)]
667struct MottoTool(Arc<AtomicUsize>);
668
669impl Tool for MottoTool {
670 const NAME: &'static str = "fetch_motto";
671 type Error = ConformanceToolError;
672 type Args = EmptyArgs;
673 type Output = String;
674
675 fn description(&self) -> String {
676 "Fetch the two-line workshop motto.".to_string()
677 }
678
679 fn parameters(&self) -> serde_json::Value {
680 serde_json::json!({ "type": "object", "properties": {}, "required": [] })
681 }
682
683 async fn call(
684 &self,
685 _context: &mut ToolContext,
686 _args: Self::Args,
687 ) -> Result<Self::Output, Self::Error> {
688 self.0.fetch_add(1, Ordering::SeqCst);
689 Ok(MOTTO_OUTPUT.to_string())
690 }
691}
692
693#[derive(Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
694struct ConfigOutput {
695 service: String,
696 max_retries: u64,
697}
698
699#[derive(Clone)]
700struct ConfigTool(Arc<AtomicUsize>);
701
702impl Tool for ConfigTool {
703 const NAME: &'static str = "fetch_config";
704 type Error = ConformanceToolError;
705 type Args = EmptyArgs;
706 type Output = ConfigOutput;
707
708 fn description(&self) -> String {
709 "Fetch the service configuration object.".to_string()
710 }
711
712 fn parameters(&self) -> serde_json::Value {
713 serde_json::json!({ "type": "object", "properties": {}, "required": [] })
714 }
715
716 async fn call(
717 &self,
718 _context: &mut ToolContext,
719 _args: Self::Args,
720 ) -> Result<Self::Output, Self::Error> {
721 self.0.fetch_add(1, Ordering::SeqCst);
722 Ok(ConfigOutput {
723 service: "cassette-lab".to_string(),
724 max_retries: 3,
725 })
726 }
727}
728
729#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
730#[serde(rename_all = "snake_case")]
731enum ComplexMode {
732 Careful,
733 Fast,
734}
735
736#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
737struct ComplexProfile {
738 name: String,
739 tags: Vec<String>,
740}
741
742#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
743struct ComplexArgs {
744 profile: ComplexProfile,
745 mode: ComplexMode,
746 note: Option<String>,
747 quote: String,
748}
749
750#[derive(Debug, Deserialize, Serialize, JsonSchema)]
751struct ExtractedPerson {
752 #[schemars(required)]
753 first_name: Option<String>,
754 #[schemars(required)]
755 last_name: Option<String>,
756 #[schemars(required)]
757 job: Option<String>,
758}
759
760#[derive(Clone)]
761struct CaptureComplexTool {
762 calls: Arc<AtomicUsize>,
763 captured: Arc<Mutex<Option<ComplexArgs>>>,
764}
765
766impl Tool for CaptureComplexTool {
767 const NAME: &'static str = "store_profile";
768 type Error = ConformanceToolError;
769 type Args = ComplexArgs;
770 type Output = ComplexArgs;
771
772 fn description(&self) -> String {
773 "Store one profile with its nested tags, mode, optional note, and exact quoted text."
774 .to_string()
775 }
776
777 fn parameters(&self) -> serde_json::Value {
778 serde_json::to_value(schemars::schema_for!(ComplexArgs)).unwrap_or_default()
779 }
780
781 async fn call(
782 &self,
783 _context: &mut ToolContext,
784 args: Self::Args,
785 ) -> Result<Self::Output, Self::Error> {
786 self.calls.fetch_add(1, Ordering::SeqCst);
787 *lock_recover(&self.captured) = Some(args.clone());
788 Ok(args)
789 }
790}
791
792fn has_tool_roundtrip(messages: &[Message]) -> bool {
793 let saw_call = messages.iter().any(|message| {
794 matches!(
795 message,
796 Message::Assistant(rig_core::message::AssistantMessage { content, .. })
797 if content.iter().any(|item| matches!(item, AssistantContent::ToolCall(_)))
798 )
799 });
800 let saw_result = messages.iter().any(|message| {
801 matches!(
802 message,
803 Message::User { content }
804 if content.iter().any(|item| matches!(item, UserContent::ToolResult(_)))
805 )
806 });
807 saw_call && saw_result
808}
809
810#[derive(Debug, Deserialize, JsonSchema)]
811struct RepeatArgs {
812 text: String,
814 times: Option<u32>,
816}
817
818#[derive(Clone)]
819struct RepeatTool {
820 calls: Arc<AtomicUsize>,
821}
822
823impl Tool for RepeatTool {
824 const NAME: &'static str = "repeat_text";
825 type Error = ConformanceToolError;
826 type Args = RepeatArgs;
827 type Output = String;
828
829 fn description(&self) -> String {
830 "Repeat `text`. `times` is optional and defaults to 2.".to_string()
831 }
832
833 fn parameters(&self) -> serde_json::Value {
834 serde_json::to_value(schemars::schema_for!(RepeatArgs)).unwrap_or_default()
835 }
836
837 async fn call(
838 &self,
839 _context: &mut ToolContext,
840 args: Self::Args,
841 ) -> Result<Self::Output, Self::Error> {
842 self.calls.fetch_add(1, Ordering::SeqCst);
843 Ok(vec![args.text.as_str(); args.times.unwrap_or(2) as usize].join(" "))
844 }
845}
846
847#[derive(Debug, Deserialize, JsonSchema)]
848struct BinOpArgs {
849 a: i64,
850 b: i64,
851}
852
853#[derive(Debug, Deserialize, JsonSchema)]
854struct ArithmeticResult {
855 answer: i64,
856 explanation: Option<String>,
857}
858
859#[derive(Clone)]
860struct AddTool(Arc<AtomicUsize>);
861
862impl Tool for AddTool {
863 const NAME: &'static str = "add";
864 type Error = ConformanceToolError;
865 type Args = BinOpArgs;
866 type Output = i64;
867
868 fn description(&self) -> String {
869 "Add two integers a and b.".to_string()
870 }
871
872 fn parameters(&self) -> serde_json::Value {
873 serde_json::to_value(schemars::schema_for!(BinOpArgs)).unwrap_or_default()
874 }
875
876 async fn call(
877 &self,
878 _context: &mut ToolContext,
879 args: Self::Args,
880 ) -> Result<Self::Output, Self::Error> {
881 self.0.fetch_add(1, Ordering::SeqCst);
882 Ok(args.a + args.b)
883 }
884}
885
886#[derive(Clone)]
887struct MultiplyTool(Arc<AtomicUsize>);
888
889impl Tool for MultiplyTool {
890 const NAME: &'static str = "multiply";
891 type Error = ConformanceToolError;
892 type Args = BinOpArgs;
893 type Output = i64;
894
895 fn description(&self) -> String {
896 "Multiply two integers a and b.".to_string()
897 }
898
899 fn parameters(&self) -> serde_json::Value {
900 serde_json::to_value(schemars::schema_for!(BinOpArgs)).unwrap_or_default()
901 }
902
903 async fn call(
904 &self,
905 _context: &mut ToolContext,
906 args: Self::Args,
907 ) -> Result<Self::Output, Self::Error> {
908 self.0.fetch_add(1, Ordering::SeqCst);
909 Ok(args.a * args.b)
910 }
911}
912
913fn report_from_response(
914 name: &'static str,
915 started: Instant,
916 tool_calls: usize,
917 response: crate::agent::PromptResponse,
918) -> Result<ScenarioReport, ScenarioError> {
919 validate_protocol_hygiene(
920 name,
921 &response.output(),
922 &response.messages,
923 &[
924 "<tool_call>",
925 "</tool_call>",
926 "<tool_response>",
927 "</tool_response>",
928 "<|im_start|>",
929 "<|im_end|>",
930 "<think>",
931 "</think>",
932 ],
933 )?;
934 Ok(ScenarioReport {
935 name,
936 tool_calls,
937 prompt_tokens: response.usage.input_tokens.unwrap_or(0),
938 generated_tokens: response.usage.output_tokens.unwrap_or(0),
939 history_messages: response.messages.len(),
940 duration: started.elapsed(),
941 response: response.output(),
942 })
943}
944
945fn correlated_messages<'a>(
948 scenario: &'static str,
949 response: &'a crate::agent::PromptResponse,
950) -> Result<&'a [Message], ScenarioError> {
951 let messages = response.messages.as_slice();
952 if messages.is_empty() {
953 return Err(ScenarioError::contract(
954 scenario,
955 "extended run omitted accumulated message history",
956 ));
957 }
958 validate_tool_correlation(scenario, messages)?;
959 Ok(messages)
960}
961
962fn correlated_result_values(
964 scenario: &'static str,
965 response: &crate::agent::PromptResponse,
966) -> Result<Vec<serde_json::Value>, ScenarioError> {
967 Ok(correlated_messages(scenario, response)?
968 .iter()
969 .flat_map(tool_result_values)
970 .collect())
971}
972
973fn value_matches_integer(value: &serde_json::Value, expected: i64) -> bool {
974 value.as_i64() == Some(expected)
975 || value
976 .as_str()
977 .and_then(|text| text.trim().parse::<i64>().ok())
978 == Some(expected)
979}
980
981pub async fn parallel_tools<W, T, F>(
987 model: rig_core::driver::Model<W, T>,
988 configure: F,
989 tool_concurrency: Option<usize>,
990) -> Result<ScenarioReport, ScenarioError>
991where
992 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
993 T: rig_core::driver::Transport<W>,
994 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
995{
996 let add_calls = Arc::new(AtomicUsize::new(0));
997 let subtract_calls = Arc::new(AtomicUsize::new(0));
998 let started = Instant::now();
999 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1000 .preamble(FORCE_TOOLS_PREAMBLE)
1001 .tool(CountingAdd(add_calls.clone()))
1002 .tool(CountingSubtract(subtract_calls.clone()))
1003 .default_max_turns(3)
1004 .build();
1005 let request = agent.prompt(PARALLEL_PROMPT).max_turns(3);
1006 let response = match tool_concurrency {
1007 Some(concurrency) => request.tool_concurrency(concurrency).await?,
1008 None => request.await?,
1009 };
1010 let scenario = if tool_concurrency == Some(1) {
1011 "parallel_tools_serial_execution"
1012 } else {
1013 "parallel_tools"
1014 };
1015 let messages = correlated_messages(scenario, &response)?;
1016
1017 let Some((call_index, calls)) = messages.iter().enumerate().find_map(|(index, message)| {
1018 let Message::Assistant(rig_core::message::AssistantMessage { content, .. }) = message
1019 else {
1020 return None;
1021 };
1022 let calls = content
1023 .iter()
1024 .filter_map(|item| match item {
1025 AssistantContent::ToolCall(call) => Some(call),
1026 _ => None,
1027 })
1028 .collect::<Vec<_>>();
1029 (calls.len() == 2).then_some((index, calls))
1030 }) else {
1031 return Err(ScenarioError::contract(
1032 scenario,
1033 format!("no assistant turn contained exactly two tool calls: {messages:?}"),
1034 ));
1035 };
1036 let mut names = calls
1037 .iter()
1038 .map(|call| call.function.name.as_str())
1039 .collect::<Vec<_>>();
1040 names.sort_unstable();
1041 if names != ["add", "subtract"] {
1042 return Err(ScenarioError::contract(
1043 scenario,
1044 format!("parallel turn called {names:?}, expected add and subtract"),
1045 ));
1046 }
1047 let results_message = messages.get(call_index + 1).ok_or_else(|| {
1048 ScenarioError::contract(
1049 scenario,
1050 "parallel call turn has no following result message",
1051 )
1052 })?;
1053 let values = tool_result_values(results_message);
1054 if !values.iter().any(|value| value_matches_integer(value, 7))
1055 || !values.iter().any(|value| value_matches_integer(value, 8))
1056 || values.len() != 2
1057 {
1058 return Err(ScenarioError::contract(
1059 scenario,
1060 format!("parallel result message did not contain exactly 7 and 8: {values:?}"),
1061 ));
1062 }
1063 let add = add_calls.load(Ordering::SeqCst);
1064 let subtract = subtract_calls.load(Ordering::SeqCst);
1065 if add != 1 || subtract != 1 {
1066 return Err(ScenarioError::contract(
1067 scenario,
1068 format!("execution counts were add={add}, subtract={subtract}, expected one each"),
1069 ));
1070 }
1071 report_from_response(scenario, started, add + subtract, response)
1072}
1073
1074pub async fn zero_argument_tool<W, T, F>(
1076 model: rig_core::driver::Model<W, T>,
1077 configure: F,
1078) -> Result<ScenarioReport, ScenarioError>
1079where
1080 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1081 T: rig_core::driver::Transport<W>,
1082 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1083{
1084 const SCENARIO: &str = "zero_argument_tool";
1085 let calls = Arc::new(AtomicUsize::new(0));
1086 let started = Instant::now();
1087 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1088 .preamble("You must use the provided tools. Report tool outputs exactly as returned.")
1089 .tool(PingTool(calls.clone()))
1090 .default_max_turns(2)
1091 .build();
1092 let response = agent
1093 .prompt("Call the ping tool, then report the exact marker it returns.")
1094 .max_turns(2)
1095 .await?;
1096 let values = correlated_result_values(SCENARIO, &response)?;
1097 if calls.load(Ordering::SeqCst) != 1
1098 || !values
1099 .iter()
1100 .any(|value| value.as_str() == Some(PING_OUTPUT))
1101 || !response.output().contains(PING_OUTPUT)
1102 {
1103 return Err(ScenarioError::contract(
1104 SCENARIO,
1105 format!(
1106 "calls={}, results={values:?}, response={:?}",
1107 calls.load(Ordering::SeqCst),
1108 response.output()
1109 ),
1110 ));
1111 }
1112 report_from_response(SCENARIO, started, 1, response)
1113}
1114
1115pub async fn tool_output_serialization<W, T, F>(
1118 model: rig_core::driver::Model<W, T>,
1119 configure: F,
1120) -> Result<ScenarioReport, ScenarioError>
1121where
1122 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1123 T: rig_core::driver::Transport<W>,
1124 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1125{
1126 const SCENARIO: &str = "tool_output_serialization";
1127 let started = Instant::now();
1128 let motto_calls = Arc::new(AtomicUsize::new(0));
1129 let config_calls = Arc::new(AtomicUsize::new(0));
1130 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1131 .preamble("You must use the provided tools before answering.")
1132 .tool(MottoTool(motto_calls.clone()))
1133 .tool(ConfigTool(config_calls.clone()))
1134 .default_max_turns(3)
1135 .build();
1136 let response = agent
1137 .prompt("Call fetch_motto and fetch_config, then summarize both outputs in one sentence.")
1138 .max_turns(3)
1139 .await?;
1140 let values = correlated_result_values(SCENARIO, &response)?;
1141 let expected_config = serde_json::to_value(ConfigOutput {
1142 service: "cassette-lab".to_string(),
1143 max_retries: 3,
1144 })?;
1145 let motto_ok = values
1146 .iter()
1147 .any(|value| value.as_str() == Some(MOTTO_OUTPUT));
1148 let config_ok = values.iter().any(|value| {
1149 value == &expected_config
1150 || value
1151 .as_str()
1152 .and_then(|text| serde_json::from_str::<serde_json::Value>(text).ok())
1153 .as_ref()
1154 == Some(&expected_config)
1155 });
1156 let motto_count = motto_calls.load(Ordering::SeqCst);
1157 let config_count = config_calls.load(Ordering::SeqCst);
1158 if !motto_ok || !config_ok || motto_count != 1 || config_count != 1 {
1159 return Err(ScenarioError::contract(
1160 SCENARIO,
1161 format!(
1162 "expected one verbatim motto and one semantic config JSON; motto_calls={motto_count}, config_calls={config_count}, values={values:?}"
1163 ),
1164 ));
1165 }
1166 report_from_response(SCENARIO, started, motto_count + config_count, response)
1167}
1168
1169pub async fn complex_tool_arguments<W, T, F>(
1172 model: rig_core::driver::Model<W, T>,
1173 configure: F,
1174) -> Result<ScenarioReport, ScenarioError>
1175where
1176 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1177 T: rig_core::driver::Transport<W>,
1178 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1179{
1180 complex_tool_arguments_with_prompt(model, configure, COMPLEX_ARGUMENTS_PROMPT).await
1181}
1182
1183pub const COMPLEX_ARGUMENTS_PROMPT: &str = "Call store_profile with profile.name exactly `Zoë \\\"Z\\\"`, profile.tags exactly [`rust`, `東京`], mode `careful`, note containing the two lines `line one` and `line two` separated by a newline, and quote exactly `path C:\\\\tmp and \\\"quoted\\\"`. Then confirm it was stored.";
1186
1187pub const COMPLEX_ARGUMENTS_RAW_PROMPT: &str = "Call store_profile once. Each value below is raw text between <<< and >>>, with no escaping: profile.name <<<Zoë \"Z\">>>, profile.tags <<<rust>>> and <<<東京>>>, mode <<<careful>>>, note the two lines <<<line one>>> and <<<line two>>> joined by one newline, and quote <<<path C:\\tmp and \"quoted\">>>. Then confirm it was stored.";
1190
1191pub async fn complex_tool_arguments_with_prompt<W, T, F>(
1194 model: rig_core::driver::Model<W, T>,
1195 configure: F,
1196 prompt: &str,
1197) -> Result<ScenarioReport, ScenarioError>
1198where
1199 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1200 T: rig_core::driver::Transport<W>,
1201 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1202{
1203 const SCENARIO: &str = "complex_tool_arguments";
1204 let expected = ComplexArgs {
1205 profile: ComplexProfile {
1206 name: "Zoë \"Z\"".to_string(),
1207 tags: vec!["rust".to_string(), "東京".to_string()],
1208 },
1209 mode: ComplexMode::Careful,
1210 note: Some("line one\nline two".to_string()),
1211 quote: "path C:\\tmp and \"quoted\"".to_string(),
1212 };
1213 let calls = Arc::new(AtomicUsize::new(0));
1214 let captured = Arc::new(Mutex::new(None));
1215 let started = Instant::now();
1216 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1217 .preamble("Use store_profile exactly once with every value supplied by the user.")
1218 .tool(CaptureComplexTool {
1219 calls: calls.clone(),
1220 captured: captured.clone(),
1221 })
1222 .default_max_turns(3)
1223 .build();
1224 let response = agent.prompt(prompt).max_turns(3).await?;
1225 let observed = lock_recover(&captured).clone();
1226 if calls.load(Ordering::SeqCst) != 1 || observed.as_ref() != Some(&expected) {
1227 return Err(ScenarioError::contract(
1228 SCENARIO,
1229 format!(
1230 "calls={}, expected={expected:?}, observed={observed:?}, response={:?}",
1231 calls.load(Ordering::SeqCst),
1232 response.output()
1233 ),
1234 ));
1235 }
1236 correlated_messages(SCENARIO, &response)?;
1237 report_from_response(SCENARIO, started, 1, response)
1238}
1239
1240pub async fn buffered_streaming_text_parity<W, T, A>(
1243 model: rig_core::driver::Model<W, T>,
1244 adjust: A,
1245) -> Result<ScenarioReport, ScenarioError>
1246where
1247 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1248 T: rig_core::driver::Transport<W>,
1249 A: Fn(CompletionRequest) -> CompletionRequest,
1250{
1251 const SCENARIO: &str = "buffered_streaming_text_parity";
1252 const PROMPT: &str = "Answer with exactly the single word Paris.";
1253 let started = Instant::now();
1254 let request = || {
1255 adjust(
1256 CompletionRequest::new(PROMPT)
1257 .temperature(0.0)
1258 .max_tokens(32),
1259 )
1260 };
1261 let buffered = model.call(request()).await?;
1262 let buffered_text = buffered
1263 .choice
1264 .iter()
1265 .filter_map(|item| match item {
1266 AssistantContent::Text(text) => Some(text.text.as_str()),
1267 _ => None,
1268 })
1269 .collect::<String>();
1270
1271 let mut stream = model.stream(request())?;
1272 let mut streamed_text = String::new();
1273 while let Some(item) = stream.next().await {
1274 if let crate::streaming::Item::Event(crate::streaming::StreamEvent::Text { text, .. }) =
1275 item?
1276 {
1277 streamed_text.push_str(&text);
1278 }
1279 }
1280 let streamed_usage = Some(stream.finish().await?.usage);
1281 let usage = streamed_usage.ok_or_else(|| {
1282 ScenarioError::contract(SCENARIO, "raw stream omitted its final response metadata")
1283 })?;
1284 let normalize = |text: &str| {
1285 text.trim()
1286 .trim_matches(|character: char| !character.is_alphanumeric())
1287 .to_string()
1288 };
1289 let buffered_answer = normalize(&buffered_text);
1290 let streamed_answer = normalize(&streamed_text);
1291 if !buffered_answer.eq_ignore_ascii_case("Paris")
1292 || !streamed_answer.eq_ignore_ascii_case("Paris")
1293 || !buffered.usage.is_reported()
1294 || !usage.is_reported()
1295 {
1296 return Err(ScenarioError::contract(
1297 SCENARIO,
1298 format!(
1299 "buffered={buffered_text:?}, streamed={streamed_text:?}, buffered_usage={:?}, streamed_usage={usage:?}",
1300 buffered.usage
1301 ),
1302 ));
1303 }
1304 Ok(ScenarioReport {
1305 name: SCENARIO,
1306 tool_calls: 0,
1307 prompt_tokens: usage.input_tokens.unwrap_or(0),
1308 generated_tokens: usage.output_tokens.unwrap_or(0),
1309 history_messages: 0,
1310 duration: started.elapsed(),
1311 response: streamed_text,
1312 })
1313}
1314
1315pub async fn structured_extraction<W, T>(
1318 model: rig_core::driver::Model<W, T>,
1319 additional_params: Option<serde_json::Value>,
1320) -> Result<ScenarioReport, ScenarioError>
1321where
1322 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1323 T: rig_core::driver::Transport<W>,
1324{
1325 const SCENARIO: &str = "structured_extraction";
1326 const INPUT: &str = "Hello, my name is Ada Lovelace and I work as a mathematician.";
1327 let started = Instant::now();
1328 let mut builder = crate::extractor::ExtractorBuilder::<ExtractedPerson>::new(model)
1329 .max_tokens(384)
1330 .retries(0);
1331 if let Some(params) = additional_params {
1332 builder = builder.additional_params(params);
1333 }
1334 let response = builder.build().extract(INPUT).await?;
1335 validate_extraction_fields(
1336 SCENARIO,
1337 response.output.first_name.as_deref(),
1338 response.output.last_name.as_deref(),
1339 response.output.job.as_deref(),
1340 response.usage,
1341 )?;
1342 Ok(ScenarioReport {
1343 name: SCENARIO,
1344 tool_calls: 1,
1345 prompt_tokens: response.usage.input_tokens.unwrap_or(0),
1346 generated_tokens: response.usage.output_tokens.unwrap_or(0),
1347 history_messages: 0,
1348 duration: started.elapsed(),
1349 response: format!(
1350 "{} {} — {}",
1351 response.output.first_name.as_deref().unwrap_or_default(),
1352 response.output.last_name.as_deref().unwrap_or_default(),
1353 response.output.job.as_deref().unwrap_or_default()
1354 ),
1355 })
1356}
1357
1358fn restricted_recovery_run(
1359 prompt: &str,
1360 turn: ModelTurn,
1361 retries: usize,
1362) -> Result<AgentRun, ScenarioError> {
1363 const SCENARIO: &str = "invalid_tool_recovery";
1364 let mut run = AgentRun::new(prompt)
1365 .max_turns(2)
1366 .max_invalid_tool_call_retries(retries);
1367 if !matches!(run.next_step()?, AgentRunStep::CallModel { .. }) {
1368 return Err(ScenarioError::contract(
1369 SCENARIO,
1370 "fresh AgentRun did not request a model turn",
1371 ));
1372 }
1373 let outcome = run.model_response(turn)?;
1374 let ModelTurnOutcome::NeedsResolution(context) = outcome else {
1375 return Err(ScenarioError::contract(
1376 SCENARIO,
1377 format!("disallowed tool call did not require resolution: {outcome:?}"),
1378 ));
1379 };
1380 if context.tool_name != CountingAdd::NAME {
1381 return Err(ScenarioError::contract(
1382 SCENARIO,
1383 format!("expected rejected add call, observed {context:?}"),
1384 ));
1385 }
1386 Ok(run)
1387}
1388
1389pub async fn invalid_tool_recovery<W, T, F>(
1392 model: rig_core::driver::Model<W, T>,
1393 configure: F,
1394) -> Result<ScenarioReport, ScenarioError>
1395where
1396 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1397 T: rig_core::driver::Transport<W>,
1398 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1399{
1400 invalid_tool_recovery_with_choice(model, configure, ToolChoice::Required).await
1401}
1402
1403pub async fn invalid_tool_recovery_with_choice<W, T, F>(
1407 model: rig_core::driver::Model<W, T>,
1408 configure: F,
1409 tool_choice: ToolChoice,
1410) -> Result<ScenarioReport, ScenarioError>
1411where
1412 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1413 T: rig_core::driver::Transport<W>,
1414 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1415{
1416 const SCENARIO: &str = "invalid_tool_recovery";
1417 const PROMPT: &str = "Call the add tool exactly once with x=2 and y=3. Do not call sum.";
1418 let started = Instant::now();
1419 let add_calls = Arc::new(AtomicUsize::new(0));
1420 let sum_calls = Arc::new(AtomicUsize::new(0));
1421 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1422 .preamble(FORCE_TOOLS_PREAMBLE)
1423 .tool(CountingAdd(add_calls.clone()))
1424 .tool(CountingSum(sum_calls.clone()))
1425 .tool_choice(tool_choice)
1426 .build();
1427 #[derive(Clone)]
1428 struct CaptureTurn(Arc<Mutex<Option<ModelTurn>>>);
1429
1430 impl AgentHook for CaptureTurn {
1431 async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
1432 let Some(response) = event.completion() else {
1433 return OutcomeAction::proceed();
1434 };
1435 *lock_recover(&self.0) = Some(ModelTurn::new(
1436 response.head(),
1437 response.choice.clone(),
1438 response.usage,
1439 BTreeSet::new(),
1440 BTreeSet::new(),
1441 response.raw.clone(),
1442 ));
1443 OutcomeAction::stop("captured conformance model turn")
1444 }
1445 }
1446
1447 let captured = Arc::new(Mutex::new(None));
1448 let stopped = agent
1449 .prompt(PROMPT)
1450 .add_hook(CaptureTurn(captured.clone()))
1451 .run()
1452 .await;
1453 if !matches!(stopped, Err(PromptError::Cancelled { .. })) {
1454 return Err(ScenarioError::contract(
1455 SCENARIO,
1456 format!("capture hook did not stop after the model response: {stopped:?}"),
1457 ));
1458 }
1459 let response = lock_recover(&captured).take().ok_or_else(|| {
1460 ScenarioError::contract(SCENARIO, "capture hook observed no model response")
1461 })?;
1462 let emitted = response
1463 .choice
1464 .iter()
1465 .filter(|item| {
1466 matches!(item, AssistantContent::ToolCall(call) if call.function.name == CountingAdd::NAME)
1467 })
1468 .count();
1469 if emitted != 1 {
1470 return Err(ScenarioError::contract(
1471 SCENARIO,
1472 format!(
1473 "model emitted {emitted} add calls, response={:?}",
1474 response.choice
1475 ),
1476 ));
1477 }
1478 let executable = BTreeSet::from([CountingAdd::NAME.to_string(), CountingSum::NAME.to_string()]);
1479 let allowed = BTreeSet::from([CountingSum::NAME.to_string()]);
1480 let turn = ModelTurn::new(
1481 response.head,
1482 response.choice,
1483 response.usage,
1484 executable,
1485 allowed,
1486 response.raw,
1487 );
1488
1489 let mut fail = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1490 let error = match fail.resolve_invalid_tool_call(InvalidToolCallAction::fail()) {
1491 Err(error) => error,
1492 Ok(outcome) => {
1493 return Err(ScenarioError::contract(
1494 SCENARIO,
1495 format!("fail action unexpectedly returned {outcome:?}"),
1496 ));
1497 }
1498 };
1499 validate_unknown_tool_failure(&error, CountingAdd::NAME, &[CountingSum::NAME])?;
1500
1501 let mut retry = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1502 let error = match retry
1503 .resolve_invalid_tool_call(InvalidToolCallAction::retry("choose an allowed tool"))
1504 {
1505 Err(error) => error,
1506 Ok(outcome) => {
1507 return Err(ScenarioError::contract(
1508 SCENARIO,
1509 format!("exhausted retry unexpectedly returned {outcome:?}"),
1510 ));
1511 }
1512 };
1513 validate_unknown_tool_failure(&error, CountingAdd::NAME, &[CountingSum::NAME])?;
1514
1515 let mut rejected_repair = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1516 let error =
1517 match rejected_repair.resolve_invalid_tool_call(InvalidToolCallAction::repair("missing")) {
1518 Err(error) => error,
1519 Ok(outcome) => {
1520 return Err(ScenarioError::contract(
1521 SCENARIO,
1522 format!("disallowed repair unexpectedly returned {outcome:?}"),
1523 ));
1524 }
1525 };
1526 validate_unknown_tool_failure(&error, "missing", &[CountingSum::NAME])?;
1527
1528 let mut repaired = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1529 if !matches!(
1530 repaired.resolve_invalid_tool_call(InvalidToolCallAction::repair(CountingSum::NAME))?,
1531 ModelTurnOutcome::Continue { .. }
1532 ) {
1533 return Err(ScenarioError::contract(
1534 SCENARIO,
1535 "valid repair did not continue",
1536 ));
1537 }
1538 let AgentRunStep::CallTools { calls } = repaired.next_step()? else {
1539 return Err(ScenarioError::contract(
1540 SCENARIO,
1541 "valid repair did not produce pending tool execution",
1542 ));
1543 };
1544 let repaired_call = calls.first();
1545 if calls.len() != 1
1546 || !repaired_call.is_some_and(|call| {
1547 call.tool_call.function.name == CountingSum::NAME && call.preresolved_result.is_none()
1548 })
1549 {
1550 return Err(ScenarioError::contract(
1551 SCENARIO,
1552 format!("repaired pending calls were incorrect: {calls:?}"),
1553 ));
1554 }
1555
1556 let mut skipped = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1557 if !matches!(
1558 skipped.resolve_invalid_tool_call(InvalidToolCallAction::skip("disabled for this turn"))?,
1559 ModelTurnOutcome::Continue { .. }
1560 ) {
1561 return Err(ScenarioError::contract(SCENARIO, "skip did not continue"));
1562 }
1563 let AgentRunStep::CallTools { calls } = skipped.next_step()? else {
1564 return Err(ScenarioError::contract(
1565 SCENARIO,
1566 "skip did not produce a pre-resolved pending call",
1567 ));
1568 };
1569 let skipped_is_preresolved = match calls.first() {
1570 Some(call) => call.preresolved_result.is_some(),
1571 None => false,
1572 };
1573 if calls.len() != 1 || !skipped_is_preresolved {
1574 return Err(ScenarioError::contract(
1575 SCENARIO,
1576 format!("skipped pending calls were incorrect: {calls:?}"),
1577 ));
1578 }
1579 if add_calls.load(Ordering::SeqCst) != 0 || sum_calls.load(Ordering::SeqCst) != 0 {
1580 return Err(ScenarioError::contract(
1581 SCENARIO,
1582 "recovery scenario executed a tool body",
1583 ));
1584 }
1585
1586 Ok(ScenarioReport {
1587 name: SCENARIO,
1588 tool_calls: emitted,
1589 prompt_tokens: turn.usage.input_tokens.unwrap_or(0),
1590 generated_tokens: turn.usage.output_tokens.unwrap_or(0),
1591 history_messages: 2,
1592 duration: started.elapsed(),
1593 response: "fail, retry, repair, rejected repair, and skip passed".to_string(),
1594 })
1595}
1596
1597pub async fn hook_rewrites_and_request_patch<W, T, F>(
1600 model: rig_core::driver::Model<W, T>,
1601 configure: F,
1602) -> Result<ScenarioReport, ScenarioError>
1603where
1604 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1605 T: rig_core::driver::Transport<W>,
1606 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1607{
1608 hook_rewrites_and_request_patch_with_choice(model, configure, ToolChoice::Required).await
1609}
1610
1611pub async fn hook_rewrites_and_request_patch_with_choice<W, T, F>(
1615 model: rig_core::driver::Model<W, T>,
1616 configure: F,
1617 first_turn_choice: ToolChoice,
1618) -> Result<ScenarioReport, ScenarioError>
1619where
1620 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1621 T: rig_core::driver::Transport<W>,
1622 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1623{
1624 const SCENARIO: &str = "hook_rewrites_and_request_patch";
1625 let started = Instant::now();
1626 let calls = Arc::new(AtomicUsize::new(0));
1627 let observed = ObserveArguments::default();
1628 let observed_probe = observed.clone();
1629 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1630 .preamble("Use add for arithmetic and report only the tool result.")
1631 .tool(CountingAdd(calls.clone()))
1632 .default_max_turns(3)
1633 .build();
1634 let response = agent
1635 .prompt("Use add once for x=1 and y=1, then report what the tool returns.")
1636 .max_turns(3)
1637 .add_hook(FirstTurnPatch(
1638 RequestPatch::new()
1639 .active_tools([CountingAdd::NAME])
1640 .tool_choice(first_turn_choice),
1641 ))
1642 .add_hook(RewriteArgument {
1643 key: "x",
1644 value: serde_json::json!(7),
1645 })
1646 .add_hook(RewriteArgument {
1647 key: "y",
1648 value: serde_json::json!(8),
1649 })
1650 .add_hook(observed)
1651 .add_hook(ReplaceResult("portable-redacted"))
1652 .add_hook(WrapResult)
1653 .await?;
1654 let observations = lock_recover(&observed_probe.0).clone();
1655 validate_rewritten_arguments(
1656 SCENARIO,
1657 &observations,
1658 &serde_json::json!({ "x": 7, "y": 8 }),
1659 )?;
1660 let messages = response.messages.as_slice();
1661 if messages.is_empty() {
1662 return Err(ScenarioError::contract(
1663 SCENARIO,
1664 "extended hook run omitted message history",
1665 ));
1666 }
1667 let results = messages
1668 .iter()
1669 .flat_map(tool_result_values)
1670 .collect::<Vec<_>>();
1671 if calls.load(Ordering::SeqCst) != 1
1672 || !results
1673 .iter()
1674 .any(|value| value == &serde_json::json!("[portable-redacted]"))
1675 || response.completion_calls.len() != 2
1676 {
1677 return Err(ScenarioError::contract(
1678 SCENARIO,
1679 format!(
1680 "calls={}, completion_calls={}, results={results:?}, output={:?}",
1681 calls.load(Ordering::SeqCst),
1682 response.completion_calls.len(),
1683 response.output()
1684 ),
1685 ));
1686 }
1687 report_from_response(SCENARIO, started, 1, response)
1688}
1689
1690pub async fn cancellation_and_max_turns<W, T, F>(
1693 model: rig_core::driver::Model<W, T>,
1694 configure: F,
1695) -> Result<ScenarioReport, ScenarioError>
1696where
1697 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1698 T: rig_core::driver::Transport<W>,
1699 F: Fn(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1700{
1701 const SCENARIO: &str = "cancellation_and_max_turns";
1702 const REASON: &str = "portable result veto";
1703 let started = Instant::now();
1704 let cancelled_calls = Arc::new(AtomicUsize::new(0));
1705 let cancelled_agent = configure(AgentBuilder::new(model.clone()).temperature(0.0))
1706 .preamble("Use add for arithmetic; never calculate by hand.")
1707 .tool(CountingAdd(cancelled_calls.clone()))
1708 .build();
1709 let cancelled = match cancelled_agent
1710 .prompt("Use add once to compute x=20 plus y=22.")
1711 .max_turns(2)
1712 .add_hook(StopAfterResult(REASON))
1713 .await
1714 {
1715 Err(error) => error,
1716 Ok(output) => {
1717 return Err(ScenarioError::contract(
1718 SCENARIO,
1719 format!("result cancellation unexpectedly completed: {output:?}"),
1720 ));
1721 }
1722 };
1723 validate_cancelled_failure(&cancelled, REASON, CountingAdd::NAME)?;
1724
1725 let max_turn_calls = Arc::new(AtomicUsize::new(0));
1726 let max_turn_agent = configure(AgentBuilder::new(model).temperature(0.0))
1727 .preamble("Use add for arithmetic; never calculate by hand.")
1728 .tool(CountingAdd(max_turn_calls.clone()))
1729 .build();
1730 let max_turn = match max_turn_agent
1731 .prompt("Use add once to compute x=20 plus y=22, then report the result.")
1732 .max_turns(1)
1733 .await
1734 {
1735 Err(error) => error,
1736 Ok(output) => {
1737 return Err(ScenarioError::contract(
1738 SCENARIO,
1739 format!("one-turn budget unexpectedly completed: {output:?}"),
1740 ));
1741 }
1742 };
1743 validate_max_turns_failure(&max_turn, 1)?;
1744 let cancelled_count = cancelled_calls.load(Ordering::SeqCst);
1745 let max_turn_count = max_turn_calls.load(Ordering::SeqCst);
1746 if cancelled_count != 1 || max_turn_count != 1 {
1747 return Err(ScenarioError::contract(
1748 SCENARIO,
1749 format!("cancelled executions={cancelled_count}, max-turn executions={max_turn_count}"),
1750 ));
1751 }
1752 Ok(ScenarioReport {
1753 name: SCENARIO,
1754 tool_calls: cancelled_count + max_turn_count,
1755 prompt_tokens: 0,
1756 generated_tokens: 0,
1757 history_messages: 4,
1758 duration: started.elapsed(),
1759 response: "post-result cancellation and max-turn diagnostics passed".to_string(),
1760 })
1761}
1762
1763pub async fn optional_argument<W, T, F>(
1769 model: rig_core::driver::Model<W, T>,
1770 configure: F,
1771) -> Result<ScenarioReport, ScenarioError>
1772where
1773 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1774 T: rig_core::driver::Transport<W>,
1775 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1776{
1777 let calls = Arc::new(AtomicUsize::new(0));
1778 let started = Instant::now();
1779 let agent = configure(AgentBuilder::new(model))
1780 .preamble("Use the repeat_text tool whenever asked to repeat text.")
1781 .tool(RepeatTool {
1782 calls: calls.clone(),
1783 })
1784 .default_max_turns(4)
1785 .build();
1786 let result = agent
1787 .prompt(
1788 "Use the repeat_text tool to repeat the word \"banana\" 3 times, then show me the exact result.",
1789 )
1790 .await?;
1791 let response = result.output();
1792 let tool_calls = calls.load(Ordering::SeqCst);
1793 if tool_calls == 0
1794 || response.matches("banana").count() < 1
1795 || !has_tool_roundtrip(&result.messages)
1796 {
1797 return Err(ScenarioError::contract(
1798 "optional_argument",
1799 format!("calls={tool_calls}, response={response:?}"),
1800 ));
1801 }
1802 report_from_response("optional_argument", started, tool_calls, result)
1803}
1804
1805pub async fn sequential_tools<W, T, F>(
1807 model: rig_core::driver::Model<W, T>,
1808 configure: F,
1809) -> Result<ScenarioReport, ScenarioError>
1810where
1811 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1812 T: rig_core::driver::Transport<W>,
1813 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1814{
1815 let add_calls = Arc::new(AtomicUsize::new(0));
1816 let multiply_calls = Arc::new(AtomicUsize::new(0));
1817 let started = Instant::now();
1818 let agent = configure(AgentBuilder::new(model))
1819 .preamble(
1820 "You are a calculator. Use the add and multiply tools for arithmetic; never compute by hand.",
1821 )
1822 .tool(AddTool(add_calls.clone()))
1823 .tool(MultiplyTool(multiply_calls.clone()))
1824 .default_max_turns(6)
1825 .build();
1826 let result = agent
1827 .prompt(
1828 "Compute (4 + 6) * 2. First call the add tool, then call the multiply tool on the result. Tell me the final number.",
1829 )
1830 .await?;
1831 let response = result.output();
1832 let add = add_calls.load(Ordering::SeqCst);
1833 let multiply = multiply_calls.load(Ordering::SeqCst);
1834 if add == 0
1835 || multiply == 0
1836 || !response.contains("20")
1837 || !has_tool_roundtrip(&result.messages)
1838 {
1839 return Err(ScenarioError::contract(
1840 "sequential_tools",
1841 format!("add={add}, multiply={multiply}, response={response:?}"),
1842 ));
1843 }
1844 report_from_response("sequential_tools", started, add + multiply, result)
1845}
1846
1847pub async fn streaming_tool<W, T, F>(
1849 model: rig_core::driver::Model<W, T>,
1850 configure: F,
1851) -> Result<ScenarioReport, ScenarioError>
1852where
1853 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1854 T: rig_core::driver::Transport<W>,
1855 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1856{
1857 let calls = Arc::new(AtomicUsize::new(0));
1858 let started = Instant::now();
1859 let agent = configure(AgentBuilder::new(model))
1860 .preamble("Use the add tool for arithmetic; do not calculate by hand.")
1861 .tool(AddTool(calls.clone()))
1862 .default_max_turns(4)
1863 .build();
1864 let mut stream = agent
1865 .prompt("Use add to calculate 17 + 25, then state the final number.")
1866 .max_turns(4)
1867 .stream();
1868 let mut final_response = None;
1869 let mut final_count = 0_usize;
1870 let mut completion_usage = crate::completion::Usage::default();
1871 let mut streamed_call_ids = Vec::new();
1872 let mut streamed_result_ids = Vec::new();
1873 while let Some(item) = stream.next().await {
1874 match item? {
1875 MultiTurnStreamItem::ToolCall { tool_call } => streamed_call_ids.push(tool_call.id),
1876 MultiTurnStreamItem::ToolResult { tool_result } => {
1877 streamed_result_ids.push(tool_result.call)
1878 }
1879 MultiTurnStreamItem::CompletionCall(call) => completion_usage += call.usage,
1880 MultiTurnStreamItem::FinalResponse(response) => {
1881 final_count += 1;
1882 final_response = Some(response);
1883 }
1884 MultiTurnStreamItem::StreamAssistantItem(_)
1885 | MultiTurnStreamItem::ToolExecutionCommitted { .. }
1886 | MultiTurnStreamItem::ModelTurnRetried { .. } => {}
1887 }
1888 }
1889 let result = final_response.ok_or_else(|| {
1890 ScenarioError::contract("streaming_tool", "stream produced no final response")
1891 })?;
1892 let response = result.output();
1893 let history_messages = result.messages.len();
1894 let tool_calls = calls.load(Ordering::SeqCst);
1895 streamed_call_ids.sort();
1896 streamed_result_ids.sort();
1897 let correlated_stream =
1898 !streamed_call_ids.is_empty() && streamed_call_ids == streamed_result_ids;
1899 let correlated_history = validate_tool_correlation("streaming_tool", &result.messages).is_ok();
1900 if tool_calls == 0
1901 || !response.contains("42")
1902 || history_messages < 4
1903 || final_count != 1
1904 || !correlated_stream
1905 || !correlated_history
1906 || completion_usage != result.usage
1907 || result.completion_calls.is_empty()
1908 {
1909 return Err(ScenarioError::contract(
1910 "streaming_tool",
1911 format!(
1912 "calls={tool_calls}, final_count={final_count}, streamed_call_ids={streamed_call_ids:?}, streamed_result_ids={streamed_result_ids:?}, completion_usage={completion_usage:?}, final_usage={:?}, history_messages={history_messages}, response={response:?}",
1913 result.usage
1914 ),
1915 ));
1916 }
1917 validate_protocol_hygiene(
1918 "streaming_tool",
1919 &response,
1920 &result.messages,
1921 &["<tool_call>", "</tool_call>", "<think>", "</think>"],
1922 )?;
1923 Ok(ScenarioReport {
1924 name: "streaming_tool",
1925 tool_calls,
1926 prompt_tokens: result.usage.input_tokens.unwrap_or(0),
1927 generated_tokens: result.usage.output_tokens.unwrap_or(0),
1928 history_messages,
1929 duration: started.elapsed(),
1930 response,
1931 })
1932}
1933
1934pub async fn structured_after_tool<W, T, F>(
1936 model: rig_core::driver::Model<W, T>,
1937 configure: F,
1938) -> Result<ScenarioReport, ScenarioError>
1939where
1940 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1941 T: rig_core::driver::Transport<W>,
1942 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1943{
1944 let calls = Arc::new(AtomicUsize::new(0));
1945 let started = Instant::now();
1946 let agent = configure(AgentBuilder::new(model))
1947 .preamble(
1948 "Use add for arithmetic, then finish by calling the structured output tool exactly once.",
1949 )
1950 .output_schema::<ArithmeticResult>()
1951 .output_mode(OutputMode::Tool)
1952 .tool(AddTool(calls.clone()))
1953 .default_max_turns(5)
1954 .build();
1955 let result = agent
1956 .prompt("Use add to calculate 19 + 23. Return answer=42 and a short optional explanation.")
1957 .await?;
1958 let response = result.output();
1959 let parsed: ArithmeticResult = serde_json::from_str(&response)?;
1960 let tool_calls = calls.load(Ordering::SeqCst);
1961 if tool_calls == 0 || parsed.answer != 42 || !has_tool_roundtrip(&result.messages) {
1962 return Err(ScenarioError::contract(
1963 "structured_after_tool",
1964 format!("calls={tool_calls}, response={response:?}"),
1965 ));
1966 }
1967 let _ = parsed.explanation;
1968 report_from_response("structured_after_tool", started, tool_calls + 1, result)
1969}
1970
1971pub async fn tool_choice_modes<W, T, A>(
1973 model: rig_core::driver::Model<W, T>,
1974 adjust: A,
1975) -> Result<ScenarioReport, ScenarioError>
1976where
1977 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1978 T: rig_core::driver::Transport<W>,
1979 A: Fn(CompletionRequest) -> CompletionRequest,
1980{
1981 let tool_name = |name: &str| {
1982 rig_core::message::ToolName::new(name)
1983 .map_err(|error| ScenarioError::contract("tool_choice_modes", error.to_string()))
1984 };
1985 let definition = |name: rig_core::message::ToolName| ToolDefinition {
1986 description: format!("Return the supplied integer using {name}."),
1987 name,
1988 parameters: serde_json::json!({
1989 "type": "object",
1990 "properties": {"value": {"type": "integer"}},
1991 "required": ["value"]
1992 }),
1993 };
1994 let tools = vec![
1995 definition(tool_name("alpha")?),
1996 definition(tool_name("beta")?),
1997 ];
1998 let started = Instant::now();
1999 let none = model
2000 .call(adjust(
2001 CompletionRequest::new("Answer with only the number 4. Do not call a function.")
2002 .tools(tools.clone())
2003 .tool_choice(ToolChoice::None)
2004 .temperature(0.0)
2005 .max_tokens(64),
2006 ))
2007 .await?;
2008 if none
2009 .choice
2010 .iter()
2011 .any(|item| matches!(item, AssistantContent::ToolCall(_)))
2012 {
2013 return Err(ScenarioError::contract(
2014 "tool_choice_modes",
2015 "tool_choice none emitted a tool call",
2016 ));
2017 }
2018
2019 let required = model
2020 .call(adjust(
2021 CompletionRequest::new("Call alpha with value 7.")
2022 .tools(tools.clone())
2023 .tool_choice(ToolChoice::Required)
2024 .temperature(0.0)
2025 .max_tokens(96),
2026 ))
2027 .await?;
2028 let required_calls = required
2029 .choice
2030 .iter()
2031 .filter(|item| matches!(item, AssistantContent::ToolCall(_)))
2032 .count();
2033 if required_calls == 0 {
2034 return Err(ScenarioError::contract(
2035 "tool_choice_modes",
2036 "tool_choice required emitted no tool call",
2037 ));
2038 }
2039
2040 let specific = model
2041 .call(adjust(
2042 CompletionRequest::new("Call beta with value 9.")
2043 .tools(tools)
2044 .tool_choice(ToolChoice::Specific {
2045 function_names: vec![tool_name("beta")?],
2046 })
2047 .temperature(0.0)
2048 .max_tokens(96),
2049 ))
2050 .await?;
2051 let specific_calls = specific
2052 .choice
2053 .iter()
2054 .filter_map(|item| match item {
2055 AssistantContent::ToolCall(call) => Some(call),
2056 _ => None,
2057 })
2058 .collect::<Vec<_>>();
2059 if specific_calls.is_empty()
2060 || specific_calls
2061 .iter()
2062 .any(|call| call.function.name != "beta")
2063 {
2064 return Err(ScenarioError::contract(
2065 "tool_choice_modes",
2066 "specific tool choice did not select only beta",
2067 ));
2068 }
2069
2070 let usage = none.usage + required.usage + specific.usage;
2071 Ok(ScenarioReport {
2072 name: "tool_choice_modes",
2073 tool_calls: required_calls + specific_calls.len(),
2074 prompt_tokens: usage.input_tokens.unwrap_or(0),
2075 generated_tokens: usage.output_tokens.unwrap_or(0),
2076 history_messages: 0,
2077 duration: started.elapsed(),
2078 response: "none, required, and specific modes passed".to_string(),
2079 })
2080}
2081
2082pub async fn streaming_structured_after_tool<W, T, F>(
2084 model: rig_core::driver::Model<W, T>,
2085 configure: F,
2086) -> Result<ScenarioReport, ScenarioError>
2087where
2088 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
2089 T: rig_core::driver::Transport<W>,
2090 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
2091{
2092 let calls = Arc::new(AtomicUsize::new(0));
2093 let started = Instant::now();
2094 let agent = configure(AgentBuilder::new(model))
2095 .preamble(
2096 "Use add for arithmetic, then finish by calling the structured output tool exactly once.",
2097 )
2098 .output_schema::<ArithmeticResult>()
2099 .output_mode(OutputMode::Tool)
2100 .tool(AddTool(calls.clone()))
2101 .default_max_turns(5)
2102 .build();
2103 let mut stream = agent
2104 .prompt("Use add to calculate 19 + 23. Return answer=42 and a short optional explanation.")
2105 .max_turns(5)
2106 .stream();
2107 let mut final_response = None;
2108 let mut final_count = 0_usize;
2109 while let Some(item) = stream.next().await {
2110 if let MultiTurnStreamItem::FinalResponse(response) = item? {
2111 final_count += 1;
2112 final_response = Some(response);
2113 }
2114 }
2115 let result = final_response.ok_or_else(|| {
2116 ScenarioError::contract(
2117 "streaming_structured_after_tool",
2118 "stream produced no final response",
2119 )
2120 })?;
2121 let parsed: ArithmeticResult = serde_json::from_str(&result.output())?;
2122 let calls = calls.load(Ordering::SeqCst);
2123 if calls == 0
2124 || final_count != 1
2125 || parsed.answer != 42
2126 || !has_tool_roundtrip(&result.messages)
2127 {
2128 return Err(ScenarioError::contract(
2129 "streaming_structured_after_tool",
2130 format!(
2131 "calls={calls}, final_count={final_count}, response={:?}",
2132 result.output()
2133 ),
2134 ));
2135 }
2136 report_from_response(
2137 "streaming_structured_after_tool",
2138 started,
2139 calls + 1,
2140 result,
2141 )
2142}
2143
2144#[cfg(test)]
2145mod tests;