1use std::{
8 collections::BTreeSet,
9 sync::{
10 Arc, Mutex, MutexGuard,
11 atomic::{AtomicUsize, Ordering},
12 },
13 time::{Duration, Instant},
14};
15
16use futures::StreamExt;
17use schemars::JsonSchema;
18use serde::{Deserialize, Serialize};
19
20use crate::{
21 agent::{
22 AgentBuilder, AgentHook, CompletionCallAction, CompletionCallEvent, DispatchAction,
23 DispatchEvent, HookContext, InvalidToolCallAction, MultiTurnStreamItem, NoToolConfig,
24 OutcomeAction, OutcomeEvent, OutputMode, RequestPatch, StreamingError,
25 run::{AgentRun, AgentRunStep, ModelTurn, ModelTurnOutcome},
26 },
27 completion::{AssistantContent, Message, PromptError, ToolDefinition},
28 tool::{Tool, ToolContext},
29};
30use rig_core::completion::CompletionRequest;
31use rig_core::error::ProviderError;
32use rig_core::message::{ToolChoice, UserContent};
33
34#[derive(Debug, thiserror::Error)]
36pub enum ScenarioError {
37 #[error(transparent)]
39 Prompt(#[from] PromptError),
40 #[error(transparent)]
42 Completion(#[from] ProviderError),
43 #[error(transparent)]
45 Streaming(#[from] StreamingError),
46 #[error(transparent)]
48 Report(#[from] rig_core::error::ErrorReport),
49 #[error(transparent)]
51 Json(#[from] serde_json::Error),
52 #[error(transparent)]
54 Extraction(#[from] crate::completion::StructuredOutputError),
55 #[error("{scenario} conformance failed: {details}")]
57 Contract {
58 scenario: &'static str,
60 details: String,
62 },
63}
64
65impl ScenarioError {
66 fn contract(scenario: &'static str, details: impl Into<String>) -> Self {
67 Self::Contract {
68 scenario,
69 details: details.into(),
70 }
71 }
72}
73
74pub fn validate_unknown_tool_failure(
77 error: &PromptError,
78 expected_tool: &str,
79 expected_allowed_tools: &[&str],
80) -> Result<(), ScenarioError> {
81 const SCENARIO: &str = "unknown_tool_failure";
82 let PromptError::UnknownToolCall {
83 tool_name,
84 allowed_tools,
85 chat_history,
86 ..
87 } = error
88 else {
89 return Err(ScenarioError::contract(
90 SCENARIO,
91 format!("expected UnknownToolCall, observed {error:?}"),
92 ));
93 };
94 let expected_allowed = expected_allowed_tools
95 .iter()
96 .map(|name| (*name).to_string())
97 .collect::<Vec<_>>();
98 let history_has_call = chat_history.iter().any(|message| {
102 matches!(
103 message,
104 Message::Assistant { content, .. }
105 if content.iter().any(|item| matches!(item, AssistantContent::ToolCall(_)))
106 )
107 });
108 if tool_name != expected_tool || allowed_tools != &expected_allowed || !history_has_call {
109 return Err(ScenarioError::contract(
110 SCENARIO,
111 format!(
112 "tool={tool_name:?}, allowed={allowed_tools:?}, expected_tool={expected_tool:?}, expected_allowed={expected_allowed:?}, history_has_call={history_has_call}, history={chat_history:?}"
113 ),
114 ));
115 }
116 Ok(())
117}
118
119pub fn validate_cancelled_failure(
122 error: &PromptError,
123 expected_reason: &str,
124 expected_tool: &str,
125) -> Result<(), ScenarioError> {
126 const SCENARIO: &str = "cancelled_failure";
127 let PromptError::PromptCancelled {
128 chat_history,
129 reason,
130 } = error
131 else {
132 return Err(ScenarioError::contract(
133 SCENARIO,
134 format!("expected PromptCancelled, observed {error:?}"),
135 ));
136 };
137 let history_has_call = chat_history.iter().any(|message| {
138 matches!(
139 message,
140 Message::Assistant { content, .. }
141 if content.iter().any(|item| matches!(
142 item,
143 AssistantContent::ToolCall(call) if call.function.name == expected_tool
144 ))
145 )
146 });
147 if reason != expected_reason || !history_has_call {
148 return Err(ScenarioError::contract(
149 SCENARIO,
150 format!(
151 "reason={reason:?}, expected={expected_reason:?}, history_has_call={history_has_call}, history={chat_history:?}"
152 ),
153 ));
154 }
155 Ok(())
156}
157
158pub fn validate_max_turns_failure(
161 error: &PromptError,
162 expected_max_turns: usize,
163) -> Result<(), ScenarioError> {
164 const SCENARIO: &str = "max_turns_failure";
165 let PromptError::MaxTurnsError {
166 max_turns,
167 chat_history,
168 prompt,
169 } = error
170 else {
171 return Err(ScenarioError::contract(
172 SCENARIO,
173 format!("expected MaxTurnsError, observed {error:?}"),
174 ));
175 };
176 let pending_prompt_retained = matches!(
177 prompt,
178 Message::User { content } if content.iter().next().is_some()
179 );
180 if *max_turns != expected_max_turns || chat_history.is_empty() || !pending_prompt_retained {
181 return Err(ScenarioError::contract(
182 SCENARIO,
183 format!(
184 "max_turns={max_turns}, expected={expected_max_turns}, history={chat_history:?}, pending_prompt_retained={pending_prompt_retained}, pending_prompt={prompt:?}"
185 ),
186 ));
187 }
188 Ok(())
189}
190
191pub fn decode_structured_output<T>(
194 scenario: &'static str,
195 response: &str,
196) -> Result<T, ScenarioError>
197where
198 T: serde::de::DeserializeOwned,
199{
200 serde_json::from_str(response).map_err(|error| {
201 ScenarioError::contract(
202 scenario,
203 format!("structured output did not decode: {error}; response={response:?}"),
204 )
205 })
206}
207
208pub fn validate_protocol_hygiene(
211 scenario: &'static str,
212 visible_output: &str,
213 messages: &[Message],
214 forbidden_markers: &[&str],
215) -> Result<(), ScenarioError> {
216 let serialized = serde_json::to_string(messages)?;
217 let leaked = forbidden_markers
218 .iter()
219 .filter(|marker| visible_output.contains(**marker) || serialized.contains(**marker))
220 .copied()
221 .collect::<Vec<_>>();
222 if !leaked.is_empty() {
223 return Err(ScenarioError::contract(
224 scenario,
225 format!(
226 "protocol markers leaked: {leaked:?}; output={visible_output:?}, history={messages:?}"
227 ),
228 ));
229 }
230 Ok(())
231}
232
233pub fn validate_rewritten_arguments(
236 scenario: &'static str,
237 observations: &[serde_json::Value],
238 expected_fields: &serde_json::Value,
239) -> Result<(), ScenarioError> {
240 let Some(expected) = expected_fields.as_object() else {
241 return Err(ScenarioError::contract(
242 scenario,
243 "expected rewritten fields must be a JSON object",
244 ));
245 };
246 if observations.is_empty() {
247 return Err(ScenarioError::contract(
248 scenario,
249 "the rewritten tool was never invoked",
250 ));
251 }
252 for observation in observations {
253 let Some(actual) = observation.as_object() else {
254 return Err(ScenarioError::contract(
255 scenario,
256 format!("observed rewritten arguments were not an object: {observation:?}"),
257 ));
258 };
259 for (key, expected_value) in expected {
260 if actual.get(key) != Some(expected_value) {
261 return Err(ScenarioError::contract(
262 scenario,
263 format!(
264 "rewritten field {key:?} expected {expected_value:?}, observed {observation:?}"
265 ),
266 ));
267 }
268 }
269 }
270 Ok(())
271}
272
273pub fn validate_result_redaction(
276 scenario: &'static str,
277 tool_produced_secret: bool,
278 visible_output: &str,
279 secret: &str,
280) -> Result<(), ScenarioError> {
281 if !tool_produced_secret || visible_output.is_empty() || visible_output.contains(secret) {
282 return Err(ScenarioError::contract(
283 scenario,
284 format!(
285 "produced_secret={tool_produced_secret}, secret_visible={}, output={visible_output:?}",
286 visible_output.contains(secret)
287 ),
288 ));
289 }
290 Ok(())
291}
292
293pub fn validate_extraction_fields(
295 scenario: &'static str,
296 first_name: Option<&str>,
297 last_name: Option<&str>,
298 job: Option<&str>,
299 usage: crate::completion::Usage,
300) -> Result<(), ScenarioError> {
301 let fields_match = first_name.is_some_and(|value| value.eq_ignore_ascii_case("Ada"))
302 && last_name.is_some_and(|value| value.eq_ignore_ascii_case("Lovelace"))
303 && job.is_some_and(|value| value.to_ascii_lowercase().contains("mathematician"));
304 if !fields_match || !usage.is_reported() {
305 return Err(ScenarioError::contract(
306 scenario,
307 format!(
308 "first_name={first_name:?}, last_name={last_name:?}, job={job:?}, usage={usage:?}"
309 ),
310 ));
311 }
312 Ok(())
313}
314
315#[derive(Debug, Clone)]
317pub struct ScenarioReport {
318 pub name: &'static str,
320 pub tool_calls: usize,
322 pub prompt_tokens: u64,
325 pub generated_tokens: u64,
328 pub history_messages: usize,
330 pub duration: Duration,
332 pub response: String,
334}
335
336#[derive(Debug, thiserror::Error)]
338#[error("model-conformance tool failed")]
339pub struct ConformanceToolError;
340
341const FORCE_TOOLS_PREAMBLE: &str = "You are a calculator assistant. You MUST use the provided tools for every arithmetic operation instead of computing results yourself. Once you have all the tool results you need, reply with the final numeric answer in plain text.";
342const PARALLEL_PROMPT: &str = "Compute 3 + 4 and 10 - 2. You MUST call the add tool and the subtract tool together in your first response, as two parallel function calls, then report both results.";
343const PING_OUTPUT: &str = "pong-crimson-7423";
344const MOTTO_OUTPUT: &str = "steady hands\ncalm waters";
345
346fn lock_recover<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
347 mutex
348 .lock()
349 .unwrap_or_else(std::sync::PoisonError::into_inner)
350}
351
352fn tool_result_values(message: &Message) -> Vec<serde_json::Value> {
353 let Message::User { content } = message else {
354 return Vec::new();
355 };
356 content
357 .iter()
358 .filter_map(|item| match item {
359 UserContent::ToolResult(result) => Some(result),
360 _ => None,
361 })
362 .flat_map(|result| result.content.iter())
363 .filter_map(|content| match content {
364 rig_core::message::ToolResultContent::Text(text) => {
365 Some(serde_json::Value::String(text.text.clone()))
366 }
367 rig_core::message::ToolResultContent::Json { value } => Some(value.clone()),
368 rig_core::message::ToolResultContent::Image(_) => None,
369 })
370 .collect()
371}
372
373fn validate_tool_correlation(
374 scenario: &'static str,
375 messages: &[Message],
376) -> Result<(), ScenarioError> {
377 let mut calls = Vec::new();
378 let mut results = Vec::new();
379 let mut turn = 0usize;
380 for message in messages {
381 match message {
382 Message::Assistant { content, .. } => {
383 turn += 1;
384 calls.extend(content.iter().filter_map(|item| match item {
385 AssistantContent::ToolCall(call) => Some((
386 turn,
387 &call.id,
388 call.id.provider().map(|provider| provider.call_id.as_str()),
389 )),
390 _ => None,
391 }));
392 }
393 Message::User { content } => {
394 results.extend(content.iter().filter_map(|item| {
395 match item {
396 UserContent::ToolResult(result) => Some((
397 turn,
398 &result.call,
399 result
400 .call
401 .provider()
402 .map(|provider| provider.call_id.as_str()),
403 )),
404 _ => None,
405 }
406 }));
407 }
408 Message::System { .. } => {}
409 }
410 }
411 if calls.is_empty() {
412 return Err(ScenarioError::contract(
413 scenario,
414 format!("history has no assistant tool calls: {messages:?}"),
415 ));
416 }
417 for (turn, id, call_id) in &calls {
418 let matches = results
419 .iter()
420 .filter(|(result_turn, result_id, result_call_id)| {
421 result_turn == turn && result_id == id && call_id == result_call_id
422 })
423 .count();
424 if matches != 1 {
425 return Err(ScenarioError::contract(
426 scenario,
427 format!(
428 "tool call id={id:?} call_id={call_id:?} has {matches} correlated results; calls={calls:?}, results={results:?}"
429 ),
430 ));
431 }
432 }
433 if results.len() != calls.len() {
434 return Err(ScenarioError::contract(
435 scenario,
436 format!(
437 "history contains dangling calls or results: calls={calls:?}, results={results:?}"
438 ),
439 ));
440 }
441 Ok(())
442}
443
444#[derive(Debug, Deserialize, JsonSchema)]
445struct OperationArgs {
446 x: i64,
447 y: i64,
448}
449
450#[derive(Clone)]
451struct CountingAdd(Arc<AtomicUsize>);
452
453impl Tool for CountingAdd {
454 const NAME: &'static str = "add";
455 type Error = ConformanceToolError;
456 type Args = OperationArgs;
457 type Output = i64;
458
459 fn description(&self) -> String {
460 "Add x and y together".to_string()
461 }
462
463 fn parameters(&self) -> serde_json::Value {
464 serde_json::json!({
465 "type": "object",
466 "properties": {
467 "x": { "type": "number", "description": "The first operand" },
468 "y": { "type": "number", "description": "The second operand" }
469 },
470 "required": ["x", "y"]
471 })
472 }
473
474 async fn call(
475 &self,
476 _context: &mut ToolContext,
477 args: Self::Args,
478 ) -> Result<Self::Output, Self::Error> {
479 self.0.fetch_add(1, Ordering::SeqCst);
480 Ok(args.x + args.y)
481 }
482}
483
484#[derive(Clone)]
485struct CountingSum(Arc<AtomicUsize>);
486
487impl Tool for CountingSum {
488 const NAME: &'static str = "sum";
489 type Error = ConformanceToolError;
490 type Args = OperationArgs;
491 type Output = i64;
492
493 fn description(&self) -> String {
494 "Add x and y together (alias of add)".to_string()
495 }
496
497 fn parameters(&self) -> serde_json::Value {
498 CountingAdd(Arc::new(AtomicUsize::new(0))).parameters()
499 }
500
501 async fn call(
502 &self,
503 _context: &mut ToolContext,
504 args: Self::Args,
505 ) -> Result<Self::Output, Self::Error> {
506 self.0.fetch_add(1, Ordering::SeqCst);
507 Ok(args.x + args.y)
508 }
509}
510
511#[derive(Clone)]
512struct CountingSubtract(Arc<AtomicUsize>);
513
514impl Tool for CountingSubtract {
515 const NAME: &'static str = "subtract";
516 type Error = ConformanceToolError;
517 type Args = OperationArgs;
518 type Output = i64;
519
520 fn description(&self) -> String {
521 "Subtract y from x (i.e. x - y)".to_string()
522 }
523
524 fn parameters(&self) -> serde_json::Value {
525 serde_json::json!({
526 "type": "object",
527 "properties": {
528 "x": { "type": "number", "description": "The first operand" },
529 "y": { "type": "number", "description": "The second operand" }
530 },
531 "required": ["x", "y"]
532 })
533 }
534
535 async fn call(
536 &self,
537 _context: &mut ToolContext,
538 args: Self::Args,
539 ) -> Result<Self::Output, Self::Error> {
540 self.0.fetch_add(1, Ordering::SeqCst);
541 Ok(args.x - args.y)
542 }
543}
544
545#[derive(Clone)]
546struct RewriteArgument {
547 key: &'static str,
548 value: serde_json::Value,
549}
550
551impl AgentHook for RewriteArgument {
552 async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
553 let (Some(name), Some(args)) = (event.tool_name(), event.tool_args()) else {
554 return DispatchAction::proceed();
555 };
556 if name != CountingAdd::NAME {
557 return DispatchAction::proceed();
558 }
559 let Ok(mut arguments) = serde_json::from_str::<serde_json::Value>(args) else {
560 return DispatchAction::proceed();
561 };
562 let Some(object) = arguments.as_object_mut() else {
563 return DispatchAction::proceed();
564 };
565 object.insert(self.key.to_string(), self.value.clone());
566 DispatchAction::rewrite_tool_args(event.kind, arguments)
567 }
568}
569
570#[derive(Clone, Default)]
571struct ObserveArguments(Arc<Mutex<Vec<serde_json::Value>>>);
572
573impl AgentHook for ObserveArguments {
574 async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
575 if let Some(args) = event.tool_args() {
576 let value = serde_json::from_str(args)
577 .unwrap_or_else(|_| serde_json::Value::String(args.to_string()));
578 lock_recover(&self.0).push(value);
579 }
580 DispatchAction::proceed()
581 }
582}
583
584#[derive(Clone)]
585struct ReplaceResult(&'static str);
586
587impl AgentHook for ReplaceResult {
588 async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
589 if event.tool_name() == Some(CountingAdd::NAME) {
590 OutcomeAction::rewrite_tool_result(&event, self.0)
591 } else {
592 OutcomeAction::proceed()
593 }
594 }
595}
596
597#[derive(Clone)]
598struct WrapResult;
599
600impl AgentHook for WrapResult {
601 async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
602 match (event.tool_name(), event.tool_result()) {
603 (Some(name), Some(result)) if name == CountingAdd::NAME => {
604 OutcomeAction::rewrite_tool_result(
605 &event,
606 format!("[{}]", result.output().render()),
607 )
608 }
609 _ => OutcomeAction::proceed(),
610 }
611 }
612}
613
614#[derive(Clone)]
615struct FirstTurnPatch(RequestPatch);
616
617impl AgentHook for FirstTurnPatch {
618 async fn on_completion_call(
619 &self,
620 ctx: &HookContext,
621 _event: CompletionCallEvent<'_>,
622 ) -> CompletionCallAction {
623 if ctx.turn() == 1 {
624 CompletionCallAction::patch(self.0.clone())
625 } else {
626 CompletionCallAction::continue_run()
627 }
628 }
629}
630
631#[derive(Clone)]
632struct StopAfterResult(&'static str);
633
634impl AgentHook for StopAfterResult {
635 async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
636 if event.tool_name() == Some(CountingAdd::NAME) {
637 OutcomeAction::stop(self.0)
638 } else {
639 OutcomeAction::proceed()
640 }
641 }
642}
643
644#[derive(Debug, Deserialize, JsonSchema)]
645struct EmptyArgs {}
646
647#[derive(Clone)]
648struct PingTool(Arc<AtomicUsize>);
649
650impl Tool for PingTool {
651 const NAME: &'static str = "ping";
652 type Error = ConformanceToolError;
653 type Args = EmptyArgs;
654 type Output = String;
655
656 fn description(&self) -> String {
657 "Return the current ping marker. Takes no arguments.".to_string()
658 }
659
660 fn parameters(&self) -> serde_json::Value {
661 serde_json::json!({ "type": "object", "properties": {}, "required": [] })
662 }
663
664 async fn call(
665 &self,
666 _context: &mut ToolContext,
667 _args: Self::Args,
668 ) -> Result<Self::Output, Self::Error> {
669 self.0.fetch_add(1, Ordering::SeqCst);
670 Ok(PING_OUTPUT.to_string())
671 }
672}
673
674#[derive(Clone)]
675struct MottoTool(Arc<AtomicUsize>);
676
677impl Tool for MottoTool {
678 const NAME: &'static str = "fetch_motto";
679 type Error = ConformanceToolError;
680 type Args = EmptyArgs;
681 type Output = String;
682
683 fn description(&self) -> String {
684 "Fetch the two-line workshop motto.".to_string()
685 }
686
687 fn parameters(&self) -> serde_json::Value {
688 serde_json::json!({ "type": "object", "properties": {}, "required": [] })
689 }
690
691 async fn call(
692 &self,
693 _context: &mut ToolContext,
694 _args: Self::Args,
695 ) -> Result<Self::Output, Self::Error> {
696 self.0.fetch_add(1, Ordering::SeqCst);
697 Ok(MOTTO_OUTPUT.to_string())
698 }
699}
700
701#[derive(Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
702struct ConfigOutput {
703 service: String,
704 max_retries: u64,
705}
706
707#[derive(Clone)]
708struct ConfigTool(Arc<AtomicUsize>);
709
710impl Tool for ConfigTool {
711 const NAME: &'static str = "fetch_config";
712 type Error = ConformanceToolError;
713 type Args = EmptyArgs;
714 type Output = ConfigOutput;
715
716 fn description(&self) -> String {
717 "Fetch the service configuration object.".to_string()
718 }
719
720 fn parameters(&self) -> serde_json::Value {
721 serde_json::json!({ "type": "object", "properties": {}, "required": [] })
722 }
723
724 async fn call(
725 &self,
726 _context: &mut ToolContext,
727 _args: Self::Args,
728 ) -> Result<Self::Output, Self::Error> {
729 self.0.fetch_add(1, Ordering::SeqCst);
730 Ok(ConfigOutput {
731 service: "cassette-lab".to_string(),
732 max_retries: 3,
733 })
734 }
735}
736
737#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
738#[serde(rename_all = "snake_case")]
739enum ComplexMode {
740 Careful,
741 Fast,
742}
743
744#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
745struct ComplexProfile {
746 name: String,
747 tags: Vec<String>,
748}
749
750#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
751struct ComplexArgs {
752 profile: ComplexProfile,
753 mode: ComplexMode,
754 note: Option<String>,
755 quote: String,
756}
757
758#[derive(Debug, Deserialize, Serialize, JsonSchema)]
759struct ExtractedPerson {
760 #[schemars(required)]
761 first_name: Option<String>,
762 #[schemars(required)]
763 last_name: Option<String>,
764 #[schemars(required)]
765 job: Option<String>,
766}
767
768#[derive(Clone)]
769struct CaptureComplexTool {
770 calls: Arc<AtomicUsize>,
771 captured: Arc<Mutex<Option<ComplexArgs>>>,
772}
773
774impl Tool for CaptureComplexTool {
775 const NAME: &'static str = "store_profile";
776 type Error = ConformanceToolError;
777 type Args = ComplexArgs;
778 type Output = ComplexArgs;
779
780 fn description(&self) -> String {
781 "Store one profile with its nested tags, mode, optional note, and exact quoted text."
782 .to_string()
783 }
784
785 fn parameters(&self) -> serde_json::Value {
786 serde_json::to_value(schemars::schema_for!(ComplexArgs)).unwrap_or_default()
787 }
788
789 async fn call(
790 &self,
791 _context: &mut ToolContext,
792 args: Self::Args,
793 ) -> Result<Self::Output, Self::Error> {
794 self.calls.fetch_add(1, Ordering::SeqCst);
795 *lock_recover(&self.captured) = Some(args.clone());
796 Ok(args)
797 }
798}
799
800fn has_tool_roundtrip(messages: Option<&[Message]>) -> bool {
801 let saw_call = messages.is_some_and(|messages| {
802 messages.iter().any(|message| {
803 matches!(
804 message,
805 Message::Assistant { content, .. }
806 if content.iter().any(|item| matches!(item, AssistantContent::ToolCall(_)))
807 )
808 })
809 });
810 let saw_result = messages.is_some_and(|messages| {
811 messages.iter().any(|message| {
812 matches!(
813 message,
814 Message::User { content }
815 if content.iter().any(|item| matches!(item, UserContent::ToolResult(_)))
816 )
817 })
818 });
819 saw_call && saw_result
820}
821
822#[derive(Debug, Deserialize, JsonSchema)]
823struct RepeatArgs {
824 text: String,
826 times: Option<u32>,
828}
829
830#[derive(Clone)]
831struct RepeatTool {
832 calls: Arc<AtomicUsize>,
833}
834
835impl Tool for RepeatTool {
836 const NAME: &'static str = "repeat_text";
837 type Error = ConformanceToolError;
838 type Args = RepeatArgs;
839 type Output = String;
840
841 fn description(&self) -> String {
842 "Repeat `text`. `times` is optional and defaults to 2.".to_string()
843 }
844
845 fn parameters(&self) -> serde_json::Value {
846 serde_json::to_value(schemars::schema_for!(RepeatArgs)).unwrap_or_default()
847 }
848
849 async fn call(
850 &self,
851 _context: &mut ToolContext,
852 args: Self::Args,
853 ) -> Result<Self::Output, Self::Error> {
854 self.calls.fetch_add(1, Ordering::SeqCst);
855 Ok(vec![args.text.as_str(); args.times.unwrap_or(2) as usize].join(" "))
856 }
857}
858
859#[derive(Debug, Deserialize, JsonSchema)]
860struct BinOpArgs {
861 a: i64,
862 b: i64,
863}
864
865#[derive(Debug, Deserialize, JsonSchema)]
866struct ArithmeticResult {
867 answer: i64,
868 explanation: Option<String>,
869}
870
871#[derive(Clone)]
872struct AddTool(Arc<AtomicUsize>);
873
874impl Tool for AddTool {
875 const NAME: &'static str = "add";
876 type Error = ConformanceToolError;
877 type Args = BinOpArgs;
878 type Output = i64;
879
880 fn description(&self) -> String {
881 "Add two integers a and b.".to_string()
882 }
883
884 fn parameters(&self) -> serde_json::Value {
885 serde_json::to_value(schemars::schema_for!(BinOpArgs)).unwrap_or_default()
886 }
887
888 async fn call(
889 &self,
890 _context: &mut ToolContext,
891 args: Self::Args,
892 ) -> Result<Self::Output, Self::Error> {
893 self.0.fetch_add(1, Ordering::SeqCst);
894 Ok(args.a + args.b)
895 }
896}
897
898#[derive(Clone)]
899struct MultiplyTool(Arc<AtomicUsize>);
900
901impl Tool for MultiplyTool {
902 const NAME: &'static str = "multiply";
903 type Error = ConformanceToolError;
904 type Args = BinOpArgs;
905 type Output = i64;
906
907 fn description(&self) -> String {
908 "Multiply two integers a and b.".to_string()
909 }
910
911 fn parameters(&self) -> serde_json::Value {
912 serde_json::to_value(schemars::schema_for!(BinOpArgs)).unwrap_or_default()
913 }
914
915 async fn call(
916 &self,
917 _context: &mut ToolContext,
918 args: Self::Args,
919 ) -> Result<Self::Output, Self::Error> {
920 self.0.fetch_add(1, Ordering::SeqCst);
921 Ok(args.a * args.b)
922 }
923}
924
925fn report_from_response(
926 name: &'static str,
927 started: Instant,
928 tool_calls: usize,
929 response: crate::agent::PromptResponse,
930) -> Result<ScenarioReport, ScenarioError> {
931 if let Some(messages) = response.messages.as_deref() {
932 validate_protocol_hygiene(
933 name,
934 &response.output,
935 messages,
936 &[
937 "<tool_call>",
938 "</tool_call>",
939 "<tool_response>",
940 "</tool_response>",
941 "<|im_start|>",
942 "<|im_end|>",
943 "<think>",
944 "</think>",
945 ],
946 )?;
947 }
948 Ok(ScenarioReport {
949 name,
950 tool_calls,
951 prompt_tokens: response.usage.input_tokens.unwrap_or(0),
952 generated_tokens: response.usage.output_tokens.unwrap_or(0),
953 history_messages: response.messages.as_ref().map_or(0, Vec::len),
954 duration: started.elapsed(),
955 response: response.output,
956 })
957}
958
959fn correlated_messages<'a>(
962 scenario: &'static str,
963 response: &'a crate::agent::PromptResponse,
964) -> Result<&'a [Message], ScenarioError> {
965 let messages = response.messages.as_deref().ok_or_else(|| {
966 ScenarioError::contract(scenario, "extended run omitted accumulated message history")
967 })?;
968 validate_tool_correlation(scenario, messages)?;
969 Ok(messages)
970}
971
972fn correlated_result_values(
974 scenario: &'static str,
975 response: &crate::agent::PromptResponse,
976) -> Result<Vec<serde_json::Value>, ScenarioError> {
977 Ok(correlated_messages(scenario, response)?
978 .iter()
979 .flat_map(tool_result_values)
980 .collect())
981}
982
983fn value_matches_integer(value: &serde_json::Value, expected: i64) -> bool {
984 value.as_i64() == Some(expected)
985 || value
986 .as_str()
987 .and_then(|text| text.trim().parse::<i64>().ok())
988 == Some(expected)
989}
990
991pub async fn parallel_tools<W, T, F>(
997 model: rig_core::driver::Model<W, T>,
998 configure: F,
999 tool_concurrency: Option<usize>,
1000) -> Result<ScenarioReport, ScenarioError>
1001where
1002 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1003 T: rig_core::driver::Transport<W>,
1004 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1005{
1006 let add_calls = Arc::new(AtomicUsize::new(0));
1007 let subtract_calls = Arc::new(AtomicUsize::new(0));
1008 let started = Instant::now();
1009 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1010 .preamble(FORCE_TOOLS_PREAMBLE)
1011 .tool(CountingAdd(add_calls.clone()))
1012 .tool(CountingSubtract(subtract_calls.clone()))
1013 .default_max_turns(3)
1014 .build();
1015 let request = agent.prompt(PARALLEL_PROMPT).max_turns(3);
1016 let response = match tool_concurrency {
1017 Some(concurrency) => request.tool_concurrency(concurrency).await?,
1018 None => request.await?,
1019 };
1020 let scenario = if tool_concurrency == Some(1) {
1021 "parallel_tools_serial_execution"
1022 } else {
1023 "parallel_tools"
1024 };
1025 let messages = correlated_messages(scenario, &response)?;
1026
1027 let Some((call_index, calls)) = messages.iter().enumerate().find_map(|(index, message)| {
1028 let Message::Assistant { content, .. } = message else {
1029 return None;
1030 };
1031 let calls = content
1032 .iter()
1033 .filter_map(|item| match item {
1034 AssistantContent::ToolCall(call) => Some(call),
1035 _ => None,
1036 })
1037 .collect::<Vec<_>>();
1038 (calls.len() == 2).then_some((index, calls))
1039 }) else {
1040 return Err(ScenarioError::contract(
1041 scenario,
1042 format!("no assistant turn contained exactly two tool calls: {messages:?}"),
1043 ));
1044 };
1045 let mut names = calls
1046 .iter()
1047 .map(|call| call.function.name.as_str())
1048 .collect::<Vec<_>>();
1049 names.sort_unstable();
1050 if names != ["add", "subtract"] {
1051 return Err(ScenarioError::contract(
1052 scenario,
1053 format!("parallel turn called {names:?}, expected add and subtract"),
1054 ));
1055 }
1056 let results_message = messages.get(call_index + 1).ok_or_else(|| {
1057 ScenarioError::contract(
1058 scenario,
1059 "parallel call turn has no following result message",
1060 )
1061 })?;
1062 let values = tool_result_values(results_message);
1063 if !values.iter().any(|value| value_matches_integer(value, 7))
1064 || !values.iter().any(|value| value_matches_integer(value, 8))
1065 || values.len() != 2
1066 {
1067 return Err(ScenarioError::contract(
1068 scenario,
1069 format!("parallel result message did not contain exactly 7 and 8: {values:?}"),
1070 ));
1071 }
1072 let add = add_calls.load(Ordering::SeqCst);
1073 let subtract = subtract_calls.load(Ordering::SeqCst);
1074 if add != 1 || subtract != 1 {
1075 return Err(ScenarioError::contract(
1076 scenario,
1077 format!("execution counts were add={add}, subtract={subtract}, expected one each"),
1078 ));
1079 }
1080 report_from_response(scenario, started, add + subtract, response)
1081}
1082
1083pub async fn zero_argument_tool<W, T, F>(
1085 model: rig_core::driver::Model<W, T>,
1086 configure: F,
1087) -> Result<ScenarioReport, ScenarioError>
1088where
1089 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1090 T: rig_core::driver::Transport<W>,
1091 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1092{
1093 const SCENARIO: &str = "zero_argument_tool";
1094 let calls = Arc::new(AtomicUsize::new(0));
1095 let started = Instant::now();
1096 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1097 .preamble("You must use the provided tools. Report tool outputs exactly as returned.")
1098 .tool(PingTool(calls.clone()))
1099 .default_max_turns(2)
1100 .build();
1101 let response = agent
1102 .prompt("Call the ping tool, then report the exact marker it returns.")
1103 .max_turns(2)
1104 .await?;
1105 let values = correlated_result_values(SCENARIO, &response)?;
1106 if calls.load(Ordering::SeqCst) != 1
1107 || !values
1108 .iter()
1109 .any(|value| value.as_str() == Some(PING_OUTPUT))
1110 || !response.output.contains(PING_OUTPUT)
1111 {
1112 return Err(ScenarioError::contract(
1113 SCENARIO,
1114 format!(
1115 "calls={}, results={values:?}, response={:?}",
1116 calls.load(Ordering::SeqCst),
1117 response.output
1118 ),
1119 ));
1120 }
1121 report_from_response(SCENARIO, started, 1, response)
1122}
1123
1124pub async fn tool_output_serialization<W, T, F>(
1127 model: rig_core::driver::Model<W, T>,
1128 configure: F,
1129) -> Result<ScenarioReport, ScenarioError>
1130where
1131 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1132 T: rig_core::driver::Transport<W>,
1133 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1134{
1135 const SCENARIO: &str = "tool_output_serialization";
1136 let started = Instant::now();
1137 let motto_calls = Arc::new(AtomicUsize::new(0));
1138 let config_calls = Arc::new(AtomicUsize::new(0));
1139 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1140 .preamble("You must use the provided tools before answering.")
1141 .tool(MottoTool(motto_calls.clone()))
1142 .tool(ConfigTool(config_calls.clone()))
1143 .default_max_turns(3)
1144 .build();
1145 let response = agent
1146 .prompt("Call fetch_motto and fetch_config, then summarize both outputs in one sentence.")
1147 .max_turns(3)
1148 .await?;
1149 let values = correlated_result_values(SCENARIO, &response)?;
1150 let expected_config = serde_json::to_value(ConfigOutput {
1151 service: "cassette-lab".to_string(),
1152 max_retries: 3,
1153 })?;
1154 let motto_ok = values
1155 .iter()
1156 .any(|value| value.as_str() == Some(MOTTO_OUTPUT));
1157 let config_ok = values.iter().any(|value| {
1158 value == &expected_config
1159 || value
1160 .as_str()
1161 .and_then(|text| serde_json::from_str::<serde_json::Value>(text).ok())
1162 .as_ref()
1163 == Some(&expected_config)
1164 });
1165 let motto_count = motto_calls.load(Ordering::SeqCst);
1166 let config_count = config_calls.load(Ordering::SeqCst);
1167 if !motto_ok || !config_ok || motto_count != 1 || config_count != 1 {
1168 return Err(ScenarioError::contract(
1169 SCENARIO,
1170 format!(
1171 "expected one verbatim motto and one semantic config JSON; motto_calls={motto_count}, config_calls={config_count}, values={values:?}"
1172 ),
1173 ));
1174 }
1175 report_from_response(SCENARIO, started, motto_count + config_count, response)
1176}
1177
1178pub async fn complex_tool_arguments<W, T, F>(
1181 model: rig_core::driver::Model<W, T>,
1182 configure: F,
1183) -> Result<ScenarioReport, ScenarioError>
1184where
1185 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1186 T: rig_core::driver::Transport<W>,
1187 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1188{
1189 complex_tool_arguments_with_prompt(model, configure, COMPLEX_ARGUMENTS_PROMPT).await
1190}
1191
1192pub const COMPLEX_ARGUMENTS_PROMPT: &str = "Call store_profile with profile.name exactly `Zoë \\\"Z\\\"`, profile.tags exactly [`rust`, `東京`], mode `careful`, note containing the two lines `line one` and `line two` separated by a newline, and quote exactly `path C:\\\\tmp and \\\"quoted\\\"`. Then confirm it was stored.";
1195
1196pub const COMPLEX_ARGUMENTS_RAW_PROMPT: &str = "Call store_profile once. Each value below is raw text between <<< and >>>, with no escaping: profile.name <<<Zoë \"Z\">>>, profile.tags <<<rust>>> and <<<東京>>>, mode <<<careful>>>, note the two lines <<<line one>>> and <<<line two>>> joined by one newline, and quote <<<path C:\\tmp and \"quoted\">>>. Then confirm it was stored.";
1199
1200pub async fn complex_tool_arguments_with_prompt<W, T, F>(
1203 model: rig_core::driver::Model<W, T>,
1204 configure: F,
1205 prompt: &str,
1206) -> Result<ScenarioReport, ScenarioError>
1207where
1208 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1209 T: rig_core::driver::Transport<W>,
1210 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1211{
1212 const SCENARIO: &str = "complex_tool_arguments";
1213 let expected = ComplexArgs {
1214 profile: ComplexProfile {
1215 name: "Zoë \"Z\"".to_string(),
1216 tags: vec!["rust".to_string(), "東京".to_string()],
1217 },
1218 mode: ComplexMode::Careful,
1219 note: Some("line one\nline two".to_string()),
1220 quote: "path C:\\tmp and \"quoted\"".to_string(),
1221 };
1222 let calls = Arc::new(AtomicUsize::new(0));
1223 let captured = Arc::new(Mutex::new(None));
1224 let started = Instant::now();
1225 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1226 .preamble("Use store_profile exactly once with every value supplied by the user.")
1227 .tool(CaptureComplexTool {
1228 calls: calls.clone(),
1229 captured: captured.clone(),
1230 })
1231 .default_max_turns(3)
1232 .build();
1233 let response = agent.prompt(prompt).max_turns(3).await?;
1234 let observed = lock_recover(&captured).clone();
1235 if calls.load(Ordering::SeqCst) != 1 || observed.as_ref() != Some(&expected) {
1236 return Err(ScenarioError::contract(
1237 SCENARIO,
1238 format!(
1239 "calls={}, expected={expected:?}, observed={observed:?}, response={:?}",
1240 calls.load(Ordering::SeqCst),
1241 response.output
1242 ),
1243 ));
1244 }
1245 correlated_messages(SCENARIO, &response)?;
1246 report_from_response(SCENARIO, started, 1, response)
1247}
1248
1249pub async fn buffered_streaming_text_parity<W, T, A>(
1252 model: rig_core::driver::Model<W, T>,
1253 adjust: A,
1254) -> Result<ScenarioReport, ScenarioError>
1255where
1256 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1257 T: rig_core::driver::Transport<W>,
1258 A: Fn(CompletionRequest) -> CompletionRequest,
1259{
1260 const SCENARIO: &str = "buffered_streaming_text_parity";
1261 const PROMPT: &str = "Answer with exactly the single word Paris.";
1262 let started = Instant::now();
1263 let request = || {
1264 adjust(
1265 CompletionRequest::new(PROMPT)
1266 .temperature(0.0)
1267 .max_tokens(32),
1268 )
1269 };
1270 let buffered = model.call(request()).await?;
1271 let buffered_text = buffered
1272 .choice
1273 .iter()
1274 .filter_map(|item| match item {
1275 AssistantContent::Text(text) => Some(text.text.as_str()),
1276 _ => None,
1277 })
1278 .collect::<String>();
1279
1280 let mut stream = model.stream(request())?;
1281 let mut streamed_text = String::new();
1282 while let Some(item) = stream.next().await {
1283 if let crate::streaming::Item::Event(crate::streaming::StreamEvent::Text { text, .. }) =
1284 item?
1285 {
1286 streamed_text.push_str(&text);
1287 }
1288 }
1289 let streamed_usage = Some(stream.finish().await?.usage);
1290 let usage = streamed_usage.ok_or_else(|| {
1291 ScenarioError::contract(SCENARIO, "raw stream omitted its final response metadata")
1292 })?;
1293 let normalize = |text: &str| {
1294 text.trim()
1295 .trim_matches(|character: char| !character.is_alphanumeric())
1296 .to_string()
1297 };
1298 let buffered_answer = normalize(&buffered_text);
1299 let streamed_answer = normalize(&streamed_text);
1300 if !buffered_answer.eq_ignore_ascii_case("Paris")
1301 || !streamed_answer.eq_ignore_ascii_case("Paris")
1302 || !buffered.usage.is_reported()
1303 || !usage.is_reported()
1304 {
1305 return Err(ScenarioError::contract(
1306 SCENARIO,
1307 format!(
1308 "buffered={buffered_text:?}, streamed={streamed_text:?}, buffered_usage={:?}, streamed_usage={usage:?}",
1309 buffered.usage
1310 ),
1311 ));
1312 }
1313 Ok(ScenarioReport {
1314 name: SCENARIO,
1315 tool_calls: 0,
1316 prompt_tokens: usage.input_tokens.unwrap_or(0),
1317 generated_tokens: usage.output_tokens.unwrap_or(0),
1318 history_messages: 0,
1319 duration: started.elapsed(),
1320 response: streamed_text,
1321 })
1322}
1323
1324pub async fn structured_extraction<W, T>(
1327 model: rig_core::driver::Model<W, T>,
1328 additional_params: Option<serde_json::Value>,
1329) -> Result<ScenarioReport, ScenarioError>
1330where
1331 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1332 T: rig_core::driver::Transport<W>,
1333{
1334 const SCENARIO: &str = "structured_extraction";
1335 const INPUT: &str = "Hello, my name is Ada Lovelace and I work as a mathematician.";
1336 let started = Instant::now();
1337 let mut builder = crate::extractor::ExtractorBuilder::<ExtractedPerson>::new(model)
1338 .max_tokens(384)
1339 .retries(0);
1340 if let Some(params) = additional_params {
1341 builder = builder.additional_params(params);
1342 }
1343 let response = builder.build().extract(INPUT).await?;
1344 validate_extraction_fields(
1345 SCENARIO,
1346 response.output.first_name.as_deref(),
1347 response.output.last_name.as_deref(),
1348 response.output.job.as_deref(),
1349 response.usage,
1350 )?;
1351 Ok(ScenarioReport {
1352 name: SCENARIO,
1353 tool_calls: 1,
1354 prompt_tokens: response.usage.input_tokens.unwrap_or(0),
1355 generated_tokens: response.usage.output_tokens.unwrap_or(0),
1356 history_messages: 0,
1357 duration: started.elapsed(),
1358 response: format!(
1359 "{} {} — {}",
1360 response.output.first_name.as_deref().unwrap_or_default(),
1361 response.output.last_name.as_deref().unwrap_or_default(),
1362 response.output.job.as_deref().unwrap_or_default()
1363 ),
1364 })
1365}
1366
1367fn restricted_recovery_run(
1368 prompt: &str,
1369 turn: ModelTurn,
1370 retries: usize,
1371) -> Result<AgentRun, ScenarioError> {
1372 const SCENARIO: &str = "invalid_tool_recovery";
1373 let mut run = AgentRun::new(prompt)
1374 .max_turns(2)
1375 .max_invalid_tool_call_retries(retries);
1376 if !matches!(run.next_step()?, AgentRunStep::CallModel { .. }) {
1377 return Err(ScenarioError::contract(
1378 SCENARIO,
1379 "fresh AgentRun did not request a model turn",
1380 ));
1381 }
1382 let outcome = run.model_response(turn)?;
1383 let ModelTurnOutcome::NeedsResolution(context) = outcome else {
1384 return Err(ScenarioError::contract(
1385 SCENARIO,
1386 format!("disallowed tool call did not require resolution: {outcome:?}"),
1387 ));
1388 };
1389 if context.tool_name != CountingAdd::NAME {
1390 return Err(ScenarioError::contract(
1391 SCENARIO,
1392 format!("expected rejected add call, observed {context:?}"),
1393 ));
1394 }
1395 Ok(run)
1396}
1397
1398pub async fn invalid_tool_recovery<W, T, F>(
1401 model: rig_core::driver::Model<W, T>,
1402 configure: F,
1403) -> Result<ScenarioReport, ScenarioError>
1404where
1405 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1406 T: rig_core::driver::Transport<W>,
1407 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1408{
1409 invalid_tool_recovery_with_choice(model, configure, ToolChoice::Required).await
1410}
1411
1412pub async fn invalid_tool_recovery_with_choice<W, T, F>(
1416 model: rig_core::driver::Model<W, T>,
1417 configure: F,
1418 tool_choice: ToolChoice,
1419) -> Result<ScenarioReport, ScenarioError>
1420where
1421 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1422 T: rig_core::driver::Transport<W>,
1423 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1424{
1425 const SCENARIO: &str = "invalid_tool_recovery";
1426 const PROMPT: &str = "Call the add tool exactly once with x=2 and y=3. Do not call sum.";
1427 let started = Instant::now();
1428 let add_calls = Arc::new(AtomicUsize::new(0));
1429 let sum_calls = Arc::new(AtomicUsize::new(0));
1430 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1431 .preamble(FORCE_TOOLS_PREAMBLE)
1432 .tool(CountingAdd(add_calls.clone()))
1433 .tool(CountingSum(sum_calls.clone()))
1434 .tool_choice(tool_choice)
1435 .build();
1436 #[derive(Clone)]
1437 struct CaptureTurn(Arc<Mutex<Option<ModelTurn>>>);
1438
1439 impl AgentHook for CaptureTurn {
1440 async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
1441 let Some(response) = event.completion() else {
1442 return OutcomeAction::proceed();
1443 };
1444 *lock_recover(&self.0) = Some(ModelTurn::new(
1445 response.message_id.clone(),
1446 response.choice.clone(),
1447 response.usage,
1448 BTreeSet::new(),
1449 BTreeSet::new(),
1450 response.raw.clone(),
1451 ));
1452 OutcomeAction::stop("captured conformance model turn")
1453 }
1454 }
1455
1456 let captured = Arc::new(Mutex::new(None));
1457 let stopped = agent
1458 .prompt(PROMPT)
1459 .add_hook(CaptureTurn(captured.clone()))
1460 .run()
1461 .await;
1462 if !matches!(stopped, Err(PromptError::PromptCancelled { .. })) {
1463 return Err(ScenarioError::contract(
1464 SCENARIO,
1465 format!("capture hook did not stop after the model response: {stopped:?}"),
1466 ));
1467 }
1468 let response = lock_recover(&captured).take().ok_or_else(|| {
1469 ScenarioError::contract(SCENARIO, "capture hook observed no model response")
1470 })?;
1471 let emitted = response
1472 .choice
1473 .iter()
1474 .filter(|item| {
1475 matches!(item, AssistantContent::ToolCall(call) if call.function.name == CountingAdd::NAME)
1476 })
1477 .count();
1478 if emitted != 1 {
1479 return Err(ScenarioError::contract(
1480 SCENARIO,
1481 format!(
1482 "model emitted {emitted} add calls, response={:?}",
1483 response.choice
1484 ),
1485 ));
1486 }
1487 let executable = BTreeSet::from([CountingAdd::NAME.to_string(), CountingSum::NAME.to_string()]);
1488 let allowed = BTreeSet::from([CountingSum::NAME.to_string()]);
1489 let turn = ModelTurn::new(
1490 response.message_id,
1491 response.choice,
1492 response.usage,
1493 executable,
1494 allowed,
1495 response.raw,
1496 );
1497
1498 let mut fail = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1499 let error = match fail.resolve_invalid_tool_call(InvalidToolCallAction::fail()) {
1500 Err(error) => error,
1501 Ok(outcome) => {
1502 return Err(ScenarioError::contract(
1503 SCENARIO,
1504 format!("fail action unexpectedly returned {outcome:?}"),
1505 ));
1506 }
1507 };
1508 validate_unknown_tool_failure(&error, CountingAdd::NAME, &[CountingSum::NAME])?;
1509
1510 let mut retry = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1511 let error = match retry
1512 .resolve_invalid_tool_call(InvalidToolCallAction::retry("choose an allowed tool"))
1513 {
1514 Err(error) => error,
1515 Ok(outcome) => {
1516 return Err(ScenarioError::contract(
1517 SCENARIO,
1518 format!("exhausted retry unexpectedly returned {outcome:?}"),
1519 ));
1520 }
1521 };
1522 validate_unknown_tool_failure(&error, CountingAdd::NAME, &[CountingSum::NAME])?;
1523
1524 let mut rejected_repair = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1525 let error =
1526 match rejected_repair.resolve_invalid_tool_call(InvalidToolCallAction::repair("missing")) {
1527 Err(error) => error,
1528 Ok(outcome) => {
1529 return Err(ScenarioError::contract(
1530 SCENARIO,
1531 format!("disallowed repair unexpectedly returned {outcome:?}"),
1532 ));
1533 }
1534 };
1535 validate_unknown_tool_failure(&error, "missing", &[CountingSum::NAME])?;
1536
1537 let mut repaired = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1538 if !matches!(
1539 repaired.resolve_invalid_tool_call(InvalidToolCallAction::repair(CountingSum::NAME))?,
1540 ModelTurnOutcome::Continue { .. }
1541 ) {
1542 return Err(ScenarioError::contract(
1543 SCENARIO,
1544 "valid repair did not continue",
1545 ));
1546 }
1547 let AgentRunStep::CallTools { calls } = repaired.next_step()? else {
1548 return Err(ScenarioError::contract(
1549 SCENARIO,
1550 "valid repair did not produce pending tool execution",
1551 ));
1552 };
1553 let repaired_call = calls.first();
1554 if calls.len() != 1
1555 || !repaired_call.is_some_and(|call| {
1556 call.tool_call.function.name == CountingSum::NAME && call.preresolved_result.is_none()
1557 })
1558 {
1559 return Err(ScenarioError::contract(
1560 SCENARIO,
1561 format!("repaired pending calls were incorrect: {calls:?}"),
1562 ));
1563 }
1564
1565 let mut skipped = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1566 if !matches!(
1567 skipped.resolve_invalid_tool_call(InvalidToolCallAction::skip("disabled for this turn"))?,
1568 ModelTurnOutcome::Continue { .. }
1569 ) {
1570 return Err(ScenarioError::contract(SCENARIO, "skip did not continue"));
1571 }
1572 let AgentRunStep::CallTools { calls } = skipped.next_step()? else {
1573 return Err(ScenarioError::contract(
1574 SCENARIO,
1575 "skip did not produce a pre-resolved pending call",
1576 ));
1577 };
1578 let skipped_is_preresolved = match calls.first() {
1579 Some(call) => call.preresolved_result.is_some(),
1580 None => false,
1581 };
1582 if calls.len() != 1 || !skipped_is_preresolved {
1583 return Err(ScenarioError::contract(
1584 SCENARIO,
1585 format!("skipped pending calls were incorrect: {calls:?}"),
1586 ));
1587 }
1588 if add_calls.load(Ordering::SeqCst) != 0 || sum_calls.load(Ordering::SeqCst) != 0 {
1589 return Err(ScenarioError::contract(
1590 SCENARIO,
1591 "recovery scenario executed a tool body",
1592 ));
1593 }
1594
1595 Ok(ScenarioReport {
1596 name: SCENARIO,
1597 tool_calls: emitted,
1598 prompt_tokens: turn.usage.input_tokens.unwrap_or(0),
1599 generated_tokens: turn.usage.output_tokens.unwrap_or(0),
1600 history_messages: 2,
1601 duration: started.elapsed(),
1602 response: "fail, retry, repair, rejected repair, and skip passed".to_string(),
1603 })
1604}
1605
1606pub async fn hook_rewrites_and_request_patch<W, T, F>(
1609 model: rig_core::driver::Model<W, T>,
1610 configure: F,
1611) -> Result<ScenarioReport, ScenarioError>
1612where
1613 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1614 T: rig_core::driver::Transport<W>,
1615 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1616{
1617 hook_rewrites_and_request_patch_with_choice(model, configure, ToolChoice::Required).await
1618}
1619
1620pub async fn hook_rewrites_and_request_patch_with_choice<W, T, F>(
1624 model: rig_core::driver::Model<W, T>,
1625 configure: F,
1626 first_turn_choice: ToolChoice,
1627) -> Result<ScenarioReport, ScenarioError>
1628where
1629 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1630 T: rig_core::driver::Transport<W>,
1631 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1632{
1633 const SCENARIO: &str = "hook_rewrites_and_request_patch";
1634 let started = Instant::now();
1635 let calls = Arc::new(AtomicUsize::new(0));
1636 let observed = ObserveArguments::default();
1637 let observed_probe = observed.clone();
1638 let agent = configure(AgentBuilder::new(model).temperature(0.0))
1639 .preamble("Use add for arithmetic and report only the tool result.")
1640 .tool(CountingAdd(calls.clone()))
1641 .default_max_turns(3)
1642 .build();
1643 let response = agent
1644 .prompt("Use add once for x=1 and y=1, then report what the tool returns.")
1645 .max_turns(3)
1646 .add_hook(FirstTurnPatch(
1647 RequestPatch::new()
1648 .active_tools([CountingAdd::NAME])
1649 .tool_choice(first_turn_choice),
1650 ))
1651 .add_hook(RewriteArgument {
1652 key: "x",
1653 value: serde_json::json!(7),
1654 })
1655 .add_hook(RewriteArgument {
1656 key: "y",
1657 value: serde_json::json!(8),
1658 })
1659 .add_hook(observed)
1660 .add_hook(ReplaceResult("portable-redacted"))
1661 .add_hook(WrapResult)
1662 .await?;
1663 let observations = lock_recover(&observed_probe.0).clone();
1664 validate_rewritten_arguments(
1665 SCENARIO,
1666 &observations,
1667 &serde_json::json!({ "x": 7, "y": 8 }),
1668 )?;
1669 let messages = response.messages.as_deref().ok_or_else(|| {
1670 ScenarioError::contract(SCENARIO, "extended hook run omitted message history")
1671 })?;
1672 let results = messages
1673 .iter()
1674 .flat_map(tool_result_values)
1675 .collect::<Vec<_>>();
1676 if calls.load(Ordering::SeqCst) != 1
1677 || !results
1678 .iter()
1679 .any(|value| value == &serde_json::json!("[portable-redacted]"))
1680 || response.completion_calls.len() != 2
1681 {
1682 return Err(ScenarioError::contract(
1683 SCENARIO,
1684 format!(
1685 "calls={}, completion_calls={}, results={results:?}, output={:?}",
1686 calls.load(Ordering::SeqCst),
1687 response.completion_calls.len(),
1688 response.output
1689 ),
1690 ));
1691 }
1692 report_from_response(SCENARIO, started, 1, response)
1693}
1694
1695pub async fn cancellation_and_max_turns<W, T, F>(
1698 model: rig_core::driver::Model<W, T>,
1699 configure: F,
1700) -> Result<ScenarioReport, ScenarioError>
1701where
1702 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1703 T: rig_core::driver::Transport<W>,
1704 F: Fn(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1705{
1706 const SCENARIO: &str = "cancellation_and_max_turns";
1707 const REASON: &str = "portable result veto";
1708 let started = Instant::now();
1709 let cancelled_calls = Arc::new(AtomicUsize::new(0));
1710 let cancelled_agent = configure(AgentBuilder::new(model.clone()).temperature(0.0))
1711 .preamble("Use add for arithmetic; never calculate by hand.")
1712 .tool(CountingAdd(cancelled_calls.clone()))
1713 .build();
1714 let cancelled = match cancelled_agent
1715 .prompt("Use add once to compute x=20 plus y=22.")
1716 .max_turns(2)
1717 .add_hook(StopAfterResult(REASON))
1718 .await
1719 {
1720 Err(error) => error,
1721 Ok(output) => {
1722 return Err(ScenarioError::contract(
1723 SCENARIO,
1724 format!("result cancellation unexpectedly completed: {output:?}"),
1725 ));
1726 }
1727 };
1728 validate_cancelled_failure(&cancelled, REASON, CountingAdd::NAME)?;
1729
1730 let max_turn_calls = Arc::new(AtomicUsize::new(0));
1731 let max_turn_agent = configure(AgentBuilder::new(model).temperature(0.0))
1732 .preamble("Use add for arithmetic; never calculate by hand.")
1733 .tool(CountingAdd(max_turn_calls.clone()))
1734 .build();
1735 let max_turn = match max_turn_agent
1736 .prompt("Use add once to compute x=20 plus y=22, then report the result.")
1737 .max_turns(1)
1738 .await
1739 {
1740 Err(error) => error,
1741 Ok(output) => {
1742 return Err(ScenarioError::contract(
1743 SCENARIO,
1744 format!("one-turn budget unexpectedly completed: {output:?}"),
1745 ));
1746 }
1747 };
1748 validate_max_turns_failure(&max_turn, 1)?;
1749 let cancelled_count = cancelled_calls.load(Ordering::SeqCst);
1750 let max_turn_count = max_turn_calls.load(Ordering::SeqCst);
1751 if cancelled_count != 1 || max_turn_count != 1 {
1752 return Err(ScenarioError::contract(
1753 SCENARIO,
1754 format!("cancelled executions={cancelled_count}, max-turn executions={max_turn_count}"),
1755 ));
1756 }
1757 Ok(ScenarioReport {
1758 name: SCENARIO,
1759 tool_calls: cancelled_count + max_turn_count,
1760 prompt_tokens: 0,
1761 generated_tokens: 0,
1762 history_messages: 4,
1763 duration: started.elapsed(),
1764 response: "post-result cancellation and max-turn diagnostics passed".to_string(),
1765 })
1766}
1767
1768pub async fn optional_argument<W, T, F>(
1774 model: rig_core::driver::Model<W, T>,
1775 configure: F,
1776) -> Result<ScenarioReport, ScenarioError>
1777where
1778 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1779 T: rig_core::driver::Transport<W>,
1780 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1781{
1782 let calls = Arc::new(AtomicUsize::new(0));
1783 let started = Instant::now();
1784 let agent = configure(AgentBuilder::new(model))
1785 .preamble("Use the repeat_text tool whenever asked to repeat text.")
1786 .tool(RepeatTool {
1787 calls: calls.clone(),
1788 })
1789 .default_max_turns(4)
1790 .build();
1791 let result = agent
1792 .prompt(
1793 "Use the repeat_text tool to repeat the word \"banana\" 3 times, then show me the exact result.",
1794 )
1795 .await?;
1796 let response = result.output.clone();
1797 let tool_calls = calls.load(Ordering::SeqCst);
1798 if tool_calls == 0
1799 || response.matches("banana").count() < 1
1800 || !has_tool_roundtrip(result.messages.as_deref())
1801 {
1802 return Err(ScenarioError::contract(
1803 "optional_argument",
1804 format!("calls={tool_calls}, response={response:?}"),
1805 ));
1806 }
1807 report_from_response("optional_argument", started, tool_calls, result)
1808}
1809
1810pub async fn sequential_tools<W, T, F>(
1812 model: rig_core::driver::Model<W, T>,
1813 configure: F,
1814) -> Result<ScenarioReport, ScenarioError>
1815where
1816 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1817 T: rig_core::driver::Transport<W>,
1818 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1819{
1820 let add_calls = Arc::new(AtomicUsize::new(0));
1821 let multiply_calls = Arc::new(AtomicUsize::new(0));
1822 let started = Instant::now();
1823 let agent = configure(AgentBuilder::new(model))
1824 .preamble(
1825 "You are a calculator. Use the add and multiply tools for arithmetic; never compute by hand.",
1826 )
1827 .tool(AddTool(add_calls.clone()))
1828 .tool(MultiplyTool(multiply_calls.clone()))
1829 .default_max_turns(6)
1830 .build();
1831 let result = agent
1832 .prompt(
1833 "Compute (4 + 6) * 2. First call the add tool, then call the multiply tool on the result. Tell me the final number.",
1834 )
1835 .await?;
1836 let response = result.output.clone();
1837 let add = add_calls.load(Ordering::SeqCst);
1838 let multiply = multiply_calls.load(Ordering::SeqCst);
1839 if add == 0
1840 || multiply == 0
1841 || !response.contains("20")
1842 || !has_tool_roundtrip(result.messages.as_deref())
1843 {
1844 return Err(ScenarioError::contract(
1845 "sequential_tools",
1846 format!("add={add}, multiply={multiply}, response={response:?}"),
1847 ));
1848 }
1849 report_from_response("sequential_tools", started, add + multiply, result)
1850}
1851
1852pub async fn streaming_tool<W, T, F>(
1854 model: rig_core::driver::Model<W, T>,
1855 configure: F,
1856) -> Result<ScenarioReport, ScenarioError>
1857where
1858 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1859 T: rig_core::driver::Transport<W>,
1860 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1861{
1862 let calls = Arc::new(AtomicUsize::new(0));
1863 let started = Instant::now();
1864 let agent = configure(AgentBuilder::new(model))
1865 .preamble("Use the add tool for arithmetic; do not calculate by hand.")
1866 .tool(AddTool(calls.clone()))
1867 .default_max_turns(4)
1868 .build();
1869 let mut stream = agent
1870 .prompt("Use add to calculate 17 + 25, then state the final number.")
1871 .max_turns(4)
1872 .stream();
1873 let mut final_response = None;
1874 let mut final_count = 0_usize;
1875 let mut completion_usage = crate::completion::Usage::default();
1876 let mut streamed_call_ids = Vec::new();
1877 let mut streamed_result_ids = Vec::new();
1878 while let Some(item) = stream.next().await {
1879 match item? {
1880 MultiTurnStreamItem::ToolCall { tool_call } => streamed_call_ids.push(tool_call.id),
1881 MultiTurnStreamItem::StreamUserItem(
1882 crate::streaming::StreamedUserContent::ToolResult { tool_result },
1883 ) => streamed_result_ids.push(tool_result.call),
1884 MultiTurnStreamItem::CompletionCall(call) => completion_usage += call.usage,
1885 MultiTurnStreamItem::FinalResponse(response) => {
1886 final_count += 1;
1887 final_response = Some(response);
1888 }
1889 MultiTurnStreamItem::StreamAssistantItem(_)
1890 | MultiTurnStreamItem::ToolExecutionCommitted { .. }
1891 | MultiTurnStreamItem::ModelTurnRetried { .. } => {}
1892 }
1893 }
1894 let result = final_response.ok_or_else(|| {
1895 ScenarioError::contract("streaming_tool", "stream produced no final response")
1896 })?;
1897 let response = result.output.clone();
1898 let history_messages = result.messages.as_ref().map_or(0, Vec::len);
1899 let tool_calls = calls.load(Ordering::SeqCst);
1900 streamed_call_ids.sort();
1901 streamed_result_ids.sort();
1902 let correlated_stream =
1903 !streamed_call_ids.is_empty() && streamed_call_ids == streamed_result_ids;
1904 let correlated_history = result
1905 .messages
1906 .as_deref()
1907 .is_some_and(|messages| validate_tool_correlation("streaming_tool", messages).is_ok());
1908 if tool_calls == 0
1909 || !response.contains("42")
1910 || history_messages < 4
1911 || final_count != 1
1912 || !correlated_stream
1913 || !correlated_history
1914 || completion_usage != result.usage
1915 || result.completion_calls.is_empty()
1916 {
1917 return Err(ScenarioError::contract(
1918 "streaming_tool",
1919 format!(
1920 "calls={tool_calls}, final_count={final_count}, streamed_call_ids={streamed_call_ids:?}, streamed_result_ids={streamed_result_ids:?}, completion_usage={completion_usage:?}, final_usage={:?}, history_messages={history_messages}, response={response:?}",
1921 result.usage
1922 ),
1923 ));
1924 }
1925 if let Some(messages) = result.messages.as_deref() {
1926 validate_protocol_hygiene(
1927 "streaming_tool",
1928 &response,
1929 messages,
1930 &["<tool_call>", "</tool_call>", "<think>", "</think>"],
1931 )?;
1932 }
1933 Ok(ScenarioReport {
1934 name: "streaming_tool",
1935 tool_calls,
1936 prompt_tokens: result.usage.input_tokens.unwrap_or(0),
1937 generated_tokens: result.usage.output_tokens.unwrap_or(0),
1938 history_messages,
1939 duration: started.elapsed(),
1940 response,
1941 })
1942}
1943
1944pub async fn structured_after_tool<W, T, F>(
1946 model: rig_core::driver::Model<W, T>,
1947 configure: F,
1948) -> Result<ScenarioReport, ScenarioError>
1949where
1950 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1951 T: rig_core::driver::Transport<W>,
1952 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1953{
1954 let calls = Arc::new(AtomicUsize::new(0));
1955 let started = Instant::now();
1956 let agent = configure(AgentBuilder::new(model))
1957 .preamble(
1958 "Use add for arithmetic, then finish by calling the structured output tool exactly once.",
1959 )
1960 .output_schema::<ArithmeticResult>()
1961 .output_mode(OutputMode::Tool)
1962 .tool(AddTool(calls.clone()))
1963 .default_max_turns(5)
1964 .build();
1965 let result = agent
1966 .prompt("Use add to calculate 19 + 23. Return answer=42 and a short optional explanation.")
1967 .await?;
1968 let response = result.output.clone();
1969 let parsed: ArithmeticResult = serde_json::from_str(&response)?;
1970 let tool_calls = calls.load(Ordering::SeqCst);
1971 if tool_calls == 0 || parsed.answer != 42 || !has_tool_roundtrip(result.messages.as_deref()) {
1972 return Err(ScenarioError::contract(
1973 "structured_after_tool",
1974 format!("calls={tool_calls}, response={response:?}"),
1975 ));
1976 }
1977 let _ = parsed.explanation;
1978 report_from_response("structured_after_tool", started, tool_calls + 1, result)
1979}
1980
1981pub async fn tool_choice_modes<W, T, A>(
1983 model: rig_core::driver::Model<W, T>,
1984 adjust: A,
1985) -> Result<ScenarioReport, ScenarioError>
1986where
1987 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1988 T: rig_core::driver::Transport<W>,
1989 A: Fn(CompletionRequest) -> CompletionRequest,
1990{
1991 let definition = |name: &str| ToolDefinition {
1992 name: name.to_string(),
1993 description: format!("Return the supplied integer using {name}."),
1994 parameters: serde_json::json!({
1995 "type": "object",
1996 "properties": {"value": {"type": "integer"}},
1997 "required": ["value"]
1998 }),
1999 };
2000 let tools = vec![definition("alpha"), definition("beta")];
2001 let started = Instant::now();
2002 let none = model
2003 .call(adjust(
2004 CompletionRequest::new("Answer with only the number 4. Do not call a function.")
2005 .tools(tools.clone())
2006 .tool_choice(ToolChoice::None)
2007 .temperature(0.0)
2008 .max_tokens(64),
2009 ))
2010 .await?;
2011 if none
2012 .choice
2013 .iter()
2014 .any(|item| matches!(item, AssistantContent::ToolCall(_)))
2015 {
2016 return Err(ScenarioError::contract(
2017 "tool_choice_modes",
2018 "tool_choice none emitted a tool call",
2019 ));
2020 }
2021
2022 let required = model
2023 .call(adjust(
2024 CompletionRequest::new("Call alpha with value 7.")
2025 .tools(tools.clone())
2026 .tool_choice(ToolChoice::Required)
2027 .temperature(0.0)
2028 .max_tokens(96),
2029 ))
2030 .await?;
2031 let required_calls = required
2032 .choice
2033 .iter()
2034 .filter(|item| matches!(item, AssistantContent::ToolCall(_)))
2035 .count();
2036 if required_calls == 0 {
2037 return Err(ScenarioError::contract(
2038 "tool_choice_modes",
2039 "tool_choice required emitted no tool call",
2040 ));
2041 }
2042
2043 let specific = model
2044 .call(adjust(
2045 CompletionRequest::new("Call beta with value 9.")
2046 .tools(tools)
2047 .tool_choice(ToolChoice::Specific {
2048 function_names: vec!["beta".to_string()],
2049 })
2050 .temperature(0.0)
2051 .max_tokens(96),
2052 ))
2053 .await?;
2054 let specific_calls = specific
2055 .choice
2056 .iter()
2057 .filter_map(|item| match item {
2058 AssistantContent::ToolCall(call) => Some(call),
2059 _ => None,
2060 })
2061 .collect::<Vec<_>>();
2062 if specific_calls.is_empty()
2063 || specific_calls
2064 .iter()
2065 .any(|call| call.function.name != "beta")
2066 {
2067 return Err(ScenarioError::contract(
2068 "tool_choice_modes",
2069 "specific tool choice did not select only beta",
2070 ));
2071 }
2072
2073 let usage = none.usage + required.usage + specific.usage;
2074 Ok(ScenarioReport {
2075 name: "tool_choice_modes",
2076 tool_calls: required_calls + specific_calls.len(),
2077 prompt_tokens: usage.input_tokens.unwrap_or(0),
2078 generated_tokens: usage.output_tokens.unwrap_or(0),
2079 history_messages: 0,
2080 duration: started.elapsed(),
2081 response: "none, required, and specific modes passed".to_string(),
2082 })
2083}
2084
2085pub async fn streaming_structured_after_tool<W, T, F>(
2087 model: rig_core::driver::Model<W, T>,
2088 configure: F,
2089) -> Result<ScenarioReport, ScenarioError>
2090where
2091 W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
2092 T: rig_core::driver::Transport<W>,
2093 F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
2094{
2095 let calls = Arc::new(AtomicUsize::new(0));
2096 let started = Instant::now();
2097 let agent = configure(AgentBuilder::new(model))
2098 .preamble(
2099 "Use add for arithmetic, then finish by calling the structured output tool exactly once.",
2100 )
2101 .output_schema::<ArithmeticResult>()
2102 .output_mode(OutputMode::Tool)
2103 .tool(AddTool(calls.clone()))
2104 .default_max_turns(5)
2105 .build();
2106 let mut stream = agent
2107 .prompt("Use add to calculate 19 + 23. Return answer=42 and a short optional explanation.")
2108 .max_turns(5)
2109 .stream();
2110 let mut final_response = None;
2111 let mut final_count = 0_usize;
2112 while let Some(item) = stream.next().await {
2113 if let MultiTurnStreamItem::FinalResponse(response) = item? {
2114 final_count += 1;
2115 final_response = Some(response);
2116 }
2117 }
2118 let result = final_response.ok_or_else(|| {
2119 ScenarioError::contract(
2120 "streaming_structured_after_tool",
2121 "stream produced no final response",
2122 )
2123 })?;
2124 let parsed: ArithmeticResult = serde_json::from_str(&result.output)?;
2125 let calls = calls.load(Ordering::SeqCst);
2126 if calls == 0
2127 || final_count != 1
2128 || parsed.answer != 42
2129 || !has_tool_roundtrip(result.messages.as_deref())
2130 {
2131 return Err(ScenarioError::contract(
2132 "streaming_structured_after_tool",
2133 format!(
2134 "calls={calls}, final_count={final_count}, response={:?}",
2135 result.output
2136 ),
2137 ));
2138 }
2139 report_from_response(
2140 "streaming_structured_after_tool",
2141 started,
2142 calls + 1,
2143 result,
2144 )
2145}
2146
2147#[cfg(test)]
2148mod tests;