Skip to main content

rig_agent/test_utils/
model_conformance.rs

1//! Provider-neutral behavioral scenarios for completion-model conformance.
2//!
3//! These helpers test the model/agent contract only. Provider wire formats,
4//! authentication, HTTP streaming, and cassette matching remain provider-suite
5//! responsibilities.
6
7use std::{
8    collections::BTreeSet,
9    sync::{
10        Arc, Mutex, MutexGuard,
11        atomic::{AtomicUsize, Ordering},
12    },
13    time::{Duration, Instant},
14};
15
16use futures::StreamExt;
17use schemars::JsonSchema;
18use serde::{Deserialize, Serialize};
19
20use crate::{
21    agent::{
22        AgentBuilder, AgentHook, CompletionCallAction, CompletionCallEvent, DispatchAction,
23        DispatchEvent, HookContext, InvalidToolCallAction, MultiTurnStreamItem, OutcomeAction,
24        OutcomeEvent, OutputMode, RequestPatch,
25    },
26    completion::{AssistantContent, Message, PromptError, ToolDefinition},
27    run::{AgentRun, AgentRunStep, ModelTurn, ModelTurnOutcome, PendingToolCall, TurnPolicy},
28    tool::{Tool, ToolContext},
29};
30use rig_core::completion::CompletionRequest;
31use rig_core::error::ProviderError;
32use rig_core::message::{ToolChoice, UserContent};
33
34/// Typed failure from a portable model-conformance scenario.
35#[derive(Debug, thiserror::Error)]
36pub enum ScenarioError {
37    /// An agent run failed.
38    #[error(transparent)]
39    Prompt(#[from] PromptError),
40    /// A direct model completion failed.
41    #[error(transparent)]
42    Completion(#[from] ProviderError),
43    /// A stream item or effect failed, as the wire reports it.
44    #[error(transparent)]
45    Report(#[from] rig_core::error::ErrorReport),
46    /// Structured content could not be decoded.
47    #[error(transparent)]
48    Json(#[from] serde_json::Error),
49    /// Rig's structured extractor failed.
50    #[error(transparent)]
51    Extraction(#[from] crate::completion::StructuredOutputError),
52    /// The model or agent violated the portable behavioral contract.
53    #[error("{scenario} conformance failed: {details}")]
54    Contract {
55        /// Stable scenario name.
56        scenario: &'static str,
57        /// Actionable observation explaining the failure.
58        details: String,
59    },
60}
61
62impl ScenarioError {
63    fn contract(scenario: &'static str, details: impl Into<String>) -> Self {
64        Self::Contract {
65            scenario,
66            details: details.into(),
67        }
68    }
69}
70
71/// Validate the portable diagnostics carried by an unknown or disallowed tool
72/// call failure.
73pub fn validate_unknown_tool_failure(
74    error: &PromptError,
75    expected_tool: &str,
76    expected_allowed_tools: &[&str],
77) -> Result<(), ScenarioError> {
78    const SCENARIO: &str = "unknown_tool_failure";
79    let PromptError::UnknownToolCall {
80        tool_name,
81        allowed_tools,
82        chat_history,
83        ..
84    } = error
85    else {
86        return Err(ScenarioError::contract(
87            SCENARIO,
88            format!("expected UnknownToolCall, observed {error:?}"),
89        ));
90    };
91    let expected_allowed = expected_allowed_tools
92        .iter()
93        .map(|name| (*name).to_string())
94        .collect::<Vec<_>>();
95    // A rejected repair target is never written back into history; diagnostics
96    // retain the model's original call while `tool_name` names the rejected
97    // target. Requiring a call-bearing assistant turn covers both paths.
98    let history_has_call = chat_history.iter().any(|message| {
99        matches!(
100            message,
101            Message::Assistant(rig_core::message::AssistantMessage { content, .. })
102                if content.iter().any(|item| matches!(item, AssistantContent::ToolCall(_)))
103        )
104    });
105    if tool_name != expected_tool || allowed_tools != &expected_allowed || !history_has_call {
106        return Err(ScenarioError::contract(
107            SCENARIO,
108            format!(
109                "tool={tool_name:?}, allowed={allowed_tools:?}, expected_tool={expected_tool:?}, expected_allowed={expected_allowed:?}, history_has_call={history_has_call}, history={chat_history:?}"
110            ),
111        ));
112    }
113    Ok(())
114}
115
116/// Validate cancellation diagnostics, including the exact reason and retained
117/// assistant tool-call history.
118pub fn validate_cancelled_failure(
119    error: &PromptError,
120    expected_reason: &str,
121    expected_tool: &str,
122) -> Result<(), ScenarioError> {
123    const SCENARIO: &str = "cancelled_failure";
124    let PromptError::Cancelled {
125        chat_history,
126        reason,
127    } = error
128    else {
129        return Err(ScenarioError::contract(
130            SCENARIO,
131            format!("expected Cancelled, observed {error:?}"),
132        ));
133    };
134    let history_has_call = chat_history.iter().any(|message| {
135        matches!(
136            message,
137            Message::Assistant(rig_core::message::AssistantMessage { content, .. })
138                if content.iter().any(|item| matches!(
139                    item,
140                    AssistantContent::ToolCall(call) if call.function.name == expected_tool
141                ))
142        )
143    });
144    if reason != expected_reason || !history_has_call {
145        return Err(ScenarioError::contract(
146            SCENARIO,
147            format!(
148                "reason={reason:?}, expected={expected_reason:?}, history_has_call={history_has_call}, history={chat_history:?}"
149            ),
150        ));
151    }
152    Ok(())
153}
154
155/// Validate max-turn diagnostics, including the exact configured budget and a
156/// retained pending prompt.
157pub fn validate_max_turns_failure(
158    error: &PromptError,
159    expected_max_turns: usize,
160) -> Result<(), ScenarioError> {
161    const SCENARIO: &str = "max_turns_failure";
162    let PromptError::MaxTurns {
163        max_turns,
164        chat_history,
165        prompt,
166    } = error
167    else {
168        return Err(ScenarioError::contract(
169            SCENARIO,
170            format!("expected MaxTurns, observed {error:?}"),
171        ));
172    };
173    let pending_prompt_retained = matches!(
174        prompt,
175        Message::User { content } if content.iter().next().is_some()
176    );
177    if *max_turns != expected_max_turns || chat_history.is_empty() || !pending_prompt_retained {
178        return Err(ScenarioError::contract(
179            SCENARIO,
180            format!(
181                "max_turns={max_turns}, expected={expected_max_turns}, history={chat_history:?}, pending_prompt_retained={pending_prompt_retained}, pending_prompt={prompt:?}"
182            ),
183        ));
184    }
185    Ok(())
186}
187
188/// Decode a structured-output response with a typed conformance failure that
189/// retains the scenario name and raw response.
190pub fn decode_structured_output<T>(
191    scenario: &'static str,
192    response: &str,
193) -> Result<T, ScenarioError>
194where
195    T: serde::de::DeserializeOwned,
196{
197    serde_json::from_str(response).map_err(|error| {
198        ScenarioError::contract(
199            scenario,
200            format!("structured output did not decode: {error}; response={response:?}"),
201        )
202    })
203}
204
205/// Validate that model-family control markers are absent from user-visible
206/// output and persisted history.
207pub fn validate_protocol_hygiene(
208    scenario: &'static str,
209    visible_output: &str,
210    messages: &[Message],
211    forbidden_markers: &[&str],
212) -> Result<(), ScenarioError> {
213    let serialized = serde_json::to_string(messages)?;
214    let leaked = forbidden_markers
215        .iter()
216        .filter(|marker| visible_output.contains(**marker) || serialized.contains(**marker))
217        .copied()
218        .collect::<Vec<_>>();
219    if !leaked.is_empty() {
220        return Err(ScenarioError::contract(
221            scenario,
222            format!(
223                "protocol markers leaked: {leaked:?}; output={visible_output:?}, history={messages:?}"
224            ),
225        ));
226    }
227    Ok(())
228}
229
230/// Validate that every observed tool invocation contains the expected rewritten
231/// argument fields while allowing unrelated original fields to remain.
232pub fn validate_rewritten_arguments(
233    scenario: &'static str,
234    observations: &[serde_json::Value],
235    expected_fields: &serde_json::Value,
236) -> Result<(), ScenarioError> {
237    let Some(expected) = expected_fields.as_object() else {
238        return Err(ScenarioError::contract(
239            scenario,
240            "expected rewritten fields must be a JSON object",
241        ));
242    };
243    if observations.is_empty() {
244        return Err(ScenarioError::contract(
245            scenario,
246            "the rewritten tool was never invoked",
247        ));
248    }
249    for observation in observations {
250        let Some(actual) = observation.as_object() else {
251            return Err(ScenarioError::contract(
252                scenario,
253                format!("observed rewritten arguments were not an object: {observation:?}"),
254            ));
255        };
256        for (key, expected_value) in expected {
257            if actual.get(key) != Some(expected_value) {
258                return Err(ScenarioError::contract(
259                    scenario,
260                    format!(
261                        "rewritten field {key:?} expected {expected_value:?}, observed {observation:?}"
262                    ),
263                ));
264            }
265        }
266    }
267    Ok(())
268}
269
270/// Validate a provider-neutral person extraction and its usage accounting.
271pub fn validate_extraction_fields(
272    scenario: &'static str,
273    first_name: Option<&str>,
274    last_name: Option<&str>,
275    job: Option<&str>,
276    usage: crate::completion::Usage,
277) -> Result<(), ScenarioError> {
278    let fields_match = first_name.is_some_and(|value| value.eq_ignore_ascii_case("Ada"))
279        && last_name.is_some_and(|value| value.eq_ignore_ascii_case("Lovelace"))
280        && job.is_some_and(|value| value.to_ascii_lowercase().contains("mathematician"));
281    if !fields_match || !usage.is_reported() {
282        return Err(ScenarioError::contract(
283            scenario,
284            format!(
285                "first_name={first_name:?}, last_name={last_name:?}, job={job:?}, usage={usage:?}"
286            ),
287        ));
288    }
289    Ok(())
290}
291
292/// Summary emitted by a portable model-conformance scenario.
293#[derive(Debug, Clone)]
294pub struct ScenarioReport {
295    /// Stable scenario name.
296    pub name: &'static str,
297    /// Number of model-invoked tool calls observed by the scenario.
298    pub tool_calls: usize,
299    /// Aggregated prompt tokens reported by the model across all turns; zero
300    /// when the model reported none.
301    pub prompt_tokens: u64,
302    /// Aggregated generated tokens reported by the model across all turns;
303    /// zero when the model reported none.
304    pub generated_tokens: u64,
305    /// Number of messages retained in the completed run history.
306    pub history_messages: usize,
307    /// End-to-end scenario duration.
308    pub duration: Duration,
309    /// The model's final user-visible response.
310    pub response: String,
311}
312
313/// Error used by the deterministic conformance tools.
314#[derive(Debug, thiserror::Error)]
315#[error("model-conformance tool failed")]
316pub struct ConformanceToolError;
317
318const FORCE_TOOLS_PREAMBLE: &str = "You are a calculator assistant. You MUST use the provided tools for every arithmetic operation instead of computing results yourself. Once you have all the tool results you need, reply with the final numeric answer in plain text.";
319const PARALLEL_PROMPT: &str = "Compute 3 + 4 and 10 - 2. You MUST call the add tool and the subtract tool together in your first response, as two parallel function calls, then report both results.";
320const PING_OUTPUT: &str = "pong-crimson-7423";
321const MOTTO_OUTPUT: &str = "steady hands\ncalm waters";
322
323fn lock_recover<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
324    mutex
325        .lock()
326        .unwrap_or_else(std::sync::PoisonError::into_inner)
327}
328
329fn tool_result_values(message: &Message) -> Vec<serde_json::Value> {
330    let Message::User { content } = message else {
331        return Vec::new();
332    };
333    content
334        .iter()
335        .filter_map(|item| match item {
336            UserContent::ToolResult(result) => Some(result),
337            _ => None,
338        })
339        .flat_map(|result| result.content.iter())
340        .filter_map(|content| match content {
341            rig_core::message::ToolResultContent::Text(text) => {
342                Some(serde_json::Value::String(text.text.clone()))
343            }
344            rig_core::message::ToolResultContent::Json { value } => Some(value.clone()),
345            rig_core::message::ToolResultContent::Image(_) => None,
346        })
347        .collect()
348}
349
350fn validate_tool_correlation(
351    scenario: &'static str,
352    messages: &[Message],
353) -> Result<(), ScenarioError> {
354    let mut calls = Vec::new();
355    let mut results = Vec::new();
356    let mut turn = 0usize;
357    for message in messages {
358        match message {
359            Message::Assistant(rig_core::message::AssistantMessage { content, .. }) => {
360                turn += 1;
361                calls.extend(content.iter().filter_map(|item| match item {
362                    AssistantContent::ToolCall(call) => Some((
363                        turn,
364                        &call.id,
365                        call.id.provider().map(|provider| provider.as_str()),
366                    )),
367                    _ => None,
368                }));
369            }
370            Message::User { content } => {
371                results.extend(content.iter().filter_map(|item| match item {
372                    UserContent::ToolResult(result) => Some((
373                        turn,
374                        &result.call,
375                        result.call.provider().map(|provider| provider.as_str()),
376                    )),
377                    _ => None,
378                }));
379            }
380            Message::System { .. } => {}
381        }
382    }
383    if calls.is_empty() {
384        return Err(ScenarioError::contract(
385            scenario,
386            format!("history has no assistant tool calls: {messages:?}"),
387        ));
388    }
389    for (turn, id, call_id) in &calls {
390        let matches = results
391            .iter()
392            .filter(|(result_turn, result_id, result_call_id)| {
393                result_turn == turn && result_id == id && call_id == result_call_id
394            })
395            .count();
396        if matches != 1 {
397            return Err(ScenarioError::contract(
398                scenario,
399                format!(
400                    "tool call id={id:?} call_id={call_id:?} has {matches} correlated results; calls={calls:?}, results={results:?}"
401                ),
402            ));
403        }
404    }
405    if results.len() != calls.len() {
406        return Err(ScenarioError::contract(
407            scenario,
408            format!(
409                "history contains dangling calls or results: calls={calls:?}, results={results:?}"
410            ),
411        ));
412    }
413    Ok(())
414}
415
416#[derive(Debug, Deserialize, JsonSchema)]
417struct OperationArgs {
418    x: i64,
419    y: i64,
420}
421
422#[derive(Clone)]
423struct CountingAdd(Arc<AtomicUsize>);
424
425impl Tool for CountingAdd {
426    const NAME: &'static str = "add";
427    type Error = ConformanceToolError;
428    type Args = OperationArgs;
429    type Output = i64;
430
431    fn description(&self) -> String {
432        "Add x and y together".to_string()
433    }
434
435    fn parameters(&self) -> serde_json::Value {
436        serde_json::json!({
437            "type": "object",
438            "properties": {
439                "x": { "type": "number", "description": "The first operand" },
440                "y": { "type": "number", "description": "The second operand" }
441            },
442            "required": ["x", "y"]
443        })
444    }
445
446    async fn call(
447        &self,
448        _context: &mut ToolContext,
449        args: Self::Args,
450    ) -> Result<Self::Output, Self::Error> {
451        self.0.fetch_add(1, Ordering::SeqCst);
452        Ok(args.x + args.y)
453    }
454}
455
456#[derive(Clone)]
457struct CountingSum(Arc<AtomicUsize>);
458
459impl Tool for CountingSum {
460    const NAME: &'static str = "sum";
461    type Error = ConformanceToolError;
462    type Args = OperationArgs;
463    type Output = i64;
464
465    fn description(&self) -> String {
466        "Add x and y together (alias of add)".to_string()
467    }
468
469    fn parameters(&self) -> serde_json::Value {
470        CountingAdd(Arc::new(AtomicUsize::new(0))).parameters()
471    }
472
473    async fn call(
474        &self,
475        _context: &mut ToolContext,
476        args: Self::Args,
477    ) -> Result<Self::Output, Self::Error> {
478        self.0.fetch_add(1, Ordering::SeqCst);
479        Ok(args.x + args.y)
480    }
481}
482
483#[derive(Clone)]
484struct CountingSubtract(Arc<AtomicUsize>);
485
486impl Tool for CountingSubtract {
487    const NAME: &'static str = "subtract";
488    type Error = ConformanceToolError;
489    type Args = OperationArgs;
490    type Output = i64;
491
492    fn description(&self) -> String {
493        "Subtract y from x (i.e. x - y)".to_string()
494    }
495
496    fn parameters(&self) -> serde_json::Value {
497        serde_json::json!({
498            "type": "object",
499            "properties": {
500                "x": { "type": "number", "description": "The first operand" },
501                "y": { "type": "number", "description": "The second operand" }
502            },
503            "required": ["x", "y"]
504        })
505    }
506
507    async fn call(
508        &self,
509        _context: &mut ToolContext,
510        args: Self::Args,
511    ) -> Result<Self::Output, Self::Error> {
512        self.0.fetch_add(1, Ordering::SeqCst);
513        Ok(args.x - args.y)
514    }
515}
516
517#[derive(Clone)]
518struct RewriteArgument {
519    key: &'static str,
520    value: serde_json::Value,
521}
522
523impl AgentHook for RewriteArgument {
524    async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
525        let (Some(name), Some(args)) = (event.tool_name(), event.tool_args()) else {
526            return DispatchAction::proceed();
527        };
528        if name != CountingAdd::NAME {
529            return DispatchAction::proceed();
530        }
531        let Ok(mut arguments) = serde_json::from_str::<serde_json::Value>(args) else {
532            return DispatchAction::proceed();
533        };
534        let Some(object) = arguments.as_object_mut() else {
535            return DispatchAction::proceed();
536        };
537        object.insert(self.key.to_string(), self.value.clone());
538        DispatchAction::rewrite_tool_args(event.kind, arguments)
539    }
540}
541
542#[derive(Clone, Default)]
543struct ObserveArguments(Arc<Mutex<Vec<serde_json::Value>>>);
544
545impl AgentHook for ObserveArguments {
546    async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
547        if let Some(args) = event.tool_args() {
548            let value = serde_json::from_str(args)
549                .unwrap_or_else(|_| serde_json::Value::String(args.to_string()));
550            lock_recover(&self.0).push(value);
551        }
552        DispatchAction::proceed()
553    }
554}
555
556#[derive(Clone)]
557struct ReplaceResult(&'static str);
558
559impl AgentHook for ReplaceResult {
560    async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
561        if event.tool_name() == Some(CountingAdd::NAME) {
562            OutcomeAction::rewrite_tool_result(&event, self.0)
563        } else {
564            OutcomeAction::proceed()
565        }
566    }
567}
568
569#[derive(Clone)]
570struct WrapResult;
571
572impl AgentHook for WrapResult {
573    async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
574        match (event.tool_name(), event.tool_result()) {
575            (Some(name), Some(result)) if name == CountingAdd::NAME => {
576                OutcomeAction::rewrite_tool_result(
577                    &event,
578                    format!("[{}]", result.output().render()),
579                )
580            }
581            _ => OutcomeAction::proceed(),
582        }
583    }
584}
585
586#[derive(Clone)]
587struct FirstTurnPatch(RequestPatch);
588
589impl AgentHook for FirstTurnPatch {
590    async fn on_completion_call(
591        &self,
592        ctx: &HookContext,
593        _event: CompletionCallEvent<'_>,
594    ) -> CompletionCallAction {
595        if ctx.turn() == 1 {
596            CompletionCallAction::patch(self.0.clone())
597        } else {
598            CompletionCallAction::continue_run()
599        }
600    }
601}
602
603#[derive(Clone)]
604struct StopAfterResult(&'static str);
605
606impl AgentHook for StopAfterResult {
607    async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
608        if event.tool_name() == Some(CountingAdd::NAME) {
609            OutcomeAction::stop(self.0)
610        } else {
611            OutcomeAction::proceed()
612        }
613    }
614}
615
616#[derive(Debug, Deserialize, JsonSchema)]
617struct EmptyArgs {}
618
619#[derive(Clone)]
620struct PingTool(Arc<AtomicUsize>);
621
622impl Tool for PingTool {
623    const NAME: &'static str = "ping";
624    type Error = ConformanceToolError;
625    type Args = EmptyArgs;
626    type Output = String;
627
628    fn description(&self) -> String {
629        "Return the current ping marker. Takes no arguments.".to_string()
630    }
631
632    fn parameters(&self) -> serde_json::Value {
633        serde_json::json!({ "type": "object", "properties": {}, "required": [] })
634    }
635
636    async fn call(
637        &self,
638        _context: &mut ToolContext,
639        _args: Self::Args,
640    ) -> Result<Self::Output, Self::Error> {
641        self.0.fetch_add(1, Ordering::SeqCst);
642        Ok(PING_OUTPUT.to_string())
643    }
644}
645
646#[derive(Clone)]
647struct MottoTool(Arc<AtomicUsize>);
648
649impl Tool for MottoTool {
650    const NAME: &'static str = "fetch_motto";
651    type Error = ConformanceToolError;
652    type Args = EmptyArgs;
653    type Output = String;
654
655    fn description(&self) -> String {
656        "Fetch the two-line workshop motto.".to_string()
657    }
658
659    fn parameters(&self) -> serde_json::Value {
660        serde_json::json!({ "type": "object", "properties": {}, "required": [] })
661    }
662
663    async fn call(
664        &self,
665        _context: &mut ToolContext,
666        _args: Self::Args,
667    ) -> Result<Self::Output, Self::Error> {
668        self.0.fetch_add(1, Ordering::SeqCst);
669        Ok(MOTTO_OUTPUT.to_string())
670    }
671}
672
673#[derive(Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
674struct ConfigOutput {
675    service: String,
676    max_retries: u64,
677}
678
679#[derive(Clone)]
680struct ConfigTool(Arc<AtomicUsize>);
681
682impl Tool for ConfigTool {
683    const NAME: &'static str = "fetch_config";
684    type Error = ConformanceToolError;
685    type Args = EmptyArgs;
686    type Output = ConfigOutput;
687
688    fn description(&self) -> String {
689        "Fetch the service configuration object.".to_string()
690    }
691
692    fn parameters(&self) -> serde_json::Value {
693        serde_json::json!({ "type": "object", "properties": {}, "required": [] })
694    }
695
696    async fn call(
697        &self,
698        _context: &mut ToolContext,
699        _args: Self::Args,
700    ) -> Result<Self::Output, Self::Error> {
701        self.0.fetch_add(1, Ordering::SeqCst);
702        Ok(ConfigOutput {
703            service: "cassette-lab".to_string(),
704            max_retries: 3,
705        })
706    }
707}
708
709#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
710#[serde(rename_all = "snake_case")]
711enum ComplexMode {
712    Careful,
713    Fast,
714}
715
716#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
717struct ComplexProfile {
718    name: String,
719    tags: Vec<String>,
720}
721
722#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
723struct ComplexArgs {
724    profile: ComplexProfile,
725    mode: ComplexMode,
726    note: Option<String>,
727    quote: String,
728}
729
730#[derive(Debug, Deserialize, Serialize, JsonSchema)]
731struct ExtractedPerson {
732    #[schemars(required)]
733    first_name: Option<String>,
734    #[schemars(required)]
735    last_name: Option<String>,
736    #[schemars(required)]
737    job: Option<String>,
738}
739
740#[derive(Clone)]
741struct CaptureComplexTool {
742    calls: Arc<AtomicUsize>,
743    captured: Arc<Mutex<Option<ComplexArgs>>>,
744}
745
746impl Tool for CaptureComplexTool {
747    const NAME: &'static str = "store_profile";
748    type Error = ConformanceToolError;
749    type Args = ComplexArgs;
750    type Output = ComplexArgs;
751
752    fn description(&self) -> String {
753        "Store one profile with its nested tags, mode, optional note, and exact quoted text."
754            .to_string()
755    }
756
757    fn parameters(&self) -> serde_json::Value {
758        serde_json::to_value(schemars::schema_for!(ComplexArgs)).unwrap_or_default()
759    }
760
761    async fn call(
762        &self,
763        _context: &mut ToolContext,
764        args: Self::Args,
765    ) -> Result<Self::Output, Self::Error> {
766        self.calls.fetch_add(1, Ordering::SeqCst);
767        *lock_recover(&self.captured) = Some(args.clone());
768        Ok(args)
769    }
770}
771
772fn has_tool_roundtrip(messages: &[Message]) -> bool {
773    let saw_call = messages.iter().any(|message| {
774        matches!(
775            message,
776            Message::Assistant(rig_core::message::AssistantMessage { content, .. })
777                if content.iter().any(|item| matches!(item, AssistantContent::ToolCall(_)))
778        )
779    });
780    let saw_result = messages.iter().any(|message| {
781        matches!(
782            message,
783            Message::User { content }
784                if content.iter().any(|item| matches!(item, UserContent::ToolResult(_)))
785        )
786    });
787    saw_call && saw_result
788}
789
790#[derive(Debug, Deserialize, JsonSchema)]
791struct RepeatArgs {
792    /// The text to repeat.
793    text: String,
794    /// Number of repetitions; defaults to 2 when omitted.
795    times: Option<u32>,
796}
797
798#[derive(Clone)]
799struct RepeatTool {
800    calls: Arc<AtomicUsize>,
801}
802
803impl Tool for RepeatTool {
804    const NAME: &'static str = "repeat_text";
805    type Error = ConformanceToolError;
806    type Args = RepeatArgs;
807    type Output = String;
808
809    fn description(&self) -> String {
810        "Repeat `text`. `times` is optional and defaults to 2.".to_string()
811    }
812
813    fn parameters(&self) -> serde_json::Value {
814        serde_json::to_value(schemars::schema_for!(RepeatArgs)).unwrap_or_default()
815    }
816
817    async fn call(
818        &self,
819        _context: &mut ToolContext,
820        args: Self::Args,
821    ) -> Result<Self::Output, Self::Error> {
822        self.calls.fetch_add(1, Ordering::SeqCst);
823        Ok(vec![args.text.as_str(); args.times.unwrap_or(2) as usize].join(" "))
824    }
825}
826
827#[derive(Debug, Deserialize, JsonSchema)]
828struct BinOpArgs {
829    a: i64,
830    b: i64,
831}
832
833#[derive(Debug, Deserialize, JsonSchema)]
834struct ArithmeticResult {
835    answer: i64,
836    explanation: Option<String>,
837}
838
839#[derive(Clone)]
840struct AddTool(Arc<AtomicUsize>);
841
842impl Tool for AddTool {
843    const NAME: &'static str = "add";
844    type Error = ConformanceToolError;
845    type Args = BinOpArgs;
846    type Output = i64;
847
848    fn description(&self) -> String {
849        "Add two integers a and b.".to_string()
850    }
851
852    fn parameters(&self) -> serde_json::Value {
853        serde_json::to_value(schemars::schema_for!(BinOpArgs)).unwrap_or_default()
854    }
855
856    async fn call(
857        &self,
858        _context: &mut ToolContext,
859        args: Self::Args,
860    ) -> Result<Self::Output, Self::Error> {
861        self.0.fetch_add(1, Ordering::SeqCst);
862        Ok(args.a + args.b)
863    }
864}
865
866#[derive(Clone)]
867struct MultiplyTool(Arc<AtomicUsize>);
868
869impl Tool for MultiplyTool {
870    const NAME: &'static str = "multiply";
871    type Error = ConformanceToolError;
872    type Args = BinOpArgs;
873    type Output = i64;
874
875    fn description(&self) -> String {
876        "Multiply two integers a and b.".to_string()
877    }
878
879    fn parameters(&self) -> serde_json::Value {
880        serde_json::to_value(schemars::schema_for!(BinOpArgs)).unwrap_or_default()
881    }
882
883    async fn call(
884        &self,
885        _context: &mut ToolContext,
886        args: Self::Args,
887    ) -> Result<Self::Output, Self::Error> {
888        self.0.fetch_add(1, Ordering::SeqCst);
889        Ok(args.a * args.b)
890    }
891}
892
893fn report_from_response(
894    name: &'static str,
895    started: Instant,
896    tool_calls: usize,
897    response: crate::agent::PromptResponse,
898) -> Result<ScenarioReport, ScenarioError> {
899    validate_protocol_hygiene(
900        name,
901        &response.output(),
902        &response.messages,
903        &[
904            "<tool_call>",
905            "</tool_call>",
906            "<tool_response>",
907            "</tool_response>",
908            "<|im_start|>",
909            "<|im_end|>",
910            "<think>",
911            "</think>",
912        ],
913    )?;
914    Ok(ScenarioReport {
915        name,
916        tool_calls,
917        prompt_tokens: response.usage.input_tokens.unwrap_or(0),
918        generated_tokens: response.usage.output_tokens.unwrap_or(0),
919        history_messages: response.messages.len(),
920        duration: started.elapsed(),
921        response: response.output(),
922    })
923}
924
925/// Require the extended run's accumulated message history and validate
926/// canonical tool call/result correlation over it.
927fn correlated_messages<'a>(
928    scenario: &'static str,
929    response: &'a crate::agent::PromptResponse,
930) -> Result<&'a [Message], ScenarioError> {
931    let messages = response.messages.as_slice();
932    if messages.is_empty() {
933        return Err(ScenarioError::contract(
934            scenario,
935            "extended run omitted accumulated message history",
936        ));
937    }
938    validate_tool_correlation(scenario, messages)?;
939    Ok(messages)
940}
941
942/// [`correlated_messages`], flattened into every tool-result value in history.
943fn correlated_result_values(
944    scenario: &'static str,
945    response: &crate::agent::PromptResponse,
946) -> Result<Vec<serde_json::Value>, ScenarioError> {
947    Ok(correlated_messages(scenario, response)?
948        .iter()
949        .flat_map(tool_result_values)
950        .collect())
951}
952
953fn value_matches_integer(value: &serde_json::Value, expected: i64) -> bool {
954    value.as_i64() == Some(expected)
955        || value
956            .as_str()
957            .and_then(|text| text.trim().parse::<i64>().ok())
958            == Some(expected)
959}
960
961/// Runs two independent calls in one assistant turn and validates canonical
962/// call/result history correlation.
963///
964/// Set `tool_concurrency` to `Some(1)` to prove that serial host execution does
965/// not split or drop the model's parallel call batch.
966pub async fn parallel_tools<W, T, F>(
967    model: rig_core::driver::Model<W, T>,
968    configure: F,
969    tool_concurrency: Option<usize>,
970) -> Result<ScenarioReport, ScenarioError>
971where
972    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
973    T: rig_core::driver::Transport<W>,
974    F: FnOnce(AgentBuilder) -> AgentBuilder,
975{
976    let add_calls = Arc::new(AtomicUsize::new(0));
977    let subtract_calls = Arc::new(AtomicUsize::new(0));
978    let started = Instant::now();
979    let agent = configure(AgentBuilder::new(model).temperature(0.0))
980        .preamble(FORCE_TOOLS_PREAMBLE)
981        .tool(CountingAdd(add_calls.clone()))
982        .tool(CountingSubtract(subtract_calls.clone()))
983        .default_max_turns(3)
984        .build();
985    let request = agent.prompt(PARALLEL_PROMPT).max_turns(3);
986    let response = match tool_concurrency {
987        Some(concurrency) => request.tool_concurrency(concurrency).await?,
988        None => request.await?,
989    };
990    let scenario = if tool_concurrency == Some(1) {
991        "parallel_tools_serial_execution"
992    } else {
993        "parallel_tools"
994    };
995    let messages = correlated_messages(scenario, &response)?;
996
997    let Some((call_index, calls)) = messages.iter().enumerate().find_map(|(index, message)| {
998        let Message::Assistant(rig_core::message::AssistantMessage { content, .. }) = message
999        else {
1000            return None;
1001        };
1002        let calls = content
1003            .iter()
1004            .filter_map(|item| match item {
1005                AssistantContent::ToolCall(call) => Some(call),
1006                _ => None,
1007            })
1008            .collect::<Vec<_>>();
1009        (calls.len() == 2).then_some((index, calls))
1010    }) else {
1011        return Err(ScenarioError::contract(
1012            scenario,
1013            format!("no assistant turn contained exactly two tool calls: {messages:?}"),
1014        ));
1015    };
1016    let mut names = calls
1017        .iter()
1018        .map(|call| call.function.name.as_str())
1019        .collect::<Vec<_>>();
1020    names.sort_unstable();
1021    if names != ["add", "subtract"] {
1022        return Err(ScenarioError::contract(
1023            scenario,
1024            format!("parallel turn called {names:?}, expected add and subtract"),
1025        ));
1026    }
1027    let results_message = messages.get(call_index + 1).ok_or_else(|| {
1028        ScenarioError::contract(
1029            scenario,
1030            "parallel call turn has no following result message",
1031        )
1032    })?;
1033    let values = tool_result_values(results_message);
1034    if !values.iter().any(|value| value_matches_integer(value, 7))
1035        || !values.iter().any(|value| value_matches_integer(value, 8))
1036        || values.len() != 2
1037    {
1038        return Err(ScenarioError::contract(
1039            scenario,
1040            format!("parallel result message did not contain exactly 7 and 8: {values:?}"),
1041        ));
1042    }
1043    let add = add_calls.load(Ordering::SeqCst);
1044    let subtract = subtract_calls.load(Ordering::SeqCst);
1045    if add != 1 || subtract != 1 {
1046        return Err(ScenarioError::contract(
1047            scenario,
1048            format!("execution counts were add={add}, subtract={subtract}, expected one each"),
1049        ));
1050    }
1051    report_from_response(scenario, started, add + subtract, response)
1052}
1053
1054/// Runs a zero-argument tool and validates verbatim string-result handling.
1055pub async fn zero_argument_tool<W, T, F>(
1056    model: rig_core::driver::Model<W, T>,
1057    configure: F,
1058) -> Result<ScenarioReport, ScenarioError>
1059where
1060    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1061    T: rig_core::driver::Transport<W>,
1062    F: FnOnce(AgentBuilder) -> AgentBuilder,
1063{
1064    const SCENARIO: &str = "zero_argument_tool";
1065    let calls = Arc::new(AtomicUsize::new(0));
1066    let started = Instant::now();
1067    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1068        .preamble("You must use the provided tools. Report tool outputs exactly as returned.")
1069        .tool(PingTool(calls.clone()))
1070        .default_max_turns(2)
1071        .build();
1072    let response = agent
1073        .prompt("Call the ping tool, then report the exact marker it returns.")
1074        .max_turns(2)
1075        .await?;
1076    let values = correlated_result_values(SCENARIO, &response)?;
1077    if calls.load(Ordering::SeqCst) != 1
1078        || !values
1079            .iter()
1080            .any(|value| value.as_str() == Some(PING_OUTPUT))
1081        || !response.output().contains(PING_OUTPUT)
1082    {
1083        return Err(ScenarioError::contract(
1084            SCENARIO,
1085            format!(
1086                "calls={}, results={values:?}, response={:?}",
1087                calls.load(Ordering::SeqCst),
1088                response.output()
1089            ),
1090        ));
1091    }
1092    report_from_response(SCENARIO, started, 1, response)
1093}
1094
1095/// Runs string- and JSON-returning tools and validates that neither output is
1096/// double encoded.
1097pub async fn tool_output_serialization<W, T, F>(
1098    model: rig_core::driver::Model<W, T>,
1099    configure: F,
1100) -> Result<ScenarioReport, ScenarioError>
1101where
1102    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1103    T: rig_core::driver::Transport<W>,
1104    F: FnOnce(AgentBuilder) -> AgentBuilder,
1105{
1106    const SCENARIO: &str = "tool_output_serialization";
1107    let started = Instant::now();
1108    let motto_calls = Arc::new(AtomicUsize::new(0));
1109    let config_calls = Arc::new(AtomicUsize::new(0));
1110    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1111        .preamble("You must use the provided tools before answering.")
1112        .tool(MottoTool(motto_calls.clone()))
1113        .tool(ConfigTool(config_calls.clone()))
1114        .default_max_turns(3)
1115        .build();
1116    let response = agent
1117        .prompt("Call fetch_motto and fetch_config, then summarize both outputs in one sentence.")
1118        .max_turns(3)
1119        .await?;
1120    let values = correlated_result_values(SCENARIO, &response)?;
1121    let expected_config = serde_json::to_value(ConfigOutput {
1122        service: "cassette-lab".to_string(),
1123        max_retries: 3,
1124    })?;
1125    let motto_ok = values
1126        .iter()
1127        .any(|value| value.as_str() == Some(MOTTO_OUTPUT));
1128    let config_ok = values.iter().any(|value| {
1129        value == &expected_config
1130            || value
1131                .as_str()
1132                .and_then(|text| serde_json::from_str::<serde_json::Value>(text).ok())
1133                .as_ref()
1134                == Some(&expected_config)
1135    });
1136    let motto_count = motto_calls.load(Ordering::SeqCst);
1137    let config_count = config_calls.load(Ordering::SeqCst);
1138    if !motto_ok || !config_ok || motto_count != 1 || config_count != 1 {
1139        return Err(ScenarioError::contract(
1140            SCENARIO,
1141            format!(
1142                "expected one verbatim motto and one semantic config JSON; motto_calls={motto_count}, config_calls={config_count}, values={values:?}"
1143            ),
1144        ));
1145    }
1146    report_from_response(SCENARIO, started, motto_count + config_count, response)
1147}
1148
1149/// Runs a nested, escaped, Unicode-bearing argument payload through typed tool
1150/// deserialization and validates the exact semantic value received by the tool.
1151pub async fn complex_tool_arguments<W, T, F>(
1152    model: rig_core::driver::Model<W, T>,
1153    configure: F,
1154) -> Result<ScenarioReport, ScenarioError>
1155where
1156    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1157    T: rig_core::driver::Transport<W>,
1158    F: FnOnce(AgentBuilder) -> AgentBuilder,
1159{
1160    complex_tool_arguments_with_prompt(model, configure, COMPLEX_ARGUMENTS_PROMPT).await
1161}
1162
1163/// The prompt [`complex_tool_arguments`] sends. It spells the values in their
1164/// JSON-escaped form, which a model may copy literally, backslashes included.
1165const COMPLEX_ARGUMENTS_PROMPT: &str = "Call store_profile with profile.name exactly `Zoë \\\"Z\\\"`, profile.tags exactly [`rust`, `東京`], mode `careful`, note containing the two lines `line one` and `line two` separated by a newline, and quote exactly `path C:\\\\tmp and \\\"quoted\\\"`. Then confirm it was stored.";
1166
1167/// The same values as [`complex_tool_arguments`] sends, spelled as raw text
1168/// between markers, for a model that copies escapes literally.
1169pub const COMPLEX_ARGUMENTS_RAW_PROMPT: &str = "Call store_profile once. Each value below is raw text between <<< and >>>, with no escaping: profile.name <<<Zoë \"Z\">>>, profile.tags <<<rust>>> and <<<東京>>>, mode <<<careful>>>, note the two lines <<<line one>>> and <<<line two>>> joined by one newline, and quote <<<path C:\\tmp and \"quoted\">>>. Then confirm it was stored.";
1170
1171/// [`complex_tool_arguments`] with the user prompt given: the same tool, the
1172/// same expected arguments, the same checks.
1173pub async fn complex_tool_arguments_with_prompt<W, T, F>(
1174    model: rig_core::driver::Model<W, T>,
1175    configure: F,
1176    prompt: &str,
1177) -> Result<ScenarioReport, ScenarioError>
1178where
1179    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1180    T: rig_core::driver::Transport<W>,
1181    F: FnOnce(AgentBuilder) -> AgentBuilder,
1182{
1183    const SCENARIO: &str = "complex_tool_arguments";
1184    let expected = ComplexArgs {
1185        profile: ComplexProfile {
1186            name: "Zoë \"Z\"".to_string(),
1187            tags: vec!["rust".to_string(), "東京".to_string()],
1188        },
1189        mode: ComplexMode::Careful,
1190        note: Some("line one\nline two".to_string()),
1191        quote: "path C:\\tmp and \"quoted\"".to_string(),
1192    };
1193    let calls = Arc::new(AtomicUsize::new(0));
1194    let captured = Arc::new(Mutex::new(None));
1195    let started = Instant::now();
1196    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1197        .preamble("Use store_profile exactly once with every value supplied by the user.")
1198        .tool(CaptureComplexTool {
1199            calls: calls.clone(),
1200            captured: captured.clone(),
1201        })
1202        .default_max_turns(3)
1203        .build();
1204    let response = agent.prompt(prompt).max_turns(3).await?;
1205    let observed = lock_recover(&captured).clone();
1206    if calls.load(Ordering::SeqCst) != 1 || observed.as_ref() != Some(&expected) {
1207        return Err(ScenarioError::contract(
1208            SCENARIO,
1209            format!(
1210                "calls={}, expected={expected:?}, observed={observed:?}, response={:?}",
1211                calls.load(Ordering::SeqCst),
1212                response.output()
1213            ),
1214        ));
1215    }
1216    correlated_messages(SCENARIO, &response)?;
1217    report_from_response(SCENARIO, started, 1, response)
1218}
1219
1220/// Runs the same deterministic text request through buffered and raw streaming
1221/// completion surfaces and validates equivalent visible content and usage.
1222pub async fn buffered_streaming_text_parity<W, T, A>(
1223    model: rig_core::driver::Model<W, T>,
1224    adjust: A,
1225) -> Result<ScenarioReport, ScenarioError>
1226where
1227    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1228    T: rig_core::driver::Transport<W>,
1229    A: Fn(CompletionRequest) -> CompletionRequest,
1230{
1231    const SCENARIO: &str = "buffered_streaming_text_parity";
1232    const PROMPT: &str = "Answer with exactly the single word Paris.";
1233    let started = Instant::now();
1234    let request = || {
1235        adjust(
1236            CompletionRequest::new(PROMPT)
1237                .temperature(0.0)
1238                .max_tokens(32),
1239        )
1240    };
1241    let buffered = model.call(request()).await?;
1242    let buffered_text = buffered
1243        .choice
1244        .iter()
1245        .filter_map(|item| match item {
1246            AssistantContent::Text(text) => Some(text.text.as_str()),
1247            _ => None,
1248        })
1249        .collect::<String>();
1250
1251    let mut stream = model.stream(request())?;
1252    let mut streamed_text = String::new();
1253    while let Some(item) = stream.next().await {
1254        if let crate::streaming::Item::Event(crate::streaming::StreamEvent::Text { text, .. }) =
1255            item?
1256        {
1257            streamed_text.push_str(&text);
1258        }
1259    }
1260    let streamed_usage = Some(stream.finish().await?.usage);
1261    let usage = streamed_usage.ok_or_else(|| {
1262        ScenarioError::contract(SCENARIO, "raw stream omitted its final response metadata")
1263    })?;
1264    let normalize = |text: &str| {
1265        text.trim()
1266            .trim_matches(|character: char| !character.is_alphanumeric())
1267            .to_string()
1268    };
1269    let buffered_answer = normalize(&buffered_text);
1270    let streamed_answer = normalize(&streamed_text);
1271    if !buffered_answer.eq_ignore_ascii_case("Paris")
1272        || !streamed_answer.eq_ignore_ascii_case("Paris")
1273        || !buffered.usage.is_reported()
1274        || !usage.is_reported()
1275    {
1276        return Err(ScenarioError::contract(
1277            SCENARIO,
1278            format!(
1279                "buffered={buffered_text:?}, streamed={streamed_text:?}, buffered_usage={:?}, streamed_usage={usage:?}",
1280                buffered.usage
1281            ),
1282        ));
1283    }
1284    Ok(ScenarioReport {
1285        name: SCENARIO,
1286        tool_calls: 0,
1287        prompt_tokens: usage.input_tokens.unwrap_or(0),
1288        generated_tokens: usage.output_tokens.unwrap_or(0),
1289        history_messages: 0,
1290        duration: started.elapsed(),
1291        response: streamed_text,
1292    })
1293}
1294
1295/// Runs Rig's structured extractor and validates both the extracted semantic
1296/// fields and accumulated usage.
1297pub async fn structured_extraction<W, T>(
1298    model: rig_core::driver::Model<W, T>,
1299    additional_params: Option<serde_json::Value>,
1300) -> Result<ScenarioReport, ScenarioError>
1301where
1302    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1303    T: rig_core::driver::Transport<W>,
1304{
1305    const SCENARIO: &str = "structured_extraction";
1306    const INPUT: &str = "Hello, my name is Ada Lovelace and I work as a mathematician.";
1307    let started = Instant::now();
1308    let mut builder = crate::extractor::ExtractorBuilder::<ExtractedPerson>::new(model)
1309        .max_tokens(384)
1310        .retries(0);
1311    if let Some(params) = additional_params {
1312        builder = builder.additional_params(params);
1313    }
1314    let response = builder.build().extract(INPUT).await?;
1315    validate_extraction_fields(
1316        SCENARIO,
1317        response.output.first_name.as_deref(),
1318        response.output.last_name.as_deref(),
1319        response.output.job.as_deref(),
1320        response.usage,
1321    )?;
1322    Ok(ScenarioReport {
1323        name: SCENARIO,
1324        tool_calls: 1,
1325        prompt_tokens: response.usage.input_tokens.unwrap_or(0),
1326        generated_tokens: response.usage.output_tokens.unwrap_or(0),
1327        history_messages: 0,
1328        duration: started.elapsed(),
1329        response: format!(
1330            "{} {} — {}",
1331            response.output.first_name.as_deref().unwrap_or_default(),
1332            response.output.last_name.as_deref().unwrap_or_default(),
1333            response.output.job.as_deref().unwrap_or_default()
1334        ),
1335    })
1336}
1337
1338fn restricted_recovery_run(
1339    prompt: &str,
1340    turn: ModelTurn,
1341    retries: usize,
1342) -> Result<AgentRun, ScenarioError> {
1343    const SCENARIO: &str = "invalid_tool_recovery";
1344    let mut run = AgentRun::new(prompt)
1345        .max_turns(2)
1346        .max_invalid_tool_call_retries(retries);
1347    if !matches!(run.next_step()?, AgentRunStep::CallModel { .. }) {
1348        return Err(ScenarioError::contract(
1349            SCENARIO,
1350            "fresh AgentRun did not request a model turn",
1351        ));
1352    }
1353    let outcome = run.model_response(turn)?;
1354    let ModelTurnOutcome::NeedsResolution(context) = outcome else {
1355        return Err(ScenarioError::contract(
1356            SCENARIO,
1357            format!("disallowed tool call did not require resolution: {outcome:?}"),
1358        ));
1359    };
1360    if context.tool_name != CountingAdd::NAME {
1361        return Err(ScenarioError::contract(
1362            SCENARIO,
1363            format!("expected rejected add call, observed {context:?}"),
1364        ));
1365    }
1366    Ok(run)
1367}
1368
1369/// Uses one real model turn to exercise fail-fast, retry exhaustion, repair,
1370/// rejected repair, and skip handling without executing a disallowed call.
1371pub async fn invalid_tool_recovery<W, T, F>(
1372    model: rig_core::driver::Model<W, T>,
1373    configure: F,
1374) -> Result<ScenarioReport, ScenarioError>
1375where
1376    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1377    T: rig_core::driver::Transport<W>,
1378    F: FnOnce(AgentBuilder) -> AgentBuilder,
1379{
1380    invalid_tool_recovery_with_choice(model, configure, ToolChoice::Required).await
1381}
1382
1383/// [`invalid_tool_recovery`] with the agent's tool choice given, for a model
1384/// that rejects a forced choice (`ToolChoice::Auto`; the prompt still asks for
1385/// the call).
1386pub async fn invalid_tool_recovery_with_choice<W, T, F>(
1387    model: rig_core::driver::Model<W, T>,
1388    configure: F,
1389    tool_choice: ToolChoice,
1390) -> Result<ScenarioReport, ScenarioError>
1391where
1392    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1393    T: rig_core::driver::Transport<W>,
1394    F: FnOnce(AgentBuilder) -> AgentBuilder,
1395{
1396    const SCENARIO: &str = "invalid_tool_recovery";
1397    const PROMPT: &str = "Call the add tool exactly once with x=2 and y=3. Do not call sum.";
1398    let started = Instant::now();
1399    let add_calls = Arc::new(AtomicUsize::new(0));
1400    let sum_calls = Arc::new(AtomicUsize::new(0));
1401    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1402        .preamble(FORCE_TOOLS_PREAMBLE)
1403        .tool(CountingAdd(add_calls.clone()))
1404        .tool(CountingSum(sum_calls.clone()))
1405        .tool_choice(tool_choice)
1406        .build();
1407    #[derive(Clone)]
1408    struct CaptureTurn(Arc<Mutex<Option<ModelTurn>>>);
1409
1410    impl AgentHook for CaptureTurn {
1411        async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
1412            let Some(response) = event.completion() else {
1413                return OutcomeAction::proceed();
1414            };
1415            *lock_recover(&self.0) = Some(ModelTurn::new(
1416                response.head(),
1417                response.choice.clone(),
1418                response.usage,
1419                TurnPolicy::default(),
1420                response.raw.clone(),
1421            ));
1422            OutcomeAction::stop("captured conformance model turn")
1423        }
1424    }
1425
1426    let captured = Arc::new(Mutex::new(None));
1427    let stopped = agent
1428        .prompt(PROMPT)
1429        .add_hook(CaptureTurn(captured.clone()))
1430        .run()
1431        .await;
1432    if !matches!(stopped, Err(PromptError::Cancelled { .. })) {
1433        return Err(ScenarioError::contract(
1434            SCENARIO,
1435            format!("capture hook did not stop after the model response: {stopped:?}"),
1436        ));
1437    }
1438    let response = lock_recover(&captured).take().ok_or_else(|| {
1439        ScenarioError::contract(SCENARIO, "capture hook observed no model response")
1440    })?;
1441    let emitted = response
1442        .choice
1443        .iter()
1444        .filter(|item| {
1445            matches!(item, AssistantContent::ToolCall(call) if call.function.name == CountingAdd::NAME)
1446        })
1447        .count();
1448    if emitted != 1 {
1449        return Err(ScenarioError::contract(
1450            SCENARIO,
1451            format!(
1452                "model emitted {emitted} add calls, response={:?}",
1453                response.choice
1454            ),
1455        ));
1456    }
1457    let executable = BTreeSet::from([CountingAdd::NAME.to_string(), CountingSum::NAME.to_string()]);
1458    let only_sum = ToolChoice::Specific {
1459        function_names: vec![
1460            rig_core::message::ToolName::new(CountingSum::NAME)
1461                .map_err(|error| ScenarioError::contract(SCENARIO, error.to_string()))?,
1462        ],
1463    };
1464    let policy = TurnPolicy::new(executable, Some(only_sum), None)
1465        .map_err(|error| ScenarioError::contract(SCENARIO, error.to_string()))?;
1466    let turn = ModelTurn::new(
1467        response.head,
1468        response.choice,
1469        response.usage,
1470        policy,
1471        response.raw,
1472    );
1473
1474    let mut fail = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1475    let error = match fail.resolve_invalid_tool_call(InvalidToolCallAction::fail()) {
1476        Err(error) => error,
1477        Ok(outcome) => {
1478            return Err(ScenarioError::contract(
1479                SCENARIO,
1480                format!("fail action unexpectedly returned {outcome:?}"),
1481            ));
1482        }
1483    };
1484    validate_unknown_tool_failure(&error, CountingAdd::NAME, &[CountingSum::NAME])?;
1485
1486    let mut retry = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1487    let error = match retry
1488        .resolve_invalid_tool_call(InvalidToolCallAction::retry("choose an allowed tool"))
1489    {
1490        Err(error) => error,
1491        Ok(outcome) => {
1492            return Err(ScenarioError::contract(
1493                SCENARIO,
1494                format!("exhausted retry unexpectedly returned {outcome:?}"),
1495            ));
1496        }
1497    };
1498    validate_unknown_tool_failure(&error, CountingAdd::NAME, &[CountingSum::NAME])?;
1499
1500    let mut rejected_repair = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1501    let error =
1502        match rejected_repair.resolve_invalid_tool_call(InvalidToolCallAction::repair("missing")) {
1503            Err(error) => error,
1504            Ok(outcome) => {
1505                return Err(ScenarioError::contract(
1506                    SCENARIO,
1507                    format!("disallowed repair unexpectedly returned {outcome:?}"),
1508                ));
1509            }
1510        };
1511    validate_unknown_tool_failure(&error, "missing", &[CountingSum::NAME])?;
1512
1513    let mut repaired = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1514    if !matches!(
1515        repaired.resolve_invalid_tool_call(InvalidToolCallAction::repair(CountingSum::NAME))?,
1516        ModelTurnOutcome::Continue { .. }
1517    ) {
1518        return Err(ScenarioError::contract(
1519            SCENARIO,
1520            "valid repair did not continue",
1521        ));
1522    }
1523    let AgentRunStep::CallTools { calls } = repaired.next_step()? else {
1524        return Err(ScenarioError::contract(
1525            SCENARIO,
1526            "valid repair did not produce pending tool execution",
1527        ));
1528    };
1529    let repaired_call = calls.first();
1530    if calls.len() != 1
1531        || !repaired_call.is_some_and(|call| {
1532            matches!(call, PendingToolCall::Execute(call) if call.name() == CountingSum::NAME)
1533        })
1534    {
1535        return Err(ScenarioError::contract(
1536            SCENARIO,
1537            format!("repaired pending calls were incorrect: {calls:?}"),
1538        ));
1539    }
1540
1541    let mut skipped = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1542    if !matches!(
1543        skipped.resolve_invalid_tool_call(InvalidToolCallAction::skip("disabled for this turn"))?,
1544        ModelTurnOutcome::Continue { .. }
1545    ) {
1546        return Err(ScenarioError::contract(SCENARIO, "skip did not continue"));
1547    }
1548    // A whole-turn skip answers every call, so the run asks the model again.
1549    let step = skipped.next_step()?;
1550    let skipped_answered = matches!(
1551        skipped.messages().last().map(|message| &**message),
1552        Some(Message::User { content }) if content.len() == 1
1553    );
1554    if !matches!(step, AgentRunStep::CallModel { .. }) || !skipped_answered {
1555        return Err(ScenarioError::contract(
1556            SCENARIO,
1557            format!("skip did not answer the turn and continue: {step:?}"),
1558        ));
1559    }
1560    if add_calls.load(Ordering::SeqCst) != 0 || sum_calls.load(Ordering::SeqCst) != 0 {
1561        return Err(ScenarioError::contract(
1562            SCENARIO,
1563            "recovery scenario executed a tool body",
1564        ));
1565    }
1566
1567    Ok(ScenarioReport {
1568        name: SCENARIO,
1569        tool_calls: emitted,
1570        prompt_tokens: turn.usage.input_tokens.unwrap_or(0),
1571        generated_tokens: turn.usage.output_tokens.unwrap_or(0),
1572        history_messages: 2,
1573        duration: started.elapsed(),
1574        response: "fail, retry, repair, rejected repair, and skip passed".to_string(),
1575    })
1576}
1577
1578/// Exercises chained argument/result rewrites and a first-turn-only request
1579/// patch. Completion proves `tool_choice=Required` did not leak to turn two.
1580pub async fn hook_rewrites_and_request_patch<W, T, F>(
1581    model: rig_core::driver::Model<W, T>,
1582    configure: F,
1583) -> Result<ScenarioReport, ScenarioError>
1584where
1585    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1586    T: rig_core::driver::Transport<W>,
1587    F: FnOnce(AgentBuilder) -> AgentBuilder,
1588{
1589    hook_rewrites_and_request_patch_with_choice(model, configure, ToolChoice::Required).await
1590}
1591
1592/// [`hook_rewrites_and_request_patch`] with the first-turn patch's tool
1593/// choice given, for a model that rejects a forced choice (`ToolChoice::Auto`:
1594/// the patch still narrows the active tools on turn one only).
1595pub async fn hook_rewrites_and_request_patch_with_choice<W, T, F>(
1596    model: rig_core::driver::Model<W, T>,
1597    configure: F,
1598    first_turn_choice: ToolChoice,
1599) -> Result<ScenarioReport, ScenarioError>
1600where
1601    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1602    T: rig_core::driver::Transport<W>,
1603    F: FnOnce(AgentBuilder) -> AgentBuilder,
1604{
1605    const SCENARIO: &str = "hook_rewrites_and_request_patch";
1606    let started = Instant::now();
1607    let calls = Arc::new(AtomicUsize::new(0));
1608    let observed = ObserveArguments::default();
1609    let observed_probe = observed.clone();
1610    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1611        .preamble("Use add for arithmetic and report only the tool result.")
1612        .tool(CountingAdd(calls.clone()))
1613        .default_max_turns(3)
1614        .build();
1615    let response = agent
1616        .prompt("Use add once for x=1 and y=1, then report what the tool returns.")
1617        .max_turns(3)
1618        .add_hook(FirstTurnPatch(
1619            RequestPatch::new()
1620                .active_tools([CountingAdd::NAME])
1621                .tool_choice(first_turn_choice),
1622        ))
1623        .add_hook(RewriteArgument {
1624            key: "x",
1625            value: serde_json::json!(7),
1626        })
1627        .add_hook(RewriteArgument {
1628            key: "y",
1629            value: serde_json::json!(8),
1630        })
1631        .add_hook(observed)
1632        .add_hook(ReplaceResult("portable-redacted"))
1633        .add_hook(WrapResult)
1634        .await?;
1635    let observations = lock_recover(&observed_probe.0).clone();
1636    validate_rewritten_arguments(
1637        SCENARIO,
1638        &observations,
1639        &serde_json::json!({ "x": 7, "y": 8 }),
1640    )?;
1641    let messages = response.messages.as_slice();
1642    if messages.is_empty() {
1643        return Err(ScenarioError::contract(
1644            SCENARIO,
1645            "extended hook run omitted message history",
1646        ));
1647    }
1648    let results = messages
1649        .iter()
1650        .flat_map(tool_result_values)
1651        .collect::<Vec<_>>();
1652    if calls.load(Ordering::SeqCst) != 1
1653        || !results
1654            .iter()
1655            .any(|value| value == &serde_json::json!("[portable-redacted]"))
1656        || response.completion_calls.len() != 2
1657    {
1658        return Err(ScenarioError::contract(
1659            SCENARIO,
1660            format!(
1661                "calls={}, completion_calls={}, results={results:?}, output={:?}",
1662                calls.load(Ordering::SeqCst),
1663                response.completion_calls.len(),
1664                response.output()
1665            ),
1666        ));
1667    }
1668    report_from_response(SCENARIO, started, 1, response)
1669}
1670
1671/// Exercises post-execution cancellation and max-turn diagnostics through the
1672/// public agent driver using real model-emitted calls.
1673pub async fn cancellation_and_max_turns<W, T, F>(
1674    model: rig_core::driver::Model<W, T>,
1675    configure: F,
1676) -> Result<ScenarioReport, ScenarioError>
1677where
1678    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1679    T: rig_core::driver::Transport<W>,
1680    F: Fn(AgentBuilder) -> AgentBuilder,
1681{
1682    const SCENARIO: &str = "cancellation_and_max_turns";
1683    const REASON: &str = "portable result veto";
1684    let started = Instant::now();
1685    let cancelled_calls = Arc::new(AtomicUsize::new(0));
1686    let cancelled_agent = configure(AgentBuilder::new(model.clone()).temperature(0.0))
1687        .preamble("Use add for arithmetic; never calculate by hand.")
1688        .tool(CountingAdd(cancelled_calls.clone()))
1689        .build();
1690    let cancelled = match cancelled_agent
1691        .prompt("Use add once to compute x=20 plus y=22.")
1692        .max_turns(2)
1693        .add_hook(StopAfterResult(REASON))
1694        .await
1695    {
1696        Err(error) => error,
1697        Ok(output) => {
1698            return Err(ScenarioError::contract(
1699                SCENARIO,
1700                format!("result cancellation unexpectedly completed: {output:?}"),
1701            ));
1702        }
1703    };
1704    validate_cancelled_failure(&cancelled, REASON, CountingAdd::NAME)?;
1705
1706    let max_turn_calls = Arc::new(AtomicUsize::new(0));
1707    let max_turn_agent = configure(AgentBuilder::new(model).temperature(0.0))
1708        .preamble("Use add for arithmetic; never calculate by hand.")
1709        .tool(CountingAdd(max_turn_calls.clone()))
1710        .build();
1711    let max_turn = match max_turn_agent
1712        .prompt("Use add once to compute x=20 plus y=22, then report the result.")
1713        .max_turns(1)
1714        .await
1715    {
1716        Err(error) => error,
1717        Ok(output) => {
1718            return Err(ScenarioError::contract(
1719                SCENARIO,
1720                format!("one-turn budget unexpectedly completed: {output:?}"),
1721            ));
1722        }
1723    };
1724    validate_max_turns_failure(&max_turn, 1)?;
1725    let cancelled_count = cancelled_calls.load(Ordering::SeqCst);
1726    let max_turn_count = max_turn_calls.load(Ordering::SeqCst);
1727    if cancelled_count != 1 || max_turn_count != 1 {
1728        return Err(ScenarioError::contract(
1729            SCENARIO,
1730            format!("cancelled executions={cancelled_count}, max-turn executions={max_turn_count}"),
1731        ));
1732    }
1733    Ok(ScenarioReport {
1734        name: SCENARIO,
1735        tool_calls: cancelled_count + max_turn_count,
1736        prompt_tokens: 0,
1737        generated_tokens: 0,
1738        history_messages: 4,
1739        duration: started.elapsed(),
1740        response: "post-result cancellation and max-turn diagnostics passed".to_string(),
1741    })
1742}
1743
1744/// Runs the portable optional-argument tool scenario.
1745///
1746/// `configure` is deliberately outside the scenario so a provider suite can
1747/// attach transport-only settings without putting them into the shared model
1748/// contract.
1749pub async fn optional_argument<W, T, F>(
1750    model: rig_core::driver::Model<W, T>,
1751    configure: F,
1752) -> Result<ScenarioReport, ScenarioError>
1753where
1754    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1755    T: rig_core::driver::Transport<W>,
1756    F: FnOnce(AgentBuilder) -> AgentBuilder,
1757{
1758    let calls = Arc::new(AtomicUsize::new(0));
1759    let started = Instant::now();
1760    let agent = configure(AgentBuilder::new(model))
1761        .preamble("Use the repeat_text tool whenever asked to repeat text.")
1762        .tool(RepeatTool {
1763            calls: calls.clone(),
1764        })
1765        .default_max_turns(4)
1766        .build();
1767    let result = agent
1768        .prompt(
1769            "Use the repeat_text tool to repeat the word \"banana\" 3 times, then show me the exact result.",
1770        )
1771        .await?;
1772    let response = result.output();
1773    let tool_calls = calls.load(Ordering::SeqCst);
1774    if tool_calls == 0
1775        || response.matches("banana").count() < 1
1776        || !has_tool_roundtrip(&result.messages)
1777    {
1778        return Err(ScenarioError::contract(
1779            "optional_argument",
1780            format!("calls={tool_calls}, response={response:?}"),
1781        ));
1782    }
1783    report_from_response("optional_argument", started, tool_calls, result)
1784}
1785
1786/// Runs a portable two-tool sequential arithmetic scenario.
1787pub async fn sequential_tools<W, T, F>(
1788    model: rig_core::driver::Model<W, T>,
1789    configure: F,
1790) -> Result<ScenarioReport, ScenarioError>
1791where
1792    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1793    T: rig_core::driver::Transport<W>,
1794    F: FnOnce(AgentBuilder) -> AgentBuilder,
1795{
1796    let add_calls = Arc::new(AtomicUsize::new(0));
1797    let multiply_calls = Arc::new(AtomicUsize::new(0));
1798    let started = Instant::now();
1799    let agent = configure(AgentBuilder::new(model))
1800        .preamble(
1801            "You are a calculator. Use the add and multiply tools for arithmetic; never compute by hand.",
1802        )
1803        .tool(AddTool(add_calls.clone()))
1804        .tool(MultiplyTool(multiply_calls.clone()))
1805        .default_max_turns(6)
1806        .build();
1807    let result = agent
1808        .prompt(
1809            "Compute (4 + 6) * 2. First call the add tool, then call the multiply tool on the result. Tell me the final number.",
1810        )
1811        .await?;
1812    let response = result.output();
1813    let add = add_calls.load(Ordering::SeqCst);
1814    let multiply = multiply_calls.load(Ordering::SeqCst);
1815    if add == 0
1816        || multiply == 0
1817        || !response.contains("20")
1818        || !has_tool_roundtrip(&result.messages)
1819    {
1820        return Err(ScenarioError::contract(
1821            "sequential_tools",
1822            format!("add={add}, multiply={multiply}, response={response:?}"),
1823        ));
1824    }
1825    report_from_response("sequential_tools", started, add + multiply, result)
1826}
1827
1828/// Runs a tool through Rig's multi-turn streaming agent driver.
1829pub async fn streaming_tool<W, T, F>(
1830    model: rig_core::driver::Model<W, T>,
1831    configure: F,
1832) -> Result<ScenarioReport, ScenarioError>
1833where
1834    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1835    T: rig_core::driver::Transport<W>,
1836    F: FnOnce(AgentBuilder) -> AgentBuilder,
1837{
1838    let calls = Arc::new(AtomicUsize::new(0));
1839    let started = Instant::now();
1840    let agent = configure(AgentBuilder::new(model))
1841        .preamble("Use the add tool for arithmetic; do not calculate by hand.")
1842        .tool(AddTool(calls.clone()))
1843        .default_max_turns(4)
1844        .build();
1845    let mut stream = agent
1846        .prompt("Use add to calculate 17 + 25, then state the final number.")
1847        .max_turns(4)
1848        .stream();
1849    let mut final_response = None;
1850    let mut final_count = 0_usize;
1851    let mut completion_usage = crate::completion::Usage::default();
1852    let mut streamed_call_ids = Vec::new();
1853    let mut streamed_result_ids = Vec::new();
1854    while let Some(item) = stream.next().await {
1855        match item? {
1856            MultiTurnStreamItem::ToolCall { tool_call } => streamed_call_ids.push(tool_call.id),
1857            MultiTurnStreamItem::ToolResult { tool_result } => {
1858                streamed_result_ids.push(tool_result.call)
1859            }
1860            MultiTurnStreamItem::CompletionCall(call) => completion_usage += call.usage,
1861            MultiTurnStreamItem::FinalResponse(response) => {
1862                final_count += 1;
1863                final_response = Some(response);
1864            }
1865            MultiTurnStreamItem::StreamAssistantItem(_)
1866            | MultiTurnStreamItem::ToolExecutionCommitted { .. }
1867            | MultiTurnStreamItem::ModelTurnRetried { .. } => {}
1868        }
1869    }
1870    let result = final_response.ok_or_else(|| {
1871        ScenarioError::contract("streaming_tool", "stream produced no final response")
1872    })?;
1873    let response = result.output();
1874    let history_messages = result.messages.len();
1875    let tool_calls = calls.load(Ordering::SeqCst);
1876    streamed_call_ids.sort();
1877    streamed_result_ids.sort();
1878    let correlated_stream =
1879        !streamed_call_ids.is_empty() && streamed_call_ids == streamed_result_ids;
1880    let correlated_history = validate_tool_correlation("streaming_tool", &result.messages).is_ok();
1881    if tool_calls == 0
1882        || !response.contains("42")
1883        || history_messages < 4
1884        || final_count != 1
1885        || !correlated_stream
1886        || !correlated_history
1887        || completion_usage != result.usage
1888        || result.completion_calls.is_empty()
1889    {
1890        return Err(ScenarioError::contract(
1891            "streaming_tool",
1892            format!(
1893                "calls={tool_calls}, final_count={final_count}, streamed_call_ids={streamed_call_ids:?}, streamed_result_ids={streamed_result_ids:?}, completion_usage={completion_usage:?}, final_usage={:?}, history_messages={history_messages}, response={response:?}",
1894                result.usage
1895            ),
1896        ));
1897    }
1898    validate_protocol_hygiene(
1899        "streaming_tool",
1900        &response,
1901        &result.messages,
1902        &["<tool_call>", "</tool_call>", "<think>", "</think>"],
1903    )?;
1904    Ok(ScenarioReport {
1905        name: "streaming_tool",
1906        tool_calls,
1907        prompt_tokens: result.usage.input_tokens.unwrap_or(0),
1908        generated_tokens: result.usage.output_tokens.unwrap_or(0),
1909        history_messages,
1910        duration: started.elapsed(),
1911        response,
1912    })
1913}
1914
1915/// Runs a normal tool followed by Rig's synthetic structured-output tool.
1916pub async fn structured_after_tool<W, T, F>(
1917    model: rig_core::driver::Model<W, T>,
1918    configure: F,
1919) -> Result<ScenarioReport, ScenarioError>
1920where
1921    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1922    T: rig_core::driver::Transport<W>,
1923    F: FnOnce(AgentBuilder) -> AgentBuilder,
1924{
1925    let calls = Arc::new(AtomicUsize::new(0));
1926    let started = Instant::now();
1927    let agent = configure(AgentBuilder::new(model))
1928        .preamble(
1929            "Use add for arithmetic, then finish by calling the structured output tool exactly once.",
1930        )
1931        .output_schema::<ArithmeticResult>()
1932        .output_mode(OutputMode::Tool)
1933        .tool(AddTool(calls.clone()))
1934        .default_max_turns(5)
1935        .build();
1936    let result = agent
1937        .prompt("Use add to calculate 19 + 23. Return answer=42 and a short optional explanation.")
1938        .await?;
1939    let response = result.output();
1940    let parsed: ArithmeticResult = serde_json::from_str(&response)?;
1941    let tool_calls = calls.load(Ordering::SeqCst);
1942    if tool_calls == 0 || parsed.answer != 42 || !has_tool_roundtrip(&result.messages) {
1943        return Err(ScenarioError::contract(
1944            "structured_after_tool",
1945            format!("calls={tool_calls}, response={response:?}"),
1946        ));
1947    }
1948    let _ = parsed.explanation;
1949    report_from_response("structured_after_tool", started, tool_calls + 1, result)
1950}
1951
1952/// Runs all portable tool-choice modes directly against a completion model.
1953pub async fn tool_choice_modes<W, T, A>(
1954    model: rig_core::driver::Model<W, T>,
1955    adjust: A,
1956) -> Result<ScenarioReport, ScenarioError>
1957where
1958    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1959    T: rig_core::driver::Transport<W>,
1960    A: Fn(CompletionRequest) -> CompletionRequest,
1961{
1962    let tool_name = |name: &str| {
1963        rig_core::message::ToolName::new(name)
1964            .map_err(|error| ScenarioError::contract("tool_choice_modes", error.to_string()))
1965    };
1966    let definition = |name: rig_core::message::ToolName| ToolDefinition {
1967        description: format!("Return the supplied integer using {name}."),
1968        name,
1969        parameters: serde_json::json!({
1970            "type": "object",
1971            "properties": {"value": {"type": "integer"}},
1972            "required": ["value"]
1973        }),
1974        annotations: Default::default(),
1975    };
1976    let tools = vec![
1977        definition(tool_name("alpha")?),
1978        definition(tool_name("beta")?),
1979    ];
1980    let started = Instant::now();
1981    let none = model
1982        .call(adjust(
1983            CompletionRequest::new("Answer with only the number 4. Do not call a function.")
1984                .tools(tools.clone())
1985                .tool_choice(ToolChoice::None)
1986                .temperature(0.0)
1987                .max_tokens(64),
1988        ))
1989        .await?;
1990    if none
1991        .choice
1992        .iter()
1993        .any(|item| matches!(item, AssistantContent::ToolCall(_)))
1994    {
1995        return Err(ScenarioError::contract(
1996            "tool_choice_modes",
1997            "tool_choice none emitted a tool call",
1998        ));
1999    }
2000
2001    let required = model
2002        .call(adjust(
2003            CompletionRequest::new("Call alpha with value 7.")
2004                .tools(tools.clone())
2005                .tool_choice(ToolChoice::Required)
2006                .temperature(0.0)
2007                .max_tokens(96),
2008        ))
2009        .await?;
2010    let required_calls = required
2011        .choice
2012        .iter()
2013        .filter(|item| matches!(item, AssistantContent::ToolCall(_)))
2014        .count();
2015    if required_calls == 0 {
2016        return Err(ScenarioError::contract(
2017            "tool_choice_modes",
2018            "tool_choice required emitted no tool call",
2019        ));
2020    }
2021
2022    let specific = model
2023        .call(adjust(
2024            CompletionRequest::new("Call beta with value 9.")
2025                .tools(tools)
2026                .tool_choice(ToolChoice::Specific {
2027                    function_names: vec![tool_name("beta")?],
2028                })
2029                .temperature(0.0)
2030                .max_tokens(96),
2031        ))
2032        .await?;
2033    let specific_calls = specific
2034        .choice
2035        .iter()
2036        .filter_map(|item| match item {
2037            AssistantContent::ToolCall(call) => Some(call),
2038            _ => None,
2039        })
2040        .collect::<Vec<_>>();
2041    if specific_calls.is_empty()
2042        || specific_calls
2043            .iter()
2044            .any(|call| call.function.name != "beta")
2045    {
2046        return Err(ScenarioError::contract(
2047            "tool_choice_modes",
2048            "specific tool choice did not select only beta",
2049        ));
2050    }
2051
2052    let usage = none.usage + required.usage + specific.usage;
2053    Ok(ScenarioReport {
2054        name: "tool_choice_modes",
2055        tool_calls: required_calls + specific_calls.len(),
2056        prompt_tokens: usage.input_tokens.unwrap_or(0),
2057        generated_tokens: usage.output_tokens.unwrap_or(0),
2058        history_messages: 0,
2059        duration: started.elapsed(),
2060        response: "none, required, and specific modes passed".to_string(),
2061    })
2062}
2063
2064/// Runs a streamed real-tool turn followed by Rig's synthetic output tool.
2065pub async fn streaming_structured_after_tool<W, T, F>(
2066    model: rig_core::driver::Model<W, T>,
2067    configure: F,
2068) -> Result<ScenarioReport, ScenarioError>
2069where
2070    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
2071    T: rig_core::driver::Transport<W>,
2072    F: FnOnce(AgentBuilder) -> AgentBuilder,
2073{
2074    let calls = Arc::new(AtomicUsize::new(0));
2075    let started = Instant::now();
2076    let agent = configure(AgentBuilder::new(model))
2077        .preamble(
2078            "Use add for arithmetic, then finish by calling the structured output tool exactly once.",
2079        )
2080        .output_schema::<ArithmeticResult>()
2081        .output_mode(OutputMode::Tool)
2082        .tool(AddTool(calls.clone()))
2083        .default_max_turns(5)
2084        .build();
2085    let mut stream = agent
2086        .prompt("Use add to calculate 19 + 23. Return answer=42 and a short optional explanation.")
2087        .max_turns(5)
2088        .stream();
2089    let mut final_response = None;
2090    let mut final_count = 0_usize;
2091    while let Some(item) = stream.next().await {
2092        if let MultiTurnStreamItem::FinalResponse(response) = item? {
2093            final_count += 1;
2094            final_response = Some(response);
2095        }
2096    }
2097    let result = final_response.ok_or_else(|| {
2098        ScenarioError::contract(
2099            "streaming_structured_after_tool",
2100            "stream produced no final response",
2101        )
2102    })?;
2103    let parsed: ArithmeticResult = serde_json::from_str(&result.output())?;
2104    let calls = calls.load(Ordering::SeqCst);
2105    if calls == 0
2106        || final_count != 1
2107        || parsed.answer != 42
2108        || !has_tool_roundtrip(&result.messages)
2109    {
2110        return Err(ScenarioError::contract(
2111            "streaming_structured_after_tool",
2112            format!(
2113                "calls={calls}, final_count={final_count}, response={:?}",
2114                result.output()
2115            ),
2116        ));
2117    }
2118    report_from_response(
2119        "streaming_structured_after_tool",
2120        started,
2121        calls + 1,
2122        result,
2123    )
2124}
2125
2126#[cfg(test)]
2127mod tests;