Skip to main content

rig_agent/test_utils/
model_conformance.rs

1//! Provider-neutral behavioral scenarios for completion-model conformance.
2//!
3//! These helpers test the model/agent contract only. Provider wire formats,
4//! authentication, HTTP streaming, and cassette matching remain provider-suite
5//! responsibilities.
6
7use std::{
8    collections::BTreeSet,
9    sync::{
10        Arc, Mutex,
11        atomic::{AtomicUsize, Ordering},
12    },
13    time::{Duration, Instant},
14};
15
16use futures::StreamExt;
17use schemars::JsonSchema;
18use serde::{Deserialize, Serialize};
19
20use crate::{
21    agent::{
22        AgentBuilder, AgentHook, CompletionCallAction, CompletionCallEvent, DispatchAction,
23        DispatchEvent, HookContext, InvalidToolCallAction, MultiTurnStreamItem, OutcomeAction,
24        OutcomeEvent, OutputMode, RequestPatch,
25    },
26    completion::{AssistantContent, Message, PromptError, ToolDefinition},
27    run::{AgentRun, AgentRunStep, ModelTurn, ModelTurnOutcome, PendingToolCall, TurnPolicy},
28    tool::{Tool, ToolContext},
29};
30use rig_core::completion::CompletionRequest;
31use rig_core::error::ProviderError;
32use rig_core::message::{ToolChoice, UserContent};
33
34/// Typed failure from a portable model-conformance scenario.
35#[derive(Debug, thiserror::Error)]
36pub enum ScenarioError {
37    /// An agent run failed.
38    #[error(transparent)]
39    Prompt(#[from] PromptError),
40    /// A direct model completion failed.
41    #[error(transparent)]
42    Completion(#[from] ProviderError),
43    /// A stream item or effect failed, as the wire reports it.
44    #[error(transparent)]
45    Report(#[from] rig_core::error::ErrorReport),
46    /// Structured content could not be decoded.
47    #[error(transparent)]
48    Json(#[from] serde_json::Error),
49    /// Rig's structured extractor failed.
50    #[error(transparent)]
51    Extraction(#[from] crate::completion::StructuredOutputError),
52    /// The model or agent violated the portable behavioral contract.
53    #[error("{scenario} conformance failed: {details}")]
54    Contract {
55        /// Stable scenario name.
56        scenario: &'static str,
57        /// Actionable observation explaining the failure.
58        details: String,
59    },
60}
61
62impl ScenarioError {
63    fn contract(scenario: &'static str, details: impl Into<String>) -> Self {
64        Self::Contract {
65            scenario,
66            details: details.into(),
67        }
68    }
69}
70
71/// Validate the portable diagnostics carried by an unknown or disallowed tool
72/// call failure.
73pub fn validate_unknown_tool_failure(
74    error: &PromptError,
75    expected_tool: &str,
76    expected_allowed_tools: &[&str],
77) -> Result<(), ScenarioError> {
78    const SCENARIO: &str = "unknown_tool_failure";
79    let PromptError::UnknownToolCall {
80        tool_name,
81        allowed_tools,
82        chat_history,
83        ..
84    } = error
85    else {
86        return Err(ScenarioError::contract(
87            SCENARIO,
88            format!("expected UnknownToolCall, observed {error:?}"),
89        ));
90    };
91    let expected_allowed = expected_allowed_tools
92        .iter()
93        .map(|name| (*name).to_string())
94        .collect::<Vec<_>>();
95    // A rejected repair target is never written back into history; diagnostics
96    // retain the model's original call while `tool_name` names the rejected
97    // target. Requiring a call-bearing assistant turn covers both paths.
98    let history_has_call = chat_history.iter().any(|message| {
99        matches!(
100            message,
101            Message::Assistant(rig_core::message::AssistantMessage { content, .. })
102                if content.iter().any(|item| matches!(item, AssistantContent::ToolCall(_)))
103        )
104    });
105    if tool_name != expected_tool || allowed_tools != &expected_allowed || !history_has_call {
106        return Err(ScenarioError::contract(
107            SCENARIO,
108            format!(
109                "tool={tool_name:?}, allowed={allowed_tools:?}, expected_tool={expected_tool:?}, expected_allowed={expected_allowed:?}, history_has_call={history_has_call}, history={chat_history:?}"
110            ),
111        ));
112    }
113    Ok(())
114}
115
116/// Validate cancellation diagnostics, including the exact reason and retained
117/// assistant tool-call history.
118pub fn validate_cancelled_failure(
119    error: &PromptError,
120    expected_reason: &str,
121    expected_tool: &str,
122) -> Result<(), ScenarioError> {
123    const SCENARIO: &str = "cancelled_failure";
124    let PromptError::Cancelled {
125        chat_history,
126        reason,
127    } = error
128    else {
129        return Err(ScenarioError::contract(
130            SCENARIO,
131            format!("expected Cancelled, observed {error:?}"),
132        ));
133    };
134    let history_has_call = chat_history.iter().any(|message| {
135        matches!(
136            message,
137            Message::Assistant(rig_core::message::AssistantMessage { content, .. })
138                if content.iter().any(|item| matches!(
139                    item,
140                    AssistantContent::ToolCall(call) if call.function.name == expected_tool
141                ))
142        )
143    });
144    if reason != expected_reason || !history_has_call {
145        return Err(ScenarioError::contract(
146            SCENARIO,
147            format!(
148                "reason={reason:?}, expected={expected_reason:?}, history_has_call={history_has_call}, history={chat_history:?}"
149            ),
150        ));
151    }
152    Ok(())
153}
154
155/// Validate max-turn diagnostics, including the exact configured budget and a
156/// retained pending prompt.
157pub fn validate_max_turns_failure(
158    error: &PromptError,
159    expected_max_turns: usize,
160) -> Result<(), ScenarioError> {
161    const SCENARIO: &str = "max_turns_failure";
162    let PromptError::MaxTurns {
163        max_turns,
164        chat_history,
165        prompt,
166    } = error
167    else {
168        return Err(ScenarioError::contract(
169            SCENARIO,
170            format!("expected MaxTurns, observed {error:?}"),
171        ));
172    };
173    let pending_prompt_retained = matches!(
174        prompt,
175        Message::User { content } if content.iter().next().is_some()
176    );
177    if *max_turns != expected_max_turns || chat_history.is_empty() || !pending_prompt_retained {
178        return Err(ScenarioError::contract(
179            SCENARIO,
180            format!(
181                "max_turns={max_turns}, expected={expected_max_turns}, history={chat_history:?}, pending_prompt_retained={pending_prompt_retained}, pending_prompt={prompt:?}"
182            ),
183        ));
184    }
185    Ok(())
186}
187
188/// Decode a structured-output response with a typed conformance failure that
189/// retains the scenario name and raw response.
190pub fn decode_structured_output<T>(
191    scenario: &'static str,
192    response: &str,
193) -> Result<T, ScenarioError>
194where
195    T: serde::de::DeserializeOwned,
196{
197    serde_json::from_str(response).map_err(|error| {
198        ScenarioError::contract(
199            scenario,
200            format!("structured output did not decode: {error}; response={response:?}"),
201        )
202    })
203}
204
205/// Validate that model-family control markers are absent from user-visible
206/// output and persisted history.
207pub fn validate_protocol_hygiene(
208    scenario: &'static str,
209    visible_output: &str,
210    messages: &[Message],
211    forbidden_markers: &[&str],
212) -> Result<(), ScenarioError> {
213    let serialized = serde_json::to_string(messages)?;
214    let leaked = forbidden_markers
215        .iter()
216        .filter(|marker| visible_output.contains(**marker) || serialized.contains(**marker))
217        .copied()
218        .collect::<Vec<_>>();
219    if !leaked.is_empty() {
220        return Err(ScenarioError::contract(
221            scenario,
222            format!(
223                "protocol markers leaked: {leaked:?}; output={visible_output:?}, history={messages:?}"
224            ),
225        ));
226    }
227    Ok(())
228}
229
230/// Validate that every observed tool invocation contains the expected rewritten
231/// argument fields while allowing unrelated original fields to remain.
232pub fn validate_rewritten_arguments(
233    scenario: &'static str,
234    observations: &[serde_json::Value],
235    expected_fields: &serde_json::Value,
236) -> Result<(), ScenarioError> {
237    let Some(expected) = expected_fields.as_object() else {
238        return Err(ScenarioError::contract(
239            scenario,
240            "expected rewritten fields must be a JSON object",
241        ));
242    };
243    if observations.is_empty() {
244        return Err(ScenarioError::contract(
245            scenario,
246            "the rewritten tool was never invoked",
247        ));
248    }
249    for observation in observations {
250        let Some(actual) = observation.as_object() else {
251            return Err(ScenarioError::contract(
252                scenario,
253                format!("observed rewritten arguments were not an object: {observation:?}"),
254            ));
255        };
256        for (key, expected_value) in expected {
257            if actual.get(key) != Some(expected_value) {
258                return Err(ScenarioError::contract(
259                    scenario,
260                    format!(
261                        "rewritten field {key:?} expected {expected_value:?}, observed {observation:?}"
262                    ),
263                ));
264            }
265        }
266    }
267    Ok(())
268}
269
270/// Validate a provider-neutral person extraction and its usage accounting.
271pub fn validate_extraction_fields(
272    scenario: &'static str,
273    first_name: Option<&str>,
274    last_name: Option<&str>,
275    job: Option<&str>,
276    usage: crate::completion::Usage,
277) -> Result<(), ScenarioError> {
278    let fields_match = first_name.is_some_and(|value| value.eq_ignore_ascii_case("Ada"))
279        && last_name.is_some_and(|value| value.eq_ignore_ascii_case("Lovelace"))
280        && job.is_some_and(|value| value.to_ascii_lowercase().contains("mathematician"));
281    if !fields_match || !usage.is_reported() {
282        return Err(ScenarioError::contract(
283            scenario,
284            format!(
285                "first_name={first_name:?}, last_name={last_name:?}, job={job:?}, usage={usage:?}"
286            ),
287        ));
288    }
289    Ok(())
290}
291
292/// Summary emitted by a portable model-conformance scenario.
293#[derive(Debug, Clone)]
294pub struct ScenarioReport {
295    /// Stable scenario name.
296    pub name: &'static str,
297    /// Number of model-invoked tool calls observed by the scenario.
298    pub tool_calls: usize,
299    /// Aggregated prompt tokens reported by the model across all turns; zero
300    /// when the model reported none.
301    pub prompt_tokens: u64,
302    /// Aggregated generated tokens reported by the model across all turns;
303    /// zero when the model reported none.
304    pub generated_tokens: u64,
305    /// Number of messages retained in the completed run history.
306    pub history_messages: usize,
307    /// End-to-end scenario duration.
308    pub duration: Duration,
309    /// The model's final user-visible response.
310    pub response: String,
311}
312
313/// Error used by the deterministic conformance tools.
314#[derive(Debug, thiserror::Error)]
315#[error("model-conformance tool failed")]
316pub struct ConformanceToolError;
317
318const FORCE_TOOLS_PREAMBLE: &str = "You are a calculator assistant. You MUST use the provided tools for every arithmetic operation instead of computing results yourself. Once you have all the tool results you need, reply with the final numeric answer in plain text.";
319const PARALLEL_PROMPT: &str = "Compute 3 + 4 and 10 - 2. You MUST call the add tool and the subtract tool together in your first response, as two parallel function calls, then report both results.";
320const PING_OUTPUT: &str = "pong-crimson-7423";
321const MOTTO_OUTPUT: &str = "steady hands\ncalm waters";
322
323fn tool_result_values(message: &Message) -> Vec<serde_json::Value> {
324    message
325        .results()
326        .flat_map(|result| result.content.iter())
327        .filter_map(|content| match content {
328            rig_core::message::ToolResultContent::Text(text) => {
329                Some(serde_json::Value::String(text.text.clone()))
330            }
331            rig_core::message::ToolResultContent::Json { value } => Some(value.clone()),
332            rig_core::message::ToolResultContent::Image(_) => None,
333        })
334        .collect()
335}
336
337fn validate_tool_correlation(
338    scenario: &'static str,
339    messages: &[Message],
340) -> Result<(), ScenarioError> {
341    let mut calls = Vec::new();
342    let mut results = Vec::new();
343    let mut turn = 0usize;
344    for message in messages {
345        match message {
346            Message::Assistant(rig_core::message::AssistantMessage { content, .. }) => {
347                turn += 1;
348                calls.extend(content.iter().filter_map(|item| match item {
349                    AssistantContent::ToolCall(call) => Some((
350                        turn,
351                        &call.id,
352                        call.id.provider().map(|provider| provider.as_str()),
353                    )),
354                    _ => None,
355                }));
356            }
357            Message::User { .. } => {
358                results.extend(message.results().map(|result| {
359                    let provider = result.call.provider().map(|provider| provider.as_str());
360                    (turn, &result.call, provider)
361                }));
362            }
363            Message::System { .. } => {}
364        }
365    }
366    if calls.is_empty() {
367        return Err(ScenarioError::contract(
368            scenario,
369            format!("history has no assistant tool calls: {messages:?}"),
370        ));
371    }
372    for (turn, id, call_id) in &calls {
373        let matches = results
374            .iter()
375            .filter(|(result_turn, result_id, result_call_id)| {
376                result_turn == turn && result_id == id && call_id == result_call_id
377            })
378            .count();
379        if matches != 1 {
380            return Err(ScenarioError::contract(
381                scenario,
382                format!(
383                    "tool call id={id:?} call_id={call_id:?} has {matches} correlated results; calls={calls:?}, results={results:?}"
384                ),
385            ));
386        }
387    }
388    if results.len() != calls.len() {
389        return Err(ScenarioError::contract(
390            scenario,
391            format!(
392                "history contains dangling calls or results: calls={calls:?}, results={results:?}"
393            ),
394        ));
395    }
396    Ok(())
397}
398
399#[derive(Debug, Deserialize, JsonSchema)]
400struct OperationArgs {
401    x: i64,
402    y: i64,
403}
404
405#[derive(Clone)]
406struct CountingAdd(Arc<AtomicUsize>);
407
408impl Tool for CountingAdd {
409    const NAME: &'static str = "add";
410    type Error = ConformanceToolError;
411    type Args = OperationArgs;
412    type Output = i64;
413
414    fn description(&self) -> String {
415        "Add x and y together".to_string()
416    }
417
418    fn parameters(&self) -> serde_json::Value {
419        serde_json::json!({
420            "type": "object",
421            "properties": {
422                "x": { "type": "number", "description": "The first operand" },
423                "y": { "type": "number", "description": "The second operand" }
424            },
425            "required": ["x", "y"]
426        })
427    }
428
429    async fn call(
430        &self,
431        _context: &mut ToolContext,
432        args: Self::Args,
433    ) -> Result<Self::Output, Self::Error> {
434        self.0.fetch_add(1, Ordering::SeqCst);
435        Ok(args.x + args.y)
436    }
437}
438
439#[derive(Clone)]
440struct CountingSum(Arc<AtomicUsize>);
441
442impl Tool for CountingSum {
443    const NAME: &'static str = "sum";
444    type Error = ConformanceToolError;
445    type Args = OperationArgs;
446    type Output = i64;
447
448    fn description(&self) -> String {
449        "Add x and y together (alias of add)".to_string()
450    }
451
452    fn parameters(&self) -> serde_json::Value {
453        CountingAdd(Arc::new(AtomicUsize::new(0))).parameters()
454    }
455
456    async fn call(
457        &self,
458        _context: &mut ToolContext,
459        args: Self::Args,
460    ) -> Result<Self::Output, Self::Error> {
461        self.0.fetch_add(1, Ordering::SeqCst);
462        Ok(args.x + args.y)
463    }
464}
465
466#[derive(Clone)]
467struct CountingSubtract(Arc<AtomicUsize>);
468
469impl Tool for CountingSubtract {
470    const NAME: &'static str = "subtract";
471    type Error = ConformanceToolError;
472    type Args = OperationArgs;
473    type Output = i64;
474
475    fn description(&self) -> String {
476        "Subtract y from x (i.e. x - y)".to_string()
477    }
478
479    fn parameters(&self) -> serde_json::Value {
480        serde_json::json!({
481            "type": "object",
482            "properties": {
483                "x": { "type": "number", "description": "The first operand" },
484                "y": { "type": "number", "description": "The second operand" }
485            },
486            "required": ["x", "y"]
487        })
488    }
489
490    async fn call(
491        &self,
492        _context: &mut ToolContext,
493        args: Self::Args,
494    ) -> Result<Self::Output, Self::Error> {
495        self.0.fetch_add(1, Ordering::SeqCst);
496        Ok(args.x - args.y)
497    }
498}
499
500#[derive(Clone)]
501struct RewriteArgument {
502    key: &'static str,
503    value: serde_json::Value,
504}
505
506impl AgentHook for RewriteArgument {
507    async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
508        let (Some(name), Some(args)) = (event.tool_name(), event.tool_args()) else {
509            return DispatchAction::proceed();
510        };
511        if name != CountingAdd::NAME {
512            return DispatchAction::proceed();
513        }
514        let Ok(mut arguments) = serde_json::from_str::<serde_json::Value>(args) else {
515            return DispatchAction::proceed();
516        };
517        let Some(object) = arguments.as_object_mut() else {
518            return DispatchAction::proceed();
519        };
520        object.insert(self.key.to_string(), self.value.clone());
521        DispatchAction::rewrite_tool_args(event.kind, arguments)
522    }
523}
524
525#[derive(Clone, Default)]
526struct ObserveArguments(Arc<Mutex<Vec<serde_json::Value>>>);
527
528impl AgentHook for ObserveArguments {
529    async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
530        if let Some(args) = event.tool_args() {
531            let value = serde_json::from_str(args)
532                .unwrap_or_else(|_| serde_json::Value::String(args.to_string()));
533            crate::lock(&self.0).push(value);
534        }
535        DispatchAction::proceed()
536    }
537}
538
539#[derive(Clone)]
540struct ReplaceResult(&'static str);
541
542impl AgentHook for ReplaceResult {
543    async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
544        if event.tool_name() == Some(CountingAdd::NAME) {
545            OutcomeAction::rewrite_tool_result(&event, self.0)
546        } else {
547            OutcomeAction::proceed()
548        }
549    }
550}
551
552#[derive(Clone)]
553struct WrapResult;
554
555impl AgentHook for WrapResult {
556    async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
557        match (event.tool_name(), event.tool_result()) {
558            (Some(name), Some(result)) if name == CountingAdd::NAME => {
559                OutcomeAction::rewrite_tool_result(
560                    &event,
561                    format!("[{}]", result.output().render()),
562                )
563            }
564            _ => OutcomeAction::proceed(),
565        }
566    }
567}
568
569#[derive(Clone)]
570struct FirstTurnPatch(RequestPatch);
571
572impl AgentHook for FirstTurnPatch {
573    async fn on_completion_call(
574        &self,
575        ctx: &HookContext,
576        _event: CompletionCallEvent<'_>,
577    ) -> CompletionCallAction {
578        if ctx.turn() == 1 {
579            CompletionCallAction::patch(self.0.clone())
580        } else {
581            CompletionCallAction::continue_run()
582        }
583    }
584}
585
586#[derive(Clone)]
587struct StopAfterResult(&'static str);
588
589impl AgentHook for StopAfterResult {
590    async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
591        if event.tool_name() == Some(CountingAdd::NAME) {
592            OutcomeAction::stop(self.0)
593        } else {
594            OutcomeAction::proceed()
595        }
596    }
597}
598
599#[derive(Debug, Deserialize, JsonSchema)]
600struct EmptyArgs {}
601
602#[derive(Clone)]
603struct PingTool(Arc<AtomicUsize>);
604
605impl Tool for PingTool {
606    const NAME: &'static str = "ping";
607    type Error = ConformanceToolError;
608    type Args = EmptyArgs;
609    type Output = String;
610
611    fn description(&self) -> String {
612        "Return the current ping marker. Takes no arguments.".to_string()
613    }
614
615    fn parameters(&self) -> serde_json::Value {
616        serde_json::json!({ "type": "object", "properties": {}, "required": [] })
617    }
618
619    async fn call(
620        &self,
621        _context: &mut ToolContext,
622        _args: Self::Args,
623    ) -> Result<Self::Output, Self::Error> {
624        self.0.fetch_add(1, Ordering::SeqCst);
625        Ok(PING_OUTPUT.to_string())
626    }
627}
628
629#[derive(Clone)]
630struct MottoTool(Arc<AtomicUsize>);
631
632impl Tool for MottoTool {
633    const NAME: &'static str = "fetch_motto";
634    type Error = ConformanceToolError;
635    type Args = EmptyArgs;
636    type Output = String;
637
638    fn description(&self) -> String {
639        "Fetch the two-line workshop motto.".to_string()
640    }
641
642    fn parameters(&self) -> serde_json::Value {
643        serde_json::json!({ "type": "object", "properties": {}, "required": [] })
644    }
645
646    async fn call(
647        &self,
648        _context: &mut ToolContext,
649        _args: Self::Args,
650    ) -> Result<Self::Output, Self::Error> {
651        self.0.fetch_add(1, Ordering::SeqCst);
652        Ok(MOTTO_OUTPUT.to_string())
653    }
654}
655
656#[derive(Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
657struct ConfigOutput {
658    service: String,
659    max_retries: u64,
660}
661
662#[derive(Clone)]
663struct ConfigTool(Arc<AtomicUsize>);
664
665impl Tool for ConfigTool {
666    const NAME: &'static str = "fetch_config";
667    type Error = ConformanceToolError;
668    type Args = EmptyArgs;
669    type Output = ConfigOutput;
670
671    fn description(&self) -> String {
672        "Fetch the service configuration object.".to_string()
673    }
674
675    fn parameters(&self) -> serde_json::Value {
676        serde_json::json!({ "type": "object", "properties": {}, "required": [] })
677    }
678
679    async fn call(
680        &self,
681        _context: &mut ToolContext,
682        _args: Self::Args,
683    ) -> Result<Self::Output, Self::Error> {
684        self.0.fetch_add(1, Ordering::SeqCst);
685        Ok(ConfigOutput {
686            service: "cassette-lab".to_string(),
687            max_retries: 3,
688        })
689    }
690}
691
692#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
693#[serde(rename_all = "snake_case")]
694enum ComplexMode {
695    Careful,
696    Fast,
697}
698
699#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
700struct ComplexProfile {
701    name: String,
702    tags: Vec<String>,
703}
704
705#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
706struct ComplexArgs {
707    profile: ComplexProfile,
708    mode: ComplexMode,
709    note: Option<String>,
710    quote: String,
711}
712
713#[derive(Debug, Deserialize, Serialize, JsonSchema)]
714struct ExtractedPerson {
715    #[schemars(required)]
716    first_name: Option<String>,
717    #[schemars(required)]
718    last_name: Option<String>,
719    #[schemars(required)]
720    job: Option<String>,
721}
722
723#[derive(Clone)]
724struct CaptureComplexTool {
725    calls: Arc<AtomicUsize>,
726    captured: Arc<Mutex<Option<ComplexArgs>>>,
727}
728
729impl Tool for CaptureComplexTool {
730    const NAME: &'static str = "store_profile";
731    type Error = ConformanceToolError;
732    type Args = ComplexArgs;
733    type Output = ComplexArgs;
734
735    fn description(&self) -> String {
736        "Store one profile with its nested tags, mode, optional note, and exact quoted text."
737            .to_string()
738    }
739
740    fn parameters(&self) -> serde_json::Value {
741        serde_json::to_value(schemars::schema_for!(ComplexArgs)).unwrap_or_default()
742    }
743
744    async fn call(
745        &self,
746        _context: &mut ToolContext,
747        args: Self::Args,
748    ) -> Result<Self::Output, Self::Error> {
749        self.calls.fetch_add(1, Ordering::SeqCst);
750        *crate::lock(&self.captured) = Some(args.clone());
751        Ok(args)
752    }
753}
754
755fn has_tool_roundtrip(messages: &[Message]) -> bool {
756    let saw_call = messages.iter().any(|message| {
757        matches!(
758            message,
759            Message::Assistant(rig_core::message::AssistantMessage { content, .. })
760                if content.iter().any(|item| matches!(item, AssistantContent::ToolCall(_)))
761        )
762    });
763    let saw_result = messages.iter().any(|message| {
764        matches!(
765            message,
766            Message::User { content }
767                if content.iter().any(|item| matches!(item, UserContent::ToolResult(_)))
768        )
769    });
770    saw_call && saw_result
771}
772
773#[derive(Debug, Deserialize, JsonSchema)]
774struct RepeatArgs {
775    /// The text to repeat.
776    text: String,
777    /// Number of repetitions; defaults to 2 when omitted.
778    times: Option<u32>,
779}
780
781#[derive(Clone)]
782struct RepeatTool {
783    calls: Arc<AtomicUsize>,
784}
785
786impl Tool for RepeatTool {
787    const NAME: &'static str = "repeat_text";
788    type Error = ConformanceToolError;
789    type Args = RepeatArgs;
790    type Output = String;
791
792    fn description(&self) -> String {
793        "Repeat `text`. `times` is optional and defaults to 2.".to_string()
794    }
795
796    fn parameters(&self) -> serde_json::Value {
797        serde_json::to_value(schemars::schema_for!(RepeatArgs)).unwrap_or_default()
798    }
799
800    async fn call(
801        &self,
802        _context: &mut ToolContext,
803        args: Self::Args,
804    ) -> Result<Self::Output, Self::Error> {
805        self.calls.fetch_add(1, Ordering::SeqCst);
806        Ok(vec![args.text.as_str(); args.times.unwrap_or(2) as usize].join(" "))
807    }
808}
809
810#[derive(Debug, Deserialize, JsonSchema)]
811struct BinOpArgs {
812    a: i64,
813    b: i64,
814}
815
816#[derive(Debug, Deserialize, JsonSchema)]
817struct ArithmeticResult {
818    answer: i64,
819    explanation: Option<String>,
820}
821
822#[derive(Clone)]
823struct AddTool(Arc<AtomicUsize>);
824
825impl Tool for AddTool {
826    const NAME: &'static str = "add";
827    type Error = ConformanceToolError;
828    type Args = BinOpArgs;
829    type Output = i64;
830
831    fn description(&self) -> String {
832        "Add two integers a and b.".to_string()
833    }
834
835    fn parameters(&self) -> serde_json::Value {
836        serde_json::to_value(schemars::schema_for!(BinOpArgs)).unwrap_or_default()
837    }
838
839    async fn call(
840        &self,
841        _context: &mut ToolContext,
842        args: Self::Args,
843    ) -> Result<Self::Output, Self::Error> {
844        self.0.fetch_add(1, Ordering::SeqCst);
845        Ok(args.a + args.b)
846    }
847}
848
849#[derive(Clone)]
850struct MultiplyTool(Arc<AtomicUsize>);
851
852impl Tool for MultiplyTool {
853    const NAME: &'static str = "multiply";
854    type Error = ConformanceToolError;
855    type Args = BinOpArgs;
856    type Output = i64;
857
858    fn description(&self) -> String {
859        "Multiply two integers a and b.".to_string()
860    }
861
862    fn parameters(&self) -> serde_json::Value {
863        serde_json::to_value(schemars::schema_for!(BinOpArgs)).unwrap_or_default()
864    }
865
866    async fn call(
867        &self,
868        _context: &mut ToolContext,
869        args: Self::Args,
870    ) -> Result<Self::Output, Self::Error> {
871        self.0.fetch_add(1, Ordering::SeqCst);
872        Ok(args.a * args.b)
873    }
874}
875
876fn report_from_response(
877    name: &'static str,
878    started: Instant,
879    tool_calls: usize,
880    response: crate::agent::PromptResponse,
881) -> Result<ScenarioReport, ScenarioError> {
882    validate_protocol_hygiene(
883        name,
884        &response.output(),
885        &response.messages,
886        &[
887            "<tool_call>",
888            "</tool_call>",
889            "<tool_response>",
890            "</tool_response>",
891            "<|im_start|>",
892            "<|im_end|>",
893            "<think>",
894            "</think>",
895        ],
896    )?;
897    Ok(ScenarioReport {
898        name,
899        tool_calls,
900        prompt_tokens: response.usage.input_tokens.unwrap_or(0),
901        generated_tokens: response.usage.output_tokens.unwrap_or(0),
902        history_messages: response.messages.len(),
903        duration: started.elapsed(),
904        response: response.output(),
905    })
906}
907
908/// Require the extended run's accumulated message history and validate
909/// canonical tool call/result correlation over it.
910fn correlated_messages<'a>(
911    scenario: &'static str,
912    response: &'a crate::agent::PromptResponse,
913) -> Result<&'a [Message], ScenarioError> {
914    let messages = response.messages.as_slice();
915    if messages.is_empty() {
916        return Err(ScenarioError::contract(
917            scenario,
918            "extended run omitted accumulated message history",
919        ));
920    }
921    validate_tool_correlation(scenario, messages)?;
922    Ok(messages)
923}
924
925/// [`correlated_messages`], flattened into every tool-result value in history.
926fn correlated_result_values(
927    scenario: &'static str,
928    response: &crate::agent::PromptResponse,
929) -> Result<Vec<serde_json::Value>, ScenarioError> {
930    Ok(correlated_messages(scenario, response)?
931        .iter()
932        .flat_map(tool_result_values)
933        .collect())
934}
935
936fn value_matches_integer(value: &serde_json::Value, expected: i64) -> bool {
937    value.as_i64() == Some(expected)
938        || value
939            .as_str()
940            .and_then(|text| text.trim().parse::<i64>().ok())
941            == Some(expected)
942}
943
944/// Runs two independent calls in one assistant turn and validates canonical
945/// call/result history correlation.
946///
947/// Set `tool_concurrency` to `Some(1)` to prove that serial host execution does
948/// not split or drop the model's parallel call batch.
949pub async fn parallel_tools<W, T, F>(
950    model: rig_core::driver::Model<W, T>,
951    configure: F,
952    tool_concurrency: Option<usize>,
953) -> Result<ScenarioReport, ScenarioError>
954where
955    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
956    T: rig_core::driver::Transport<W>,
957    F: FnOnce(AgentBuilder) -> AgentBuilder,
958{
959    let add_calls = Arc::new(AtomicUsize::new(0));
960    let subtract_calls = Arc::new(AtomicUsize::new(0));
961    let started = Instant::now();
962    let agent = configure(AgentBuilder::new(model).temperature(0.0))
963        .preamble(FORCE_TOOLS_PREAMBLE)
964        .tool(CountingAdd(add_calls.clone()))
965        .tool(CountingSubtract(subtract_calls.clone()))
966        .default_max_turns(3)
967        .build();
968    let request = agent.prompt(PARALLEL_PROMPT).max_turns(3);
969    let response = match tool_concurrency {
970        Some(concurrency) => request.tool_concurrency(concurrency).await?,
971        None => request.await?,
972    };
973    let scenario = if tool_concurrency == Some(1) {
974        "parallel_tools_serial_execution"
975    } else {
976        "parallel_tools"
977    };
978    let messages = correlated_messages(scenario, &response)?;
979
980    let Some((call_index, calls)) = messages.iter().enumerate().find_map(|(index, message)| {
981        let Message::Assistant(rig_core::message::AssistantMessage { content, .. }) = message
982        else {
983            return None;
984        };
985        let calls = content
986            .iter()
987            .filter_map(|item| match item {
988                AssistantContent::ToolCall(call) => Some(call),
989                _ => None,
990            })
991            .collect::<Vec<_>>();
992        (calls.len() == 2).then_some((index, calls))
993    }) else {
994        return Err(ScenarioError::contract(
995            scenario,
996            format!("no assistant turn contained exactly two tool calls: {messages:?}"),
997        ));
998    };
999    let mut names = calls
1000        .iter()
1001        .map(|call| call.function.name.as_str())
1002        .collect::<Vec<_>>();
1003    names.sort_unstable();
1004    if names != ["add", "subtract"] {
1005        return Err(ScenarioError::contract(
1006            scenario,
1007            format!("parallel turn called {names:?}, expected add and subtract"),
1008        ));
1009    }
1010    let results_message = messages.get(call_index + 1).ok_or_else(|| {
1011        ScenarioError::contract(
1012            scenario,
1013            "parallel call turn has no following result message",
1014        )
1015    })?;
1016    let values = tool_result_values(results_message);
1017    if !values.iter().any(|value| value_matches_integer(value, 7))
1018        || !values.iter().any(|value| value_matches_integer(value, 8))
1019        || values.len() != 2
1020    {
1021        return Err(ScenarioError::contract(
1022            scenario,
1023            format!("parallel result message did not contain exactly 7 and 8: {values:?}"),
1024        ));
1025    }
1026    let add = add_calls.load(Ordering::SeqCst);
1027    let subtract = subtract_calls.load(Ordering::SeqCst);
1028    if add != 1 || subtract != 1 {
1029        return Err(ScenarioError::contract(
1030            scenario,
1031            format!("execution counts were add={add}, subtract={subtract}, expected one each"),
1032        ));
1033    }
1034    report_from_response(scenario, started, add + subtract, response)
1035}
1036
1037/// Runs a zero-argument tool and validates verbatim string-result handling.
1038pub async fn zero_argument_tool<W, T, F>(
1039    model: rig_core::driver::Model<W, T>,
1040    configure: F,
1041) -> Result<ScenarioReport, ScenarioError>
1042where
1043    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1044    T: rig_core::driver::Transport<W>,
1045    F: FnOnce(AgentBuilder) -> AgentBuilder,
1046{
1047    const SCENARIO: &str = "zero_argument_tool";
1048    let calls = Arc::new(AtomicUsize::new(0));
1049    let started = Instant::now();
1050    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1051        .preamble("You must use the provided tools. Report tool outputs exactly as returned.")
1052        .tool(PingTool(calls.clone()))
1053        .default_max_turns(2)
1054        .build();
1055    let response = agent
1056        .prompt("Call the ping tool, then report the exact marker it returns.")
1057        .max_turns(2)
1058        .await?;
1059    let values = correlated_result_values(SCENARIO, &response)?;
1060    if calls.load(Ordering::SeqCst) != 1
1061        || !values
1062            .iter()
1063            .any(|value| value.as_str() == Some(PING_OUTPUT))
1064        || !response.output().contains(PING_OUTPUT)
1065    {
1066        return Err(ScenarioError::contract(
1067            SCENARIO,
1068            format!(
1069                "calls={}, results={values:?}, response={:?}",
1070                calls.load(Ordering::SeqCst),
1071                response.output()
1072            ),
1073        ));
1074    }
1075    report_from_response(SCENARIO, started, 1, response)
1076}
1077
1078/// Runs string- and JSON-returning tools and validates that neither output is
1079/// double encoded.
1080pub async fn tool_output_serialization<W, T, F>(
1081    model: rig_core::driver::Model<W, T>,
1082    configure: F,
1083) -> Result<ScenarioReport, ScenarioError>
1084where
1085    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1086    T: rig_core::driver::Transport<W>,
1087    F: FnOnce(AgentBuilder) -> AgentBuilder,
1088{
1089    const SCENARIO: &str = "tool_output_serialization";
1090    let started = Instant::now();
1091    let motto_calls = Arc::new(AtomicUsize::new(0));
1092    let config_calls = Arc::new(AtomicUsize::new(0));
1093    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1094        .preamble("You must use the provided tools before answering.")
1095        .tool(MottoTool(motto_calls.clone()))
1096        .tool(ConfigTool(config_calls.clone()))
1097        .default_max_turns(3)
1098        .build();
1099    let response = agent
1100        .prompt("Call fetch_motto and fetch_config, then summarize both outputs in one sentence.")
1101        .max_turns(3)
1102        .await?;
1103    let values = correlated_result_values(SCENARIO, &response)?;
1104    let expected_config = serde_json::to_value(ConfigOutput {
1105        service: "cassette-lab".to_string(),
1106        max_retries: 3,
1107    })?;
1108    let motto_ok = values
1109        .iter()
1110        .any(|value| value.as_str() == Some(MOTTO_OUTPUT));
1111    let config_ok = values.iter().any(|value| {
1112        value == &expected_config
1113            || value
1114                .as_str()
1115                .and_then(|text| serde_json::from_str::<serde_json::Value>(text).ok())
1116                .as_ref()
1117                == Some(&expected_config)
1118    });
1119    let motto_count = motto_calls.load(Ordering::SeqCst);
1120    let config_count = config_calls.load(Ordering::SeqCst);
1121    if !motto_ok || !config_ok || motto_count != 1 || config_count != 1 {
1122        return Err(ScenarioError::contract(
1123            SCENARIO,
1124            format!(
1125                "expected one verbatim motto and one semantic config JSON; motto_calls={motto_count}, config_calls={config_count}, values={values:?}"
1126            ),
1127        ));
1128    }
1129    report_from_response(SCENARIO, started, motto_count + config_count, response)
1130}
1131
1132/// Runs a nested, escaped, Unicode-bearing argument payload through typed tool
1133/// deserialization and validates the exact semantic value received by the tool.
1134pub async fn complex_tool_arguments<W, T, F>(
1135    model: rig_core::driver::Model<W, T>,
1136    configure: F,
1137) -> Result<ScenarioReport, ScenarioError>
1138where
1139    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1140    T: rig_core::driver::Transport<W>,
1141    F: FnOnce(AgentBuilder) -> AgentBuilder,
1142{
1143    complex_tool_arguments_with_prompt(model, configure, COMPLEX_ARGUMENTS_PROMPT).await
1144}
1145
1146/// The prompt [`complex_tool_arguments`] sends. It spells the values in their
1147/// JSON-escaped form, which a model may copy literally, backslashes included.
1148const COMPLEX_ARGUMENTS_PROMPT: &str = "Call store_profile with profile.name exactly `Zoë \\\"Z\\\"`, profile.tags exactly [`rust`, `東京`], mode `careful`, note containing the two lines `line one` and `line two` separated by a newline, and quote exactly `path C:\\\\tmp and \\\"quoted\\\"`. Then confirm it was stored.";
1149
1150/// The same values as [`complex_tool_arguments`] sends, spelled as raw text
1151/// between markers, for a model that copies escapes literally.
1152pub const COMPLEX_ARGUMENTS_RAW_PROMPT: &str = "Call store_profile once. Each value below is raw text between <<< and >>>, with no escaping: profile.name <<<Zoë \"Z\">>>, profile.tags <<<rust>>> and <<<東京>>>, mode <<<careful>>>, note the two lines <<<line one>>> and <<<line two>>> joined by one newline, and quote <<<path C:\\tmp and \"quoted\">>>. Then confirm it was stored.";
1153
1154/// [`complex_tool_arguments`] with the user prompt given: the same tool, the
1155/// same expected arguments, the same checks.
1156pub async fn complex_tool_arguments_with_prompt<W, T, F>(
1157    model: rig_core::driver::Model<W, T>,
1158    configure: F,
1159    prompt: &str,
1160) -> Result<ScenarioReport, ScenarioError>
1161where
1162    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1163    T: rig_core::driver::Transport<W>,
1164    F: FnOnce(AgentBuilder) -> AgentBuilder,
1165{
1166    const SCENARIO: &str = "complex_tool_arguments";
1167    let expected = ComplexArgs {
1168        profile: ComplexProfile {
1169            name: "Zoë \"Z\"".to_string(),
1170            tags: vec!["rust".to_string(), "東京".to_string()],
1171        },
1172        mode: ComplexMode::Careful,
1173        note: Some("line one\nline two".to_string()),
1174        quote: "path C:\\tmp and \"quoted\"".to_string(),
1175    };
1176    let calls = Arc::new(AtomicUsize::new(0));
1177    let captured = Arc::new(Mutex::new(None));
1178    let started = Instant::now();
1179    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1180        .preamble("Use store_profile exactly once with every value supplied by the user.")
1181        .tool(CaptureComplexTool {
1182            calls: calls.clone(),
1183            captured: captured.clone(),
1184        })
1185        .default_max_turns(3)
1186        .build();
1187    let response = agent.prompt(prompt).max_turns(3).await?;
1188    let observed = crate::lock(&captured).clone();
1189    if calls.load(Ordering::SeqCst) != 1 || observed.as_ref() != Some(&expected) {
1190        return Err(ScenarioError::contract(
1191            SCENARIO,
1192            format!(
1193                "calls={}, expected={expected:?}, observed={observed:?}, response={:?}",
1194                calls.load(Ordering::SeqCst),
1195                response.output()
1196            ),
1197        ));
1198    }
1199    correlated_messages(SCENARIO, &response)?;
1200    report_from_response(SCENARIO, started, 1, response)
1201}
1202
1203/// Runs the same deterministic text request through buffered and raw streaming
1204/// completion surfaces and validates equivalent visible content and usage.
1205pub async fn buffered_streaming_text_parity<W, T, A>(
1206    model: rig_core::driver::Model<W, T>,
1207    adjust: A,
1208) -> Result<ScenarioReport, ScenarioError>
1209where
1210    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1211    T: rig_core::driver::Transport<W>,
1212    A: Fn(CompletionRequest) -> CompletionRequest,
1213{
1214    const SCENARIO: &str = "buffered_streaming_text_parity";
1215    const PROMPT: &str = "Answer with exactly the single word Paris.";
1216    let started = Instant::now();
1217    let request = || {
1218        adjust(
1219            CompletionRequest::new(PROMPT)
1220                .temperature(0.0)
1221                .max_tokens(32),
1222        )
1223    };
1224    let buffered = model.call(request()).await?;
1225    let buffered_text = buffered
1226        .choice
1227        .iter()
1228        .filter_map(|item| match item {
1229            AssistantContent::Text(text) => Some(text.text.as_str()),
1230            _ => None,
1231        })
1232        .collect::<String>();
1233
1234    let mut stream = model.stream(request())?;
1235    let mut streamed_text = String::new();
1236    while let Some(item) = stream.next().await {
1237        if let crate::streaming::Item::Event(crate::streaming::StreamEvent::Text { text, .. }) =
1238            item?
1239        {
1240            streamed_text.push_str(&text);
1241        }
1242    }
1243    let streamed_usage = Some(stream.finish().await?.usage);
1244    let usage = streamed_usage.ok_or_else(|| {
1245        ScenarioError::contract(SCENARIO, "raw stream omitted its final response metadata")
1246    })?;
1247    let normalize = |text: &str| {
1248        text.trim()
1249            .trim_matches(|character: char| !character.is_alphanumeric())
1250            .to_string()
1251    };
1252    let buffered_answer = normalize(&buffered_text);
1253    let streamed_answer = normalize(&streamed_text);
1254    if !buffered_answer.eq_ignore_ascii_case("Paris")
1255        || !streamed_answer.eq_ignore_ascii_case("Paris")
1256        || !buffered.usage.is_reported()
1257        || !usage.is_reported()
1258    {
1259        return Err(ScenarioError::contract(
1260            SCENARIO,
1261            format!(
1262                "buffered={buffered_text:?}, streamed={streamed_text:?}, buffered_usage={:?}, streamed_usage={usage:?}",
1263                buffered.usage
1264            ),
1265        ));
1266    }
1267    Ok(ScenarioReport {
1268        name: SCENARIO,
1269        tool_calls: 0,
1270        prompt_tokens: usage.input_tokens.unwrap_or(0),
1271        generated_tokens: usage.output_tokens.unwrap_or(0),
1272        history_messages: 0,
1273        duration: started.elapsed(),
1274        response: streamed_text,
1275    })
1276}
1277
1278/// Runs Rig's structured extractor and validates both the extracted semantic
1279/// fields and accumulated usage.
1280pub async fn structured_extraction<W, T>(
1281    model: rig_core::driver::Model<W, T>,
1282    additional_params: Option<serde_json::Value>,
1283) -> Result<ScenarioReport, ScenarioError>
1284where
1285    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1286    T: rig_core::driver::Transport<W>,
1287{
1288    const SCENARIO: &str = "structured_extraction";
1289    const INPUT: &str = "Hello, my name is Ada Lovelace and I work as a mathematician.";
1290    let started = Instant::now();
1291    let mut builder = crate::extractor::ExtractorBuilder::<ExtractedPerson>::new(model)
1292        .max_tokens(384)
1293        .retries(0);
1294    if let Some(params) = additional_params {
1295        builder = builder.additional_params(params);
1296    }
1297    let response = builder.build().extract(INPUT).await?;
1298    validate_extraction_fields(
1299        SCENARIO,
1300        response.output.first_name.as_deref(),
1301        response.output.last_name.as_deref(),
1302        response.output.job.as_deref(),
1303        response.usage,
1304    )?;
1305    Ok(ScenarioReport {
1306        name: SCENARIO,
1307        tool_calls: 1,
1308        prompt_tokens: response.usage.input_tokens.unwrap_or(0),
1309        generated_tokens: response.usage.output_tokens.unwrap_or(0),
1310        history_messages: 0,
1311        duration: started.elapsed(),
1312        response: format!(
1313            "{} {} — {}",
1314            response.output.first_name.as_deref().unwrap_or_default(),
1315            response.output.last_name.as_deref().unwrap_or_default(),
1316            response.output.job.as_deref().unwrap_or_default()
1317        ),
1318    })
1319}
1320
1321fn restricted_recovery_run(
1322    prompt: &str,
1323    turn: ModelTurn,
1324    retries: usize,
1325) -> Result<AgentRun, ScenarioError> {
1326    const SCENARIO: &str = "invalid_tool_recovery";
1327    let mut run = AgentRun::new(prompt)
1328        .max_turns(2)
1329        .max_invalid_tool_call_retries(retries);
1330    if !matches!(run.next_step()?, AgentRunStep::CallModel { .. }) {
1331        return Err(ScenarioError::contract(
1332            SCENARIO,
1333            "fresh AgentRun did not request a model turn",
1334        ));
1335    }
1336    let outcome = run.model_response(turn)?;
1337    let ModelTurnOutcome::NeedsResolution(context) = outcome else {
1338        return Err(ScenarioError::contract(
1339            SCENARIO,
1340            format!("disallowed tool call did not require resolution: {outcome:?}"),
1341        ));
1342    };
1343    if context.tool_name != CountingAdd::NAME {
1344        return Err(ScenarioError::contract(
1345            SCENARIO,
1346            format!("expected rejected add call, observed {context:?}"),
1347        ));
1348    }
1349    Ok(run)
1350}
1351
1352/// Uses one real model turn to exercise fail-fast, retry exhaustion, repair,
1353/// rejected repair, and skip handling without executing a disallowed call.
1354pub async fn invalid_tool_recovery<W, T, F>(
1355    model: rig_core::driver::Model<W, T>,
1356    configure: F,
1357) -> Result<ScenarioReport, ScenarioError>
1358where
1359    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1360    T: rig_core::driver::Transport<W>,
1361    F: FnOnce(AgentBuilder) -> AgentBuilder,
1362{
1363    invalid_tool_recovery_with_choice(model, configure, ToolChoice::Required).await
1364}
1365
1366/// [`invalid_tool_recovery`] with the agent's tool choice given, for a model
1367/// that rejects a forced choice (`ToolChoice::Auto`; the prompt still asks for
1368/// the call).
1369pub async fn invalid_tool_recovery_with_choice<W, T, F>(
1370    model: rig_core::driver::Model<W, T>,
1371    configure: F,
1372    tool_choice: ToolChoice,
1373) -> Result<ScenarioReport, ScenarioError>
1374where
1375    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1376    T: rig_core::driver::Transport<W>,
1377    F: FnOnce(AgentBuilder) -> AgentBuilder,
1378{
1379    const SCENARIO: &str = "invalid_tool_recovery";
1380    const PROMPT: &str = "Call the add tool exactly once with x=2 and y=3. Do not call sum.";
1381    let started = Instant::now();
1382    let add_calls = Arc::new(AtomicUsize::new(0));
1383    let sum_calls = Arc::new(AtomicUsize::new(0));
1384    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1385        .preamble(FORCE_TOOLS_PREAMBLE)
1386        .tool(CountingAdd(add_calls.clone()))
1387        .tool(CountingSum(sum_calls.clone()))
1388        .tool_choice(tool_choice)
1389        .build();
1390    #[derive(Clone)]
1391    struct CaptureTurn(Arc<Mutex<Option<ModelTurn>>>);
1392
1393    impl AgentHook for CaptureTurn {
1394        async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
1395            let Some(response) = event.completion() else {
1396                return OutcomeAction::proceed();
1397            };
1398            *crate::lock(&self.0) = Some(ModelTurn::new(
1399                response.head(),
1400                response.choice.clone(),
1401                response.usage,
1402                TurnPolicy::default(),
1403                response.raw.clone(),
1404            ));
1405            OutcomeAction::stop("captured conformance model turn")
1406        }
1407    }
1408
1409    let captured = Arc::new(Mutex::new(None));
1410    let stopped = agent
1411        .prompt(PROMPT)
1412        .add_hook(CaptureTurn(captured.clone()))
1413        .run()
1414        .await;
1415    if !matches!(stopped, Err(PromptError::Cancelled { .. })) {
1416        return Err(ScenarioError::contract(
1417            SCENARIO,
1418            format!("capture hook did not stop after the model response: {stopped:?}"),
1419        ));
1420    }
1421    let response = crate::lock(&captured).take().ok_or_else(|| {
1422        ScenarioError::contract(SCENARIO, "capture hook observed no model response")
1423    })?;
1424    let emitted = response
1425        .choice
1426        .iter()
1427        .filter(|item| {
1428            matches!(item, AssistantContent::ToolCall(call) if call.function.name == CountingAdd::NAME)
1429        })
1430        .count();
1431    if emitted != 1 {
1432        return Err(ScenarioError::contract(
1433            SCENARIO,
1434            format!(
1435                "model emitted {emitted} add calls, response={:?}",
1436                response.choice
1437            ),
1438        ));
1439    }
1440    let executable = BTreeSet::from([CountingAdd::NAME.to_string(), CountingSum::NAME.to_string()]);
1441    let only_sum = ToolChoice::Specific {
1442        function_names: vec![
1443            rig_core::message::ToolName::new(CountingSum::NAME)
1444                .map_err(|error| ScenarioError::contract(SCENARIO, error.to_string()))?,
1445        ],
1446    };
1447    let policy = TurnPolicy::new(executable, Some(only_sum), None)
1448        .map_err(|error| ScenarioError::contract(SCENARIO, error.to_string()))?;
1449    let turn = ModelTurn::new(
1450        response.head,
1451        response.choice,
1452        response.usage,
1453        policy,
1454        response.raw,
1455    );
1456
1457    let mut fail = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1458    let error = match fail.resolve_invalid_tool_call(InvalidToolCallAction::fail()) {
1459        Err(error) => error,
1460        Ok(outcome) => {
1461            return Err(ScenarioError::contract(
1462                SCENARIO,
1463                format!("fail action unexpectedly returned {outcome:?}"),
1464            ));
1465        }
1466    };
1467    validate_unknown_tool_failure(&error, CountingAdd::NAME, &[CountingSum::NAME])?;
1468
1469    let mut retry = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1470    let error = match retry
1471        .resolve_invalid_tool_call(InvalidToolCallAction::retry("choose an allowed tool"))
1472    {
1473        Err(error) => error,
1474        Ok(outcome) => {
1475            return Err(ScenarioError::contract(
1476                SCENARIO,
1477                format!("exhausted retry unexpectedly returned {outcome:?}"),
1478            ));
1479        }
1480    };
1481    validate_unknown_tool_failure(&error, CountingAdd::NAME, &[CountingSum::NAME])?;
1482
1483    let mut rejected_repair = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1484    let error =
1485        match rejected_repair.resolve_invalid_tool_call(InvalidToolCallAction::repair("missing")) {
1486            Err(error) => error,
1487            Ok(outcome) => {
1488                return Err(ScenarioError::contract(
1489                    SCENARIO,
1490                    format!("disallowed repair unexpectedly returned {outcome:?}"),
1491                ));
1492            }
1493        };
1494    validate_unknown_tool_failure(&error, "missing", &[CountingSum::NAME])?;
1495
1496    let mut repaired = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1497    if !matches!(
1498        repaired.resolve_invalid_tool_call(InvalidToolCallAction::repair(CountingSum::NAME))?,
1499        ModelTurnOutcome::Continue { .. }
1500    ) {
1501        return Err(ScenarioError::contract(
1502            SCENARIO,
1503            "valid repair did not continue",
1504        ));
1505    }
1506    let AgentRunStep::CallTools { calls } = repaired.next_step()? else {
1507        return Err(ScenarioError::contract(
1508            SCENARIO,
1509            "valid repair did not produce pending tool execution",
1510        ));
1511    };
1512    let repaired_call = calls.first();
1513    if calls.len() != 1
1514        || !repaired_call.is_some_and(|call| {
1515            matches!(call, PendingToolCall::Execute(call) if call.name() == CountingSum::NAME)
1516        })
1517    {
1518        return Err(ScenarioError::contract(
1519            SCENARIO,
1520            format!("repaired pending calls were incorrect: {calls:?}"),
1521        ));
1522    }
1523
1524    let mut skipped = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1525    if !matches!(
1526        skipped.resolve_invalid_tool_call(InvalidToolCallAction::skip("disabled for this turn"))?,
1527        ModelTurnOutcome::Continue { .. }
1528    ) {
1529        return Err(ScenarioError::contract(SCENARIO, "skip did not continue"));
1530    }
1531    // A whole-turn skip answers every call, so the run asks the model again.
1532    let step = skipped.next_step()?;
1533    let skipped_answered = matches!(
1534        skipped.messages().last().map(|message| &**message),
1535        Some(Message::User { content }) if content.len() == 1
1536    );
1537    if !matches!(step, AgentRunStep::CallModel { .. }) || !skipped_answered {
1538        return Err(ScenarioError::contract(
1539            SCENARIO,
1540            format!("skip did not answer the turn and continue: {step:?}"),
1541        ));
1542    }
1543    if add_calls.load(Ordering::SeqCst) != 0 || sum_calls.load(Ordering::SeqCst) != 0 {
1544        return Err(ScenarioError::contract(
1545            SCENARIO,
1546            "recovery scenario executed a tool body",
1547        ));
1548    }
1549
1550    Ok(ScenarioReport {
1551        name: SCENARIO,
1552        tool_calls: emitted,
1553        prompt_tokens: turn.usage.input_tokens.unwrap_or(0),
1554        generated_tokens: turn.usage.output_tokens.unwrap_or(0),
1555        history_messages: 2,
1556        duration: started.elapsed(),
1557        response: "fail, retry, repair, rejected repair, and skip passed".to_string(),
1558    })
1559}
1560
1561/// Exercises chained argument/result rewrites and a first-turn-only request
1562/// patch. Completion proves `tool_choice=Required` did not leak to turn two.
1563pub async fn hook_rewrites_and_request_patch<W, T, F>(
1564    model: rig_core::driver::Model<W, T>,
1565    configure: F,
1566) -> Result<ScenarioReport, ScenarioError>
1567where
1568    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1569    T: rig_core::driver::Transport<W>,
1570    F: FnOnce(AgentBuilder) -> AgentBuilder,
1571{
1572    hook_rewrites_and_request_patch_with_choice(model, configure, ToolChoice::Required).await
1573}
1574
1575/// [`hook_rewrites_and_request_patch`] with the first-turn patch's tool
1576/// choice given, for a model that rejects a forced choice (`ToolChoice::Auto`:
1577/// the patch still narrows the active tools on turn one only).
1578pub async fn hook_rewrites_and_request_patch_with_choice<W, T, F>(
1579    model: rig_core::driver::Model<W, T>,
1580    configure: F,
1581    first_turn_choice: ToolChoice,
1582) -> Result<ScenarioReport, ScenarioError>
1583where
1584    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1585    T: rig_core::driver::Transport<W>,
1586    F: FnOnce(AgentBuilder) -> AgentBuilder,
1587{
1588    const SCENARIO: &str = "hook_rewrites_and_request_patch";
1589    let started = Instant::now();
1590    let calls = Arc::new(AtomicUsize::new(0));
1591    let observed = ObserveArguments::default();
1592    let observed_probe = observed.clone();
1593    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1594        .preamble("Use add for arithmetic and report only the tool result.")
1595        .tool(CountingAdd(calls.clone()))
1596        .default_max_turns(3)
1597        .build();
1598    let response = agent
1599        .prompt("Use add once for x=1 and y=1, then report what the tool returns.")
1600        .max_turns(3)
1601        .add_hook(FirstTurnPatch(
1602            RequestPatch::new()
1603                .active_tools([CountingAdd::NAME])
1604                .tool_choice(first_turn_choice),
1605        ))
1606        .add_hook(RewriteArgument {
1607            key: "x",
1608            value: serde_json::json!(7),
1609        })
1610        .add_hook(RewriteArgument {
1611            key: "y",
1612            value: serde_json::json!(8),
1613        })
1614        .add_hook(observed)
1615        .add_hook(ReplaceResult("portable-redacted"))
1616        .add_hook(WrapResult)
1617        .await?;
1618    let observations = crate::lock(&observed_probe.0).clone();
1619    validate_rewritten_arguments(
1620        SCENARIO,
1621        &observations,
1622        &serde_json::json!({ "x": 7, "y": 8 }),
1623    )?;
1624    let messages = response.messages.as_slice();
1625    if messages.is_empty() {
1626        return Err(ScenarioError::contract(
1627            SCENARIO,
1628            "extended hook run omitted message history",
1629        ));
1630    }
1631    let results = messages
1632        .iter()
1633        .flat_map(tool_result_values)
1634        .collect::<Vec<_>>();
1635    if calls.load(Ordering::SeqCst) != 1
1636        || !results
1637            .iter()
1638            .any(|value| value == &serde_json::json!("[portable-redacted]"))
1639        || response.completion_calls.len() != 2
1640    {
1641        return Err(ScenarioError::contract(
1642            SCENARIO,
1643            format!(
1644                "calls={}, completion_calls={}, results={results:?}, output={:?}",
1645                calls.load(Ordering::SeqCst),
1646                response.completion_calls.len(),
1647                response.output()
1648            ),
1649        ));
1650    }
1651    report_from_response(SCENARIO, started, 1, response)
1652}
1653
1654/// Exercises post-execution cancellation and max-turn diagnostics through the
1655/// public agent driver using real model-emitted calls.
1656pub async fn cancellation_and_max_turns<W, T, F>(
1657    model: rig_core::driver::Model<W, T>,
1658    configure: F,
1659) -> Result<ScenarioReport, ScenarioError>
1660where
1661    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1662    T: rig_core::driver::Transport<W>,
1663    F: Fn(AgentBuilder) -> AgentBuilder,
1664{
1665    const SCENARIO: &str = "cancellation_and_max_turns";
1666    const REASON: &str = "portable result veto";
1667    let started = Instant::now();
1668    let cancelled_calls = Arc::new(AtomicUsize::new(0));
1669    let cancelled_agent = configure(AgentBuilder::new(model.clone()).temperature(0.0))
1670        .preamble("Use add for arithmetic; never calculate by hand.")
1671        .tool(CountingAdd(cancelled_calls.clone()))
1672        .build();
1673    let cancelled = match cancelled_agent
1674        .prompt("Use add once to compute x=20 plus y=22.")
1675        .max_turns(2)
1676        .add_hook(StopAfterResult(REASON))
1677        .await
1678    {
1679        Err(error) => error,
1680        Ok(output) => {
1681            return Err(ScenarioError::contract(
1682                SCENARIO,
1683                format!("result cancellation unexpectedly completed: {output:?}"),
1684            ));
1685        }
1686    };
1687    validate_cancelled_failure(&cancelled, REASON, CountingAdd::NAME)?;
1688
1689    let max_turn_calls = Arc::new(AtomicUsize::new(0));
1690    let max_turn_agent = configure(AgentBuilder::new(model).temperature(0.0))
1691        .preamble("Use add for arithmetic; never calculate by hand.")
1692        .tool(CountingAdd(max_turn_calls.clone()))
1693        .build();
1694    let max_turn = match max_turn_agent
1695        .prompt("Use add once to compute x=20 plus y=22, then report the result.")
1696        .max_turns(1)
1697        .await
1698    {
1699        Err(error) => error,
1700        Ok(output) => {
1701            return Err(ScenarioError::contract(
1702                SCENARIO,
1703                format!("one-turn budget unexpectedly completed: {output:?}"),
1704            ));
1705        }
1706    };
1707    validate_max_turns_failure(&max_turn, 1)?;
1708    let cancelled_count = cancelled_calls.load(Ordering::SeqCst);
1709    let max_turn_count = max_turn_calls.load(Ordering::SeqCst);
1710    if cancelled_count != 1 || max_turn_count != 1 {
1711        return Err(ScenarioError::contract(
1712            SCENARIO,
1713            format!("cancelled executions={cancelled_count}, max-turn executions={max_turn_count}"),
1714        ));
1715    }
1716    Ok(ScenarioReport {
1717        name: SCENARIO,
1718        tool_calls: cancelled_count + max_turn_count,
1719        prompt_tokens: 0,
1720        generated_tokens: 0,
1721        history_messages: 4,
1722        duration: started.elapsed(),
1723        response: "post-result cancellation and max-turn diagnostics passed".to_string(),
1724    })
1725}
1726
1727/// Runs the portable optional-argument tool scenario.
1728///
1729/// `configure` is deliberately outside the scenario so a provider suite can
1730/// attach transport-only settings without putting them into the shared model
1731/// contract.
1732pub async fn optional_argument<W, T, F>(
1733    model: rig_core::driver::Model<W, T>,
1734    configure: F,
1735) -> Result<ScenarioReport, ScenarioError>
1736where
1737    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1738    T: rig_core::driver::Transport<W>,
1739    F: FnOnce(AgentBuilder) -> AgentBuilder,
1740{
1741    let calls = Arc::new(AtomicUsize::new(0));
1742    let started = Instant::now();
1743    let agent = configure(AgentBuilder::new(model))
1744        .preamble("Use the repeat_text tool whenever asked to repeat text.")
1745        .tool(RepeatTool {
1746            calls: calls.clone(),
1747        })
1748        .default_max_turns(4)
1749        .build();
1750    let result = agent
1751        .prompt(
1752            "Use the repeat_text tool to repeat the word \"banana\" 3 times, then show me the exact result.",
1753        )
1754        .await?;
1755    let response = result.output();
1756    let tool_calls = calls.load(Ordering::SeqCst);
1757    if tool_calls == 0
1758        || response.matches("banana").count() < 1
1759        || !has_tool_roundtrip(&result.messages)
1760    {
1761        return Err(ScenarioError::contract(
1762            "optional_argument",
1763            format!("calls={tool_calls}, response={response:?}"),
1764        ));
1765    }
1766    report_from_response("optional_argument", started, tool_calls, result)
1767}
1768
1769/// Runs a portable two-tool sequential arithmetic scenario.
1770pub async fn sequential_tools<W, T, F>(
1771    model: rig_core::driver::Model<W, T>,
1772    configure: F,
1773) -> Result<ScenarioReport, ScenarioError>
1774where
1775    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1776    T: rig_core::driver::Transport<W>,
1777    F: FnOnce(AgentBuilder) -> AgentBuilder,
1778{
1779    let add_calls = Arc::new(AtomicUsize::new(0));
1780    let multiply_calls = Arc::new(AtomicUsize::new(0));
1781    let started = Instant::now();
1782    let agent = configure(AgentBuilder::new(model))
1783        .preamble(
1784            "You are a calculator. Use the add and multiply tools for arithmetic; never compute by hand.",
1785        )
1786        .tool(AddTool(add_calls.clone()))
1787        .tool(MultiplyTool(multiply_calls.clone()))
1788        .default_max_turns(6)
1789        .build();
1790    let result = agent
1791        .prompt(
1792            "Compute (4 + 6) * 2. First call the add tool, then call the multiply tool on the result. Tell me the final number.",
1793        )
1794        .await?;
1795    let response = result.output();
1796    let add = add_calls.load(Ordering::SeqCst);
1797    let multiply = multiply_calls.load(Ordering::SeqCst);
1798    if add == 0
1799        || multiply == 0
1800        || !response.contains("20")
1801        || !has_tool_roundtrip(&result.messages)
1802    {
1803        return Err(ScenarioError::contract(
1804            "sequential_tools",
1805            format!("add={add}, multiply={multiply}, response={response:?}"),
1806        ));
1807    }
1808    report_from_response("sequential_tools", started, add + multiply, result)
1809}
1810
1811/// Runs a tool through Rig's multi-turn streaming agent driver.
1812pub async fn streaming_tool<W, T, F>(
1813    model: rig_core::driver::Model<W, T>,
1814    configure: F,
1815) -> Result<ScenarioReport, ScenarioError>
1816where
1817    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1818    T: rig_core::driver::Transport<W>,
1819    F: FnOnce(AgentBuilder) -> AgentBuilder,
1820{
1821    let calls = Arc::new(AtomicUsize::new(0));
1822    let started = Instant::now();
1823    let agent = configure(AgentBuilder::new(model))
1824        .preamble("Use the add tool for arithmetic; do not calculate by hand.")
1825        .tool(AddTool(calls.clone()))
1826        .default_max_turns(4)
1827        .build();
1828    let mut stream = agent
1829        .prompt("Use add to calculate 17 + 25, then state the final number.")
1830        .max_turns(4)
1831        .stream();
1832    let mut final_response = None;
1833    let mut final_count = 0_usize;
1834    let mut completion_usage = crate::completion::Usage::default();
1835    let mut streamed_call_ids = Vec::new();
1836    let mut streamed_result_ids = Vec::new();
1837    while let Some(item) = stream.next().await {
1838        match item? {
1839            MultiTurnStreamItem::ToolCall { tool_call } => streamed_call_ids.push(tool_call.id),
1840            MultiTurnStreamItem::ToolResult { tool_result } => {
1841                streamed_result_ids.push(tool_result.call)
1842            }
1843            MultiTurnStreamItem::CompletionCall(call) => completion_usage += call.usage,
1844            MultiTurnStreamItem::FinalResponse(response) => {
1845                final_count += 1;
1846                final_response = Some(response);
1847            }
1848            MultiTurnStreamItem::StreamAssistantItem(_)
1849            | MultiTurnStreamItem::ToolExecutionStarted { .. }
1850            | MultiTurnStreamItem::ToolResultItem { .. }
1851            | MultiTurnStreamItem::ToolExecutionCommitted { .. }
1852            | MultiTurnStreamItem::ModelTurnRetried { .. } => {}
1853        }
1854    }
1855    let result = final_response.ok_or_else(|| {
1856        ScenarioError::contract("streaming_tool", "stream produced no final response")
1857    })?;
1858    let response = result.output();
1859    let history_messages = result.messages.len();
1860    let tool_calls = calls.load(Ordering::SeqCst);
1861    streamed_call_ids.sort();
1862    streamed_result_ids.sort();
1863    let correlated_stream =
1864        !streamed_call_ids.is_empty() && streamed_call_ids == streamed_result_ids;
1865    let correlated_history = validate_tool_correlation("streaming_tool", &result.messages).is_ok();
1866    if tool_calls == 0
1867        || !response.contains("42")
1868        || history_messages < 4
1869        || final_count != 1
1870        || !correlated_stream
1871        || !correlated_history
1872        || completion_usage != result.usage
1873        || result.completion_calls.is_empty()
1874    {
1875        return Err(ScenarioError::contract(
1876            "streaming_tool",
1877            format!(
1878                "calls={tool_calls}, final_count={final_count}, streamed_call_ids={streamed_call_ids:?}, streamed_result_ids={streamed_result_ids:?}, completion_usage={completion_usage:?}, final_usage={:?}, history_messages={history_messages}, response={response:?}",
1879                result.usage
1880            ),
1881        ));
1882    }
1883    validate_protocol_hygiene(
1884        "streaming_tool",
1885        &response,
1886        &result.messages,
1887        &["<tool_call>", "</tool_call>", "<think>", "</think>"],
1888    )?;
1889    Ok(ScenarioReport {
1890        name: "streaming_tool",
1891        tool_calls,
1892        prompt_tokens: result.usage.input_tokens.unwrap_or(0),
1893        generated_tokens: result.usage.output_tokens.unwrap_or(0),
1894        history_messages,
1895        duration: started.elapsed(),
1896        response,
1897    })
1898}
1899
1900/// Runs a normal tool followed by Rig's synthetic structured-output tool.
1901pub async fn structured_after_tool<W, T, F>(
1902    model: rig_core::driver::Model<W, T>,
1903    configure: F,
1904) -> Result<ScenarioReport, ScenarioError>
1905where
1906    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1907    T: rig_core::driver::Transport<W>,
1908    F: FnOnce(AgentBuilder) -> AgentBuilder,
1909{
1910    let calls = Arc::new(AtomicUsize::new(0));
1911    let started = Instant::now();
1912    let agent = configure(AgentBuilder::new(model))
1913        .preamble(
1914            "Use add for arithmetic, then finish by calling the structured output tool exactly once.",
1915        )
1916        .output_schema::<ArithmeticResult>()
1917        .output_mode(OutputMode::Tool)
1918        .tool(AddTool(calls.clone()))
1919        .default_max_turns(5)
1920        .build();
1921    let result = agent
1922        .prompt("Use add to calculate 19 + 23. Return answer=42 and a short optional explanation.")
1923        .await?;
1924    let response = result.output();
1925    let parsed: ArithmeticResult = serde_json::from_str(&response)?;
1926    let tool_calls = calls.load(Ordering::SeqCst);
1927    if tool_calls == 0 || parsed.answer != 42 || !has_tool_roundtrip(&result.messages) {
1928        return Err(ScenarioError::contract(
1929            "structured_after_tool",
1930            format!("calls={tool_calls}, response={response:?}"),
1931        ));
1932    }
1933    let _ = parsed.explanation;
1934    report_from_response("structured_after_tool", started, tool_calls + 1, result)
1935}
1936
1937/// Runs all portable tool-choice modes directly against a completion model.
1938pub async fn tool_choice_modes<W, T, A>(
1939    model: rig_core::driver::Model<W, T>,
1940    adjust: A,
1941) -> Result<ScenarioReport, ScenarioError>
1942where
1943    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1944    T: rig_core::driver::Transport<W>,
1945    A: Fn(CompletionRequest) -> CompletionRequest,
1946{
1947    let tool_name = |name: &str| {
1948        rig_core::message::ToolName::new(name)
1949            .map_err(|error| ScenarioError::contract("tool_choice_modes", error.to_string()))
1950    };
1951    let definition = |name: rig_core::message::ToolName| ToolDefinition {
1952        description: format!("Return the supplied integer using {name}."),
1953        name,
1954        parameters: serde_json::json!({
1955            "type": "object",
1956            "properties": {"value": {"type": "integer"}},
1957            "required": ["value"]
1958        }),
1959        annotations: Default::default(),
1960    };
1961    let tools = vec![
1962        definition(tool_name("alpha")?),
1963        definition(tool_name("beta")?),
1964    ];
1965    let started = Instant::now();
1966    let none = model
1967        .call(adjust(
1968            CompletionRequest::new("Answer with only the number 4. Do not call a function.")
1969                .tools(tools.clone())
1970                .tool_choice(ToolChoice::None)
1971                .temperature(0.0)
1972                .max_tokens(64),
1973        ))
1974        .await?;
1975    if none
1976        .choice
1977        .iter()
1978        .any(|item| matches!(item, AssistantContent::ToolCall(_)))
1979    {
1980        return Err(ScenarioError::contract(
1981            "tool_choice_modes",
1982            "tool_choice none emitted a tool call",
1983        ));
1984    }
1985
1986    let required = model
1987        .call(adjust(
1988            CompletionRequest::new("Call alpha with value 7.")
1989                .tools(tools.clone())
1990                .tool_choice(ToolChoice::Required)
1991                .temperature(0.0)
1992                .max_tokens(96),
1993        ))
1994        .await?;
1995    let required_calls = required
1996        .choice
1997        .iter()
1998        .filter(|item| matches!(item, AssistantContent::ToolCall(_)))
1999        .count();
2000    if required_calls == 0 {
2001        return Err(ScenarioError::contract(
2002            "tool_choice_modes",
2003            "tool_choice required emitted no tool call",
2004        ));
2005    }
2006
2007    let specific = model
2008        .call(adjust(
2009            CompletionRequest::new("Call beta with value 9.")
2010                .tools(tools)
2011                .tool_choice(ToolChoice::Specific {
2012                    function_names: vec![tool_name("beta")?],
2013                })
2014                .temperature(0.0)
2015                .max_tokens(96),
2016        ))
2017        .await?;
2018    let specific_calls = specific
2019        .choice
2020        .iter()
2021        .filter_map(|item| match item {
2022            AssistantContent::ToolCall(call) => Some(call),
2023            _ => None,
2024        })
2025        .collect::<Vec<_>>();
2026    if specific_calls.is_empty()
2027        || specific_calls
2028            .iter()
2029            .any(|call| call.function.name != "beta")
2030    {
2031        return Err(ScenarioError::contract(
2032            "tool_choice_modes",
2033            "specific tool choice did not select only beta",
2034        ));
2035    }
2036
2037    let usage = none.usage + required.usage + specific.usage;
2038    Ok(ScenarioReport {
2039        name: "tool_choice_modes",
2040        tool_calls: required_calls + specific_calls.len(),
2041        prompt_tokens: usage.input_tokens.unwrap_or(0),
2042        generated_tokens: usage.output_tokens.unwrap_or(0),
2043        history_messages: 0,
2044        duration: started.elapsed(),
2045        response: "none, required, and specific modes passed".to_string(),
2046    })
2047}
2048
2049/// Runs a streamed real-tool turn followed by Rig's synthetic output tool.
2050pub async fn streaming_structured_after_tool<W, T, F>(
2051    model: rig_core::driver::Model<W, T>,
2052    configure: F,
2053) -> Result<ScenarioReport, ScenarioError>
2054where
2055    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
2056    T: rig_core::driver::Transport<W>,
2057    F: FnOnce(AgentBuilder) -> AgentBuilder,
2058{
2059    let calls = Arc::new(AtomicUsize::new(0));
2060    let started = Instant::now();
2061    let agent = configure(AgentBuilder::new(model))
2062        .preamble(
2063            "Use add for arithmetic, then finish by calling the structured output tool exactly once.",
2064        )
2065        .output_schema::<ArithmeticResult>()
2066        .output_mode(OutputMode::Tool)
2067        .tool(AddTool(calls.clone()))
2068        .default_max_turns(5)
2069        .build();
2070    let mut stream = agent
2071        .prompt("Use add to calculate 19 + 23. Return answer=42 and a short optional explanation.")
2072        .max_turns(5)
2073        .stream();
2074    let mut final_response = None;
2075    let mut final_count = 0_usize;
2076    while let Some(item) = stream.next().await {
2077        if let MultiTurnStreamItem::FinalResponse(response) = item? {
2078            final_count += 1;
2079            final_response = Some(response);
2080        }
2081    }
2082    let result = final_response.ok_or_else(|| {
2083        ScenarioError::contract(
2084            "streaming_structured_after_tool",
2085            "stream produced no final response",
2086        )
2087    })?;
2088    let parsed: ArithmeticResult = serde_json::from_str(&result.output())?;
2089    let calls = calls.load(Ordering::SeqCst);
2090    if calls == 0
2091        || final_count != 1
2092        || parsed.answer != 42
2093        || !has_tool_roundtrip(&result.messages)
2094    {
2095        return Err(ScenarioError::contract(
2096            "streaming_structured_after_tool",
2097            format!(
2098                "calls={calls}, final_count={final_count}, response={:?}",
2099                result.output()
2100            ),
2101        ));
2102    }
2103    report_from_response(
2104        "streaming_structured_after_tool",
2105        started,
2106        calls + 1,
2107        result,
2108    )
2109}
2110
2111#[cfg(test)]
2112mod tests;