Skip to main content

rig_agent/test_utils/
model_conformance.rs

1//! Provider-neutral behavioral scenarios for completion-model conformance.
2//!
3//! These helpers test the model/agent contract only. Provider wire formats,
4//! authentication, HTTP streaming, and cassette matching remain provider-suite
5//! responsibilities.
6
7use std::{
8    collections::BTreeSet,
9    sync::{
10        Arc, Mutex, MutexGuard,
11        atomic::{AtomicUsize, Ordering},
12    },
13    time::{Duration, Instant},
14};
15
16use futures::StreamExt;
17use schemars::JsonSchema;
18use serde::{Deserialize, Serialize};
19
20use crate::{
21    agent::{
22        AgentBuilder, AgentHook, CompletionCallAction, CompletionCallEvent, DispatchAction,
23        DispatchEvent, HookContext, InvalidToolCallAction, MultiTurnStreamItem, NoToolConfig,
24        OutcomeAction, OutcomeEvent, OutputMode, RequestPatch,
25        run::{AgentRun, AgentRunStep, ModelTurn, ModelTurnOutcome},
26    },
27    completion::{AssistantContent, Message, PromptError, ToolDefinition},
28    tool::{Tool, ToolContext},
29};
30use rig_core::completion::CompletionRequest;
31use rig_core::error::ProviderError;
32use rig_core::message::{ToolChoice, UserContent};
33
34/// Typed failure from a portable model-conformance scenario.
35#[derive(Debug, thiserror::Error)]
36pub enum ScenarioError {
37    /// An agent run failed.
38    #[error(transparent)]
39    Prompt(#[from] PromptError),
40    /// A direct model completion failed.
41    #[error(transparent)]
42    Completion(#[from] ProviderError),
43    /// A stream item or bus effect failed, as the wire reports it.
44    #[error(transparent)]
45    Report(#[from] rig_core::error::ErrorReport),
46    /// Structured content could not be decoded.
47    #[error(transparent)]
48    Json(#[from] serde_json::Error),
49    /// Rig's structured extractor failed.
50    #[error(transparent)]
51    Extraction(#[from] crate::completion::StructuredOutputError),
52    /// The model or agent violated the portable behavioral contract.
53    #[error("{scenario} conformance failed: {details}")]
54    Contract {
55        /// Stable scenario name.
56        scenario: &'static str,
57        /// Actionable observation explaining the failure.
58        details: String,
59    },
60}
61
62impl ScenarioError {
63    fn contract(scenario: &'static str, details: impl Into<String>) -> Self {
64        Self::Contract {
65            scenario,
66            details: details.into(),
67        }
68    }
69}
70
71/// Validate the portable diagnostics carried by an unknown or disallowed tool
72/// call failure.
73pub fn validate_unknown_tool_failure(
74    error: &PromptError,
75    expected_tool: &str,
76    expected_allowed_tools: &[&str],
77) -> Result<(), ScenarioError> {
78    const SCENARIO: &str = "unknown_tool_failure";
79    let PromptError::UnknownToolCall {
80        tool_name,
81        allowed_tools,
82        chat_history,
83        ..
84    } = error
85    else {
86        return Err(ScenarioError::contract(
87            SCENARIO,
88            format!("expected UnknownToolCall, observed {error:?}"),
89        ));
90    };
91    let expected_allowed = expected_allowed_tools
92        .iter()
93        .map(|name| (*name).to_string())
94        .collect::<Vec<_>>();
95    // A rejected repair target is never written back into history; diagnostics
96    // retain the model's original call while `tool_name` names the rejected
97    // target. Requiring a call-bearing assistant turn covers both paths.
98    let history_has_call = chat_history.iter().any(|message| {
99        matches!(
100            message,
101            Message::Assistant(rig_core::message::AssistantMessage { content, .. })
102                if content.iter().any(|item| matches!(item, AssistantContent::ToolCall(_)))
103        )
104    });
105    if tool_name != expected_tool || allowed_tools != &expected_allowed || !history_has_call {
106        return Err(ScenarioError::contract(
107            SCENARIO,
108            format!(
109                "tool={tool_name:?}, allowed={allowed_tools:?}, expected_tool={expected_tool:?}, expected_allowed={expected_allowed:?}, history_has_call={history_has_call}, history={chat_history:?}"
110            ),
111        ));
112    }
113    Ok(())
114}
115
116/// Validate cancellation diagnostics, including the exact reason and retained
117/// assistant tool-call history.
118pub fn validate_cancelled_failure(
119    error: &PromptError,
120    expected_reason: &str,
121    expected_tool: &str,
122) -> Result<(), ScenarioError> {
123    const SCENARIO: &str = "cancelled_failure";
124    let PromptError::Cancelled {
125        chat_history,
126        reason,
127    } = error
128    else {
129        return Err(ScenarioError::contract(
130            SCENARIO,
131            format!("expected Cancelled, observed {error:?}"),
132        ));
133    };
134    let history_has_call = chat_history.iter().any(|message| {
135        matches!(
136            message,
137            Message::Assistant(rig_core::message::AssistantMessage { content, .. })
138                if content.iter().any(|item| matches!(
139                    item,
140                    AssistantContent::ToolCall(call) if call.function.name == expected_tool
141                ))
142        )
143    });
144    if reason != expected_reason || !history_has_call {
145        return Err(ScenarioError::contract(
146            SCENARIO,
147            format!(
148                "reason={reason:?}, expected={expected_reason:?}, history_has_call={history_has_call}, history={chat_history:?}"
149            ),
150        ));
151    }
152    Ok(())
153}
154
155/// Validate max-turn diagnostics, including the exact configured budget and a
156/// retained pending prompt.
157pub fn validate_max_turns_failure(
158    error: &PromptError,
159    expected_max_turns: usize,
160) -> Result<(), ScenarioError> {
161    const SCENARIO: &str = "max_turns_failure";
162    let PromptError::MaxTurns {
163        max_turns,
164        chat_history,
165        prompt,
166    } = error
167    else {
168        return Err(ScenarioError::contract(
169            SCENARIO,
170            format!("expected MaxTurns, observed {error:?}"),
171        ));
172    };
173    let pending_prompt_retained = matches!(
174        prompt,
175        Message::User { content } if content.iter().next().is_some()
176    );
177    if *max_turns != expected_max_turns || chat_history.is_empty() || !pending_prompt_retained {
178        return Err(ScenarioError::contract(
179            SCENARIO,
180            format!(
181                "max_turns={max_turns}, expected={expected_max_turns}, history={chat_history:?}, pending_prompt_retained={pending_prompt_retained}, pending_prompt={prompt:?}"
182            ),
183        ));
184    }
185    Ok(())
186}
187
188/// Decode a structured-output response with a typed conformance failure that
189/// retains the scenario name and raw response.
190pub fn decode_structured_output<T>(
191    scenario: &'static str,
192    response: &str,
193) -> Result<T, ScenarioError>
194where
195    T: serde::de::DeserializeOwned,
196{
197    serde_json::from_str(response).map_err(|error| {
198        ScenarioError::contract(
199            scenario,
200            format!("structured output did not decode: {error}; response={response:?}"),
201        )
202    })
203}
204
205/// Validate that model-family control markers are absent from user-visible
206/// output and persisted history.
207pub fn validate_protocol_hygiene(
208    scenario: &'static str,
209    visible_output: &str,
210    messages: &[Message],
211    forbidden_markers: &[&str],
212) -> Result<(), ScenarioError> {
213    let serialized = serde_json::to_string(messages)?;
214    let leaked = forbidden_markers
215        .iter()
216        .filter(|marker| visible_output.contains(**marker) || serialized.contains(**marker))
217        .copied()
218        .collect::<Vec<_>>();
219    if !leaked.is_empty() {
220        return Err(ScenarioError::contract(
221            scenario,
222            format!(
223                "protocol markers leaked: {leaked:?}; output={visible_output:?}, history={messages:?}"
224            ),
225        ));
226    }
227    Ok(())
228}
229
230/// Validate that every observed tool invocation contains the expected rewritten
231/// argument fields while allowing unrelated original fields to remain.
232pub fn validate_rewritten_arguments(
233    scenario: &'static str,
234    observations: &[serde_json::Value],
235    expected_fields: &serde_json::Value,
236) -> Result<(), ScenarioError> {
237    let Some(expected) = expected_fields.as_object() else {
238        return Err(ScenarioError::contract(
239            scenario,
240            "expected rewritten fields must be a JSON object",
241        ));
242    };
243    if observations.is_empty() {
244        return Err(ScenarioError::contract(
245            scenario,
246            "the rewritten tool was never invoked",
247        ));
248    }
249    for observation in observations {
250        let Some(actual) = observation.as_object() else {
251            return Err(ScenarioError::contract(
252                scenario,
253                format!("observed rewritten arguments were not an object: {observation:?}"),
254            ));
255        };
256        for (key, expected_value) in expected {
257            if actual.get(key) != Some(expected_value) {
258                return Err(ScenarioError::contract(
259                    scenario,
260                    format!(
261                        "rewritten field {key:?} expected {expected_value:?}, observed {observation:?}"
262                    ),
263                ));
264            }
265        }
266    }
267    Ok(())
268}
269
270/// Validate that a sensitive raw tool result was produced but did not reach the
271/// model's user-visible response after result hooks ran.
272pub fn validate_result_redaction(
273    scenario: &'static str,
274    tool_produced_secret: bool,
275    visible_output: &str,
276    secret: &str,
277) -> Result<(), ScenarioError> {
278    if !tool_produced_secret || visible_output.is_empty() || visible_output.contains(secret) {
279        return Err(ScenarioError::contract(
280            scenario,
281            format!(
282                "produced_secret={tool_produced_secret}, secret_visible={}, output={visible_output:?}",
283                visible_output.contains(secret)
284            ),
285        ));
286    }
287    Ok(())
288}
289
290/// Validate a provider-neutral person extraction and its usage accounting.
291pub fn validate_extraction_fields(
292    scenario: &'static str,
293    first_name: Option<&str>,
294    last_name: Option<&str>,
295    job: Option<&str>,
296    usage: crate::completion::Usage,
297) -> Result<(), ScenarioError> {
298    let fields_match = first_name.is_some_and(|value| value.eq_ignore_ascii_case("Ada"))
299        && last_name.is_some_and(|value| value.eq_ignore_ascii_case("Lovelace"))
300        && job.is_some_and(|value| value.to_ascii_lowercase().contains("mathematician"));
301    if !fields_match || !usage.is_reported() {
302        return Err(ScenarioError::contract(
303            scenario,
304            format!(
305                "first_name={first_name:?}, last_name={last_name:?}, job={job:?}, usage={usage:?}"
306            ),
307        ));
308    }
309    Ok(())
310}
311
312/// Summary emitted by a portable model-conformance scenario.
313#[derive(Debug, Clone)]
314pub struct ScenarioReport {
315    /// Stable scenario name.
316    pub name: &'static str,
317    /// Number of model-invoked tool calls observed by the scenario.
318    pub tool_calls: usize,
319    /// Aggregated prompt tokens reported by the model across all turns; zero
320    /// when the model reported none.
321    pub prompt_tokens: u64,
322    /// Aggregated generated tokens reported by the model across all turns;
323    /// zero when the model reported none.
324    pub generated_tokens: u64,
325    /// Number of messages retained in the completed run history.
326    pub history_messages: usize,
327    /// End-to-end scenario duration.
328    pub duration: Duration,
329    /// The model's final user-visible response.
330    pub response: String,
331}
332
333/// Error used by the deterministic conformance tools.
334#[derive(Debug, thiserror::Error)]
335#[error("model-conformance tool failed")]
336pub struct ConformanceToolError;
337
338const FORCE_TOOLS_PREAMBLE: &str = "You are a calculator assistant. You MUST use the provided tools for every arithmetic operation instead of computing results yourself. Once you have all the tool results you need, reply with the final numeric answer in plain text.";
339const PARALLEL_PROMPT: &str = "Compute 3 + 4 and 10 - 2. You MUST call the add tool and the subtract tool together in your first response, as two parallel function calls, then report both results.";
340const PING_OUTPUT: &str = "pong-crimson-7423";
341const MOTTO_OUTPUT: &str = "steady hands\ncalm waters";
342
343fn lock_recover<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
344    mutex
345        .lock()
346        .unwrap_or_else(std::sync::PoisonError::into_inner)
347}
348
349fn tool_result_values(message: &Message) -> Vec<serde_json::Value> {
350    let Message::User { content } = message else {
351        return Vec::new();
352    };
353    content
354        .iter()
355        .filter_map(|item| match item {
356            UserContent::ToolResult(result) => Some(result),
357            _ => None,
358        })
359        .flat_map(|result| result.content.iter())
360        .filter_map(|content| match content {
361            rig_core::message::ToolResultContent::Text(text) => {
362                Some(serde_json::Value::String(text.text.clone()))
363            }
364            rig_core::message::ToolResultContent::Json { value } => Some(value.clone()),
365            rig_core::message::ToolResultContent::Image(_) => None,
366        })
367        .collect()
368}
369
370fn validate_tool_correlation(
371    scenario: &'static str,
372    messages: &[Message],
373) -> Result<(), ScenarioError> {
374    let mut calls = Vec::new();
375    let mut results = Vec::new();
376    let mut turn = 0usize;
377    for message in messages {
378        match message {
379            Message::Assistant(rig_core::message::AssistantMessage { content, .. }) => {
380                turn += 1;
381                calls.extend(content.iter().filter_map(|item| match item {
382                    AssistantContent::ToolCall(call) => Some((
383                        turn,
384                        &call.id,
385                        call.id.provider().map(|provider| provider.as_str()),
386                    )),
387                    _ => None,
388                }));
389            }
390            Message::User { content } => {
391                results.extend(content.iter().filter_map(|item| match item {
392                    UserContent::ToolResult(result) => Some((
393                        turn,
394                        &result.call,
395                        result.call.provider().map(|provider| provider.as_str()),
396                    )),
397                    _ => None,
398                }));
399            }
400            Message::System { .. } => {}
401        }
402    }
403    if calls.is_empty() {
404        return Err(ScenarioError::contract(
405            scenario,
406            format!("history has no assistant tool calls: {messages:?}"),
407        ));
408    }
409    for (turn, id, call_id) in &calls {
410        let matches = results
411            .iter()
412            .filter(|(result_turn, result_id, result_call_id)| {
413                result_turn == turn && result_id == id && call_id == result_call_id
414            })
415            .count();
416        if matches != 1 {
417            return Err(ScenarioError::contract(
418                scenario,
419                format!(
420                    "tool call id={id:?} call_id={call_id:?} has {matches} correlated results; calls={calls:?}, results={results:?}"
421                ),
422            ));
423        }
424    }
425    if results.len() != calls.len() {
426        return Err(ScenarioError::contract(
427            scenario,
428            format!(
429                "history contains dangling calls or results: calls={calls:?}, results={results:?}"
430            ),
431        ));
432    }
433    Ok(())
434}
435
436#[derive(Debug, Deserialize, JsonSchema)]
437struct OperationArgs {
438    x: i64,
439    y: i64,
440}
441
442#[derive(Clone)]
443struct CountingAdd(Arc<AtomicUsize>);
444
445impl Tool for CountingAdd {
446    const NAME: &'static str = "add";
447    type Error = ConformanceToolError;
448    type Args = OperationArgs;
449    type Output = i64;
450
451    fn description(&self) -> String {
452        "Add x and y together".to_string()
453    }
454
455    fn parameters(&self) -> serde_json::Value {
456        serde_json::json!({
457            "type": "object",
458            "properties": {
459                "x": { "type": "number", "description": "The first operand" },
460                "y": { "type": "number", "description": "The second operand" }
461            },
462            "required": ["x", "y"]
463        })
464    }
465
466    async fn call(
467        &self,
468        _context: &mut ToolContext,
469        args: Self::Args,
470    ) -> Result<Self::Output, Self::Error> {
471        self.0.fetch_add(1, Ordering::SeqCst);
472        Ok(args.x + args.y)
473    }
474}
475
476#[derive(Clone)]
477struct CountingSum(Arc<AtomicUsize>);
478
479impl Tool for CountingSum {
480    const NAME: &'static str = "sum";
481    type Error = ConformanceToolError;
482    type Args = OperationArgs;
483    type Output = i64;
484
485    fn description(&self) -> String {
486        "Add x and y together (alias of add)".to_string()
487    }
488
489    fn parameters(&self) -> serde_json::Value {
490        CountingAdd(Arc::new(AtomicUsize::new(0))).parameters()
491    }
492
493    async fn call(
494        &self,
495        _context: &mut ToolContext,
496        args: Self::Args,
497    ) -> Result<Self::Output, Self::Error> {
498        self.0.fetch_add(1, Ordering::SeqCst);
499        Ok(args.x + args.y)
500    }
501}
502
503#[derive(Clone)]
504struct CountingSubtract(Arc<AtomicUsize>);
505
506impl Tool for CountingSubtract {
507    const NAME: &'static str = "subtract";
508    type Error = ConformanceToolError;
509    type Args = OperationArgs;
510    type Output = i64;
511
512    fn description(&self) -> String {
513        "Subtract y from x (i.e. x - y)".to_string()
514    }
515
516    fn parameters(&self) -> serde_json::Value {
517        serde_json::json!({
518            "type": "object",
519            "properties": {
520                "x": { "type": "number", "description": "The first operand" },
521                "y": { "type": "number", "description": "The second operand" }
522            },
523            "required": ["x", "y"]
524        })
525    }
526
527    async fn call(
528        &self,
529        _context: &mut ToolContext,
530        args: Self::Args,
531    ) -> Result<Self::Output, Self::Error> {
532        self.0.fetch_add(1, Ordering::SeqCst);
533        Ok(args.x - args.y)
534    }
535}
536
537#[derive(Clone)]
538struct RewriteArgument {
539    key: &'static str,
540    value: serde_json::Value,
541}
542
543impl AgentHook for RewriteArgument {
544    async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
545        let (Some(name), Some(args)) = (event.tool_name(), event.tool_args()) else {
546            return DispatchAction::proceed();
547        };
548        if name != CountingAdd::NAME {
549            return DispatchAction::proceed();
550        }
551        let Ok(mut arguments) = serde_json::from_str::<serde_json::Value>(args) else {
552            return DispatchAction::proceed();
553        };
554        let Some(object) = arguments.as_object_mut() else {
555            return DispatchAction::proceed();
556        };
557        object.insert(self.key.to_string(), self.value.clone());
558        DispatchAction::rewrite_tool_args(event.kind, arguments)
559    }
560}
561
562#[derive(Clone, Default)]
563struct ObserveArguments(Arc<Mutex<Vec<serde_json::Value>>>);
564
565impl AgentHook for ObserveArguments {
566    async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
567        if let Some(args) = event.tool_args() {
568            let value = serde_json::from_str(args)
569                .unwrap_or_else(|_| serde_json::Value::String(args.to_string()));
570            lock_recover(&self.0).push(value);
571        }
572        DispatchAction::proceed()
573    }
574}
575
576#[derive(Clone)]
577struct ReplaceResult(&'static str);
578
579impl AgentHook for ReplaceResult {
580    async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
581        if event.tool_name() == Some(CountingAdd::NAME) {
582            OutcomeAction::rewrite_tool_result(&event, self.0)
583        } else {
584            OutcomeAction::proceed()
585        }
586    }
587}
588
589#[derive(Clone)]
590struct WrapResult;
591
592impl AgentHook for WrapResult {
593    async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
594        match (event.tool_name(), event.tool_result()) {
595            (Some(name), Some(result)) if name == CountingAdd::NAME => {
596                OutcomeAction::rewrite_tool_result(
597                    &event,
598                    format!("[{}]", result.output().render()),
599                )
600            }
601            _ => OutcomeAction::proceed(),
602        }
603    }
604}
605
606#[derive(Clone)]
607struct FirstTurnPatch(RequestPatch);
608
609impl AgentHook for FirstTurnPatch {
610    async fn on_completion_call(
611        &self,
612        ctx: &HookContext,
613        _event: CompletionCallEvent<'_>,
614    ) -> CompletionCallAction {
615        if ctx.turn() == 1 {
616            CompletionCallAction::patch(self.0.clone())
617        } else {
618            CompletionCallAction::continue_run()
619        }
620    }
621}
622
623#[derive(Clone)]
624struct StopAfterResult(&'static str);
625
626impl AgentHook for StopAfterResult {
627    async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
628        if event.tool_name() == Some(CountingAdd::NAME) {
629            OutcomeAction::stop(self.0)
630        } else {
631            OutcomeAction::proceed()
632        }
633    }
634}
635
636#[derive(Debug, Deserialize, JsonSchema)]
637struct EmptyArgs {}
638
639#[derive(Clone)]
640struct PingTool(Arc<AtomicUsize>);
641
642impl Tool for PingTool {
643    const NAME: &'static str = "ping";
644    type Error = ConformanceToolError;
645    type Args = EmptyArgs;
646    type Output = String;
647
648    fn description(&self) -> String {
649        "Return the current ping marker. Takes no arguments.".to_string()
650    }
651
652    fn parameters(&self) -> serde_json::Value {
653        serde_json::json!({ "type": "object", "properties": {}, "required": [] })
654    }
655
656    async fn call(
657        &self,
658        _context: &mut ToolContext,
659        _args: Self::Args,
660    ) -> Result<Self::Output, Self::Error> {
661        self.0.fetch_add(1, Ordering::SeqCst);
662        Ok(PING_OUTPUT.to_string())
663    }
664}
665
666#[derive(Clone)]
667struct MottoTool(Arc<AtomicUsize>);
668
669impl Tool for MottoTool {
670    const NAME: &'static str = "fetch_motto";
671    type Error = ConformanceToolError;
672    type Args = EmptyArgs;
673    type Output = String;
674
675    fn description(&self) -> String {
676        "Fetch the two-line workshop motto.".to_string()
677    }
678
679    fn parameters(&self) -> serde_json::Value {
680        serde_json::json!({ "type": "object", "properties": {}, "required": [] })
681    }
682
683    async fn call(
684        &self,
685        _context: &mut ToolContext,
686        _args: Self::Args,
687    ) -> Result<Self::Output, Self::Error> {
688        self.0.fetch_add(1, Ordering::SeqCst);
689        Ok(MOTTO_OUTPUT.to_string())
690    }
691}
692
693#[derive(Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
694struct ConfigOutput {
695    service: String,
696    max_retries: u64,
697}
698
699#[derive(Clone)]
700struct ConfigTool(Arc<AtomicUsize>);
701
702impl Tool for ConfigTool {
703    const NAME: &'static str = "fetch_config";
704    type Error = ConformanceToolError;
705    type Args = EmptyArgs;
706    type Output = ConfigOutput;
707
708    fn description(&self) -> String {
709        "Fetch the service configuration object.".to_string()
710    }
711
712    fn parameters(&self) -> serde_json::Value {
713        serde_json::json!({ "type": "object", "properties": {}, "required": [] })
714    }
715
716    async fn call(
717        &self,
718        _context: &mut ToolContext,
719        _args: Self::Args,
720    ) -> Result<Self::Output, Self::Error> {
721        self.0.fetch_add(1, Ordering::SeqCst);
722        Ok(ConfigOutput {
723            service: "cassette-lab".to_string(),
724            max_retries: 3,
725        })
726    }
727}
728
729#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
730#[serde(rename_all = "snake_case")]
731enum ComplexMode {
732    Careful,
733    Fast,
734}
735
736#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
737struct ComplexProfile {
738    name: String,
739    tags: Vec<String>,
740}
741
742#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
743struct ComplexArgs {
744    profile: ComplexProfile,
745    mode: ComplexMode,
746    note: Option<String>,
747    quote: String,
748}
749
750#[derive(Debug, Deserialize, Serialize, JsonSchema)]
751struct ExtractedPerson {
752    #[schemars(required)]
753    first_name: Option<String>,
754    #[schemars(required)]
755    last_name: Option<String>,
756    #[schemars(required)]
757    job: Option<String>,
758}
759
760#[derive(Clone)]
761struct CaptureComplexTool {
762    calls: Arc<AtomicUsize>,
763    captured: Arc<Mutex<Option<ComplexArgs>>>,
764}
765
766impl Tool for CaptureComplexTool {
767    const NAME: &'static str = "store_profile";
768    type Error = ConformanceToolError;
769    type Args = ComplexArgs;
770    type Output = ComplexArgs;
771
772    fn description(&self) -> String {
773        "Store one profile with its nested tags, mode, optional note, and exact quoted text."
774            .to_string()
775    }
776
777    fn parameters(&self) -> serde_json::Value {
778        serde_json::to_value(schemars::schema_for!(ComplexArgs)).unwrap_or_default()
779    }
780
781    async fn call(
782        &self,
783        _context: &mut ToolContext,
784        args: Self::Args,
785    ) -> Result<Self::Output, Self::Error> {
786        self.calls.fetch_add(1, Ordering::SeqCst);
787        *lock_recover(&self.captured) = Some(args.clone());
788        Ok(args)
789    }
790}
791
792fn has_tool_roundtrip(messages: &[Message]) -> bool {
793    let saw_call = messages.iter().any(|message| {
794        matches!(
795            message,
796            Message::Assistant(rig_core::message::AssistantMessage { content, .. })
797                if content.iter().any(|item| matches!(item, AssistantContent::ToolCall(_)))
798        )
799    });
800    let saw_result = messages.iter().any(|message| {
801        matches!(
802            message,
803            Message::User { content }
804                if content.iter().any(|item| matches!(item, UserContent::ToolResult(_)))
805        )
806    });
807    saw_call && saw_result
808}
809
810#[derive(Debug, Deserialize, JsonSchema)]
811struct RepeatArgs {
812    /// The text to repeat.
813    text: String,
814    /// Number of repetitions; defaults to 2 when omitted.
815    times: Option<u32>,
816}
817
818#[derive(Clone)]
819struct RepeatTool {
820    calls: Arc<AtomicUsize>,
821}
822
823impl Tool for RepeatTool {
824    const NAME: &'static str = "repeat_text";
825    type Error = ConformanceToolError;
826    type Args = RepeatArgs;
827    type Output = String;
828
829    fn description(&self) -> String {
830        "Repeat `text`. `times` is optional and defaults to 2.".to_string()
831    }
832
833    fn parameters(&self) -> serde_json::Value {
834        serde_json::to_value(schemars::schema_for!(RepeatArgs)).unwrap_or_default()
835    }
836
837    async fn call(
838        &self,
839        _context: &mut ToolContext,
840        args: Self::Args,
841    ) -> Result<Self::Output, Self::Error> {
842        self.calls.fetch_add(1, Ordering::SeqCst);
843        Ok(vec![args.text.as_str(); args.times.unwrap_or(2) as usize].join(" "))
844    }
845}
846
847#[derive(Debug, Deserialize, JsonSchema)]
848struct BinOpArgs {
849    a: i64,
850    b: i64,
851}
852
853#[derive(Debug, Deserialize, JsonSchema)]
854struct ArithmeticResult {
855    answer: i64,
856    explanation: Option<String>,
857}
858
859#[derive(Clone)]
860struct AddTool(Arc<AtomicUsize>);
861
862impl Tool for AddTool {
863    const NAME: &'static str = "add";
864    type Error = ConformanceToolError;
865    type Args = BinOpArgs;
866    type Output = i64;
867
868    fn description(&self) -> String {
869        "Add two integers a and b.".to_string()
870    }
871
872    fn parameters(&self) -> serde_json::Value {
873        serde_json::to_value(schemars::schema_for!(BinOpArgs)).unwrap_or_default()
874    }
875
876    async fn call(
877        &self,
878        _context: &mut ToolContext,
879        args: Self::Args,
880    ) -> Result<Self::Output, Self::Error> {
881        self.0.fetch_add(1, Ordering::SeqCst);
882        Ok(args.a + args.b)
883    }
884}
885
886#[derive(Clone)]
887struct MultiplyTool(Arc<AtomicUsize>);
888
889impl Tool for MultiplyTool {
890    const NAME: &'static str = "multiply";
891    type Error = ConformanceToolError;
892    type Args = BinOpArgs;
893    type Output = i64;
894
895    fn description(&self) -> String {
896        "Multiply two integers a and b.".to_string()
897    }
898
899    fn parameters(&self) -> serde_json::Value {
900        serde_json::to_value(schemars::schema_for!(BinOpArgs)).unwrap_or_default()
901    }
902
903    async fn call(
904        &self,
905        _context: &mut ToolContext,
906        args: Self::Args,
907    ) -> Result<Self::Output, Self::Error> {
908        self.0.fetch_add(1, Ordering::SeqCst);
909        Ok(args.a * args.b)
910    }
911}
912
913fn report_from_response(
914    name: &'static str,
915    started: Instant,
916    tool_calls: usize,
917    response: crate::agent::PromptResponse,
918) -> Result<ScenarioReport, ScenarioError> {
919    validate_protocol_hygiene(
920        name,
921        &response.output(),
922        &response.messages,
923        &[
924            "<tool_call>",
925            "</tool_call>",
926            "<tool_response>",
927            "</tool_response>",
928            "<|im_start|>",
929            "<|im_end|>",
930            "<think>",
931            "</think>",
932        ],
933    )?;
934    Ok(ScenarioReport {
935        name,
936        tool_calls,
937        prompt_tokens: response.usage.input_tokens.unwrap_or(0),
938        generated_tokens: response.usage.output_tokens.unwrap_or(0),
939        history_messages: response.messages.len(),
940        duration: started.elapsed(),
941        response: response.output(),
942    })
943}
944
945/// Require the extended run's accumulated message history and validate
946/// canonical tool call/result correlation over it.
947fn correlated_messages<'a>(
948    scenario: &'static str,
949    response: &'a crate::agent::PromptResponse,
950) -> Result<&'a [Message], ScenarioError> {
951    let messages = response.messages.as_slice();
952    if messages.is_empty() {
953        return Err(ScenarioError::contract(
954            scenario,
955            "extended run omitted accumulated message history",
956        ));
957    }
958    validate_tool_correlation(scenario, messages)?;
959    Ok(messages)
960}
961
962/// [`correlated_messages`], flattened into every tool-result value in history.
963fn correlated_result_values(
964    scenario: &'static str,
965    response: &crate::agent::PromptResponse,
966) -> Result<Vec<serde_json::Value>, ScenarioError> {
967    Ok(correlated_messages(scenario, response)?
968        .iter()
969        .flat_map(tool_result_values)
970        .collect())
971}
972
973fn value_matches_integer(value: &serde_json::Value, expected: i64) -> bool {
974    value.as_i64() == Some(expected)
975        || value
976            .as_str()
977            .and_then(|text| text.trim().parse::<i64>().ok())
978            == Some(expected)
979}
980
981/// Runs two independent calls in one assistant turn and validates canonical
982/// call/result history correlation.
983///
984/// Set `tool_concurrency` to `Some(1)` to prove that serial host execution does
985/// not split or drop the model's parallel call batch.
986pub async fn parallel_tools<W, T, F>(
987    model: rig_core::driver::Model<W, T>,
988    configure: F,
989    tool_concurrency: Option<usize>,
990) -> Result<ScenarioReport, ScenarioError>
991where
992    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
993    T: rig_core::driver::Transport<W>,
994    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
995{
996    let add_calls = Arc::new(AtomicUsize::new(0));
997    let subtract_calls = Arc::new(AtomicUsize::new(0));
998    let started = Instant::now();
999    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1000        .preamble(FORCE_TOOLS_PREAMBLE)
1001        .tool(CountingAdd(add_calls.clone()))
1002        .tool(CountingSubtract(subtract_calls.clone()))
1003        .default_max_turns(3)
1004        .build();
1005    let request = agent.prompt(PARALLEL_PROMPT).max_turns(3);
1006    let response = match tool_concurrency {
1007        Some(concurrency) => request.tool_concurrency(concurrency).await?,
1008        None => request.await?,
1009    };
1010    let scenario = if tool_concurrency == Some(1) {
1011        "parallel_tools_serial_execution"
1012    } else {
1013        "parallel_tools"
1014    };
1015    let messages = correlated_messages(scenario, &response)?;
1016
1017    let Some((call_index, calls)) = messages.iter().enumerate().find_map(|(index, message)| {
1018        let Message::Assistant(rig_core::message::AssistantMessage { content, .. }) = message
1019        else {
1020            return None;
1021        };
1022        let calls = content
1023            .iter()
1024            .filter_map(|item| match item {
1025                AssistantContent::ToolCall(call) => Some(call),
1026                _ => None,
1027            })
1028            .collect::<Vec<_>>();
1029        (calls.len() == 2).then_some((index, calls))
1030    }) else {
1031        return Err(ScenarioError::contract(
1032            scenario,
1033            format!("no assistant turn contained exactly two tool calls: {messages:?}"),
1034        ));
1035    };
1036    let mut names = calls
1037        .iter()
1038        .map(|call| call.function.name.as_str())
1039        .collect::<Vec<_>>();
1040    names.sort_unstable();
1041    if names != ["add", "subtract"] {
1042        return Err(ScenarioError::contract(
1043            scenario,
1044            format!("parallel turn called {names:?}, expected add and subtract"),
1045        ));
1046    }
1047    let results_message = messages.get(call_index + 1).ok_or_else(|| {
1048        ScenarioError::contract(
1049            scenario,
1050            "parallel call turn has no following result message",
1051        )
1052    })?;
1053    let values = tool_result_values(results_message);
1054    if !values.iter().any(|value| value_matches_integer(value, 7))
1055        || !values.iter().any(|value| value_matches_integer(value, 8))
1056        || values.len() != 2
1057    {
1058        return Err(ScenarioError::contract(
1059            scenario,
1060            format!("parallel result message did not contain exactly 7 and 8: {values:?}"),
1061        ));
1062    }
1063    let add = add_calls.load(Ordering::SeqCst);
1064    let subtract = subtract_calls.load(Ordering::SeqCst);
1065    if add != 1 || subtract != 1 {
1066        return Err(ScenarioError::contract(
1067            scenario,
1068            format!("execution counts were add={add}, subtract={subtract}, expected one each"),
1069        ));
1070    }
1071    report_from_response(scenario, started, add + subtract, response)
1072}
1073
1074/// Runs a zero-argument tool and validates verbatim string-result handling.
1075pub async fn zero_argument_tool<W, T, F>(
1076    model: rig_core::driver::Model<W, T>,
1077    configure: F,
1078) -> Result<ScenarioReport, ScenarioError>
1079where
1080    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1081    T: rig_core::driver::Transport<W>,
1082    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1083{
1084    const SCENARIO: &str = "zero_argument_tool";
1085    let calls = Arc::new(AtomicUsize::new(0));
1086    let started = Instant::now();
1087    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1088        .preamble("You must use the provided tools. Report tool outputs exactly as returned.")
1089        .tool(PingTool(calls.clone()))
1090        .default_max_turns(2)
1091        .build();
1092    let response = agent
1093        .prompt("Call the ping tool, then report the exact marker it returns.")
1094        .max_turns(2)
1095        .await?;
1096    let values = correlated_result_values(SCENARIO, &response)?;
1097    if calls.load(Ordering::SeqCst) != 1
1098        || !values
1099            .iter()
1100            .any(|value| value.as_str() == Some(PING_OUTPUT))
1101        || !response.output().contains(PING_OUTPUT)
1102    {
1103        return Err(ScenarioError::contract(
1104            SCENARIO,
1105            format!(
1106                "calls={}, results={values:?}, response={:?}",
1107                calls.load(Ordering::SeqCst),
1108                response.output()
1109            ),
1110        ));
1111    }
1112    report_from_response(SCENARIO, started, 1, response)
1113}
1114
1115/// Runs string- and JSON-returning tools and validates that neither output is
1116/// double encoded.
1117pub async fn tool_output_serialization<W, T, F>(
1118    model: rig_core::driver::Model<W, T>,
1119    configure: F,
1120) -> Result<ScenarioReport, ScenarioError>
1121where
1122    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1123    T: rig_core::driver::Transport<W>,
1124    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1125{
1126    const SCENARIO: &str = "tool_output_serialization";
1127    let started = Instant::now();
1128    let motto_calls = Arc::new(AtomicUsize::new(0));
1129    let config_calls = Arc::new(AtomicUsize::new(0));
1130    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1131        .preamble("You must use the provided tools before answering.")
1132        .tool(MottoTool(motto_calls.clone()))
1133        .tool(ConfigTool(config_calls.clone()))
1134        .default_max_turns(3)
1135        .build();
1136    let response = agent
1137        .prompt("Call fetch_motto and fetch_config, then summarize both outputs in one sentence.")
1138        .max_turns(3)
1139        .await?;
1140    let values = correlated_result_values(SCENARIO, &response)?;
1141    let expected_config = serde_json::to_value(ConfigOutput {
1142        service: "cassette-lab".to_string(),
1143        max_retries: 3,
1144    })?;
1145    let motto_ok = values
1146        .iter()
1147        .any(|value| value.as_str() == Some(MOTTO_OUTPUT));
1148    let config_ok = values.iter().any(|value| {
1149        value == &expected_config
1150            || value
1151                .as_str()
1152                .and_then(|text| serde_json::from_str::<serde_json::Value>(text).ok())
1153                .as_ref()
1154                == Some(&expected_config)
1155    });
1156    let motto_count = motto_calls.load(Ordering::SeqCst);
1157    let config_count = config_calls.load(Ordering::SeqCst);
1158    if !motto_ok || !config_ok || motto_count != 1 || config_count != 1 {
1159        return Err(ScenarioError::contract(
1160            SCENARIO,
1161            format!(
1162                "expected one verbatim motto and one semantic config JSON; motto_calls={motto_count}, config_calls={config_count}, values={values:?}"
1163            ),
1164        ));
1165    }
1166    report_from_response(SCENARIO, started, motto_count + config_count, response)
1167}
1168
1169/// Runs a nested, escaped, Unicode-bearing argument payload through typed tool
1170/// deserialization and validates the exact semantic value received by the tool.
1171pub async fn complex_tool_arguments<W, T, F>(
1172    model: rig_core::driver::Model<W, T>,
1173    configure: F,
1174) -> Result<ScenarioReport, ScenarioError>
1175where
1176    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1177    T: rig_core::driver::Transport<W>,
1178    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1179{
1180    complex_tool_arguments_with_prompt(model, configure, COMPLEX_ARGUMENTS_PROMPT).await
1181}
1182
1183/// The prompt [`complex_tool_arguments`] sends. It spells the values in their
1184/// JSON-escaped form, which a model may copy literally, backslashes included.
1185pub const COMPLEX_ARGUMENTS_PROMPT: &str = "Call store_profile with profile.name exactly `Zoë \\\"Z\\\"`, profile.tags exactly [`rust`, `東京`], mode `careful`, note containing the two lines `line one` and `line two` separated by a newline, and quote exactly `path C:\\\\tmp and \\\"quoted\\\"`. Then confirm it was stored.";
1186
1187/// The same values as [`COMPLEX_ARGUMENTS_PROMPT`], spelled as raw text
1188/// between markers, for a model that copies escapes literally.
1189pub const COMPLEX_ARGUMENTS_RAW_PROMPT: &str = "Call store_profile once. Each value below is raw text between <<< and >>>, with no escaping: profile.name <<<Zoë \"Z\">>>, profile.tags <<<rust>>> and <<<東京>>>, mode <<<careful>>>, note the two lines <<<line one>>> and <<<line two>>> joined by one newline, and quote <<<path C:\\tmp and \"quoted\">>>. Then confirm it was stored.";
1190
1191/// [`complex_tool_arguments`] with the user prompt given: the same tool, the
1192/// same expected arguments, the same checks.
1193pub async fn complex_tool_arguments_with_prompt<W, T, F>(
1194    model: rig_core::driver::Model<W, T>,
1195    configure: F,
1196    prompt: &str,
1197) -> Result<ScenarioReport, ScenarioError>
1198where
1199    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1200    T: rig_core::driver::Transport<W>,
1201    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1202{
1203    const SCENARIO: &str = "complex_tool_arguments";
1204    let expected = ComplexArgs {
1205        profile: ComplexProfile {
1206            name: "Zoë \"Z\"".to_string(),
1207            tags: vec!["rust".to_string(), "東京".to_string()],
1208        },
1209        mode: ComplexMode::Careful,
1210        note: Some("line one\nline two".to_string()),
1211        quote: "path C:\\tmp and \"quoted\"".to_string(),
1212    };
1213    let calls = Arc::new(AtomicUsize::new(0));
1214    let captured = Arc::new(Mutex::new(None));
1215    let started = Instant::now();
1216    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1217        .preamble("Use store_profile exactly once with every value supplied by the user.")
1218        .tool(CaptureComplexTool {
1219            calls: calls.clone(),
1220            captured: captured.clone(),
1221        })
1222        .default_max_turns(3)
1223        .build();
1224    let response = agent.prompt(prompt).max_turns(3).await?;
1225    let observed = lock_recover(&captured).clone();
1226    if calls.load(Ordering::SeqCst) != 1 || observed.as_ref() != Some(&expected) {
1227        return Err(ScenarioError::contract(
1228            SCENARIO,
1229            format!(
1230                "calls={}, expected={expected:?}, observed={observed:?}, response={:?}",
1231                calls.load(Ordering::SeqCst),
1232                response.output()
1233            ),
1234        ));
1235    }
1236    correlated_messages(SCENARIO, &response)?;
1237    report_from_response(SCENARIO, started, 1, response)
1238}
1239
1240/// Runs the same deterministic text request through buffered and raw streaming
1241/// completion surfaces and validates equivalent visible content and usage.
1242pub async fn buffered_streaming_text_parity<W, T, A>(
1243    model: rig_core::driver::Model<W, T>,
1244    adjust: A,
1245) -> Result<ScenarioReport, ScenarioError>
1246where
1247    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1248    T: rig_core::driver::Transport<W>,
1249    A: Fn(CompletionRequest) -> CompletionRequest,
1250{
1251    const SCENARIO: &str = "buffered_streaming_text_parity";
1252    const PROMPT: &str = "Answer with exactly the single word Paris.";
1253    let started = Instant::now();
1254    let request = || {
1255        adjust(
1256            CompletionRequest::new(PROMPT)
1257                .temperature(0.0)
1258                .max_tokens(32),
1259        )
1260    };
1261    let buffered = model.call(request()).await?;
1262    let buffered_text = buffered
1263        .choice
1264        .iter()
1265        .filter_map(|item| match item {
1266            AssistantContent::Text(text) => Some(text.text.as_str()),
1267            _ => None,
1268        })
1269        .collect::<String>();
1270
1271    let mut stream = model.stream(request())?;
1272    let mut streamed_text = String::new();
1273    while let Some(item) = stream.next().await {
1274        if let crate::streaming::Item::Event(crate::streaming::StreamEvent::Text { text, .. }) =
1275            item?
1276        {
1277            streamed_text.push_str(&text);
1278        }
1279    }
1280    let streamed_usage = Some(stream.finish().await?.usage);
1281    let usage = streamed_usage.ok_or_else(|| {
1282        ScenarioError::contract(SCENARIO, "raw stream omitted its final response metadata")
1283    })?;
1284    let normalize = |text: &str| {
1285        text.trim()
1286            .trim_matches(|character: char| !character.is_alphanumeric())
1287            .to_string()
1288    };
1289    let buffered_answer = normalize(&buffered_text);
1290    let streamed_answer = normalize(&streamed_text);
1291    if !buffered_answer.eq_ignore_ascii_case("Paris")
1292        || !streamed_answer.eq_ignore_ascii_case("Paris")
1293        || !buffered.usage.is_reported()
1294        || !usage.is_reported()
1295    {
1296        return Err(ScenarioError::contract(
1297            SCENARIO,
1298            format!(
1299                "buffered={buffered_text:?}, streamed={streamed_text:?}, buffered_usage={:?}, streamed_usage={usage:?}",
1300                buffered.usage
1301            ),
1302        ));
1303    }
1304    Ok(ScenarioReport {
1305        name: SCENARIO,
1306        tool_calls: 0,
1307        prompt_tokens: usage.input_tokens.unwrap_or(0),
1308        generated_tokens: usage.output_tokens.unwrap_or(0),
1309        history_messages: 0,
1310        duration: started.elapsed(),
1311        response: streamed_text,
1312    })
1313}
1314
1315/// Runs Rig's structured extractor and validates both the extracted semantic
1316/// fields and accumulated usage.
1317pub async fn structured_extraction<W, T>(
1318    model: rig_core::driver::Model<W, T>,
1319    additional_params: Option<serde_json::Value>,
1320) -> Result<ScenarioReport, ScenarioError>
1321where
1322    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1323    T: rig_core::driver::Transport<W>,
1324{
1325    const SCENARIO: &str = "structured_extraction";
1326    const INPUT: &str = "Hello, my name is Ada Lovelace and I work as a mathematician.";
1327    let started = Instant::now();
1328    let mut builder = crate::extractor::ExtractorBuilder::<ExtractedPerson>::new(model)
1329        .max_tokens(384)
1330        .retries(0);
1331    if let Some(params) = additional_params {
1332        builder = builder.additional_params(params);
1333    }
1334    let response = builder.build().extract(INPUT).await?;
1335    validate_extraction_fields(
1336        SCENARIO,
1337        response.output.first_name.as_deref(),
1338        response.output.last_name.as_deref(),
1339        response.output.job.as_deref(),
1340        response.usage,
1341    )?;
1342    Ok(ScenarioReport {
1343        name: SCENARIO,
1344        tool_calls: 1,
1345        prompt_tokens: response.usage.input_tokens.unwrap_or(0),
1346        generated_tokens: response.usage.output_tokens.unwrap_or(0),
1347        history_messages: 0,
1348        duration: started.elapsed(),
1349        response: format!(
1350            "{} {} — {}",
1351            response.output.first_name.as_deref().unwrap_or_default(),
1352            response.output.last_name.as_deref().unwrap_or_default(),
1353            response.output.job.as_deref().unwrap_or_default()
1354        ),
1355    })
1356}
1357
1358fn restricted_recovery_run(
1359    prompt: &str,
1360    turn: ModelTurn,
1361    retries: usize,
1362) -> Result<AgentRun, ScenarioError> {
1363    const SCENARIO: &str = "invalid_tool_recovery";
1364    let mut run = AgentRun::new(prompt)
1365        .max_turns(2)
1366        .max_invalid_tool_call_retries(retries);
1367    if !matches!(run.next_step()?, AgentRunStep::CallModel { .. }) {
1368        return Err(ScenarioError::contract(
1369            SCENARIO,
1370            "fresh AgentRun did not request a model turn",
1371        ));
1372    }
1373    let outcome = run.model_response(turn)?;
1374    let ModelTurnOutcome::NeedsResolution(context) = outcome else {
1375        return Err(ScenarioError::contract(
1376            SCENARIO,
1377            format!("disallowed tool call did not require resolution: {outcome:?}"),
1378        ));
1379    };
1380    if context.tool_name != CountingAdd::NAME {
1381        return Err(ScenarioError::contract(
1382            SCENARIO,
1383            format!("expected rejected add call, observed {context:?}"),
1384        ));
1385    }
1386    Ok(run)
1387}
1388
1389/// Uses one real model turn to exercise fail-fast, retry exhaustion, repair,
1390/// rejected repair, and skip handling without executing a disallowed call.
1391pub async fn invalid_tool_recovery<W, T, F>(
1392    model: rig_core::driver::Model<W, T>,
1393    configure: F,
1394) -> Result<ScenarioReport, ScenarioError>
1395where
1396    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1397    T: rig_core::driver::Transport<W>,
1398    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1399{
1400    invalid_tool_recovery_with_choice(model, configure, ToolChoice::Required).await
1401}
1402
1403/// [`invalid_tool_recovery`] with the agent's tool choice given, for a model
1404/// that rejects a forced choice (`ToolChoice::Auto`; the prompt still asks for
1405/// the call).
1406pub async fn invalid_tool_recovery_with_choice<W, T, F>(
1407    model: rig_core::driver::Model<W, T>,
1408    configure: F,
1409    tool_choice: ToolChoice,
1410) -> Result<ScenarioReport, ScenarioError>
1411where
1412    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1413    T: rig_core::driver::Transport<W>,
1414    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1415{
1416    const SCENARIO: &str = "invalid_tool_recovery";
1417    const PROMPT: &str = "Call the add tool exactly once with x=2 and y=3. Do not call sum.";
1418    let started = Instant::now();
1419    let add_calls = Arc::new(AtomicUsize::new(0));
1420    let sum_calls = Arc::new(AtomicUsize::new(0));
1421    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1422        .preamble(FORCE_TOOLS_PREAMBLE)
1423        .tool(CountingAdd(add_calls.clone()))
1424        .tool(CountingSum(sum_calls.clone()))
1425        .tool_choice(tool_choice)
1426        .build();
1427    #[derive(Clone)]
1428    struct CaptureTurn(Arc<Mutex<Option<ModelTurn>>>);
1429
1430    impl AgentHook for CaptureTurn {
1431        async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
1432            let Some(response) = event.completion() else {
1433                return OutcomeAction::proceed();
1434            };
1435            *lock_recover(&self.0) = Some(ModelTurn::new(
1436                response.head(),
1437                response.choice.clone(),
1438                response.usage,
1439                BTreeSet::new(),
1440                BTreeSet::new(),
1441                response.raw.clone(),
1442            ));
1443            OutcomeAction::stop("captured conformance model turn")
1444        }
1445    }
1446
1447    let captured = Arc::new(Mutex::new(None));
1448    let stopped = agent
1449        .prompt(PROMPT)
1450        .add_hook(CaptureTurn(captured.clone()))
1451        .run()
1452        .await;
1453    if !matches!(stopped, Err(PromptError::Cancelled { .. })) {
1454        return Err(ScenarioError::contract(
1455            SCENARIO,
1456            format!("capture hook did not stop after the model response: {stopped:?}"),
1457        ));
1458    }
1459    let response = lock_recover(&captured).take().ok_or_else(|| {
1460        ScenarioError::contract(SCENARIO, "capture hook observed no model response")
1461    })?;
1462    let emitted = response
1463        .choice
1464        .iter()
1465        .filter(|item| {
1466            matches!(item, AssistantContent::ToolCall(call) if call.function.name == CountingAdd::NAME)
1467        })
1468        .count();
1469    if emitted != 1 {
1470        return Err(ScenarioError::contract(
1471            SCENARIO,
1472            format!(
1473                "model emitted {emitted} add calls, response={:?}",
1474                response.choice
1475            ),
1476        ));
1477    }
1478    let executable = BTreeSet::from([CountingAdd::NAME.to_string(), CountingSum::NAME.to_string()]);
1479    let allowed = BTreeSet::from([CountingSum::NAME.to_string()]);
1480    let turn = ModelTurn::new(
1481        response.head,
1482        response.choice,
1483        response.usage,
1484        executable,
1485        allowed,
1486        response.raw,
1487    );
1488
1489    let mut fail = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1490    let error = match fail.resolve_invalid_tool_call(InvalidToolCallAction::fail()) {
1491        Err(error) => error,
1492        Ok(outcome) => {
1493            return Err(ScenarioError::contract(
1494                SCENARIO,
1495                format!("fail action unexpectedly returned {outcome:?}"),
1496            ));
1497        }
1498    };
1499    validate_unknown_tool_failure(&error, CountingAdd::NAME, &[CountingSum::NAME])?;
1500
1501    let mut retry = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1502    let error = match retry
1503        .resolve_invalid_tool_call(InvalidToolCallAction::retry("choose an allowed tool"))
1504    {
1505        Err(error) => error,
1506        Ok(outcome) => {
1507            return Err(ScenarioError::contract(
1508                SCENARIO,
1509                format!("exhausted retry unexpectedly returned {outcome:?}"),
1510            ));
1511        }
1512    };
1513    validate_unknown_tool_failure(&error, CountingAdd::NAME, &[CountingSum::NAME])?;
1514
1515    let mut rejected_repair = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1516    let error =
1517        match rejected_repair.resolve_invalid_tool_call(InvalidToolCallAction::repair("missing")) {
1518            Err(error) => error,
1519            Ok(outcome) => {
1520                return Err(ScenarioError::contract(
1521                    SCENARIO,
1522                    format!("disallowed repair unexpectedly returned {outcome:?}"),
1523                ));
1524            }
1525        };
1526    validate_unknown_tool_failure(&error, "missing", &[CountingSum::NAME])?;
1527
1528    let mut repaired = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1529    if !matches!(
1530        repaired.resolve_invalid_tool_call(InvalidToolCallAction::repair(CountingSum::NAME))?,
1531        ModelTurnOutcome::Continue { .. }
1532    ) {
1533        return Err(ScenarioError::contract(
1534            SCENARIO,
1535            "valid repair did not continue",
1536        ));
1537    }
1538    let AgentRunStep::CallTools { calls } = repaired.next_step()? else {
1539        return Err(ScenarioError::contract(
1540            SCENARIO,
1541            "valid repair did not produce pending tool execution",
1542        ));
1543    };
1544    let repaired_call = calls.first();
1545    if calls.len() != 1
1546        || !repaired_call.is_some_and(|call| {
1547            call.tool_call.function.name == CountingSum::NAME && call.preresolved_result.is_none()
1548        })
1549    {
1550        return Err(ScenarioError::contract(
1551            SCENARIO,
1552            format!("repaired pending calls were incorrect: {calls:?}"),
1553        ));
1554    }
1555
1556    let mut skipped = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1557    if !matches!(
1558        skipped.resolve_invalid_tool_call(InvalidToolCallAction::skip("disabled for this turn"))?,
1559        ModelTurnOutcome::Continue { .. }
1560    ) {
1561        return Err(ScenarioError::contract(SCENARIO, "skip did not continue"));
1562    }
1563    let AgentRunStep::CallTools { calls } = skipped.next_step()? else {
1564        return Err(ScenarioError::contract(
1565            SCENARIO,
1566            "skip did not produce a pre-resolved pending call",
1567        ));
1568    };
1569    let skipped_is_preresolved = match calls.first() {
1570        Some(call) => call.preresolved_result.is_some(),
1571        None => false,
1572    };
1573    if calls.len() != 1 || !skipped_is_preresolved {
1574        return Err(ScenarioError::contract(
1575            SCENARIO,
1576            format!("skipped pending calls were incorrect: {calls:?}"),
1577        ));
1578    }
1579    if add_calls.load(Ordering::SeqCst) != 0 || sum_calls.load(Ordering::SeqCst) != 0 {
1580        return Err(ScenarioError::contract(
1581            SCENARIO,
1582            "recovery scenario executed a tool body",
1583        ));
1584    }
1585
1586    Ok(ScenarioReport {
1587        name: SCENARIO,
1588        tool_calls: emitted,
1589        prompt_tokens: turn.usage.input_tokens.unwrap_or(0),
1590        generated_tokens: turn.usage.output_tokens.unwrap_or(0),
1591        history_messages: 2,
1592        duration: started.elapsed(),
1593        response: "fail, retry, repair, rejected repair, and skip passed".to_string(),
1594    })
1595}
1596
1597/// Exercises chained argument/result rewrites and a first-turn-only request
1598/// patch. Completion proves `tool_choice=Required` did not leak to turn two.
1599pub async fn hook_rewrites_and_request_patch<W, T, F>(
1600    model: rig_core::driver::Model<W, T>,
1601    configure: F,
1602) -> Result<ScenarioReport, ScenarioError>
1603where
1604    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1605    T: rig_core::driver::Transport<W>,
1606    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1607{
1608    hook_rewrites_and_request_patch_with_choice(model, configure, ToolChoice::Required).await
1609}
1610
1611/// [`hook_rewrites_and_request_patch`] with the first-turn patch's tool
1612/// choice given, for a model that rejects a forced choice (`ToolChoice::Auto`:
1613/// the patch still narrows the active tools on turn one only).
1614pub async fn hook_rewrites_and_request_patch_with_choice<W, T, F>(
1615    model: rig_core::driver::Model<W, T>,
1616    configure: F,
1617    first_turn_choice: ToolChoice,
1618) -> Result<ScenarioReport, ScenarioError>
1619where
1620    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1621    T: rig_core::driver::Transport<W>,
1622    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1623{
1624    const SCENARIO: &str = "hook_rewrites_and_request_patch";
1625    let started = Instant::now();
1626    let calls = Arc::new(AtomicUsize::new(0));
1627    let observed = ObserveArguments::default();
1628    let observed_probe = observed.clone();
1629    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1630        .preamble("Use add for arithmetic and report only the tool result.")
1631        .tool(CountingAdd(calls.clone()))
1632        .default_max_turns(3)
1633        .build();
1634    let response = agent
1635        .prompt("Use add once for x=1 and y=1, then report what the tool returns.")
1636        .max_turns(3)
1637        .add_hook(FirstTurnPatch(
1638            RequestPatch::new()
1639                .active_tools([CountingAdd::NAME])
1640                .tool_choice(first_turn_choice),
1641        ))
1642        .add_hook(RewriteArgument {
1643            key: "x",
1644            value: serde_json::json!(7),
1645        })
1646        .add_hook(RewriteArgument {
1647            key: "y",
1648            value: serde_json::json!(8),
1649        })
1650        .add_hook(observed)
1651        .add_hook(ReplaceResult("portable-redacted"))
1652        .add_hook(WrapResult)
1653        .await?;
1654    let observations = lock_recover(&observed_probe.0).clone();
1655    validate_rewritten_arguments(
1656        SCENARIO,
1657        &observations,
1658        &serde_json::json!({ "x": 7, "y": 8 }),
1659    )?;
1660    let messages = response.messages.as_slice();
1661    if messages.is_empty() {
1662        return Err(ScenarioError::contract(
1663            SCENARIO,
1664            "extended hook run omitted message history",
1665        ));
1666    }
1667    let results = messages
1668        .iter()
1669        .flat_map(tool_result_values)
1670        .collect::<Vec<_>>();
1671    if calls.load(Ordering::SeqCst) != 1
1672        || !results
1673            .iter()
1674            .any(|value| value == &serde_json::json!("[portable-redacted]"))
1675        || response.completion_calls.len() != 2
1676    {
1677        return Err(ScenarioError::contract(
1678            SCENARIO,
1679            format!(
1680                "calls={}, completion_calls={}, results={results:?}, output={:?}",
1681                calls.load(Ordering::SeqCst),
1682                response.completion_calls.len(),
1683                response.output()
1684            ),
1685        ));
1686    }
1687    report_from_response(SCENARIO, started, 1, response)
1688}
1689
1690/// Exercises post-execution cancellation and max-turn diagnostics through the
1691/// public agent driver using real model-emitted calls.
1692pub async fn cancellation_and_max_turns<W, T, F>(
1693    model: rig_core::driver::Model<W, T>,
1694    configure: F,
1695) -> Result<ScenarioReport, ScenarioError>
1696where
1697    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1698    T: rig_core::driver::Transport<W>,
1699    F: Fn(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1700{
1701    const SCENARIO: &str = "cancellation_and_max_turns";
1702    const REASON: &str = "portable result veto";
1703    let started = Instant::now();
1704    let cancelled_calls = Arc::new(AtomicUsize::new(0));
1705    let cancelled_agent = configure(AgentBuilder::new(model.clone()).temperature(0.0))
1706        .preamble("Use add for arithmetic; never calculate by hand.")
1707        .tool(CountingAdd(cancelled_calls.clone()))
1708        .build();
1709    let cancelled = match cancelled_agent
1710        .prompt("Use add once to compute x=20 plus y=22.")
1711        .max_turns(2)
1712        .add_hook(StopAfterResult(REASON))
1713        .await
1714    {
1715        Err(error) => error,
1716        Ok(output) => {
1717            return Err(ScenarioError::contract(
1718                SCENARIO,
1719                format!("result cancellation unexpectedly completed: {output:?}"),
1720            ));
1721        }
1722    };
1723    validate_cancelled_failure(&cancelled, REASON, CountingAdd::NAME)?;
1724
1725    let max_turn_calls = Arc::new(AtomicUsize::new(0));
1726    let max_turn_agent = configure(AgentBuilder::new(model).temperature(0.0))
1727        .preamble("Use add for arithmetic; never calculate by hand.")
1728        .tool(CountingAdd(max_turn_calls.clone()))
1729        .build();
1730    let max_turn = match max_turn_agent
1731        .prompt("Use add once to compute x=20 plus y=22, then report the result.")
1732        .max_turns(1)
1733        .await
1734    {
1735        Err(error) => error,
1736        Ok(output) => {
1737            return Err(ScenarioError::contract(
1738                SCENARIO,
1739                format!("one-turn budget unexpectedly completed: {output:?}"),
1740            ));
1741        }
1742    };
1743    validate_max_turns_failure(&max_turn, 1)?;
1744    let cancelled_count = cancelled_calls.load(Ordering::SeqCst);
1745    let max_turn_count = max_turn_calls.load(Ordering::SeqCst);
1746    if cancelled_count != 1 || max_turn_count != 1 {
1747        return Err(ScenarioError::contract(
1748            SCENARIO,
1749            format!("cancelled executions={cancelled_count}, max-turn executions={max_turn_count}"),
1750        ));
1751    }
1752    Ok(ScenarioReport {
1753        name: SCENARIO,
1754        tool_calls: cancelled_count + max_turn_count,
1755        prompt_tokens: 0,
1756        generated_tokens: 0,
1757        history_messages: 4,
1758        duration: started.elapsed(),
1759        response: "post-result cancellation and max-turn diagnostics passed".to_string(),
1760    })
1761}
1762
1763/// Runs the portable optional-argument tool scenario.
1764///
1765/// `configure` is deliberately outside the scenario so a provider suite can
1766/// attach transport-only settings without putting them into the shared model
1767/// contract.
1768pub async fn optional_argument<W, T, F>(
1769    model: rig_core::driver::Model<W, T>,
1770    configure: F,
1771) -> Result<ScenarioReport, ScenarioError>
1772where
1773    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1774    T: rig_core::driver::Transport<W>,
1775    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1776{
1777    let calls = Arc::new(AtomicUsize::new(0));
1778    let started = Instant::now();
1779    let agent = configure(AgentBuilder::new(model))
1780        .preamble("Use the repeat_text tool whenever asked to repeat text.")
1781        .tool(RepeatTool {
1782            calls: calls.clone(),
1783        })
1784        .default_max_turns(4)
1785        .build();
1786    let result = agent
1787        .prompt(
1788            "Use the repeat_text tool to repeat the word \"banana\" 3 times, then show me the exact result.",
1789        )
1790        .await?;
1791    let response = result.output();
1792    let tool_calls = calls.load(Ordering::SeqCst);
1793    if tool_calls == 0
1794        || response.matches("banana").count() < 1
1795        || !has_tool_roundtrip(&result.messages)
1796    {
1797        return Err(ScenarioError::contract(
1798            "optional_argument",
1799            format!("calls={tool_calls}, response={response:?}"),
1800        ));
1801    }
1802    report_from_response("optional_argument", started, tool_calls, result)
1803}
1804
1805/// Runs a portable two-tool sequential arithmetic scenario.
1806pub async fn sequential_tools<W, T, F>(
1807    model: rig_core::driver::Model<W, T>,
1808    configure: F,
1809) -> Result<ScenarioReport, ScenarioError>
1810where
1811    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1812    T: rig_core::driver::Transport<W>,
1813    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1814{
1815    let add_calls = Arc::new(AtomicUsize::new(0));
1816    let multiply_calls = Arc::new(AtomicUsize::new(0));
1817    let started = Instant::now();
1818    let agent = configure(AgentBuilder::new(model))
1819        .preamble(
1820            "You are a calculator. Use the add and multiply tools for arithmetic; never compute by hand.",
1821        )
1822        .tool(AddTool(add_calls.clone()))
1823        .tool(MultiplyTool(multiply_calls.clone()))
1824        .default_max_turns(6)
1825        .build();
1826    let result = agent
1827        .prompt(
1828            "Compute (4 + 6) * 2. First call the add tool, then call the multiply tool on the result. Tell me the final number.",
1829        )
1830        .await?;
1831    let response = result.output();
1832    let add = add_calls.load(Ordering::SeqCst);
1833    let multiply = multiply_calls.load(Ordering::SeqCst);
1834    if add == 0
1835        || multiply == 0
1836        || !response.contains("20")
1837        || !has_tool_roundtrip(&result.messages)
1838    {
1839        return Err(ScenarioError::contract(
1840            "sequential_tools",
1841            format!("add={add}, multiply={multiply}, response={response:?}"),
1842        ));
1843    }
1844    report_from_response("sequential_tools", started, add + multiply, result)
1845}
1846
1847/// Runs a tool through Rig's multi-turn streaming agent driver.
1848pub async fn streaming_tool<W, T, F>(
1849    model: rig_core::driver::Model<W, T>,
1850    configure: F,
1851) -> Result<ScenarioReport, ScenarioError>
1852where
1853    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1854    T: rig_core::driver::Transport<W>,
1855    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1856{
1857    let calls = Arc::new(AtomicUsize::new(0));
1858    let started = Instant::now();
1859    let agent = configure(AgentBuilder::new(model))
1860        .preamble("Use the add tool for arithmetic; do not calculate by hand.")
1861        .tool(AddTool(calls.clone()))
1862        .default_max_turns(4)
1863        .build();
1864    let mut stream = agent
1865        .prompt("Use add to calculate 17 + 25, then state the final number.")
1866        .max_turns(4)
1867        .stream();
1868    let mut final_response = None;
1869    let mut final_count = 0_usize;
1870    let mut completion_usage = crate::completion::Usage::default();
1871    let mut streamed_call_ids = Vec::new();
1872    let mut streamed_result_ids = Vec::new();
1873    while let Some(item) = stream.next().await {
1874        match item? {
1875            MultiTurnStreamItem::ToolCall { tool_call } => streamed_call_ids.push(tool_call.id),
1876            MultiTurnStreamItem::ToolResult { tool_result } => {
1877                streamed_result_ids.push(tool_result.call)
1878            }
1879            MultiTurnStreamItem::CompletionCall(call) => completion_usage += call.usage,
1880            MultiTurnStreamItem::FinalResponse(response) => {
1881                final_count += 1;
1882                final_response = Some(response);
1883            }
1884            MultiTurnStreamItem::StreamAssistantItem(_)
1885            | MultiTurnStreamItem::ToolExecutionCommitted { .. }
1886            | MultiTurnStreamItem::ModelTurnRetried { .. } => {}
1887        }
1888    }
1889    let result = final_response.ok_or_else(|| {
1890        ScenarioError::contract("streaming_tool", "stream produced no final response")
1891    })?;
1892    let response = result.output();
1893    let history_messages = result.messages.len();
1894    let tool_calls = calls.load(Ordering::SeqCst);
1895    streamed_call_ids.sort();
1896    streamed_result_ids.sort();
1897    let correlated_stream =
1898        !streamed_call_ids.is_empty() && streamed_call_ids == streamed_result_ids;
1899    let correlated_history = validate_tool_correlation("streaming_tool", &result.messages).is_ok();
1900    if tool_calls == 0
1901        || !response.contains("42")
1902        || history_messages < 4
1903        || final_count != 1
1904        || !correlated_stream
1905        || !correlated_history
1906        || completion_usage != result.usage
1907        || result.completion_calls.is_empty()
1908    {
1909        return Err(ScenarioError::contract(
1910            "streaming_tool",
1911            format!(
1912                "calls={tool_calls}, final_count={final_count}, streamed_call_ids={streamed_call_ids:?}, streamed_result_ids={streamed_result_ids:?}, completion_usage={completion_usage:?}, final_usage={:?}, history_messages={history_messages}, response={response:?}",
1913                result.usage
1914            ),
1915        ));
1916    }
1917    validate_protocol_hygiene(
1918        "streaming_tool",
1919        &response,
1920        &result.messages,
1921        &["<tool_call>", "</tool_call>", "<think>", "</think>"],
1922    )?;
1923    Ok(ScenarioReport {
1924        name: "streaming_tool",
1925        tool_calls,
1926        prompt_tokens: result.usage.input_tokens.unwrap_or(0),
1927        generated_tokens: result.usage.output_tokens.unwrap_or(0),
1928        history_messages,
1929        duration: started.elapsed(),
1930        response,
1931    })
1932}
1933
1934/// Runs a normal tool followed by Rig's synthetic structured-output tool.
1935pub async fn structured_after_tool<W, T, F>(
1936    model: rig_core::driver::Model<W, T>,
1937    configure: F,
1938) -> Result<ScenarioReport, ScenarioError>
1939where
1940    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1941    T: rig_core::driver::Transport<W>,
1942    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1943{
1944    let calls = Arc::new(AtomicUsize::new(0));
1945    let started = Instant::now();
1946    let agent = configure(AgentBuilder::new(model))
1947        .preamble(
1948            "Use add for arithmetic, then finish by calling the structured output tool exactly once.",
1949        )
1950        .output_schema::<ArithmeticResult>()
1951        .output_mode(OutputMode::Tool)
1952        .tool(AddTool(calls.clone()))
1953        .default_max_turns(5)
1954        .build();
1955    let result = agent
1956        .prompt("Use add to calculate 19 + 23. Return answer=42 and a short optional explanation.")
1957        .await?;
1958    let response = result.output();
1959    let parsed: ArithmeticResult = serde_json::from_str(&response)?;
1960    let tool_calls = calls.load(Ordering::SeqCst);
1961    if tool_calls == 0 || parsed.answer != 42 || !has_tool_roundtrip(&result.messages) {
1962        return Err(ScenarioError::contract(
1963            "structured_after_tool",
1964            format!("calls={tool_calls}, response={response:?}"),
1965        ));
1966    }
1967    let _ = parsed.explanation;
1968    report_from_response("structured_after_tool", started, tool_calls + 1, result)
1969}
1970
1971/// Runs all portable tool-choice modes directly against a completion model.
1972pub async fn tool_choice_modes<W, T, A>(
1973    model: rig_core::driver::Model<W, T>,
1974    adjust: A,
1975) -> Result<ScenarioReport, ScenarioError>
1976where
1977    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1978    T: rig_core::driver::Transport<W>,
1979    A: Fn(CompletionRequest) -> CompletionRequest,
1980{
1981    let tool_name = |name: &str| {
1982        rig_core::message::ToolName::new(name)
1983            .map_err(|error| ScenarioError::contract("tool_choice_modes", error.to_string()))
1984    };
1985    let definition = |name: rig_core::message::ToolName| ToolDefinition {
1986        description: format!("Return the supplied integer using {name}."),
1987        name,
1988        parameters: serde_json::json!({
1989            "type": "object",
1990            "properties": {"value": {"type": "integer"}},
1991            "required": ["value"]
1992        }),
1993    };
1994    let tools = vec![
1995        definition(tool_name("alpha")?),
1996        definition(tool_name("beta")?),
1997    ];
1998    let started = Instant::now();
1999    let none = model
2000        .call(adjust(
2001            CompletionRequest::new("Answer with only the number 4. Do not call a function.")
2002                .tools(tools.clone())
2003                .tool_choice(ToolChoice::None)
2004                .temperature(0.0)
2005                .max_tokens(64),
2006        ))
2007        .await?;
2008    if none
2009        .choice
2010        .iter()
2011        .any(|item| matches!(item, AssistantContent::ToolCall(_)))
2012    {
2013        return Err(ScenarioError::contract(
2014            "tool_choice_modes",
2015            "tool_choice none emitted a tool call",
2016        ));
2017    }
2018
2019    let required = model
2020        .call(adjust(
2021            CompletionRequest::new("Call alpha with value 7.")
2022                .tools(tools.clone())
2023                .tool_choice(ToolChoice::Required)
2024                .temperature(0.0)
2025                .max_tokens(96),
2026        ))
2027        .await?;
2028    let required_calls = required
2029        .choice
2030        .iter()
2031        .filter(|item| matches!(item, AssistantContent::ToolCall(_)))
2032        .count();
2033    if required_calls == 0 {
2034        return Err(ScenarioError::contract(
2035            "tool_choice_modes",
2036            "tool_choice required emitted no tool call",
2037        ));
2038    }
2039
2040    let specific = model
2041        .call(adjust(
2042            CompletionRequest::new("Call beta with value 9.")
2043                .tools(tools)
2044                .tool_choice(ToolChoice::Specific {
2045                    function_names: vec![tool_name("beta")?],
2046                })
2047                .temperature(0.0)
2048                .max_tokens(96),
2049        ))
2050        .await?;
2051    let specific_calls = specific
2052        .choice
2053        .iter()
2054        .filter_map(|item| match item {
2055            AssistantContent::ToolCall(call) => Some(call),
2056            _ => None,
2057        })
2058        .collect::<Vec<_>>();
2059    if specific_calls.is_empty()
2060        || specific_calls
2061            .iter()
2062            .any(|call| call.function.name != "beta")
2063    {
2064        return Err(ScenarioError::contract(
2065            "tool_choice_modes",
2066            "specific tool choice did not select only beta",
2067        ));
2068    }
2069
2070    let usage = none.usage + required.usage + specific.usage;
2071    Ok(ScenarioReport {
2072        name: "tool_choice_modes",
2073        tool_calls: required_calls + specific_calls.len(),
2074        prompt_tokens: usage.input_tokens.unwrap_or(0),
2075        generated_tokens: usage.output_tokens.unwrap_or(0),
2076        history_messages: 0,
2077        duration: started.elapsed(),
2078        response: "none, required, and specific modes passed".to_string(),
2079    })
2080}
2081
2082/// Runs a streamed real-tool turn followed by Rig's synthetic output tool.
2083pub async fn streaming_structured_after_tool<W, T, F>(
2084    model: rig_core::driver::Model<W, T>,
2085    configure: F,
2086) -> Result<ScenarioReport, ScenarioError>
2087where
2088    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
2089    T: rig_core::driver::Transport<W>,
2090    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
2091{
2092    let calls = Arc::new(AtomicUsize::new(0));
2093    let started = Instant::now();
2094    let agent = configure(AgentBuilder::new(model))
2095        .preamble(
2096            "Use add for arithmetic, then finish by calling the structured output tool exactly once.",
2097        )
2098        .output_schema::<ArithmeticResult>()
2099        .output_mode(OutputMode::Tool)
2100        .tool(AddTool(calls.clone()))
2101        .default_max_turns(5)
2102        .build();
2103    let mut stream = agent
2104        .prompt("Use add to calculate 19 + 23. Return answer=42 and a short optional explanation.")
2105        .max_turns(5)
2106        .stream();
2107    let mut final_response = None;
2108    let mut final_count = 0_usize;
2109    while let Some(item) = stream.next().await {
2110        if let MultiTurnStreamItem::FinalResponse(response) = item? {
2111            final_count += 1;
2112            final_response = Some(response);
2113        }
2114    }
2115    let result = final_response.ok_or_else(|| {
2116        ScenarioError::contract(
2117            "streaming_structured_after_tool",
2118            "stream produced no final response",
2119        )
2120    })?;
2121    let parsed: ArithmeticResult = serde_json::from_str(&result.output())?;
2122    let calls = calls.load(Ordering::SeqCst);
2123    if calls == 0
2124        || final_count != 1
2125        || parsed.answer != 42
2126        || !has_tool_roundtrip(&result.messages)
2127    {
2128        return Err(ScenarioError::contract(
2129            "streaming_structured_after_tool",
2130            format!(
2131                "calls={calls}, final_count={final_count}, response={:?}",
2132                result.output()
2133            ),
2134        ));
2135    }
2136    report_from_response(
2137        "streaming_structured_after_tool",
2138        started,
2139        calls + 1,
2140        result,
2141    )
2142}
2143
2144#[cfg(test)]
2145mod tests;