Skip to main content

rig_agent/test_utils/
model_conformance.rs

1//! Provider-neutral behavioral scenarios for completion-model conformance.
2//!
3//! These helpers test the model/agent contract only. Provider wire formats,
4//! authentication, HTTP streaming, and cassette matching remain provider-suite
5//! responsibilities.
6
7use std::{
8    collections::BTreeSet,
9    sync::{
10        Arc, Mutex, MutexGuard,
11        atomic::{AtomicUsize, Ordering},
12    },
13    time::{Duration, Instant},
14};
15
16use futures::StreamExt;
17use schemars::JsonSchema;
18use serde::{Deserialize, Serialize};
19
20use crate::{
21    agent::{
22        AgentBuilder, AgentHook, CompletionCallAction, CompletionCallEvent, DispatchAction,
23        DispatchEvent, HookContext, InvalidToolCallAction, MultiTurnStreamItem, NoToolConfig,
24        OutcomeAction, OutcomeEvent, OutputMode, RequestPatch, StreamingError,
25        run::{AgentRun, AgentRunStep, ModelTurn, ModelTurnOutcome},
26    },
27    completion::{AssistantContent, Message, PromptError, ToolDefinition},
28    tool::{Tool, ToolContext},
29};
30use rig_core::completion::CompletionRequest;
31use rig_core::error::ProviderError;
32use rig_core::message::{ToolChoice, UserContent};
33
34/// Typed failure from a portable model-conformance scenario.
35#[derive(Debug, thiserror::Error)]
36pub enum ScenarioError {
37    /// A buffered agent run failed.
38    #[error(transparent)]
39    Prompt(#[from] PromptError),
40    /// A direct model completion failed.
41    #[error(transparent)]
42    Completion(#[from] ProviderError),
43    /// A streaming agent run failed.
44    #[error(transparent)]
45    Streaming(#[from] StreamingError),
46    /// A stream item or bus effect failed, as the wire reports it.
47    #[error(transparent)]
48    Report(#[from] rig_core::error::ErrorReport),
49    /// Structured content could not be decoded.
50    #[error(transparent)]
51    Json(#[from] serde_json::Error),
52    /// Rig's structured extractor failed.
53    #[error(transparent)]
54    Extraction(#[from] crate::completion::StructuredOutputError),
55    /// The model or agent violated the portable behavioral contract.
56    #[error("{scenario} conformance failed: {details}")]
57    Contract {
58        /// Stable scenario name.
59        scenario: &'static str,
60        /// Actionable observation explaining the failure.
61        details: String,
62    },
63}
64
65impl ScenarioError {
66    fn contract(scenario: &'static str, details: impl Into<String>) -> Self {
67        Self::Contract {
68            scenario,
69            details: details.into(),
70        }
71    }
72}
73
74/// Validate the portable diagnostics carried by an unknown or disallowed tool
75/// call failure.
76pub fn validate_unknown_tool_failure(
77    error: &PromptError,
78    expected_tool: &str,
79    expected_allowed_tools: &[&str],
80) -> Result<(), ScenarioError> {
81    const SCENARIO: &str = "unknown_tool_failure";
82    let PromptError::UnknownToolCall {
83        tool_name,
84        allowed_tools,
85        chat_history,
86        ..
87    } = error
88    else {
89        return Err(ScenarioError::contract(
90            SCENARIO,
91            format!("expected UnknownToolCall, observed {error:?}"),
92        ));
93    };
94    let expected_allowed = expected_allowed_tools
95        .iter()
96        .map(|name| (*name).to_string())
97        .collect::<Vec<_>>();
98    // A rejected repair target is never written back into history; diagnostics
99    // retain the model's original call while `tool_name` names the rejected
100    // target. Requiring a call-bearing assistant turn covers both paths.
101    let history_has_call = chat_history.iter().any(|message| {
102        matches!(
103            message,
104            Message::Assistant { content, .. }
105                if content.iter().any(|item| matches!(item, AssistantContent::ToolCall(_)))
106        )
107    });
108    if tool_name != expected_tool || allowed_tools != &expected_allowed || !history_has_call {
109        return Err(ScenarioError::contract(
110            SCENARIO,
111            format!(
112                "tool={tool_name:?}, allowed={allowed_tools:?}, expected_tool={expected_tool:?}, expected_allowed={expected_allowed:?}, history_has_call={history_has_call}, history={chat_history:?}"
113            ),
114        ));
115    }
116    Ok(())
117}
118
119/// Validate cancellation diagnostics, including the exact reason and retained
120/// assistant tool-call history.
121pub fn validate_cancelled_failure(
122    error: &PromptError,
123    expected_reason: &str,
124    expected_tool: &str,
125) -> Result<(), ScenarioError> {
126    const SCENARIO: &str = "cancelled_failure";
127    let PromptError::PromptCancelled {
128        chat_history,
129        reason,
130    } = error
131    else {
132        return Err(ScenarioError::contract(
133            SCENARIO,
134            format!("expected PromptCancelled, observed {error:?}"),
135        ));
136    };
137    let history_has_call = chat_history.iter().any(|message| {
138        matches!(
139            message,
140            Message::Assistant { content, .. }
141                if content.iter().any(|item| matches!(
142                    item,
143                    AssistantContent::ToolCall(call) if call.function.name == expected_tool
144                ))
145        )
146    });
147    if reason != expected_reason || !history_has_call {
148        return Err(ScenarioError::contract(
149            SCENARIO,
150            format!(
151                "reason={reason:?}, expected={expected_reason:?}, history_has_call={history_has_call}, history={chat_history:?}"
152            ),
153        ));
154    }
155    Ok(())
156}
157
158/// Validate max-turn diagnostics, including the exact configured budget and a
159/// retained pending prompt.
160pub fn validate_max_turns_failure(
161    error: &PromptError,
162    expected_max_turns: usize,
163) -> Result<(), ScenarioError> {
164    const SCENARIO: &str = "max_turns_failure";
165    let PromptError::MaxTurnsError {
166        max_turns,
167        chat_history,
168        prompt,
169    } = error
170    else {
171        return Err(ScenarioError::contract(
172            SCENARIO,
173            format!("expected MaxTurnsError, observed {error:?}"),
174        ));
175    };
176    let pending_prompt_retained = matches!(
177        prompt,
178        Message::User { content } if content.iter().next().is_some()
179    );
180    if *max_turns != expected_max_turns || chat_history.is_empty() || !pending_prompt_retained {
181        return Err(ScenarioError::contract(
182            SCENARIO,
183            format!(
184                "max_turns={max_turns}, expected={expected_max_turns}, history={chat_history:?}, pending_prompt_retained={pending_prompt_retained}, pending_prompt={prompt:?}"
185            ),
186        ));
187    }
188    Ok(())
189}
190
191/// Decode a structured-output response with a typed conformance failure that
192/// retains the scenario name and raw response.
193pub fn decode_structured_output<T>(
194    scenario: &'static str,
195    response: &str,
196) -> Result<T, ScenarioError>
197where
198    T: serde::de::DeserializeOwned,
199{
200    serde_json::from_str(response).map_err(|error| {
201        ScenarioError::contract(
202            scenario,
203            format!("structured output did not decode: {error}; response={response:?}"),
204        )
205    })
206}
207
208/// Validate that model-family control markers are absent from user-visible
209/// output and persisted history.
210pub fn validate_protocol_hygiene(
211    scenario: &'static str,
212    visible_output: &str,
213    messages: &[Message],
214    forbidden_markers: &[&str],
215) -> Result<(), ScenarioError> {
216    let serialized = serde_json::to_string(messages)?;
217    let leaked = forbidden_markers
218        .iter()
219        .filter(|marker| visible_output.contains(**marker) || serialized.contains(**marker))
220        .copied()
221        .collect::<Vec<_>>();
222    if !leaked.is_empty() {
223        return Err(ScenarioError::contract(
224            scenario,
225            format!(
226                "protocol markers leaked: {leaked:?}; output={visible_output:?}, history={messages:?}"
227            ),
228        ));
229    }
230    Ok(())
231}
232
233/// Validate that every observed tool invocation contains the expected rewritten
234/// argument fields while allowing unrelated original fields to remain.
235pub fn validate_rewritten_arguments(
236    scenario: &'static str,
237    observations: &[serde_json::Value],
238    expected_fields: &serde_json::Value,
239) -> Result<(), ScenarioError> {
240    let Some(expected) = expected_fields.as_object() else {
241        return Err(ScenarioError::contract(
242            scenario,
243            "expected rewritten fields must be a JSON object",
244        ));
245    };
246    if observations.is_empty() {
247        return Err(ScenarioError::contract(
248            scenario,
249            "the rewritten tool was never invoked",
250        ));
251    }
252    for observation in observations {
253        let Some(actual) = observation.as_object() else {
254            return Err(ScenarioError::contract(
255                scenario,
256                format!("observed rewritten arguments were not an object: {observation:?}"),
257            ));
258        };
259        for (key, expected_value) in expected {
260            if actual.get(key) != Some(expected_value) {
261                return Err(ScenarioError::contract(
262                    scenario,
263                    format!(
264                        "rewritten field {key:?} expected {expected_value:?}, observed {observation:?}"
265                    ),
266                ));
267            }
268        }
269    }
270    Ok(())
271}
272
273/// Validate that a sensitive raw tool result was produced but did not reach the
274/// model's user-visible response after result hooks ran.
275pub fn validate_result_redaction(
276    scenario: &'static str,
277    tool_produced_secret: bool,
278    visible_output: &str,
279    secret: &str,
280) -> Result<(), ScenarioError> {
281    if !tool_produced_secret || visible_output.is_empty() || visible_output.contains(secret) {
282        return Err(ScenarioError::contract(
283            scenario,
284            format!(
285                "produced_secret={tool_produced_secret}, secret_visible={}, output={visible_output:?}",
286                visible_output.contains(secret)
287            ),
288        ));
289    }
290    Ok(())
291}
292
293/// Validate a provider-neutral person extraction and its usage accounting.
294pub fn validate_extraction_fields(
295    scenario: &'static str,
296    first_name: Option<&str>,
297    last_name: Option<&str>,
298    job: Option<&str>,
299    usage: crate::completion::Usage,
300) -> Result<(), ScenarioError> {
301    let fields_match = first_name.is_some_and(|value| value.eq_ignore_ascii_case("Ada"))
302        && last_name.is_some_and(|value| value.eq_ignore_ascii_case("Lovelace"))
303        && job.is_some_and(|value| value.to_ascii_lowercase().contains("mathematician"));
304    if !fields_match || !usage.is_reported() {
305        return Err(ScenarioError::contract(
306            scenario,
307            format!(
308                "first_name={first_name:?}, last_name={last_name:?}, job={job:?}, usage={usage:?}"
309            ),
310        ));
311    }
312    Ok(())
313}
314
315/// Summary emitted by a portable model-conformance scenario.
316#[derive(Debug, Clone)]
317pub struct ScenarioReport {
318    /// Stable scenario name.
319    pub name: &'static str,
320    /// Number of model-invoked tool calls observed by the scenario.
321    pub tool_calls: usize,
322    /// Aggregated prompt tokens reported by the model across all turns; zero
323    /// when the model reported none.
324    pub prompt_tokens: u64,
325    /// Aggregated generated tokens reported by the model across all turns;
326    /// zero when the model reported none.
327    pub generated_tokens: u64,
328    /// Number of messages retained in the completed run history.
329    pub history_messages: usize,
330    /// End-to-end scenario duration.
331    pub duration: Duration,
332    /// The model's final user-visible response.
333    pub response: String,
334}
335
336/// Error used by the deterministic conformance tools.
337#[derive(Debug, thiserror::Error)]
338#[error("model-conformance tool failed")]
339pub struct ConformanceToolError;
340
341const FORCE_TOOLS_PREAMBLE: &str = "You are a calculator assistant. You MUST use the provided tools for every arithmetic operation instead of computing results yourself. Once you have all the tool results you need, reply with the final numeric answer in plain text.";
342const PARALLEL_PROMPT: &str = "Compute 3 + 4 and 10 - 2. You MUST call the add tool and the subtract tool together in your first response, as two parallel function calls, then report both results.";
343const PING_OUTPUT: &str = "pong-crimson-7423";
344const MOTTO_OUTPUT: &str = "steady hands\ncalm waters";
345
346fn lock_recover<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
347    mutex
348        .lock()
349        .unwrap_or_else(std::sync::PoisonError::into_inner)
350}
351
352fn tool_result_values(message: &Message) -> Vec<serde_json::Value> {
353    let Message::User { content } = message else {
354        return Vec::new();
355    };
356    content
357        .iter()
358        .filter_map(|item| match item {
359            UserContent::ToolResult(result) => Some(result),
360            _ => None,
361        })
362        .flat_map(|result| result.content.iter())
363        .filter_map(|content| match content {
364            rig_core::message::ToolResultContent::Text(text) => {
365                Some(serde_json::Value::String(text.text.clone()))
366            }
367            rig_core::message::ToolResultContent::Json { value } => Some(value.clone()),
368            rig_core::message::ToolResultContent::Image(_) => None,
369        })
370        .collect()
371}
372
373fn validate_tool_correlation(
374    scenario: &'static str,
375    messages: &[Message],
376) -> Result<(), ScenarioError> {
377    let mut calls = Vec::new();
378    let mut results = Vec::new();
379    let mut turn = 0usize;
380    for message in messages {
381        match message {
382            Message::Assistant { content, .. } => {
383                turn += 1;
384                calls.extend(content.iter().filter_map(|item| match item {
385                    AssistantContent::ToolCall(call) => Some((
386                        turn,
387                        &call.id,
388                        call.id.provider().map(|provider| provider.call_id.as_str()),
389                    )),
390                    _ => None,
391                }));
392            }
393            Message::User { content } => {
394                results.extend(content.iter().filter_map(|item| {
395                    match item {
396                        UserContent::ToolResult(result) => Some((
397                            turn,
398                            &result.call,
399                            result
400                                .call
401                                .provider()
402                                .map(|provider| provider.call_id.as_str()),
403                        )),
404                        _ => None,
405                    }
406                }));
407            }
408            Message::System { .. } => {}
409        }
410    }
411    if calls.is_empty() {
412        return Err(ScenarioError::contract(
413            scenario,
414            format!("history has no assistant tool calls: {messages:?}"),
415        ));
416    }
417    for (turn, id, call_id) in &calls {
418        let matches = results
419            .iter()
420            .filter(|(result_turn, result_id, result_call_id)| {
421                result_turn == turn && result_id == id && call_id == result_call_id
422            })
423            .count();
424        if matches != 1 {
425            return Err(ScenarioError::contract(
426                scenario,
427                format!(
428                    "tool call id={id:?} call_id={call_id:?} has {matches} correlated results; calls={calls:?}, results={results:?}"
429                ),
430            ));
431        }
432    }
433    if results.len() != calls.len() {
434        return Err(ScenarioError::contract(
435            scenario,
436            format!(
437                "history contains dangling calls or results: calls={calls:?}, results={results:?}"
438            ),
439        ));
440    }
441    Ok(())
442}
443
444#[derive(Debug, Deserialize, JsonSchema)]
445struct OperationArgs {
446    x: i64,
447    y: i64,
448}
449
450#[derive(Clone)]
451struct CountingAdd(Arc<AtomicUsize>);
452
453impl Tool for CountingAdd {
454    const NAME: &'static str = "add";
455    type Error = ConformanceToolError;
456    type Args = OperationArgs;
457    type Output = i64;
458
459    fn description(&self) -> String {
460        "Add x and y together".to_string()
461    }
462
463    fn parameters(&self) -> serde_json::Value {
464        serde_json::json!({
465            "type": "object",
466            "properties": {
467                "x": { "type": "number", "description": "The first operand" },
468                "y": { "type": "number", "description": "The second operand" }
469            },
470            "required": ["x", "y"]
471        })
472    }
473
474    async fn call(
475        &self,
476        _context: &mut ToolContext,
477        args: Self::Args,
478    ) -> Result<Self::Output, Self::Error> {
479        self.0.fetch_add(1, Ordering::SeqCst);
480        Ok(args.x + args.y)
481    }
482}
483
484#[derive(Clone)]
485struct CountingSum(Arc<AtomicUsize>);
486
487impl Tool for CountingSum {
488    const NAME: &'static str = "sum";
489    type Error = ConformanceToolError;
490    type Args = OperationArgs;
491    type Output = i64;
492
493    fn description(&self) -> String {
494        "Add x and y together (alias of add)".to_string()
495    }
496
497    fn parameters(&self) -> serde_json::Value {
498        CountingAdd(Arc::new(AtomicUsize::new(0))).parameters()
499    }
500
501    async fn call(
502        &self,
503        _context: &mut ToolContext,
504        args: Self::Args,
505    ) -> Result<Self::Output, Self::Error> {
506        self.0.fetch_add(1, Ordering::SeqCst);
507        Ok(args.x + args.y)
508    }
509}
510
511#[derive(Clone)]
512struct CountingSubtract(Arc<AtomicUsize>);
513
514impl Tool for CountingSubtract {
515    const NAME: &'static str = "subtract";
516    type Error = ConformanceToolError;
517    type Args = OperationArgs;
518    type Output = i64;
519
520    fn description(&self) -> String {
521        "Subtract y from x (i.e. x - y)".to_string()
522    }
523
524    fn parameters(&self) -> serde_json::Value {
525        serde_json::json!({
526            "type": "object",
527            "properties": {
528                "x": { "type": "number", "description": "The first operand" },
529                "y": { "type": "number", "description": "The second operand" }
530            },
531            "required": ["x", "y"]
532        })
533    }
534
535    async fn call(
536        &self,
537        _context: &mut ToolContext,
538        args: Self::Args,
539    ) -> Result<Self::Output, Self::Error> {
540        self.0.fetch_add(1, Ordering::SeqCst);
541        Ok(args.x - args.y)
542    }
543}
544
545#[derive(Clone)]
546struct RewriteArgument {
547    key: &'static str,
548    value: serde_json::Value,
549}
550
551impl AgentHook for RewriteArgument {
552    async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
553        let (Some(name), Some(args)) = (event.tool_name(), event.tool_args()) else {
554            return DispatchAction::proceed();
555        };
556        if name != CountingAdd::NAME {
557            return DispatchAction::proceed();
558        }
559        let Ok(mut arguments) = serde_json::from_str::<serde_json::Value>(args) else {
560            return DispatchAction::proceed();
561        };
562        let Some(object) = arguments.as_object_mut() else {
563            return DispatchAction::proceed();
564        };
565        object.insert(self.key.to_string(), self.value.clone());
566        DispatchAction::rewrite_tool_args(event.kind, arguments)
567    }
568}
569
570#[derive(Clone, Default)]
571struct ObserveArguments(Arc<Mutex<Vec<serde_json::Value>>>);
572
573impl AgentHook for ObserveArguments {
574    async fn on_dispatch(&self, _ctx: &HookContext, event: DispatchEvent<'_>) -> DispatchAction {
575        if let Some(args) = event.tool_args() {
576            let value = serde_json::from_str(args)
577                .unwrap_or_else(|_| serde_json::Value::String(args.to_string()));
578            lock_recover(&self.0).push(value);
579        }
580        DispatchAction::proceed()
581    }
582}
583
584#[derive(Clone)]
585struct ReplaceResult(&'static str);
586
587impl AgentHook for ReplaceResult {
588    async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
589        if event.tool_name() == Some(CountingAdd::NAME) {
590            OutcomeAction::rewrite_tool_result(&event, self.0)
591        } else {
592            OutcomeAction::proceed()
593        }
594    }
595}
596
597#[derive(Clone)]
598struct WrapResult;
599
600impl AgentHook for WrapResult {
601    async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
602        match (event.tool_name(), event.tool_result()) {
603            (Some(name), Some(result)) if name == CountingAdd::NAME => {
604                OutcomeAction::rewrite_tool_result(
605                    &event,
606                    format!("[{}]", result.output().render()),
607                )
608            }
609            _ => OutcomeAction::proceed(),
610        }
611    }
612}
613
614#[derive(Clone)]
615struct FirstTurnPatch(RequestPatch);
616
617impl AgentHook for FirstTurnPatch {
618    async fn on_completion_call(
619        &self,
620        ctx: &HookContext,
621        _event: CompletionCallEvent<'_>,
622    ) -> CompletionCallAction {
623        if ctx.turn() == 1 {
624            CompletionCallAction::patch(self.0.clone())
625        } else {
626            CompletionCallAction::continue_run()
627        }
628    }
629}
630
631#[derive(Clone)]
632struct StopAfterResult(&'static str);
633
634impl AgentHook for StopAfterResult {
635    async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
636        if event.tool_name() == Some(CountingAdd::NAME) {
637            OutcomeAction::stop(self.0)
638        } else {
639            OutcomeAction::proceed()
640        }
641    }
642}
643
644#[derive(Debug, Deserialize, JsonSchema)]
645struct EmptyArgs {}
646
647#[derive(Clone)]
648struct PingTool(Arc<AtomicUsize>);
649
650impl Tool for PingTool {
651    const NAME: &'static str = "ping";
652    type Error = ConformanceToolError;
653    type Args = EmptyArgs;
654    type Output = String;
655
656    fn description(&self) -> String {
657        "Return the current ping marker. Takes no arguments.".to_string()
658    }
659
660    fn parameters(&self) -> serde_json::Value {
661        serde_json::json!({ "type": "object", "properties": {}, "required": [] })
662    }
663
664    async fn call(
665        &self,
666        _context: &mut ToolContext,
667        _args: Self::Args,
668    ) -> Result<Self::Output, Self::Error> {
669        self.0.fetch_add(1, Ordering::SeqCst);
670        Ok(PING_OUTPUT.to_string())
671    }
672}
673
674#[derive(Clone)]
675struct MottoTool(Arc<AtomicUsize>);
676
677impl Tool for MottoTool {
678    const NAME: &'static str = "fetch_motto";
679    type Error = ConformanceToolError;
680    type Args = EmptyArgs;
681    type Output = String;
682
683    fn description(&self) -> String {
684        "Fetch the two-line workshop motto.".to_string()
685    }
686
687    fn parameters(&self) -> serde_json::Value {
688        serde_json::json!({ "type": "object", "properties": {}, "required": [] })
689    }
690
691    async fn call(
692        &self,
693        _context: &mut ToolContext,
694        _args: Self::Args,
695    ) -> Result<Self::Output, Self::Error> {
696        self.0.fetch_add(1, Ordering::SeqCst);
697        Ok(MOTTO_OUTPUT.to_string())
698    }
699}
700
701#[derive(Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
702struct ConfigOutput {
703    service: String,
704    max_retries: u64,
705}
706
707#[derive(Clone)]
708struct ConfigTool(Arc<AtomicUsize>);
709
710impl Tool for ConfigTool {
711    const NAME: &'static str = "fetch_config";
712    type Error = ConformanceToolError;
713    type Args = EmptyArgs;
714    type Output = ConfigOutput;
715
716    fn description(&self) -> String {
717        "Fetch the service configuration object.".to_string()
718    }
719
720    fn parameters(&self) -> serde_json::Value {
721        serde_json::json!({ "type": "object", "properties": {}, "required": [] })
722    }
723
724    async fn call(
725        &self,
726        _context: &mut ToolContext,
727        _args: Self::Args,
728    ) -> Result<Self::Output, Self::Error> {
729        self.0.fetch_add(1, Ordering::SeqCst);
730        Ok(ConfigOutput {
731            service: "cassette-lab".to_string(),
732            max_retries: 3,
733        })
734    }
735}
736
737#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
738#[serde(rename_all = "snake_case")]
739enum ComplexMode {
740    Careful,
741    Fast,
742}
743
744#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
745struct ComplexProfile {
746    name: String,
747    tags: Vec<String>,
748}
749
750#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
751struct ComplexArgs {
752    profile: ComplexProfile,
753    mode: ComplexMode,
754    note: Option<String>,
755    quote: String,
756}
757
758#[derive(Debug, Deserialize, Serialize, JsonSchema)]
759struct ExtractedPerson {
760    #[schemars(required)]
761    first_name: Option<String>,
762    #[schemars(required)]
763    last_name: Option<String>,
764    #[schemars(required)]
765    job: Option<String>,
766}
767
768#[derive(Clone)]
769struct CaptureComplexTool {
770    calls: Arc<AtomicUsize>,
771    captured: Arc<Mutex<Option<ComplexArgs>>>,
772}
773
774impl Tool for CaptureComplexTool {
775    const NAME: &'static str = "store_profile";
776    type Error = ConformanceToolError;
777    type Args = ComplexArgs;
778    type Output = ComplexArgs;
779
780    fn description(&self) -> String {
781        "Store one profile with its nested tags, mode, optional note, and exact quoted text."
782            .to_string()
783    }
784
785    fn parameters(&self) -> serde_json::Value {
786        serde_json::to_value(schemars::schema_for!(ComplexArgs)).unwrap_or_default()
787    }
788
789    async fn call(
790        &self,
791        _context: &mut ToolContext,
792        args: Self::Args,
793    ) -> Result<Self::Output, Self::Error> {
794        self.calls.fetch_add(1, Ordering::SeqCst);
795        *lock_recover(&self.captured) = Some(args.clone());
796        Ok(args)
797    }
798}
799
800fn has_tool_roundtrip(messages: Option<&[Message]>) -> bool {
801    let saw_call = messages.is_some_and(|messages| {
802        messages.iter().any(|message| {
803            matches!(
804                message,
805                Message::Assistant { content, .. }
806                    if content.iter().any(|item| matches!(item, AssistantContent::ToolCall(_)))
807            )
808        })
809    });
810    let saw_result = messages.is_some_and(|messages| {
811        messages.iter().any(|message| {
812            matches!(
813                message,
814                Message::User { content }
815                    if content.iter().any(|item| matches!(item, UserContent::ToolResult(_)))
816            )
817        })
818    });
819    saw_call && saw_result
820}
821
822#[derive(Debug, Deserialize, JsonSchema)]
823struct RepeatArgs {
824    /// The text to repeat.
825    text: String,
826    /// Number of repetitions; defaults to 2 when omitted.
827    times: Option<u32>,
828}
829
830#[derive(Clone)]
831struct RepeatTool {
832    calls: Arc<AtomicUsize>,
833}
834
835impl Tool for RepeatTool {
836    const NAME: &'static str = "repeat_text";
837    type Error = ConformanceToolError;
838    type Args = RepeatArgs;
839    type Output = String;
840
841    fn description(&self) -> String {
842        "Repeat `text`. `times` is optional and defaults to 2.".to_string()
843    }
844
845    fn parameters(&self) -> serde_json::Value {
846        serde_json::to_value(schemars::schema_for!(RepeatArgs)).unwrap_or_default()
847    }
848
849    async fn call(
850        &self,
851        _context: &mut ToolContext,
852        args: Self::Args,
853    ) -> Result<Self::Output, Self::Error> {
854        self.calls.fetch_add(1, Ordering::SeqCst);
855        Ok(vec![args.text.as_str(); args.times.unwrap_or(2) as usize].join(" "))
856    }
857}
858
859#[derive(Debug, Deserialize, JsonSchema)]
860struct BinOpArgs {
861    a: i64,
862    b: i64,
863}
864
865#[derive(Debug, Deserialize, JsonSchema)]
866struct ArithmeticResult {
867    answer: i64,
868    explanation: Option<String>,
869}
870
871#[derive(Clone)]
872struct AddTool(Arc<AtomicUsize>);
873
874impl Tool for AddTool {
875    const NAME: &'static str = "add";
876    type Error = ConformanceToolError;
877    type Args = BinOpArgs;
878    type Output = i64;
879
880    fn description(&self) -> String {
881        "Add two integers a and b.".to_string()
882    }
883
884    fn parameters(&self) -> serde_json::Value {
885        serde_json::to_value(schemars::schema_for!(BinOpArgs)).unwrap_or_default()
886    }
887
888    async fn call(
889        &self,
890        _context: &mut ToolContext,
891        args: Self::Args,
892    ) -> Result<Self::Output, Self::Error> {
893        self.0.fetch_add(1, Ordering::SeqCst);
894        Ok(args.a + args.b)
895    }
896}
897
898#[derive(Clone)]
899struct MultiplyTool(Arc<AtomicUsize>);
900
901impl Tool for MultiplyTool {
902    const NAME: &'static str = "multiply";
903    type Error = ConformanceToolError;
904    type Args = BinOpArgs;
905    type Output = i64;
906
907    fn description(&self) -> String {
908        "Multiply two integers a and b.".to_string()
909    }
910
911    fn parameters(&self) -> serde_json::Value {
912        serde_json::to_value(schemars::schema_for!(BinOpArgs)).unwrap_or_default()
913    }
914
915    async fn call(
916        &self,
917        _context: &mut ToolContext,
918        args: Self::Args,
919    ) -> Result<Self::Output, Self::Error> {
920        self.0.fetch_add(1, Ordering::SeqCst);
921        Ok(args.a * args.b)
922    }
923}
924
925fn report_from_response(
926    name: &'static str,
927    started: Instant,
928    tool_calls: usize,
929    response: crate::agent::PromptResponse,
930) -> Result<ScenarioReport, ScenarioError> {
931    if let Some(messages) = response.messages.as_deref() {
932        validate_protocol_hygiene(
933            name,
934            &response.output,
935            messages,
936            &[
937                "<tool_call>",
938                "</tool_call>",
939                "<tool_response>",
940                "</tool_response>",
941                "<|im_start|>",
942                "<|im_end|>",
943                "<think>",
944                "</think>",
945            ],
946        )?;
947    }
948    Ok(ScenarioReport {
949        name,
950        tool_calls,
951        prompt_tokens: response.usage.input_tokens.unwrap_or(0),
952        generated_tokens: response.usage.output_tokens.unwrap_or(0),
953        history_messages: response.messages.as_ref().map_or(0, Vec::len),
954        duration: started.elapsed(),
955        response: response.output,
956    })
957}
958
959/// Require the extended run's accumulated message history and validate
960/// canonical tool call/result correlation over it.
961fn correlated_messages<'a>(
962    scenario: &'static str,
963    response: &'a crate::agent::PromptResponse,
964) -> Result<&'a [Message], ScenarioError> {
965    let messages = response.messages.as_deref().ok_or_else(|| {
966        ScenarioError::contract(scenario, "extended run omitted accumulated message history")
967    })?;
968    validate_tool_correlation(scenario, messages)?;
969    Ok(messages)
970}
971
972/// [`correlated_messages`], flattened into every tool-result value in history.
973fn correlated_result_values(
974    scenario: &'static str,
975    response: &crate::agent::PromptResponse,
976) -> Result<Vec<serde_json::Value>, ScenarioError> {
977    Ok(correlated_messages(scenario, response)?
978        .iter()
979        .flat_map(tool_result_values)
980        .collect())
981}
982
983fn value_matches_integer(value: &serde_json::Value, expected: i64) -> bool {
984    value.as_i64() == Some(expected)
985        || value
986            .as_str()
987            .and_then(|text| text.trim().parse::<i64>().ok())
988            == Some(expected)
989}
990
991/// Runs two independent calls in one assistant turn and validates canonical
992/// call/result history correlation.
993///
994/// Set `tool_concurrency` to `Some(1)` to prove that serial host execution does
995/// not split or drop the model's parallel call batch.
996pub async fn parallel_tools<W, T, F>(
997    model: rig_core::driver::Model<W, T>,
998    configure: F,
999    tool_concurrency: Option<usize>,
1000) -> Result<ScenarioReport, ScenarioError>
1001where
1002    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1003    T: rig_core::driver::Transport<W>,
1004    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1005{
1006    let add_calls = Arc::new(AtomicUsize::new(0));
1007    let subtract_calls = Arc::new(AtomicUsize::new(0));
1008    let started = Instant::now();
1009    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1010        .preamble(FORCE_TOOLS_PREAMBLE)
1011        .tool(CountingAdd(add_calls.clone()))
1012        .tool(CountingSubtract(subtract_calls.clone()))
1013        .default_max_turns(3)
1014        .build();
1015    let request = agent.prompt(PARALLEL_PROMPT).max_turns(3);
1016    let response = match tool_concurrency {
1017        Some(concurrency) => request.tool_concurrency(concurrency).await?,
1018        None => request.await?,
1019    };
1020    let scenario = if tool_concurrency == Some(1) {
1021        "parallel_tools_serial_execution"
1022    } else {
1023        "parallel_tools"
1024    };
1025    let messages = correlated_messages(scenario, &response)?;
1026
1027    let Some((call_index, calls)) = messages.iter().enumerate().find_map(|(index, message)| {
1028        let Message::Assistant { content, .. } = message else {
1029            return None;
1030        };
1031        let calls = content
1032            .iter()
1033            .filter_map(|item| match item {
1034                AssistantContent::ToolCall(call) => Some(call),
1035                _ => None,
1036            })
1037            .collect::<Vec<_>>();
1038        (calls.len() == 2).then_some((index, calls))
1039    }) else {
1040        return Err(ScenarioError::contract(
1041            scenario,
1042            format!("no assistant turn contained exactly two tool calls: {messages:?}"),
1043        ));
1044    };
1045    let mut names = calls
1046        .iter()
1047        .map(|call| call.function.name.as_str())
1048        .collect::<Vec<_>>();
1049    names.sort_unstable();
1050    if names != ["add", "subtract"] {
1051        return Err(ScenarioError::contract(
1052            scenario,
1053            format!("parallel turn called {names:?}, expected add and subtract"),
1054        ));
1055    }
1056    let results_message = messages.get(call_index + 1).ok_or_else(|| {
1057        ScenarioError::contract(
1058            scenario,
1059            "parallel call turn has no following result message",
1060        )
1061    })?;
1062    let values = tool_result_values(results_message);
1063    if !values.iter().any(|value| value_matches_integer(value, 7))
1064        || !values.iter().any(|value| value_matches_integer(value, 8))
1065        || values.len() != 2
1066    {
1067        return Err(ScenarioError::contract(
1068            scenario,
1069            format!("parallel result message did not contain exactly 7 and 8: {values:?}"),
1070        ));
1071    }
1072    let add = add_calls.load(Ordering::SeqCst);
1073    let subtract = subtract_calls.load(Ordering::SeqCst);
1074    if add != 1 || subtract != 1 {
1075        return Err(ScenarioError::contract(
1076            scenario,
1077            format!("execution counts were add={add}, subtract={subtract}, expected one each"),
1078        ));
1079    }
1080    report_from_response(scenario, started, add + subtract, response)
1081}
1082
1083/// Runs a zero-argument tool and validates verbatim string-result handling.
1084pub async fn zero_argument_tool<W, T, F>(
1085    model: rig_core::driver::Model<W, T>,
1086    configure: F,
1087) -> Result<ScenarioReport, ScenarioError>
1088where
1089    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1090    T: rig_core::driver::Transport<W>,
1091    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1092{
1093    const SCENARIO: &str = "zero_argument_tool";
1094    let calls = Arc::new(AtomicUsize::new(0));
1095    let started = Instant::now();
1096    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1097        .preamble("You must use the provided tools. Report tool outputs exactly as returned.")
1098        .tool(PingTool(calls.clone()))
1099        .default_max_turns(2)
1100        .build();
1101    let response = agent
1102        .prompt("Call the ping tool, then report the exact marker it returns.")
1103        .max_turns(2)
1104        .await?;
1105    let values = correlated_result_values(SCENARIO, &response)?;
1106    if calls.load(Ordering::SeqCst) != 1
1107        || !values
1108            .iter()
1109            .any(|value| value.as_str() == Some(PING_OUTPUT))
1110        || !response.output.contains(PING_OUTPUT)
1111    {
1112        return Err(ScenarioError::contract(
1113            SCENARIO,
1114            format!(
1115                "calls={}, results={values:?}, response={:?}",
1116                calls.load(Ordering::SeqCst),
1117                response.output
1118            ),
1119        ));
1120    }
1121    report_from_response(SCENARIO, started, 1, response)
1122}
1123
1124/// Runs string- and JSON-returning tools and validates that neither output is
1125/// double encoded.
1126pub async fn tool_output_serialization<W, T, F>(
1127    model: rig_core::driver::Model<W, T>,
1128    configure: F,
1129) -> Result<ScenarioReport, ScenarioError>
1130where
1131    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1132    T: rig_core::driver::Transport<W>,
1133    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1134{
1135    const SCENARIO: &str = "tool_output_serialization";
1136    let started = Instant::now();
1137    let motto_calls = Arc::new(AtomicUsize::new(0));
1138    let config_calls = Arc::new(AtomicUsize::new(0));
1139    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1140        .preamble("You must use the provided tools before answering.")
1141        .tool(MottoTool(motto_calls.clone()))
1142        .tool(ConfigTool(config_calls.clone()))
1143        .default_max_turns(3)
1144        .build();
1145    let response = agent
1146        .prompt("Call fetch_motto and fetch_config, then summarize both outputs in one sentence.")
1147        .max_turns(3)
1148        .await?;
1149    let values = correlated_result_values(SCENARIO, &response)?;
1150    let expected_config = serde_json::to_value(ConfigOutput {
1151        service: "cassette-lab".to_string(),
1152        max_retries: 3,
1153    })?;
1154    let motto_ok = values
1155        .iter()
1156        .any(|value| value.as_str() == Some(MOTTO_OUTPUT));
1157    let config_ok = values.iter().any(|value| {
1158        value == &expected_config
1159            || value
1160                .as_str()
1161                .and_then(|text| serde_json::from_str::<serde_json::Value>(text).ok())
1162                .as_ref()
1163                == Some(&expected_config)
1164    });
1165    let motto_count = motto_calls.load(Ordering::SeqCst);
1166    let config_count = config_calls.load(Ordering::SeqCst);
1167    if !motto_ok || !config_ok || motto_count != 1 || config_count != 1 {
1168        return Err(ScenarioError::contract(
1169            SCENARIO,
1170            format!(
1171                "expected one verbatim motto and one semantic config JSON; motto_calls={motto_count}, config_calls={config_count}, values={values:?}"
1172            ),
1173        ));
1174    }
1175    report_from_response(SCENARIO, started, motto_count + config_count, response)
1176}
1177
1178/// Runs a nested, escaped, Unicode-bearing argument payload through typed tool
1179/// deserialization and validates the exact semantic value received by the tool.
1180pub async fn complex_tool_arguments<W, T, F>(
1181    model: rig_core::driver::Model<W, T>,
1182    configure: F,
1183) -> Result<ScenarioReport, ScenarioError>
1184where
1185    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1186    T: rig_core::driver::Transport<W>,
1187    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1188{
1189    complex_tool_arguments_with_prompt(model, configure, COMPLEX_ARGUMENTS_PROMPT).await
1190}
1191
1192/// The prompt [`complex_tool_arguments`] sends. It spells the values in their
1193/// JSON-escaped form, which a model may copy literally, backslashes included.
1194pub const COMPLEX_ARGUMENTS_PROMPT: &str = "Call store_profile with profile.name exactly `Zoë \\\"Z\\\"`, profile.tags exactly [`rust`, `東京`], mode `careful`, note containing the two lines `line one` and `line two` separated by a newline, and quote exactly `path C:\\\\tmp and \\\"quoted\\\"`. Then confirm it was stored.";
1195
1196/// The same values as [`COMPLEX_ARGUMENTS_PROMPT`], spelled as raw text
1197/// between markers, for a model that copies escapes literally.
1198pub const COMPLEX_ARGUMENTS_RAW_PROMPT: &str = "Call store_profile once. Each value below is raw text between <<< and >>>, with no escaping: profile.name <<<Zoë \"Z\">>>, profile.tags <<<rust>>> and <<<東京>>>, mode <<<careful>>>, note the two lines <<<line one>>> and <<<line two>>> joined by one newline, and quote <<<path C:\\tmp and \"quoted\">>>. Then confirm it was stored.";
1199
1200/// [`complex_tool_arguments`] with the user prompt given: the same tool, the
1201/// same expected arguments, the same checks.
1202pub async fn complex_tool_arguments_with_prompt<W, T, F>(
1203    model: rig_core::driver::Model<W, T>,
1204    configure: F,
1205    prompt: &str,
1206) -> Result<ScenarioReport, ScenarioError>
1207where
1208    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1209    T: rig_core::driver::Transport<W>,
1210    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1211{
1212    const SCENARIO: &str = "complex_tool_arguments";
1213    let expected = ComplexArgs {
1214        profile: ComplexProfile {
1215            name: "Zoë \"Z\"".to_string(),
1216            tags: vec!["rust".to_string(), "東京".to_string()],
1217        },
1218        mode: ComplexMode::Careful,
1219        note: Some("line one\nline two".to_string()),
1220        quote: "path C:\\tmp and \"quoted\"".to_string(),
1221    };
1222    let calls = Arc::new(AtomicUsize::new(0));
1223    let captured = Arc::new(Mutex::new(None));
1224    let started = Instant::now();
1225    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1226        .preamble("Use store_profile exactly once with every value supplied by the user.")
1227        .tool(CaptureComplexTool {
1228            calls: calls.clone(),
1229            captured: captured.clone(),
1230        })
1231        .default_max_turns(3)
1232        .build();
1233    let response = agent.prompt(prompt).max_turns(3).await?;
1234    let observed = lock_recover(&captured).clone();
1235    if calls.load(Ordering::SeqCst) != 1 || observed.as_ref() != Some(&expected) {
1236        return Err(ScenarioError::contract(
1237            SCENARIO,
1238            format!(
1239                "calls={}, expected={expected:?}, observed={observed:?}, response={:?}",
1240                calls.load(Ordering::SeqCst),
1241                response.output
1242            ),
1243        ));
1244    }
1245    correlated_messages(SCENARIO, &response)?;
1246    report_from_response(SCENARIO, started, 1, response)
1247}
1248
1249/// Runs the same deterministic text request through buffered and raw streaming
1250/// completion surfaces and validates equivalent visible content and usage.
1251pub async fn buffered_streaming_text_parity<W, T, A>(
1252    model: rig_core::driver::Model<W, T>,
1253    adjust: A,
1254) -> Result<ScenarioReport, ScenarioError>
1255where
1256    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1257    T: rig_core::driver::Transport<W>,
1258    A: Fn(CompletionRequest) -> CompletionRequest,
1259{
1260    const SCENARIO: &str = "buffered_streaming_text_parity";
1261    const PROMPT: &str = "Answer with exactly the single word Paris.";
1262    let started = Instant::now();
1263    let request = || {
1264        adjust(
1265            CompletionRequest::new(PROMPT)
1266                .temperature(0.0)
1267                .max_tokens(32),
1268        )
1269    };
1270    let buffered = model.call(request()).await?;
1271    let buffered_text = buffered
1272        .choice
1273        .iter()
1274        .filter_map(|item| match item {
1275            AssistantContent::Text(text) => Some(text.text.as_str()),
1276            _ => None,
1277        })
1278        .collect::<String>();
1279
1280    let mut stream = model.stream(request())?;
1281    let mut streamed_text = String::new();
1282    while let Some(item) = stream.next().await {
1283        if let crate::streaming::Item::Event(crate::streaming::StreamEvent::Text { text, .. }) =
1284            item?
1285        {
1286            streamed_text.push_str(&text);
1287        }
1288    }
1289    let streamed_usage = Some(stream.finish().await?.usage);
1290    let usage = streamed_usage.ok_or_else(|| {
1291        ScenarioError::contract(SCENARIO, "raw stream omitted its final response metadata")
1292    })?;
1293    let normalize = |text: &str| {
1294        text.trim()
1295            .trim_matches(|character: char| !character.is_alphanumeric())
1296            .to_string()
1297    };
1298    let buffered_answer = normalize(&buffered_text);
1299    let streamed_answer = normalize(&streamed_text);
1300    if !buffered_answer.eq_ignore_ascii_case("Paris")
1301        || !streamed_answer.eq_ignore_ascii_case("Paris")
1302        || !buffered.usage.is_reported()
1303        || !usage.is_reported()
1304    {
1305        return Err(ScenarioError::contract(
1306            SCENARIO,
1307            format!(
1308                "buffered={buffered_text:?}, streamed={streamed_text:?}, buffered_usage={:?}, streamed_usage={usage:?}",
1309                buffered.usage
1310            ),
1311        ));
1312    }
1313    Ok(ScenarioReport {
1314        name: SCENARIO,
1315        tool_calls: 0,
1316        prompt_tokens: usage.input_tokens.unwrap_or(0),
1317        generated_tokens: usage.output_tokens.unwrap_or(0),
1318        history_messages: 0,
1319        duration: started.elapsed(),
1320        response: streamed_text,
1321    })
1322}
1323
1324/// Runs Rig's structured extractor and validates both the extracted semantic
1325/// fields and accumulated usage.
1326pub async fn structured_extraction<W, T>(
1327    model: rig_core::driver::Model<W, T>,
1328    additional_params: Option<serde_json::Value>,
1329) -> Result<ScenarioReport, ScenarioError>
1330where
1331    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1332    T: rig_core::driver::Transport<W>,
1333{
1334    const SCENARIO: &str = "structured_extraction";
1335    const INPUT: &str = "Hello, my name is Ada Lovelace and I work as a mathematician.";
1336    let started = Instant::now();
1337    let mut builder = crate::extractor::ExtractorBuilder::<ExtractedPerson>::new(model)
1338        .max_tokens(384)
1339        .retries(0);
1340    if let Some(params) = additional_params {
1341        builder = builder.additional_params(params);
1342    }
1343    let response = builder.build().extract(INPUT).await?;
1344    validate_extraction_fields(
1345        SCENARIO,
1346        response.output.first_name.as_deref(),
1347        response.output.last_name.as_deref(),
1348        response.output.job.as_deref(),
1349        response.usage,
1350    )?;
1351    Ok(ScenarioReport {
1352        name: SCENARIO,
1353        tool_calls: 1,
1354        prompt_tokens: response.usage.input_tokens.unwrap_or(0),
1355        generated_tokens: response.usage.output_tokens.unwrap_or(0),
1356        history_messages: 0,
1357        duration: started.elapsed(),
1358        response: format!(
1359            "{} {} — {}",
1360            response.output.first_name.as_deref().unwrap_or_default(),
1361            response.output.last_name.as_deref().unwrap_or_default(),
1362            response.output.job.as_deref().unwrap_or_default()
1363        ),
1364    })
1365}
1366
1367fn restricted_recovery_run(
1368    prompt: &str,
1369    turn: ModelTurn,
1370    retries: usize,
1371) -> Result<AgentRun, ScenarioError> {
1372    const SCENARIO: &str = "invalid_tool_recovery";
1373    let mut run = AgentRun::new(prompt)
1374        .max_turns(2)
1375        .max_invalid_tool_call_retries(retries);
1376    if !matches!(run.next_step()?, AgentRunStep::CallModel { .. }) {
1377        return Err(ScenarioError::contract(
1378            SCENARIO,
1379            "fresh AgentRun did not request a model turn",
1380        ));
1381    }
1382    let outcome = run.model_response(turn)?;
1383    let ModelTurnOutcome::NeedsResolution(context) = outcome else {
1384        return Err(ScenarioError::contract(
1385            SCENARIO,
1386            format!("disallowed tool call did not require resolution: {outcome:?}"),
1387        ));
1388    };
1389    if context.tool_name != CountingAdd::NAME {
1390        return Err(ScenarioError::contract(
1391            SCENARIO,
1392            format!("expected rejected add call, observed {context:?}"),
1393        ));
1394    }
1395    Ok(run)
1396}
1397
1398/// Uses one real model turn to exercise fail-fast, retry exhaustion, repair,
1399/// rejected repair, and skip handling without executing a disallowed call.
1400pub async fn invalid_tool_recovery<W, T, F>(
1401    model: rig_core::driver::Model<W, T>,
1402    configure: F,
1403) -> Result<ScenarioReport, ScenarioError>
1404where
1405    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1406    T: rig_core::driver::Transport<W>,
1407    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1408{
1409    invalid_tool_recovery_with_choice(model, configure, ToolChoice::Required).await
1410}
1411
1412/// [`invalid_tool_recovery`] with the agent's tool choice given, for a model
1413/// that rejects a forced choice (`ToolChoice::Auto`; the prompt still asks for
1414/// the call).
1415pub async fn invalid_tool_recovery_with_choice<W, T, F>(
1416    model: rig_core::driver::Model<W, T>,
1417    configure: F,
1418    tool_choice: ToolChoice,
1419) -> Result<ScenarioReport, ScenarioError>
1420where
1421    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1422    T: rig_core::driver::Transport<W>,
1423    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1424{
1425    const SCENARIO: &str = "invalid_tool_recovery";
1426    const PROMPT: &str = "Call the add tool exactly once with x=2 and y=3. Do not call sum.";
1427    let started = Instant::now();
1428    let add_calls = Arc::new(AtomicUsize::new(0));
1429    let sum_calls = Arc::new(AtomicUsize::new(0));
1430    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1431        .preamble(FORCE_TOOLS_PREAMBLE)
1432        .tool(CountingAdd(add_calls.clone()))
1433        .tool(CountingSum(sum_calls.clone()))
1434        .tool_choice(tool_choice)
1435        .build();
1436    #[derive(Clone)]
1437    struct CaptureTurn(Arc<Mutex<Option<ModelTurn>>>);
1438
1439    impl AgentHook for CaptureTurn {
1440        async fn on_outcome(&self, _ctx: &HookContext, event: OutcomeEvent<'_>) -> OutcomeAction {
1441            let Some(response) = event.completion() else {
1442                return OutcomeAction::proceed();
1443            };
1444            *lock_recover(&self.0) = Some(ModelTurn::new(
1445                response.message_id.clone(),
1446                response.choice.clone(),
1447                response.usage,
1448                BTreeSet::new(),
1449                BTreeSet::new(),
1450                response.raw.clone(),
1451            ));
1452            OutcomeAction::stop("captured conformance model turn")
1453        }
1454    }
1455
1456    let captured = Arc::new(Mutex::new(None));
1457    let stopped = agent
1458        .prompt(PROMPT)
1459        .add_hook(CaptureTurn(captured.clone()))
1460        .run()
1461        .await;
1462    if !matches!(stopped, Err(PromptError::PromptCancelled { .. })) {
1463        return Err(ScenarioError::contract(
1464            SCENARIO,
1465            format!("capture hook did not stop after the model response: {stopped:?}"),
1466        ));
1467    }
1468    let response = lock_recover(&captured).take().ok_or_else(|| {
1469        ScenarioError::contract(SCENARIO, "capture hook observed no model response")
1470    })?;
1471    let emitted = response
1472        .choice
1473        .iter()
1474        .filter(|item| {
1475            matches!(item, AssistantContent::ToolCall(call) if call.function.name == CountingAdd::NAME)
1476        })
1477        .count();
1478    if emitted != 1 {
1479        return Err(ScenarioError::contract(
1480            SCENARIO,
1481            format!(
1482                "model emitted {emitted} add calls, response={:?}",
1483                response.choice
1484            ),
1485        ));
1486    }
1487    let executable = BTreeSet::from([CountingAdd::NAME.to_string(), CountingSum::NAME.to_string()]);
1488    let allowed = BTreeSet::from([CountingSum::NAME.to_string()]);
1489    let turn = ModelTurn::new(
1490        response.message_id,
1491        response.choice,
1492        response.usage,
1493        executable,
1494        allowed,
1495        response.raw,
1496    );
1497
1498    let mut fail = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1499    let error = match fail.resolve_invalid_tool_call(InvalidToolCallAction::fail()) {
1500        Err(error) => error,
1501        Ok(outcome) => {
1502            return Err(ScenarioError::contract(
1503                SCENARIO,
1504                format!("fail action unexpectedly returned {outcome:?}"),
1505            ));
1506        }
1507    };
1508    validate_unknown_tool_failure(&error, CountingAdd::NAME, &[CountingSum::NAME])?;
1509
1510    let mut retry = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1511    let error = match retry
1512        .resolve_invalid_tool_call(InvalidToolCallAction::retry("choose an allowed tool"))
1513    {
1514        Err(error) => error,
1515        Ok(outcome) => {
1516            return Err(ScenarioError::contract(
1517                SCENARIO,
1518                format!("exhausted retry unexpectedly returned {outcome:?}"),
1519            ));
1520        }
1521    };
1522    validate_unknown_tool_failure(&error, CountingAdd::NAME, &[CountingSum::NAME])?;
1523
1524    let mut rejected_repair = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1525    let error =
1526        match rejected_repair.resolve_invalid_tool_call(InvalidToolCallAction::repair("missing")) {
1527            Err(error) => error,
1528            Ok(outcome) => {
1529                return Err(ScenarioError::contract(
1530                    SCENARIO,
1531                    format!("disallowed repair unexpectedly returned {outcome:?}"),
1532                ));
1533            }
1534        };
1535    validate_unknown_tool_failure(&error, "missing", &[CountingSum::NAME])?;
1536
1537    let mut repaired = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1538    if !matches!(
1539        repaired.resolve_invalid_tool_call(InvalidToolCallAction::repair(CountingSum::NAME))?,
1540        ModelTurnOutcome::Continue { .. }
1541    ) {
1542        return Err(ScenarioError::contract(
1543            SCENARIO,
1544            "valid repair did not continue",
1545        ));
1546    }
1547    let AgentRunStep::CallTools { calls } = repaired.next_step()? else {
1548        return Err(ScenarioError::contract(
1549            SCENARIO,
1550            "valid repair did not produce pending tool execution",
1551        ));
1552    };
1553    let repaired_call = calls.first();
1554    if calls.len() != 1
1555        || !repaired_call.is_some_and(|call| {
1556            call.tool_call.function.name == CountingSum::NAME && call.preresolved_result.is_none()
1557        })
1558    {
1559        return Err(ScenarioError::contract(
1560            SCENARIO,
1561            format!("repaired pending calls were incorrect: {calls:?}"),
1562        ));
1563    }
1564
1565    let mut skipped = restricted_recovery_run(PROMPT, turn.clone(), 0)?;
1566    if !matches!(
1567        skipped.resolve_invalid_tool_call(InvalidToolCallAction::skip("disabled for this turn"))?,
1568        ModelTurnOutcome::Continue { .. }
1569    ) {
1570        return Err(ScenarioError::contract(SCENARIO, "skip did not continue"));
1571    }
1572    let AgentRunStep::CallTools { calls } = skipped.next_step()? else {
1573        return Err(ScenarioError::contract(
1574            SCENARIO,
1575            "skip did not produce a pre-resolved pending call",
1576        ));
1577    };
1578    let skipped_is_preresolved = match calls.first() {
1579        Some(call) => call.preresolved_result.is_some(),
1580        None => false,
1581    };
1582    if calls.len() != 1 || !skipped_is_preresolved {
1583        return Err(ScenarioError::contract(
1584            SCENARIO,
1585            format!("skipped pending calls were incorrect: {calls:?}"),
1586        ));
1587    }
1588    if add_calls.load(Ordering::SeqCst) != 0 || sum_calls.load(Ordering::SeqCst) != 0 {
1589        return Err(ScenarioError::contract(
1590            SCENARIO,
1591            "recovery scenario executed a tool body",
1592        ));
1593    }
1594
1595    Ok(ScenarioReport {
1596        name: SCENARIO,
1597        tool_calls: emitted,
1598        prompt_tokens: turn.usage.input_tokens.unwrap_or(0),
1599        generated_tokens: turn.usage.output_tokens.unwrap_or(0),
1600        history_messages: 2,
1601        duration: started.elapsed(),
1602        response: "fail, retry, repair, rejected repair, and skip passed".to_string(),
1603    })
1604}
1605
1606/// Exercises chained argument/result rewrites and a first-turn-only request
1607/// patch. Completion proves `tool_choice=Required` did not leak to turn two.
1608pub async fn hook_rewrites_and_request_patch<W, T, F>(
1609    model: rig_core::driver::Model<W, T>,
1610    configure: F,
1611) -> Result<ScenarioReport, ScenarioError>
1612where
1613    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1614    T: rig_core::driver::Transport<W>,
1615    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1616{
1617    hook_rewrites_and_request_patch_with_choice(model, configure, ToolChoice::Required).await
1618}
1619
1620/// [`hook_rewrites_and_request_patch`] with the first-turn patch's tool
1621/// choice given, for a model that rejects a forced choice (`ToolChoice::Auto`:
1622/// the patch still narrows the active tools on turn one only).
1623pub async fn hook_rewrites_and_request_patch_with_choice<W, T, F>(
1624    model: rig_core::driver::Model<W, T>,
1625    configure: F,
1626    first_turn_choice: ToolChoice,
1627) -> Result<ScenarioReport, ScenarioError>
1628where
1629    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1630    T: rig_core::driver::Transport<W>,
1631    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1632{
1633    const SCENARIO: &str = "hook_rewrites_and_request_patch";
1634    let started = Instant::now();
1635    let calls = Arc::new(AtomicUsize::new(0));
1636    let observed = ObserveArguments::default();
1637    let observed_probe = observed.clone();
1638    let agent = configure(AgentBuilder::new(model).temperature(0.0))
1639        .preamble("Use add for arithmetic and report only the tool result.")
1640        .tool(CountingAdd(calls.clone()))
1641        .default_max_turns(3)
1642        .build();
1643    let response = agent
1644        .prompt("Use add once for x=1 and y=1, then report what the tool returns.")
1645        .max_turns(3)
1646        .add_hook(FirstTurnPatch(
1647            RequestPatch::new()
1648                .active_tools([CountingAdd::NAME])
1649                .tool_choice(first_turn_choice),
1650        ))
1651        .add_hook(RewriteArgument {
1652            key: "x",
1653            value: serde_json::json!(7),
1654        })
1655        .add_hook(RewriteArgument {
1656            key: "y",
1657            value: serde_json::json!(8),
1658        })
1659        .add_hook(observed)
1660        .add_hook(ReplaceResult("portable-redacted"))
1661        .add_hook(WrapResult)
1662        .await?;
1663    let observations = lock_recover(&observed_probe.0).clone();
1664    validate_rewritten_arguments(
1665        SCENARIO,
1666        &observations,
1667        &serde_json::json!({ "x": 7, "y": 8 }),
1668    )?;
1669    let messages = response.messages.as_deref().ok_or_else(|| {
1670        ScenarioError::contract(SCENARIO, "extended hook run omitted message history")
1671    })?;
1672    let results = messages
1673        .iter()
1674        .flat_map(tool_result_values)
1675        .collect::<Vec<_>>();
1676    if calls.load(Ordering::SeqCst) != 1
1677        || !results
1678            .iter()
1679            .any(|value| value == &serde_json::json!("[portable-redacted]"))
1680        || response.completion_calls.len() != 2
1681    {
1682        return Err(ScenarioError::contract(
1683            SCENARIO,
1684            format!(
1685                "calls={}, completion_calls={}, results={results:?}, output={:?}",
1686                calls.load(Ordering::SeqCst),
1687                response.completion_calls.len(),
1688                response.output
1689            ),
1690        ));
1691    }
1692    report_from_response(SCENARIO, started, 1, response)
1693}
1694
1695/// Exercises post-execution cancellation and max-turn diagnostics through the
1696/// public agent driver using real model-emitted calls.
1697pub async fn cancellation_and_max_turns<W, T, F>(
1698    model: rig_core::driver::Model<W, T>,
1699    configure: F,
1700) -> Result<ScenarioReport, ScenarioError>
1701where
1702    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1703    T: rig_core::driver::Transport<W>,
1704    F: Fn(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1705{
1706    const SCENARIO: &str = "cancellation_and_max_turns";
1707    const REASON: &str = "portable result veto";
1708    let started = Instant::now();
1709    let cancelled_calls = Arc::new(AtomicUsize::new(0));
1710    let cancelled_agent = configure(AgentBuilder::new(model.clone()).temperature(0.0))
1711        .preamble("Use add for arithmetic; never calculate by hand.")
1712        .tool(CountingAdd(cancelled_calls.clone()))
1713        .build();
1714    let cancelled = match cancelled_agent
1715        .prompt("Use add once to compute x=20 plus y=22.")
1716        .max_turns(2)
1717        .add_hook(StopAfterResult(REASON))
1718        .await
1719    {
1720        Err(error) => error,
1721        Ok(output) => {
1722            return Err(ScenarioError::contract(
1723                SCENARIO,
1724                format!("result cancellation unexpectedly completed: {output:?}"),
1725            ));
1726        }
1727    };
1728    validate_cancelled_failure(&cancelled, REASON, CountingAdd::NAME)?;
1729
1730    let max_turn_calls = Arc::new(AtomicUsize::new(0));
1731    let max_turn_agent = configure(AgentBuilder::new(model).temperature(0.0))
1732        .preamble("Use add for arithmetic; never calculate by hand.")
1733        .tool(CountingAdd(max_turn_calls.clone()))
1734        .build();
1735    let max_turn = match max_turn_agent
1736        .prompt("Use add once to compute x=20 plus y=22, then report the result.")
1737        .max_turns(1)
1738        .await
1739    {
1740        Err(error) => error,
1741        Ok(output) => {
1742            return Err(ScenarioError::contract(
1743                SCENARIO,
1744                format!("one-turn budget unexpectedly completed: {output:?}"),
1745            ));
1746        }
1747    };
1748    validate_max_turns_failure(&max_turn, 1)?;
1749    let cancelled_count = cancelled_calls.load(Ordering::SeqCst);
1750    let max_turn_count = max_turn_calls.load(Ordering::SeqCst);
1751    if cancelled_count != 1 || max_turn_count != 1 {
1752        return Err(ScenarioError::contract(
1753            SCENARIO,
1754            format!("cancelled executions={cancelled_count}, max-turn executions={max_turn_count}"),
1755        ));
1756    }
1757    Ok(ScenarioReport {
1758        name: SCENARIO,
1759        tool_calls: cancelled_count + max_turn_count,
1760        prompt_tokens: 0,
1761        generated_tokens: 0,
1762        history_messages: 4,
1763        duration: started.elapsed(),
1764        response: "post-result cancellation and max-turn diagnostics passed".to_string(),
1765    })
1766}
1767
1768/// Runs the portable optional-argument tool scenario.
1769///
1770/// `configure` is deliberately outside the scenario so a provider suite can
1771/// attach transport-only settings without putting them into the shared model
1772/// contract.
1773pub async fn optional_argument<W, T, F>(
1774    model: rig_core::driver::Model<W, T>,
1775    configure: F,
1776) -> Result<ScenarioReport, ScenarioError>
1777where
1778    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1779    T: rig_core::driver::Transport<W>,
1780    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1781{
1782    let calls = Arc::new(AtomicUsize::new(0));
1783    let started = Instant::now();
1784    let agent = configure(AgentBuilder::new(model))
1785        .preamble("Use the repeat_text tool whenever asked to repeat text.")
1786        .tool(RepeatTool {
1787            calls: calls.clone(),
1788        })
1789        .default_max_turns(4)
1790        .build();
1791    let result = agent
1792        .prompt(
1793            "Use the repeat_text tool to repeat the word \"banana\" 3 times, then show me the exact result.",
1794        )
1795        .await?;
1796    let response = result.output.clone();
1797    let tool_calls = calls.load(Ordering::SeqCst);
1798    if tool_calls == 0
1799        || response.matches("banana").count() < 1
1800        || !has_tool_roundtrip(result.messages.as_deref())
1801    {
1802        return Err(ScenarioError::contract(
1803            "optional_argument",
1804            format!("calls={tool_calls}, response={response:?}"),
1805        ));
1806    }
1807    report_from_response("optional_argument", started, tool_calls, result)
1808}
1809
1810/// Runs a portable two-tool sequential arithmetic scenario.
1811pub async fn sequential_tools<W, T, F>(
1812    model: rig_core::driver::Model<W, T>,
1813    configure: F,
1814) -> Result<ScenarioReport, ScenarioError>
1815where
1816    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1817    T: rig_core::driver::Transport<W>,
1818    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1819{
1820    let add_calls = Arc::new(AtomicUsize::new(0));
1821    let multiply_calls = Arc::new(AtomicUsize::new(0));
1822    let started = Instant::now();
1823    let agent = configure(AgentBuilder::new(model))
1824        .preamble(
1825            "You are a calculator. Use the add and multiply tools for arithmetic; never compute by hand.",
1826        )
1827        .tool(AddTool(add_calls.clone()))
1828        .tool(MultiplyTool(multiply_calls.clone()))
1829        .default_max_turns(6)
1830        .build();
1831    let result = agent
1832        .prompt(
1833            "Compute (4 + 6) * 2. First call the add tool, then call the multiply tool on the result. Tell me the final number.",
1834        )
1835        .await?;
1836    let response = result.output.clone();
1837    let add = add_calls.load(Ordering::SeqCst);
1838    let multiply = multiply_calls.load(Ordering::SeqCst);
1839    if add == 0
1840        || multiply == 0
1841        || !response.contains("20")
1842        || !has_tool_roundtrip(result.messages.as_deref())
1843    {
1844        return Err(ScenarioError::contract(
1845            "sequential_tools",
1846            format!("add={add}, multiply={multiply}, response={response:?}"),
1847        ));
1848    }
1849    report_from_response("sequential_tools", started, add + multiply, result)
1850}
1851
1852/// Runs a tool through Rig's multi-turn streaming agent driver.
1853pub async fn streaming_tool<W, T, F>(
1854    model: rig_core::driver::Model<W, T>,
1855    configure: F,
1856) -> Result<ScenarioReport, ScenarioError>
1857where
1858    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1859    T: rig_core::driver::Transport<W>,
1860    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1861{
1862    let calls = Arc::new(AtomicUsize::new(0));
1863    let started = Instant::now();
1864    let agent = configure(AgentBuilder::new(model))
1865        .preamble("Use the add tool for arithmetic; do not calculate by hand.")
1866        .tool(AddTool(calls.clone()))
1867        .default_max_turns(4)
1868        .build();
1869    let mut stream = agent
1870        .prompt("Use add to calculate 17 + 25, then state the final number.")
1871        .max_turns(4)
1872        .stream();
1873    let mut final_response = None;
1874    let mut final_count = 0_usize;
1875    let mut completion_usage = crate::completion::Usage::default();
1876    let mut streamed_call_ids = Vec::new();
1877    let mut streamed_result_ids = Vec::new();
1878    while let Some(item) = stream.next().await {
1879        match item? {
1880            MultiTurnStreamItem::ToolCall { tool_call } => streamed_call_ids.push(tool_call.id),
1881            MultiTurnStreamItem::StreamUserItem(
1882                crate::streaming::StreamedUserContent::ToolResult { tool_result },
1883            ) => streamed_result_ids.push(tool_result.call),
1884            MultiTurnStreamItem::CompletionCall(call) => completion_usage += call.usage,
1885            MultiTurnStreamItem::FinalResponse(response) => {
1886                final_count += 1;
1887                final_response = Some(response);
1888            }
1889            MultiTurnStreamItem::StreamAssistantItem(_)
1890            | MultiTurnStreamItem::ToolExecutionCommitted { .. }
1891            | MultiTurnStreamItem::ModelTurnRetried { .. } => {}
1892        }
1893    }
1894    let result = final_response.ok_or_else(|| {
1895        ScenarioError::contract("streaming_tool", "stream produced no final response")
1896    })?;
1897    let response = result.output.clone();
1898    let history_messages = result.messages.as_ref().map_or(0, Vec::len);
1899    let tool_calls = calls.load(Ordering::SeqCst);
1900    streamed_call_ids.sort();
1901    streamed_result_ids.sort();
1902    let correlated_stream =
1903        !streamed_call_ids.is_empty() && streamed_call_ids == streamed_result_ids;
1904    let correlated_history = result
1905        .messages
1906        .as_deref()
1907        .is_some_and(|messages| validate_tool_correlation("streaming_tool", messages).is_ok());
1908    if tool_calls == 0
1909        || !response.contains("42")
1910        || history_messages < 4
1911        || final_count != 1
1912        || !correlated_stream
1913        || !correlated_history
1914        || completion_usage != result.usage
1915        || result.completion_calls.is_empty()
1916    {
1917        return Err(ScenarioError::contract(
1918            "streaming_tool",
1919            format!(
1920                "calls={tool_calls}, final_count={final_count}, streamed_call_ids={streamed_call_ids:?}, streamed_result_ids={streamed_result_ids:?}, completion_usage={completion_usage:?}, final_usage={:?}, history_messages={history_messages}, response={response:?}",
1921                result.usage
1922            ),
1923        ));
1924    }
1925    if let Some(messages) = result.messages.as_deref() {
1926        validate_protocol_hygiene(
1927            "streaming_tool",
1928            &response,
1929            messages,
1930            &["<tool_call>", "</tool_call>", "<think>", "</think>"],
1931        )?;
1932    }
1933    Ok(ScenarioReport {
1934        name: "streaming_tool",
1935        tool_calls,
1936        prompt_tokens: result.usage.input_tokens.unwrap_or(0),
1937        generated_tokens: result.usage.output_tokens.unwrap_or(0),
1938        history_messages,
1939        duration: started.elapsed(),
1940        response,
1941    })
1942}
1943
1944/// Runs a normal tool followed by Rig's synthetic structured-output tool.
1945pub async fn structured_after_tool<W, T, F>(
1946    model: rig_core::driver::Model<W, T>,
1947    configure: F,
1948) -> Result<ScenarioReport, ScenarioError>
1949where
1950    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1951    T: rig_core::driver::Transport<W>,
1952    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
1953{
1954    let calls = Arc::new(AtomicUsize::new(0));
1955    let started = Instant::now();
1956    let agent = configure(AgentBuilder::new(model))
1957        .preamble(
1958            "Use add for arithmetic, then finish by calling the structured output tool exactly once.",
1959        )
1960        .output_schema::<ArithmeticResult>()
1961        .output_mode(OutputMode::Tool)
1962        .tool(AddTool(calls.clone()))
1963        .default_max_turns(5)
1964        .build();
1965    let result = agent
1966        .prompt("Use add to calculate 19 + 23. Return answer=42 and a short optional explanation.")
1967        .await?;
1968    let response = result.output.clone();
1969    let parsed: ArithmeticResult = serde_json::from_str(&response)?;
1970    let tool_calls = calls.load(Ordering::SeqCst);
1971    if tool_calls == 0 || parsed.answer != 42 || !has_tool_roundtrip(result.messages.as_deref()) {
1972        return Err(ScenarioError::contract(
1973            "structured_after_tool",
1974            format!("calls={tool_calls}, response={response:?}"),
1975        ));
1976    }
1977    let _ = parsed.explanation;
1978    report_from_response("structured_after_tool", started, tool_calls + 1, result)
1979}
1980
1981/// Runs all portable tool-choice modes directly against a completion model.
1982pub async fn tool_choice_modes<W, T, A>(
1983    model: rig_core::driver::Model<W, T>,
1984    adjust: A,
1985) -> Result<ScenarioReport, ScenarioError>
1986where
1987    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
1988    T: rig_core::driver::Transport<W>,
1989    A: Fn(CompletionRequest) -> CompletionRequest,
1990{
1991    let definition = |name: &str| ToolDefinition {
1992        name: name.to_string(),
1993        description: format!("Return the supplied integer using {name}."),
1994        parameters: serde_json::json!({
1995            "type": "object",
1996            "properties": {"value": {"type": "integer"}},
1997            "required": ["value"]
1998        }),
1999    };
2000    let tools = vec![definition("alpha"), definition("beta")];
2001    let started = Instant::now();
2002    let none = model
2003        .call(adjust(
2004            CompletionRequest::new("Answer with only the number 4. Do not call a function.")
2005                .tools(tools.clone())
2006                .tool_choice(ToolChoice::None)
2007                .temperature(0.0)
2008                .max_tokens(64),
2009        ))
2010        .await?;
2011    if none
2012        .choice
2013        .iter()
2014        .any(|item| matches!(item, AssistantContent::ToolCall(_)))
2015    {
2016        return Err(ScenarioError::contract(
2017            "tool_choice_modes",
2018            "tool_choice none emitted a tool call",
2019        ));
2020    }
2021
2022    let required = model
2023        .call(adjust(
2024            CompletionRequest::new("Call alpha with value 7.")
2025                .tools(tools.clone())
2026                .tool_choice(ToolChoice::Required)
2027                .temperature(0.0)
2028                .max_tokens(96),
2029        ))
2030        .await?;
2031    let required_calls = required
2032        .choice
2033        .iter()
2034        .filter(|item| matches!(item, AssistantContent::ToolCall(_)))
2035        .count();
2036    if required_calls == 0 {
2037        return Err(ScenarioError::contract(
2038            "tool_choice_modes",
2039            "tool_choice required emitted no tool call",
2040        ));
2041    }
2042
2043    let specific = model
2044        .call(adjust(
2045            CompletionRequest::new("Call beta with value 9.")
2046                .tools(tools)
2047                .tool_choice(ToolChoice::Specific {
2048                    function_names: vec!["beta".to_string()],
2049                })
2050                .temperature(0.0)
2051                .max_tokens(96),
2052        ))
2053        .await?;
2054    let specific_calls = specific
2055        .choice
2056        .iter()
2057        .filter_map(|item| match item {
2058            AssistantContent::ToolCall(call) => Some(call),
2059            _ => None,
2060        })
2061        .collect::<Vec<_>>();
2062    if specific_calls.is_empty()
2063        || specific_calls
2064            .iter()
2065            .any(|call| call.function.name != "beta")
2066    {
2067        return Err(ScenarioError::contract(
2068            "tool_choice_modes",
2069            "specific tool choice did not select only beta",
2070        ));
2071    }
2072
2073    let usage = none.usage + required.usage + specific.usage;
2074    Ok(ScenarioReport {
2075        name: "tool_choice_modes",
2076        tool_calls: required_calls + specific_calls.len(),
2077        prompt_tokens: usage.input_tokens.unwrap_or(0),
2078        generated_tokens: usage.output_tokens.unwrap_or(0),
2079        history_messages: 0,
2080        duration: started.elapsed(),
2081        response: "none, required, and specific modes passed".to_string(),
2082    })
2083}
2084
2085/// Runs a streamed real-tool turn followed by Rig's synthetic output tool.
2086pub async fn streaming_structured_after_tool<W, T, F>(
2087    model: rig_core::driver::Model<W, T>,
2088    configure: F,
2089) -> Result<ScenarioReport, ScenarioError>
2090where
2091    W: rig_core::wire::Wire<Op = rig_core::operation::Completion>,
2092    T: rig_core::driver::Transport<W>,
2093    F: FnOnce(AgentBuilder<NoToolConfig>) -> AgentBuilder<NoToolConfig>,
2094{
2095    let calls = Arc::new(AtomicUsize::new(0));
2096    let started = Instant::now();
2097    let agent = configure(AgentBuilder::new(model))
2098        .preamble(
2099            "Use add for arithmetic, then finish by calling the structured output tool exactly once.",
2100        )
2101        .output_schema::<ArithmeticResult>()
2102        .output_mode(OutputMode::Tool)
2103        .tool(AddTool(calls.clone()))
2104        .default_max_turns(5)
2105        .build();
2106    let mut stream = agent
2107        .prompt("Use add to calculate 19 + 23. Return answer=42 and a short optional explanation.")
2108        .max_turns(5)
2109        .stream();
2110    let mut final_response = None;
2111    let mut final_count = 0_usize;
2112    while let Some(item) = stream.next().await {
2113        if let MultiTurnStreamItem::FinalResponse(response) = item? {
2114            final_count += 1;
2115            final_response = Some(response);
2116        }
2117    }
2118    let result = final_response.ok_or_else(|| {
2119        ScenarioError::contract(
2120            "streaming_structured_after_tool",
2121            "stream produced no final response",
2122        )
2123    })?;
2124    let parsed: ArithmeticResult = serde_json::from_str(&result.output)?;
2125    let calls = calls.load(Ordering::SeqCst);
2126    if calls == 0
2127        || final_count != 1
2128        || parsed.answer != 42
2129        || !has_tool_roundtrip(result.messages.as_deref())
2130    {
2131        return Err(ScenarioError::contract(
2132            "streaming_structured_after_tool",
2133            format!(
2134                "calls={calls}, final_count={final_count}, response={:?}",
2135                result.output
2136            ),
2137        ));
2138    }
2139    report_from_response(
2140        "streaming_structured_after_tool",
2141        started,
2142        calls + 1,
2143        result,
2144    )
2145}
2146
2147#[cfg(test)]
2148mod tests;