Skip to main content

rig_core/test_utils/
completion.rs

1//! Completion helpers for deterministic agent-loop tests.
2
3use std::{
4    collections::VecDeque,
5    sync::{Arc, Mutex, MutexGuard},
6};
7
8use crate::driver::{Exchange, Model, Opened, Opening, Transport};
9use crate::error::{EncodeError, ProviderError};
10use crate::lock;
11use crate::operation::Completion;
12use crate::wire::{Capabilities, Descriptor, Mode, Wire};
13use crate::{
14    completion::{AssistantContent, CompletionRequest, CompletionResponse, Usage},
15    message::{ToolCall, ToolFunction},
16};
17
18use super::streaming::{MOCK_PROVIDER, MockDecoder, MockDocument, MockFrame, MockStreamEvent};
19
20/// Scripted error returned by [`MockCompletionModel`].
21#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
22pub enum MockError {
23    /// Provider error.
24    Provider(String),
25    /// Request construction error.
26    Request(String),
27    /// A preserved provider error response, id included.
28    ProviderResponse(crate::provider_response::ProviderResponseError),
29    /// A streamed reply that sent no frame for this long.
30    Stalled(std::time::Duration),
31}
32
33impl MockError {
34    /// Create a provider error.
35    pub fn provider(message: impl Into<String>) -> Self {
36        Self::Provider(message.into())
37    }
38
39    /// Create a request error.
40    pub fn request(message: impl Into<String>) -> Self {
41        Self::Request(message.into())
42    }
43
44    pub(crate) fn into_completion_error(self) -> ProviderError {
45        match self {
46            Self::Provider(message) => ProviderError::Provider(message),
47            Self::Request(message) => ProviderError::request(message),
48            Self::ProviderResponse(response) => ProviderError::ProviderResponse(response),
49            Self::Stalled(idle) => ProviderError::Stalled(idle),
50        }
51    }
52}
53
54/// A scripted non-streaming mock completion turn.
55///
56/// A turn is data: a script serializes, so a scripted model can be written
57/// to a fixture and read back (see `MockCompletionModel::script`).
58#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
59pub struct MockTurn {
60    response: Result<MockTurnResponse, MockError>,
61}
62
63#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
64struct MockTurnResponse {
65    choice: Vec<AssistantContent>,
66    usage: Usage,
67    response_id: Option<String>,
68    provider_request_id: Option<String>,
69    finish_reason: Option<crate::completion::FinishReason>,
70    /// A scripted provider document, when the test supplies one; otherwise
71    /// the turn itself, serialized, is the mock's document. Absent from the
72    /// serialized turn when unscripted, so that document never nests
73    /// itself; a scripted one survives a serde round trip of the script.
74    #[serde(
75        default,
76        skip_serializing_if = "Option::is_none",
77        deserialize_with = "deserialize_scripted_raw"
78    )]
79    raw: Option<serde_json::Value>,
80}
81
82fn deserialize_scripted_raw<'de, D: serde::Deserializer<'de>>(
83    deserializer: D,
84) -> Result<Option<serde_json::Value>, D::Error> {
85    // Only a missing field means unscripted; explicit JSON null is a document.
86    serde::Deserialize::deserialize(deserializer).map(Some)
87}
88
89impl MockTurn {
90    /// Create a text response turn.
91    pub fn text(text: impl Into<String>) -> Self {
92        Self::from_content(AssistantContent::text(text.into()))
93    }
94
95    /// Create a tool-call response turn. An empty `name` scripts a provider
96    /// error instead: no call can be built without a name.
97    pub fn tool_call(
98        id: impl Into<String>,
99        name: impl Into<String>,
100        arguments: serde_json::Value,
101    ) -> Self {
102        match crate::message::ToolName::new(name) {
103            Ok(name) => Self::from_content(AssistantContent::ToolCall(ToolCall::from_wire(
104                id,
105                ToolFunction::new(name, arguments),
106            ))),
107            Err(error) => Self::error(error.to_string()),
108        }
109    }
110
111    /// Create a provider-error response turn.
112    pub fn error(message: impl Into<String>) -> Self {
113        Self {
114            response: Err(MockError::provider(message)),
115        }
116    }
117
118    /// Create a provider-response error turn carrying a transport request id:
119    /// the scripted failure a test uses to assert error-identity
120    /// attribution.
121    pub fn provider_response_error(
122        status: http::StatusCode,
123        body: impl Into<String>,
124        request_id: impl Into<String>,
125    ) -> Self {
126        Self {
127            response: Err(MockError::ProviderResponse(
128                crate::provider_response::ProviderResponseError::new(status, body)
129                    .with_provider_request_id(Some(request_id.into())),
130            )),
131        }
132    }
133
134    /// Create a request-error response turn.
135    pub fn request_error(message: impl Into<String>) -> Self {
136        Self {
137            response: Err(MockError::request(message)),
138        }
139    }
140
141    /// Create a response turn from one assistant content item.
142    pub fn from_content(content: AssistantContent) -> Self {
143        Self {
144            response: Ok(MockTurnResponse {
145                choice: vec![content],
146                usage: Usage::default(),
147                response_id: None,
148                provider_request_id: None,
149                finish_reason: None,
150                raw: None,
151            }),
152        }
153    }
154
155    /// Create a response turn from assistant content items.
156    ///
157    /// Infallible now that content is a `Vec`: an empty turn is a shape a
158    /// provider can genuinely return, so it is a value to build, not an error.
159    pub fn from_contents(content: impl IntoIterator<Item = AssistantContent>) -> Self {
160        Self {
161            response: Ok(MockTurnResponse {
162                choice: content.into_iter().collect(),
163                usage: Usage::default(),
164                response_id: None,
165                provider_request_id: None,
166                finish_reason: None,
167                raw: None,
168            }),
169        }
170    }
171
172    /// Attach a provider-specific call ID to a tool-call response turn.
173    pub fn with_call_id(mut self, call_id: impl Into<String>) -> Self {
174        let call_id = call_id.into();
175        if let Ok(response) = &mut self.response {
176            for content in response.choice.iter_mut() {
177                if let AssistantContent::ToolCall(tool_call) = content {
178                    tool_call.id = crate::message::CallId::from_wire(call_id);
179                    break;
180                }
181            }
182        }
183        self
184    }
185
186    /// Override usage for this turn.
187    pub fn with_usage(mut self, usage: Usage) -> Self {
188        if let Ok(response) = &mut self.response {
189            response.usage = usage;
190        }
191        self
192    }
193
194    /// Set a provider-assigned response-scoped ID for this turn.
195    pub fn with_response_id(mut self, response_id: impl Into<String>) -> Self {
196        if let Ok(response) = &mut self.response {
197            response.response_id = Some(response_id.into());
198        }
199        self
200    }
201
202    /// Set a provider transport request id for this turn.
203    pub fn with_provider_request_id(mut self, request_id: impl Into<String>) -> Self {
204        if let Ok(response) = &mut self.response {
205            response.provider_request_id = Some(request_id.into());
206        }
207        self
208    }
209
210    /// Set the terminal finish reason for this turn.
211    ///
212    /// Without this, a mocked blocking turn always reports `None`, which
213    /// leaves the whole blocking half of the truncation contract
214    /// unexercisable — the streamed mock could script a reason and the
215    /// blocking one could not.
216    pub fn with_finish_reason(mut self, finish_reason: crate::completion::FinishReason) -> Self {
217        if let Ok(response) = &mut self.response {
218            response.finish_reason = Some(finish_reason);
219        }
220        self
221    }
222
223    /// The provider document the mock attaches to this turn's response: the
224    /// scripted payload when one was supplied, otherwise the turn itself,
225    /// serialized. Public so a test can state the expected `raw` of a
226    /// recorded call without repeating the mock's serialization. An error
227    /// turn has no document.
228    pub fn raw(&self) -> Result<serde_json::Value, ProviderError> {
229        let response = self
230            .response
231            .as_ref()
232            .map_err(|error| error.clone().into_completion_error())?;
233        match &response.raw {
234            Some(raw) => Ok(raw.clone()),
235            None => Ok(serde_json::to_value(response)?),
236        }
237    }
238
239    fn into_completion_response(self) -> Result<CompletionResponse, ProviderError> {
240        let raw = self.raw()?;
241        let response = self.response.map_err(MockError::into_completion_error)?;
242        let mut origin = crate::message::Origin::new(MOCK_API, MOCK_PROVIDER, "");
243        origin.response_id = response.response_id;
244        let mut completion = CompletionResponse::new(response.choice, response.usage, origin, raw)
245            .with_optional_finish_reason(response.finish_reason);
246        completion.provider_request_id = response.provider_request_id;
247        Ok(completion)
248    }
249}
250
251#[derive(Default)]
252struct MockScriptState {
253    turns: Mutex<VecDeque<MockTurn>>,
254    stream_turns: Mutex<VecDeque<Vec<MockStreamEvent>>>,
255    requests: Mutex<Vec<CompletionRequest>>,
256}
257
258/// The scripted completion wire: its payload is the request, and each frame
259/// is a scripted step its decoder writes through the completion writer's
260/// part handles, as any wire's decoder does. A runtime that answers from a
261/// script ([`MockRuntime`], or a test's own) is its transport.
262#[derive(Clone, Debug, PartialEq, Eq)]
263pub struct MockScript {
264    name: String,
265    id: Option<String>,
266    capabilities: Capabilities,
267    /// The facts its replies are priced by, as a real wire's are.
268    facts: crate::catalog::ModelFacts,
269}
270
271impl MockScript {
272    /// A scripted wire named `name`, as records and telemetry name it, with
273    /// default capabilities.
274    pub fn new(name: impl Into<String>) -> Self {
275        Self {
276            name: name.into(),
277            id: None,
278            capabilities: Capabilities::default(),
279            facts: crate::catalog::ModelFacts::default(),
280        }
281    }
282
283    /// The same wire, addressing the model `id`.
284    pub fn with_id(mut self, id: impl Into<String>) -> Self {
285        self.id = Some(id.into());
286        self
287    }
288
289    /// The same wire, reporting `capabilities`.
290    pub fn with_capabilities(mut self, capabilities: Capabilities) -> Self {
291        self.capabilities = capabilities;
292        self
293    }
294
295    /// The same wire answering as `spec`'s model: named as its provider and
296    /// id, and pricing each reply's usage at the spec's prices, as the
297    /// provider's own wire would. A reply whose usage states its cost keeps
298    /// it.
299    pub fn priced_as(self, spec: impl Into<std::sync::Arc<crate::catalog::ModelSpec>>) -> Self {
300        let spec = spec.into();
301        Self {
302            name: spec.provider.vendor().to_owned(),
303            id: Some(spec.id.clone()),
304            facts: crate::catalog::ModelFacts::new(spec),
305            ..self
306        }
307    }
308}
309
310/// The wire format the scripted wire names.
311pub const MOCK_API: crate::message::Api = crate::message::Api::from_static("mock.script");
312
313/// The model a scripted wire addresses when it names none.
314pub const MOCK_MODEL: &str = "mock-model";
315
316impl crate::completion::ReplayTarget for MockScript {
317    fn api(&self) -> crate::message::Api {
318        MOCK_API
319    }
320
321    /// A script takes every option and sends nothing for it, so a test
322    /// reads the options it set on the request the script records.
323    fn map_options(
324        &self,
325        _request: &crate::completion::CompletionRequest,
326        fields: crate::completion::options::OptionFields<'_>,
327    ) -> crate::completion::options::OptionMap {
328        each_set(fields, || {
329            crate::completion::options::Mapping::Omit("a scripted reply ignores options")
330        })
331    }
332
333    // Scripts state a finish reason only when a test is about one.
334    fn states_finish_reason(&self) -> bool {
335        false
336    }
337
338    fn provider(&self) -> &str {
339        &self.name
340    }
341
342    /// The model the script addresses: its id, or [`MOCK_MODEL`].
343    fn model(&self) -> &str {
344        self.id.as_deref().unwrap_or(MOCK_MODEL)
345    }
346
347    fn accepts(&self, _model: &str) -> crate::completion::Accepts {
348        crate::completion::Accepts::ALL
349    }
350}
351
352impl Default for MockScript {
353    fn default() -> Self {
354        Self::new(MOCK_PROVIDER)
355    }
356}
357
358impl Wire for MockScript {
359    type Op = Completion;
360    type Payload = CompletionRequest;
361    type Frame = MockFrame;
362    type Event = MockFrame;
363    type Decoder<'id> = MockDecoder<'id>;
364    type Reassembler = MockDocument;
365
366    fn describe(&self) -> Descriptor<'_> {
367        Descriptor::new(&self.name)
368            .model(self.id.as_deref())
369            .capabilities(self.capabilities)
370            .facts(&self.facts)
371            .replay(self)
372    }
373
374    /// The request as the runtime received it, its history already shaped
375    /// for this wire.
376    fn encode(
377        &self,
378        request: CompletionRequest,
379        _mode: Mode,
380    ) -> Result<CompletionRequest, EncodeError> {
381        Ok(request)
382    }
383
384    fn decoder<'id>(&self) -> MockDecoder<'id> {
385        MockDecoder::default()
386    }
387}
388
389/// The scripted runtime behind [`MockCompletionModel`]: the transport of a
390/// [`MockScript`] wire named [`MOCK_PROVIDER`].
391///
392/// Each call consumes exactly one scripted turn. If no turn is available,
393/// the call fails with [`ProviderError::Provider`] and a clear message
394/// instead of repeating previous responses.
395#[derive(Clone, Default)]
396pub struct MockRuntime {
397    state: Arc<MockScriptState>,
398}
399
400/// A cloneable scripted completion model for tests: the scripted wire over
401/// its runtime. Clones share the script and the recorded requests.
402pub type MockCompletionModel = Model<MockScript, MockRuntime>;
403
404impl MockCompletionModel {
405    /// Create a mock model that returns one text completion.
406    pub fn text(text: impl Into<String>) -> Self {
407        Self::from_turns([MockTurn::text(text)])
408    }
409
410    /// Create a mock model from scripted non-streaming turns.
411    pub fn from_turns(turns: impl IntoIterator<Item = MockTurn>) -> Self {
412        Self::scripted(turns.into_iter().collect(), VecDeque::new())
413    }
414
415    /// Create a mock model from scripted streaming turns.
416    pub fn from_stream_turns(
417        stream_turns: impl IntoIterator<Item = impl IntoIterator<Item = MockStreamEvent>>,
418    ) -> Self {
419        Self::scripted(
420            VecDeque::new(),
421            stream_turns
422                .into_iter()
423                .map(|turn| turn.into_iter().collect())
424                .collect(),
425        )
426    }
427
428    fn scripted(turns: VecDeque<MockTurn>, stream_turns: VecDeque<Vec<MockStreamEvent>>) -> Self {
429        Model::new(
430            MockScript::default(),
431            MockRuntime {
432                state: Arc::new(MockScriptState {
433                    turns: Mutex::new(turns),
434                    stream_turns: Mutex::new(stream_turns),
435                    requests: Mutex::new(Vec::new()),
436                }),
437            },
438        )
439    }
440
441    /// The same scripted model answering as `spec`'s model and pricing its
442    /// replies at the spec's prices ([`MockScript::priced_as`]), for a test
443    /// of what spending is counted.
444    pub fn priced_as(mut self, spec: impl Into<std::sync::Arc<crate::catalog::ModelSpec>>) -> Self {
445        self.wire = self.wire.priced_as(spec);
446        self
447    }
448
449    /// Return cloned requests received by this model.
450    pub fn requests(&self) -> Vec<CompletionRequest> {
451        self.transport.requests_guard().clone()
452    }
453
454    /// Return the number of requests received by this model.
455    pub fn request_count(&self) -> usize {
456        self.transport.requests_guard().len()
457    }
458
459    /// The non-streaming turns not yet consumed, in order — the read-back
460    /// half of the script, so a script is serde in and serde out.
461    pub fn script(&self) -> Vec<MockTurn> {
462        lock(&self.transport.state.turns).iter().cloned().collect()
463    }
464}
465
466impl MockRuntime {
467    fn requests_guard(&self) -> MutexGuard<'_, Vec<CompletionRequest>> {
468        lock(&self.state.requests)
469    }
470}
471
472impl Transport<MockScript> for MockRuntime {
473    fn send(&self, request: CompletionRequest, exchange: Exchange) -> Opening<MockFrame> {
474        let mode = exchange.mode;
475        self.requests_guard().push(request);
476        match mode {
477            // A whole turn is one response, its document the response's
478            // `raw`; a scripted failure fails the reply.
479            Mode::Unary => {
480                let Some(turn) = lock(&self.state.turns).pop_front() else {
481                    return Opening::failed(ProviderError::Provider(
482                        "mock completion model has no scripted completion turn".to_string(),
483                    ));
484                };
485                match turn.into_completion_response() {
486                    Ok(response) => {
487                        let document = response.raw.clone();
488                        let request_id = response.provider_request_id.clone();
489                        Opening::ready(
490                            Opened::new(futures::stream::iter([Ok(MockFrame::Response(
491                                Box::new(response),
492                            ))]))
493                            .with_document(document)
494                            .with_request_id(request_id),
495                        )
496                    }
497                    Err(error) => Opening::ready(Opened::failed(error)),
498                }
499            }
500            Mode::Streaming => {
501                let Some(turn) = lock(&self.state.stream_turns).pop_front() else {
502                    return Opening::failed(ProviderError::Provider(
503                        "mock completion model has no scripted streaming turn".to_string(),
504                    ));
505                };
506                let request_id = turn.iter().find_map(|event| match event {
507                    MockStreamEvent::RequestId(id) => Some(id.clone()),
508                    _ => None,
509                });
510                Opening::ready(
511                    Opened::new(futures::stream::iter(
512                        turn.into_iter()
513                            .filter(|event| !matches!(event, MockStreamEvent::RequestId(_)))
514                            .map(|event| Ok(MockFrame::Event(event))),
515                    ))
516                    .with_request_id(request_id),
517                )
518            }
519        }
520    }
521}
522
523#[cfg(test)]
524mod tests;
525
526/// Every set option refused, for a test target that takes none.
527pub fn refuse_options(
528    fields: crate::completion::options::OptionFields<'_>,
529) -> crate::completion::options::OptionMap {
530    each_set(fields, || {
531        crate::completion::options::Mapping::unsupported("the test target takes no options")
532    })
533}
534
535/// `answer` for every set option but the cache key, which is unrouted.
536fn each_set(
537    fields: crate::completion::options::OptionFields<'_>,
538    answer: impl Fn() -> crate::completion::options::Mapping,
539) -> crate::completion::options::OptionMap {
540    use crate::completion::options::{Mapping, OptionMap, OptionName};
541    let each = |option| match fields.is_set(option) {
542        true => answer(),
543        false => Mapping::Nothing,
544    };
545    OptionMap {
546        reasoning: each(OptionName::Reasoning),
547        cache: each(OptionName::Cache),
548        service_tier: each(OptionName::ServiceTier),
549        verbosity: each(OptionName::Verbosity),
550        parallel_tool_calls: each(OptionName::ParallelToolCalls),
551        top_p: each(OptionName::TopP),
552        seed: each(OptionName::Seed),
553        stop: each(OptionName::Stop),
554        cache_key: Mapping::unrouted(fields.cache_key),
555    }
556}