runifold_provider_testkit/
conformance.rs

1//! Provider-neutral acceptance checks over the canonical model boundary.
2
3use runifold_core::RetrySafety;
4use runifold_model::{
5    ContentPart, Model, ModelCallContext, ModelError, ModelErrorKind, ModelRequest, ModelUsage,
6};
7use thiserror::Error;
8
9/// One successfully verified provider behavior.
10#[derive(Clone, Copy, Debug, Eq, PartialEq)]
11#[non_exhaustive]
12pub enum ConformanceCheck {
13    /// The response retained the configured provider identity.
14    ProviderIdentity,
15    /// Visible text matched without including reasoning.
16    VisibleText,
17    /// Reasoning was normalized into canonical reasoning blocks.
18    Reasoning,
19    /// Detailed token usage matched.
20    Usage,
21    /// Raw provider events were retained and correctly namespaced.
22    ProviderEvents,
23    /// A failure had the expected kind, provider, and retry safety.
24    ErrorClassification,
25}
26
27/// Evidence produced by a conformance run.
28#[derive(Clone, Debug, Eq, PartialEq)]
29pub struct ProviderConformanceReport {
30    provider: String,
31    checks: Vec<ConformanceCheck>,
32}
33
34impl ProviderConformanceReport {
35    /// Returns the provider namespace that was verified.
36    pub fn provider(&self) -> &str {
37        &self.provider
38    }
39
40    /// Returns the checks completed by this run.
41    pub fn checks(&self) -> &[ConformanceCheck] {
42        &self.checks
43    }
44}
45
46/// Expected canonical result of a successful provider invocation.
47#[derive(Clone, Debug, Eq, PartialEq)]
48pub struct SuccessContract {
49    provider: String,
50    visible_text: Option<String>,
51    reasoning: Option<String>,
52    usage: Option<ModelUsage>,
53    require_provider_events: bool,
54}
55
56impl SuccessContract {
57    /// Starts a contract for one canonical provider namespace.
58    pub fn new(provider: impl Into<String>) -> Self {
59        Self {
60            provider: provider.into(),
61            visible_text: None,
62            reasoning: None,
63            usage: None,
64            require_provider_events: false,
65        }
66    }
67
68    /// Requires exact model-visible text.
69    #[must_use]
70    pub fn visible_text(mut self, text: impl Into<String>) -> Self {
71        self.visible_text = Some(text.into());
72        self
73    }
74
75    /// Requires exact concatenated canonical reasoning text.
76    #[must_use]
77    pub fn reasoning(mut self, reasoning: impl Into<String>) -> Self {
78        self.reasoning = Some(reasoning.into());
79        self
80    }
81
82    /// Requires exact normalized usage.
83    #[must_use]
84    pub const fn usage(mut self, usage: ModelUsage) -> Self {
85        self.usage = Some(usage);
86        self
87    }
88
89    /// Requires at least one correctly namespaced raw provider event.
90    #[must_use]
91    pub const fn provider_events(mut self) -> Self {
92        self.require_provider_events = true;
93        self
94    }
95}
96
97/// Expected classification of a failed provider invocation.
98#[derive(Clone, Debug, Eq, PartialEq)]
99pub struct ErrorContract {
100    provider: String,
101    kind: ModelErrorKind,
102    retry_safety: RetrySafety,
103}
104
105impl ErrorContract {
106    /// Creates an exact normalized error contract.
107    pub fn new(
108        provider: impl Into<String>,
109        kind: ModelErrorKind,
110        retry_safety: RetrySafety,
111    ) -> Self {
112        Self {
113            provider: provider.into(),
114            kind,
115            retry_safety,
116        }
117    }
118}
119
120/// A provider violated its canonical acceptance contract.
121#[derive(Debug, Error)]
122#[non_exhaustive]
123pub enum ProviderConformanceError {
124    /// The provider invocation failed before success checks could run.
125    #[error("provider invocation failed during success conformance: {0}")]
126    Invocation(#[source] ModelError),
127    /// A failure contract unexpectedly produced a response.
128    #[error("provider invocation unexpectedly succeeded during error conformance")]
129    UnexpectedSuccess,
130    /// The canonical response used another provider identity.
131    #[error("expected provider `{expected}`, received `{actual}`")]
132    ProviderIdentity {
133        /// Required provider.
134        expected: String,
135        /// Actual provider.
136        actual: String,
137    },
138    /// Model-visible text differed.
139    #[error("provider visible text did not match the contract")]
140    VisibleText,
141    /// Canonical reasoning differed.
142    #[error("provider reasoning did not match the contract")]
143    Reasoning,
144    /// Normalized usage differed.
145    #[error("provider usage did not match the contract")]
146    Usage,
147    /// No raw provider event was retained.
148    #[error("provider response did not retain a raw provider event")]
149    MissingProviderEvent,
150    /// A retained raw event used another namespace.
151    #[error("raw provider event used `{actual}` instead of `{expected}`")]
152    ProviderEventIdentity {
153        /// Required provider.
154        expected: String,
155        /// Actual provider.
156        actual: String,
157    },
158    /// The normalized error category differed.
159    #[error("expected error kind {expected:?}, received {actual:?}")]
160    ErrorKind {
161        /// Required kind.
162        expected: ModelErrorKind,
163        /// Actual kind.
164        actual: ModelErrorKind,
165    },
166    /// The failed invocation omitted or changed provider identity.
167    #[error("expected error provider `{expected}`, received {actual:?}")]
168    ErrorProvider {
169        /// Required provider.
170        expected: String,
171        /// Actual provider.
172        actual: Option<String>,
173    },
174    /// Retry safety differed.
175    #[error("expected retry safety {expected:?}, received {actual:?}")]
176    RetrySafety {
177        /// Required classification.
178        expected: RetrySafety,
179        /// Actual classification.
180        actual: RetrySafety,
181    },
182}
183
184/// Executes and verifies one successful canonical provider invocation.
185///
186/// # Errors
187///
188/// Returns [`ProviderConformanceError`] when invocation or any requested
189/// acceptance check fails.
190pub async fn verify_success(
191    model: &dyn Model,
192    request: ModelRequest,
193    context: ModelCallContext,
194    contract: &SuccessContract,
195) -> Result<ProviderConformanceReport, ProviderConformanceError> {
196    let response = model
197        .invoke(request, context)
198        .await
199        .map_err(ProviderConformanceError::Invocation)?;
200    let mut checks = Vec::new();
201    if response.model.provider != contract.provider {
202        return Err(ProviderConformanceError::ProviderIdentity {
203            expected: contract.provider.clone(),
204            actual: response.model.provider,
205        });
206    }
207    checks.push(ConformanceCheck::ProviderIdentity);
208
209    if let Some(expected) = &contract.visible_text {
210        if response.text() != *expected {
211            return Err(ProviderConformanceError::VisibleText);
212        }
213        checks.push(ConformanceCheck::VisibleText);
214    }
215    if let Some(expected) = &contract.reasoning {
216        let reasoning = response
217            .content
218            .iter()
219            .filter_map(|part| match part {
220                ContentPart::Reasoning(reasoning) => reasoning.text.as_deref(),
221                _ => None,
222            })
223            .collect::<String>();
224        if reasoning != *expected {
225            return Err(ProviderConformanceError::Reasoning);
226        }
227        checks.push(ConformanceCheck::Reasoning);
228    }
229    if let Some(expected) = contract.usage {
230        if response.usage != expected {
231            return Err(ProviderConformanceError::Usage);
232        }
233        checks.push(ConformanceCheck::Usage);
234    }
235    if contract.require_provider_events {
236        if response.provider_events.is_empty() {
237            return Err(ProviderConformanceError::MissingProviderEvent);
238        }
239        if let Some(event) = response
240            .provider_events
241            .iter()
242            .find(|event| event.provider != contract.provider)
243        {
244            return Err(ProviderConformanceError::ProviderEventIdentity {
245                expected: contract.provider.clone(),
246                actual: event.provider.clone(),
247            });
248        }
249        checks.push(ConformanceCheck::ProviderEvents);
250    }
251    Ok(ProviderConformanceReport {
252        provider: contract.provider.clone(),
253        checks,
254    })
255}
256
257/// Executes and verifies one failed canonical provider invocation.
258///
259/// # Errors
260///
261/// Returns [`ProviderConformanceError`] when the invocation succeeds or its
262/// error classification differs from the contract.
263pub async fn verify_error(
264    model: &dyn Model,
265    request: ModelRequest,
266    context: ModelCallContext,
267    contract: &ErrorContract,
268) -> Result<ProviderConformanceReport, ProviderConformanceError> {
269    let Err(error) = model.invoke(request, context).await else {
270        return Err(ProviderConformanceError::UnexpectedSuccess);
271    };
272    if error.kind != contract.kind {
273        return Err(ProviderConformanceError::ErrorKind {
274            expected: contract.kind.clone(),
275            actual: error.kind,
276        });
277    }
278    if error.provider.as_deref() != Some(contract.provider.as_str()) {
279        return Err(ProviderConformanceError::ErrorProvider {
280            expected: contract.provider.clone(),
281            actual: error.provider,
282        });
283    }
284    if error.retry_safety != contract.retry_safety {
285        return Err(ProviderConformanceError::RetrySafety {
286            expected: contract.retry_safety,
287            actual: error.retry_safety,
288        });
289    }
290    Ok(ProviderConformanceReport {
291        provider: contract.provider.clone(),
292        checks: vec![ConformanceCheck::ErrorClassification],
293    })
294}
295
296#[cfg(test)]
297mod tests {
298    use std::collections::BTreeMap;
299
300    use futures_executor::block_on;
301    use runifold_model::{
302        ContentBlockKind, FinishReason, Message, ModelCapabilities, ModelEventStream, ModelFuture,
303        ModelRef, ModelStreamEvent, ProviderEvent,
304    };
305    use serde_json::json;
306
307    use super::*;
308
309    struct CanonicalModel;
310
311    impl Model for CanonicalModel {
312        fn capabilities<'a>(
313            &'a self,
314            _model: &'a ModelRef,
315        ) -> ModelFuture<'a, Result<ModelCapabilities, ModelError>> {
316            Box::pin(async { Ok(ModelCapabilities::default()) })
317        }
318
319        fn stream(
320            &self,
321            _request: ModelRequest,
322            _context: ModelCallContext,
323        ) -> ModelFuture<'_, Result<ModelEventStream, ModelError>> {
324            let events = vec![
325                ModelStreamEvent::ResponseStarted {
326                    id: Some("response-1".into()),
327                    model: ModelRef::new("test-provider", "test-model"),
328                },
329                ModelStreamEvent::ContentBlockStarted {
330                    index: 0,
331                    kind: ContentBlockKind::Reasoning {
332                        signature: None,
333                        redacted: false,
334                    },
335                },
336                ModelStreamEvent::ReasoningDelta {
337                    index: 0,
338                    text: "think".into(),
339                },
340                ModelStreamEvent::ContentBlockCompleted { index: 0 },
341                ModelStreamEvent::ContentBlockStarted {
342                    index: 1,
343                    kind: ContentBlockKind::Text,
344                },
345                ModelStreamEvent::TextDelta {
346                    index: 1,
347                    text: "answer".into(),
348                },
349                ModelStreamEvent::ContentBlockCompleted { index: 1 },
350                ModelStreamEvent::UsageUpdated {
351                    usage: ModelUsage {
352                        input_tokens: 2,
353                        output_tokens: 3,
354                        reasoning_tokens: 1,
355                        ..ModelUsage::default()
356                    },
357                },
358                ModelStreamEvent::Provider {
359                    event: ProviderEvent {
360                        provider: "test-provider".into(),
361                        name: "raw.chunk".into(),
362                        payload: json!({"chunk": 1}),
363                    },
364                },
365                ModelStreamEvent::ResponseCompleted {
366                    finish_reason: FinishReason::Stop,
367                    provider_metadata: BTreeMap::new(),
368                },
369            ];
370            Box::pin(async move {
371                Ok(
372                    Box::pin(futures_util::stream::iter(events.into_iter().map(Ok)))
373                        as ModelEventStream,
374                )
375            })
376        }
377    }
378
379    struct FailingModel;
380
381    impl Model for FailingModel {
382        fn capabilities<'a>(
383            &'a self,
384            _model: &'a ModelRef,
385        ) -> ModelFuture<'a, Result<ModelCapabilities, ModelError>> {
386            Box::pin(async { Ok(ModelCapabilities::default()) })
387        }
388
389        fn stream(
390            &self,
391            _request: ModelRequest,
392            _context: ModelCallContext,
393        ) -> ModelFuture<'_, Result<ModelEventStream, ModelError>> {
394            Box::pin(async {
395                let mut error = ModelError::local(ModelErrorKind::Provider, "provider unavailable");
396                error.provider = Some("test-provider".into());
397                error.retry_safety = RetrySafety::Safe;
398                Err(error)
399            })
400        }
401    }
402
403    fn request() -> ModelRequest {
404        ModelRequest::new(
405            ModelRef::new("test-provider", "test-model"),
406            Message::user("test"),
407        )
408    }
409
410    #[test]
411    fn verifies_the_full_success_contract() {
412        let usage = ModelUsage {
413            input_tokens: 2,
414            output_tokens: 3,
415            reasoning_tokens: 1,
416            ..ModelUsage::default()
417        };
418        let report = block_on(verify_success(
419            &CanonicalModel,
420            request(),
421            ModelCallContext::new(),
422            &SuccessContract::new("test-provider")
423                .visible_text("answer")
424                .reasoning("think")
425                .usage(usage)
426                .provider_events(),
427        ))
428        .unwrap();
429
430        assert_eq!(report.checks().len(), 5);
431    }
432
433    #[test]
434    fn verifies_error_kind_identity_and_retry_safety() {
435        let report = block_on(verify_error(
436            &FailingModel,
437            request(),
438            ModelCallContext::new(),
439            &ErrorContract::new("test-provider", ModelErrorKind::Provider, RetrySafety::Safe),
440        ))
441        .unwrap();
442
443        assert_eq!(report.checks(), &[ConformanceCheck::ErrorClassification]);
444    }
445}
runifold_provider_testkit/conformance.rs

runifold_provider_testkit/
conformance.rs
