1use chrono::{DateTime, Utc};
20use serde::{Deserialize, Serialize};
21
22pub const CONFORMANCE_SUITE_VERSION: &str = "0.0.1";
27
28pub const MAX_EXCERPT_CHARS: usize = 512;
30
31const MIN_ACCEPTANCE_RUNS: u32 = 100;
33
34const MAX_TRANSPORT_FAILURE_FRACTION: f64 = 0.02;
36
37const MIN_SINGLE_TOOL_SUCCESS_RATE: f64 = 0.95;
39
40const MIN_MULTI_TURN_TOOL_SUCCESS_RATE: f64 = 0.93;
42
43const DEGRADED_RATE_THRESHOLD: f64 = 0.9;
45
46#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
52#[serde(rename_all = "snake_case")]
53pub enum FailureClass {
54 AuthMissing,
56 AuthRejected,
58 AccountNotFound,
60 ModelUnavailable,
62 UnsupportedParameter,
64 ContextLimitExceeded,
66 ConnectTimeout,
68 ReadTimeout,
70 StreamIdleTimeout,
72 ConnectionReset,
74 TlsFailure,
76 RateLimited,
78 ProviderServerError,
80 CloudflareEdgeError,
82 EmptyCompletion,
84 TruncatedCompletion,
86 InvalidJson,
88 InvalidChatCompletionShape,
90 InvalidSseEvent,
92 MissingFinishReason,
94 NoToolCall,
96 InvalidToolName,
98 InvalidToolArguments,
100 DuplicateToolCall,
102 ToolLoopDidNotConverge,
104 Unknown,
106}
107
108#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
116#[serde(deny_unknown_fields)]
117pub struct FailureEvidence {
118 pub failure_class: FailureClass,
119 pub http_status: Option<u16>,
120 pub cloudflare_ray_id: Option<String>,
121 pub elapsed_ms: Option<u64>,
122 pub model_id: String,
123 pub request_id: Option<String>,
124 pub response_excerpt: Option<String>,
125}
126
127impl FailureEvidence {
128 pub fn new(
131 failure_class: FailureClass,
132 http_status: Option<u16>,
133 cloudflare_ray_id: Option<String>,
134 elapsed_ms: Option<u64>,
135 model_id: impl Into<String>,
136 request_id: Option<String>,
137 response_excerpt: Option<String>,
138 ) -> Self {
139 Self {
140 failure_class,
141 http_status,
142 cloudflare_ray_id,
143 elapsed_ms,
144 model_id: model_id.into(),
145 request_id,
146 response_excerpt: Self::sanitize_excerpt(response_excerpt.as_deref()),
147 }
148 }
149
150 pub fn sanitize_excerpt(raw: Option<&str>) -> Option<String> {
156 raw.map(|excerpt| {
157 if excerpt.chars().count() <= MAX_EXCERPT_CHARS {
158 excerpt.to_string()
159 } else {
160 excerpt.chars().take(MAX_EXCERPT_CHARS).collect()
161 }
162 })
163 }
164}
165
166#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
169pub struct ModelHealthRecord {
170 pub model_id: String,
171 pub window_started_at: DateTime<Utc>,
172 pub request_count: u64,
173 pub successful_responses: u64,
174 pub transport_failures: u64,
175 pub upstream_5xx_failures: u64,
176 pub timeout_failures: u64,
177 pub malformed_tool_calls: u64,
178 pub median_latency_ms: Option<u64>,
179 pub last_failure_at: Option<DateTime<Utc>>,
180 pub last_failure_class: Option<FailureClass>,
181}
182
183impl ModelHealthRecord {
184 pub fn delivery_success_rate(&self) -> Option<f64> {
187 if self.request_count == 0 {
188 return None;
189 }
190 Some((self.successful_responses as f64 / self.request_count as f64).min(1.0))
191 }
192
193 pub fn is_degraded(&self) -> bool {
197 let failure_total =
198 self.transport_failures + self.upstream_5xx_failures + self.timeout_failures + self.malformed_tool_calls;
199 if failure_total > 0 {
200 return true;
201 }
202 self.delivery_success_rate().is_some_and(|rate| rate < DEGRADED_RATE_THRESHOLD)
203 }
204}
205
206pub fn health_warning(record: &ModelHealthRecord, stable_alternative: &str) -> Option<String> {
213 if !record.is_degraded() {
214 return None;
215 }
216 let percent = record.delivery_success_rate().unwrap_or(0.0) * 100.0;
217 Some(format!(
218 "model {} delivery is degraded ({:.1} percent successful over {} requests); recommended stable alternative: {}",
219 record.model_id, percent, record.request_count, stable_alternative
220 ))
221}
222
223pub fn recommended_stable_alternative(model_id: &str) -> &'static str {
227 if model_id == crate::DEFAULT_MODEL { "" } else { crate::DEFAULT_MODEL }
228}
229
230#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
232#[serde(rename_all = "snake_case")]
233pub enum VerificationStatus {
234 Untested,
236 Passing,
238 Degraded,
240 Failing,
242 Expired,
244}
245
246#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
249#[serde(rename_all = "snake_case")]
250pub enum VerificationConfidence {
251 None,
253 StaticMetadataOnly,
255 SmokeTested,
257 ConformanceTested,
259 RegressionTested,
261}
262
263#[derive(Debug, Clone, Serialize, Deserialize)]
266pub struct ModelVerification {
267 pub model_id: String,
268 pub latest_run_at: Option<DateTime<Utc>>,
269 pub expires_at: Option<DateTime<Utc>>,
270 pub suite_version: String,
271 pub runner_version: String,
272 pub status: VerificationStatus,
273 pub agent_eligible: bool,
274 pub confidence: VerificationConfidence,
275 pub total_runs: u32,
276 pub successful_runs: u32,
277 pub text_completion_success_rate: Option<f64>,
278 pub stream_completion_success_rate: Option<f64>,
279 pub single_tool_success_rate: Option<f64>,
280 pub multi_turn_tool_success_rate: Option<f64>,
281 pub structured_output_success_rate: Option<f64>,
282 pub median_latency_ms: Option<u64>,
283 pub p95_latency_ms: Option<u64>,
284 pub total_failures: u32,
285 pub timeout_failures: u32,
286 pub transport_failures: u32,
287 pub provider_5xx_failures: u32,
288 pub malformed_response_failures: u32,
289 pub malformed_tool_call_failures: u32,
290 pub tool_loop_failures: u32,
291 pub last_failure: Option<FailureEvidence>,
292}
293
294impl ModelVerification {
295 pub fn delivery_success_rate(&self) -> Option<f64> {
298 if self.total_runs == 0 {
299 return None;
300 }
301 Some((self.successful_runs as f64 / self.total_runs as f64).min(1.0))
302 }
303
304 pub fn passes_acceptance_gate(&self) -> bool {
314 if self.total_runs < MIN_ACCEPTANCE_RUNS {
315 return false;
316 }
317 let transport_failures = self.transport_failures + self.timeout_failures + self.provider_5xx_failures;
318 let transport_fraction = transport_failures as f64 / self.total_runs as f64;
319 if transport_fraction > MAX_TRANSPORT_FAILURE_FRACTION {
320 return false;
321 }
322 match (self.single_tool_success_rate, self.multi_turn_tool_success_rate) {
323 (Some(single), Some(multi)) => {
324 single >= MIN_SINGLE_TOOL_SUCCESS_RATE && multi >= MIN_MULTI_TURN_TOOL_SUCCESS_RATE
325 },
326 _ => false,
327 }
328 }
329}
330
331#[cfg(test)]
332mod tests {
333 use super::*;
334
335 fn timestamp() -> DateTime<Utc> {
336 DateTime::from_timestamp(1_788_800_000, 0).expect("valid timestamp")
337 }
338
339 fn health_record(
340 requests: u64,
341 successful: u64,
342 transport: u64,
343 timeout: u64,
344 fivexx: u64,
345 tool_calls: u64,
346 ) -> ModelHealthRecord {
347 ModelHealthRecord {
348 model_id: "@cf/deepseek-ai/deepseek-v4-flash-0731".to_string(),
349 window_started_at: timestamp(),
350 request_count: requests,
351 successful_responses: successful,
352 transport_failures: transport,
353 upstream_5xx_failures: fivexx,
354 timeout_failures: timeout,
355 malformed_tool_calls: tool_calls,
356 median_latency_ms: None,
357 last_failure_at: None,
358 last_failure_class: None,
359 }
360 }
361
362 fn verification_record(
363 total: u32,
364 successful: u32,
365 transport: u32,
366 timeout: u32,
367 fivexx: u32,
368 single: Option<f64>,
369 multi: Option<f64>,
370 ) -> ModelVerification {
371 ModelVerification {
372 model_id: "@cf/deepseek-ai/deepseek-v4-flash-0731".to_string(),
373 latest_run_at: Some(timestamp()),
374 expires_at: None,
375 suite_version: CONFORMANCE_SUITE_VERSION.to_string(),
376 runner_version: "test".to_string(),
377 status: VerificationStatus::Untested,
378 agent_eligible: true,
379 confidence: VerificationConfidence::ConformanceTested,
380 total_runs: total,
381 successful_runs: successful,
382 text_completion_success_rate: None,
383 stream_completion_success_rate: None,
384 single_tool_success_rate: single,
385 multi_turn_tool_success_rate: multi,
386 structured_output_success_rate: None,
387 median_latency_ms: None,
388 p95_latency_ms: None,
389 total_failures: total - successful,
390 timeout_failures: timeout,
391 transport_failures: transport,
392 provider_5xx_failures: fivexx,
393 malformed_response_failures: 0,
394 malformed_tool_call_failures: 0,
395 tool_loop_failures: 0,
396 last_failure: None,
397 }
398 }
399
400 #[test]
401 fn success_rate_math() {
402 let rate = health_record(31, 22, 9, 0, 0, 0).delivery_success_rate().expect("rate present");
404 let expected = 22.0 / 31.0;
405 assert!((rate - expected).abs() < 1e-9, "rate {rate} != {expected}");
406 assert_eq!(health_record(100, 100, 0, 0, 0, 0).delivery_success_rate(), Some(1.0));
407 assert_eq!(health_record(0, 0, 0, 0, 0, 0).delivery_success_rate(), None);
408 }
409
410 #[test]
411 fn degraded_threshold() {
412 assert!(health_record(31, 22, 9, 0, 0, 0).is_degraded());
414 assert!(health_record(100, 99, 0, 1, 0, 0).is_degraded());
416 assert!(health_record(100, 99, 0, 0, 1, 0).is_degraded());
417 assert!(health_record(100, 99, 0, 0, 0, 1).is_degraded());
418 assert!(!health_record(100, 100, 0, 0, 0, 0).is_degraded());
420 assert!(!health_record(0, 0, 0, 0, 0, 0).is_degraded());
422 }
423
424 #[test]
425 fn acceptance_gate_passes_for_conformant_record() {
426 let record = verification_record(100, 100, 0, 0, 0, Some(0.96), Some(0.94));
427 assert!(record.passes_acceptance_gate());
428 }
429
430 #[test]
431 fn acceptance_gate_allows_boundary_transport_failures() {
432 let record = verification_record(100, 98, 2, 0, 0, Some(0.95), Some(0.93));
434 assert!(record.passes_acceptance_gate());
435 }
436
437 #[test]
438 fn acceptance_gate_rejects_glm_style_record() {
439 let record = verification_record(31, 22, 9, 0, 0, Some(0.709), Some(0.709));
441 assert!(!record.passes_acceptance_gate());
442 }
443
444 #[test]
445 fn acceptance_gate_rejects_missing_rates() {
446 assert!(!verification_record(100, 100, 0, 0, 0, None, None).passes_acceptance_gate());
447 assert!(!verification_record(100, 100, 0, 0, 0, Some(0.96), None).passes_acceptance_gate());
448 }
449
450 #[test]
451 fn acceptance_gate_rejects_low_tool_rates() {
452 assert!(!verification_record(100, 100, 0, 0, 0, Some(0.90), Some(0.94)).passes_acceptance_gate());
453 assert!(!verification_record(100, 100, 0, 0, 0, Some(0.96), Some(0.90)).passes_acceptance_gate());
454 }
455
456 #[test]
457 fn failure_evidence_truncates_long_excerpts() {
458 let evidence = FailureEvidence::new(
459 FailureClass::StreamIdleTimeout,
460 Some(200),
461 Some("ray-1".to_string()),
462 Some(120_000),
463 "@cf/zai-org/glm-5.3-flash",
464 Some("uuid".to_string()),
465 Some("e".repeat(600)),
466 );
467 let excerpt = evidence.response_excerpt.expect("excerpt present");
468 assert_eq!(excerpt.chars().count(), MAX_EXCERPT_CHARS);
469 assert_eq!(excerpt, "e".repeat(MAX_EXCERPT_CHARS));
470 }
471
472 #[test]
473 fn failure_evidence_truncation_is_char_safe() {
474 let sanitized = FailureEvidence::sanitize_excerpt(Some(&"€".repeat(600))).expect("sanitized");
476 assert_eq!(sanitized.chars().count(), MAX_EXCERPT_CHARS);
477 assert_eq!(sanitized, "€".repeat(MAX_EXCERPT_CHARS));
478 }
479
480 #[test]
481 fn failure_evidence_keeps_short_excerpts() {
482 assert_eq!(FailureEvidence::sanitize_excerpt(Some("ok")), Some("ok".to_string()));
483 assert_eq!(FailureEvidence::sanitize_excerpt(None), None);
484 }
485
486 #[test]
487 fn failure_evidence_has_no_token_field() {
488 let evidence = FailureEvidence::new(
489 FailureClass::AuthRejected,
490 Some(401),
491 None,
492 Some(100),
493 "@cf/deepseek-ai/deepseek-v4-flash-0731",
494 None,
495 Some("unauthorized".to_string()),
496 );
497 let json = serde_json::to_string(&evidence).expect("serializes");
498 let lowered = json.to_lowercase();
499 for forbidden in ["token", "authorization", "secret", "api_key", "bearer"] {
500 assert!(!lowered.contains(forbidden), "JSON must not contain {forbidden}: {json}");
501 }
502 }
503
504 #[test]
505 fn failure_evidence_rejects_unknown_fields() {
506 let json = serde_json::json!({
508 "failure_class": "auth_missing",
509 "http_status": null,
510 "cloudflare_ray_id": null,
511 "elapsed_ms": null,
512 "model_id": "@cf/deepseek-ai/deepseek-v4-flash-0731",
513 "request_id": null,
514 "response_excerpt": null,
515 "token": "leaked",
516 });
517 let result = serde_json::from_value::<FailureEvidence>(json);
518 assert!(result.is_err(), "unknown field must be rejected: {result:?}");
519 }
520
521 #[test]
522 fn failure_class_serde_snake_case() {
523 let cases = [
524 (FailureClass::AuthMissing, "\"auth_missing\""),
525 (FailureClass::ToolLoopDidNotConverge, "\"tool_loop_did_not_converge\""),
526 (FailureClass::StreamIdleTimeout, "\"stream_idle_timeout\""),
527 (FailureClass::InvalidChatCompletionShape, "\"invalid_chat_completion_shape\""),
528 (FailureClass::ProviderServerError, "\"provider_server_error\""),
529 ];
530 for (class, name) in cases {
531 let json = serde_json::to_string(&class).expect("ser");
532 assert_eq!(json, name);
533 assert_eq!(serde_json::from_str::<FailureClass>(&json).expect("de"), class);
534 }
535 let back: FailureClass = serde_json::from_str("\"cloudflare_edge_error\"").expect("de");
536 assert_eq!(back, FailureClass::CloudflareEdgeError);
537 }
538
539 #[test]
540 fn status_and_confidence_serde_snake_case() {
541 let statuses = [
542 (VerificationStatus::Untested, "untested"),
543 (VerificationStatus::Passing, "passing"),
544 (VerificationStatus::Degraded, "degraded"),
545 (VerificationStatus::Failing, "failing"),
546 (VerificationStatus::Expired, "expired"),
547 ];
548 for (status, name) in statuses {
549 let json = serde_json::to_string(&status).expect("ser");
550 assert_eq!(json, format!("\"{name}\""));
551 assert_eq!(serde_json::from_str::<VerificationStatus>(&json).expect("de"), status);
552 }
553 let confidences = [
554 (VerificationConfidence::None, "none"),
555 (VerificationConfidence::StaticMetadataOnly, "static_metadata_only"),
556 (VerificationConfidence::SmokeTested, "smoke_tested"),
557 (VerificationConfidence::ConformanceTested, "conformance_tested"),
558 (VerificationConfidence::RegressionTested, "regression_tested"),
559 ];
560 for (confidence, name) in confidences {
561 let json = serde_json::to_string(&confidence).expect("ser");
562 assert_eq!(json, format!("\"{name}\""));
563 assert_eq!(serde_json::from_str::<VerificationConfidence>(&json).expect("de"), confidence);
564 }
565 }
566
567 #[test]
568 fn health_record_serde_roundtrip() {
569 let record = health_record(100, 98, 2, 0, 0, 1);
570 let json = serde_json::to_string(&record).expect("ser");
571 let back: ModelHealthRecord = serde_json::from_str(&json).expect("de");
572 assert_eq!(back, record);
573 }
574
575 #[test]
576 fn verification_serde_roundtrip_with_evidence() {
577 let mut record = verification_record(100, 98, 2, 0, 0, Some(0.96), Some(0.94));
578 record.status = VerificationStatus::Passing;
579 record.last_failure = Some(FailureEvidence::new(
580 FailureClass::RateLimited,
581 Some(429),
582 Some("ray-abc".to_string()),
583 Some(1_200),
584 "@cf/deepseek-ai/deepseek-v4-flash-0731",
585 Some("req-7".to_string()),
586 Some("rate limited".to_string()),
587 ));
588 let json = serde_json::to_string(&record).expect("ser");
589 let back: ModelVerification = serde_json::from_str(&json).expect("de");
590 assert_eq!(back.model_id, record.model_id);
591 assert_eq!(back.status, VerificationStatus::Passing);
592 assert_eq!(back.total_runs, 100);
593 assert_eq!(back.single_tool_success_rate, Some(0.96));
594 let evidence = back.last_failure.expect("evidence present");
595 assert_eq!(evidence.failure_class, FailureClass::RateLimited);
596 assert_eq!(evidence.http_status, Some(429));
597 assert_eq!(evidence.cloudflare_ray_id.as_deref(), Some("ray-abc"));
598 }
599
600 #[test]
601 fn health_warning_renders_for_degraded_record() {
602 let record = health_record(100, 89, 11, 0, 0, 0);
603 let warning = health_warning(&record, crate::DEFAULT_MODEL).expect("warning present");
604 assert!(warning.contains("89.0"), "warning must contain the rate: {warning}");
605 assert!(
606 warning.contains(crate::DEFAULT_MODEL),
607 "warning must contain the alternative: {warning}"
608 );
609 assert!(warning.contains("100"), "warning must contain the request count: {warning}");
610 assert!(
611 warning.contains(&record.model_id),
612 "warning must contain the model id: {warning}"
613 );
614 }
615
616 #[test]
617 fn health_warning_none_for_clean_record() {
618 assert_eq!(health_warning(&health_record(100, 100, 0, 0, 0, 0), "alt"), None);
619 }
620
621 #[test]
622 fn health_warning_none_for_empty_record() {
623 assert_eq!(health_warning(&health_record(0, 0, 0, 0, 0, 0), "alt"), None);
624 }
625
626 #[test]
627 fn recommended_stable_alternative_empty_sentinel_for_default() {
628 assert_eq!(recommended_stable_alternative(crate::DEFAULT_MODEL), "");
629 }
630
631 #[test]
632 fn recommended_stable_alternative_returns_default_for_glm() {
633 assert_eq!(
634 recommended_stable_alternative("@cf/zai-org/glm-5.3-flash"),
635 crate::DEFAULT_MODEL
636 );
637 }
638
639 #[test]
640 fn health_warning_is_token_free() {
641 let record = health_record(31, 22, 9, 0, 0, 0);
642 let warning = health_warning(&record, crate::DEFAULT_MODEL).expect("warning present");
643 let lowered = warning.to_lowercase();
644 for forbidden in ["token", "secret", "bearer"] {
645 assert!(!lowered.contains(forbidden), "warning must not contain {forbidden}: {warning}");
646 }
647 }
648}