Skip to main content

auth_cloudflare/
health.rs

1//! Health - delivery health records, conformance verification, and typed
2//! failure evidence for Cloudflare Workers AI models.
3//!
4//! Feedback 01 requires the provider to capture delivery-failure evidence
5//! (circuit-breaker record) instead of silently switching models, and
6//! feedback 02 adds the canonical verification record (`ModelVerification`),
7//! the delivery-failure taxonomy (`FailureClass`), and sanitized-evidence
8//! rules: never persist Authorization headers, API tokens, full prompts, or
9//! tool outputs; response excerpts are capped at `MAX_EXCERPT_CHARS` (512).
10//!
11//! The three statements a model can make must never be conflated:
12//! *Available* (Cloudflare says the account can invoke it), *Capable*
13//! (schema/docs say a feature is supported), and *Verified* (this project
14//! recently tested it successfully with Hermes-style tools).
15//! `ModelVerification` owns the *Verified* statement, and
16//! `passes_acceptance_gate` implements the feedback-01 acceptance criteria
17//! for recommended models.
18
19use chrono::{DateTime, Utc};
20use serde::{Deserialize, Serialize};
21
22/// Version of the conformance suite that produced verification records.
23///
24/// Bump when suite scenarios or acceptance criteria change so stale
25/// `ModelVerification` artifacts cannot be mistaken for fresh ones.
26pub const CONFORMANCE_SUITE_VERSION: &str = "0.0.1";
27
28/// Maximum length of a sanitized `response_excerpt` (feedback 02).
29pub const MAX_EXCERPT_CHARS: usize = 512;
30
31/// Minimum completed runs before the acceptance gate may pass (feedback 01).
32const MIN_ACCEPTANCE_RUNS: u32 = 100;
33
34/// Maximum tolerated fraction of transport/timeout/provider-5xx failures.
35const MAX_TRANSPORT_FAILURE_FRACTION: f64 = 0.02;
36
37/// Minimum single-tool-call success rate for the acceptance gate.
38const MIN_SINGLE_TOOL_SUCCESS_RATE: f64 = 0.95;
39
40/// Minimum multi-turn tool-loop success rate for the acceptance gate.
41const MIN_MULTI_TURN_TOOL_SUCCESS_RATE: f64 = 0.93;
42
43/// Delivery-rate threshold below which a health window is degraded.
44const DEGRADED_RATE_THRESHOLD: f64 = 0.9;
45
46/// Typed delivery-failure taxonomy (feedback 02).
47///
48/// Every failure recorded for a Cloudflare Workers AI model must map to
49/// exactly one class. `Unknown` is the last resort for classes not yet in
50/// the taxonomy - never guess a specific class without evidence.
51#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
52#[serde(rename_all = "snake_case")]
53pub enum FailureClass {
54	/// Credentials could not be resolved (missing env var or empty token).
55	AuthMissing,
56	/// Cloudflare rejected the presented credentials (401).
57	AuthRejected,
58	/// The configured account could not be found or is out of scope.
59	AccountNotFound,
60	/// The requested model is not available on this account or route.
61	ModelUnavailable,
62	/// A request parameter is not supported by the model or endpoint.
63	UnsupportedParameter,
64	/// The context window was exceeded by the request.
65	ContextLimitExceeded,
66	/// Connection establishment timed out.
67	ConnectTimeout,
68	/// The upstream did not respond within the read timeout.
69	ReadTimeout,
70	/// The stream produced no event within the idle timeout.
71	StreamIdleTimeout,
72	/// The connection was reset mid-request or mid-stream.
73	ConnectionReset,
74	/// TLS handshake or certificate failure.
75	TlsFailure,
76	/// Cloudflare rate-limited the request (429).
77	RateLimited,
78	/// The provider returned a 5xx error.
79	ProviderServerError,
80	/// Cloudflare edge error envelope (cf-ray present, success: false).
81	CloudflareEdgeError,
82	/// The completion was delivered with empty content.
83	EmptyCompletion,
84	/// The completion was cut off before a terminal state.
85	TruncatedCompletion,
86	/// The response body was not valid JSON.
87	InvalidJson,
88	/// The JSON did not match the Chat Completions shape.
89	InvalidChatCompletionShape,
90	/// An SSE event was malformed.
91	InvalidSseEvent,
92	/// A terminal chunk was missing its finish reason.
93	MissingFinishReason,
94	/// The model answered without issuing the required tool call.
95	NoToolCall,
96	/// The tool call named an unknown or disallowed tool.
97	InvalidToolName,
98	/// The tool-call arguments were not valid JSON for the schema.
99	InvalidToolArguments,
100	/// The same tool call was issued more than once.
101	DuplicateToolCall,
102	/// The multi-turn tool loop did not converge to a final answer.
103	ToolLoopDidNotConverge,
104	/// Failure class not yet classified - never guess a specific class.
105	Unknown,
106}
107
108/// One sanitized failure observation (feedback 02).
109///
110/// Security contract: this type has NO field for Authorization headers, API
111/// tokens, prompts, or tool outputs - and `deny_unknown_fields` rejects any
112/// JSON carrying such a field, so a leaked credential cannot round-trip
113/// through serde silently. `response_excerpt` is capped at
114/// `MAX_EXCERPT_CHARS` (512) characters by `new`/`sanitize_excerpt`.
115#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
116#[serde(deny_unknown_fields)]
117pub struct FailureEvidence {
118	pub failure_class: FailureClass,
119	pub http_status: Option<u16>,
120	pub cloudflare_ray_id: Option<String>,
121	pub elapsed_ms: Option<u64>,
122	pub model_id: String,
123	pub request_id: Option<String>,
124	pub response_excerpt: Option<String>,
125}
126
127impl FailureEvidence {
128	/// Build sanitized evidence. `response_excerpt` is truncated to
129	/// `MAX_EXCERPT_CHARS` characters; everything else passes through as-is.
130	pub fn new(
131		failure_class: FailureClass,
132		http_status: Option<u16>,
133		cloudflare_ray_id: Option<String>,
134		elapsed_ms: Option<u64>,
135		model_id: impl Into<String>,
136		request_id: Option<String>,
137		response_excerpt: Option<String>,
138	) -> Self {
139		Self {
140			failure_class,
141			http_status,
142			cloudflare_ray_id,
143			elapsed_ms,
144			model_id: model_id.into(),
145			request_id,
146			response_excerpt: Self::sanitize_excerpt(response_excerpt.as_deref()),
147		}
148	}
149
150	/// Cap a raw response excerpt at `MAX_EXCERPT_CHARS` characters
151	/// (char-safe: never splits a UTF-8 scalar mid-sequence).
152	///
153	/// Callers must still avoid passing Authorization/token content - this
154	/// caps size, it does not redact secrets.
155	pub fn sanitize_excerpt(raw: Option<&str>) -> Option<String> {
156		raw.map(|excerpt| {
157			if excerpt.chars().count() <= MAX_EXCERPT_CHARS {
158				excerpt.to_string()
159			} else {
160				excerpt.chars().take(MAX_EXCERPT_CHARS).collect()
161			}
162		})
163	}
164}
165
166/// Rolling delivery-health window for one model (feedback 01 circuit-breaker
167/// record: captures failure evidence; automatic failover stays disabled).
168#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
169pub struct ModelHealthRecord {
170	pub model_id: String,
171	pub window_started_at: DateTime<Utc>,
172	pub request_count: u64,
173	pub successful_responses: u64,
174	pub transport_failures: u64,
175	pub upstream_5xx_failures: u64,
176	pub timeout_failures: u64,
177	pub malformed_tool_calls: u64,
178	pub median_latency_ms: Option<u64>,
179	pub last_failure_at: Option<DateTime<Utc>>,
180	pub last_failure_class: Option<FailureClass>,
181}
182
183impl ModelHealthRecord {
184	/// Fraction of requests that delivered successfully
185	/// (`successful / request_count`), `None` when the window is empty.
186	pub fn delivery_success_rate(&self) -> Option<f64> {
187		if self.request_count == 0 {
188			return None;
189		}
190		Some((self.successful_responses as f64 / self.request_count as f64).min(1.0))
191	}
192
193	/// True when the window is degraded: delivery rate below 0.9 or any
194	/// recorded failure in the window (feedback 01 policy - GLM-5.3 Flash
195	/// delivered 22/31 = 70.9% before it was demoted to experimental).
196	pub fn is_degraded(&self) -> bool {
197		let failure_total =
198			self.transport_failures + self.upstream_5xx_failures + self.timeout_failures + self.malformed_tool_calls;
199		if failure_total > 0 {
200			return true;
201		}
202		self.delivery_success_rate().is_some_and(|rate| rate < DEGRADED_RATE_THRESHOLD)
203	}
204}
205
206/// Render a token-free, human-readable delivery-health warning for a
207/// degraded model (feedback 01), or `None` for a clean/empty window.
208///
209/// The message states the model id, the percent delivery rate, the request
210/// count, and the recommended stable alternative - no tokens, secrets, or
211/// Authorization material are ever embedded.
212pub fn health_warning(record: &ModelHealthRecord, stable_alternative: &str) -> Option<String> {
213	if !record.is_degraded() {
214		return None;
215	}
216	let percent = record.delivery_success_rate().unwrap_or(0.0) * 100.0;
217	Some(format!(
218		"model {} delivery is degraded ({:.1} percent successful over {} requests); recommended stable alternative: {}",
219		record.model_id, percent, record.request_count, stable_alternative
220	))
221}
222
223/// The stable fallback to recommend for a model: the default model when the
224/// given id differs from it, or an empty-string sentinel when it already is
225/// the default (no change to recommend).
226pub fn recommended_stable_alternative(model_id: &str) -> &'static str {
227	if model_id == crate::DEFAULT_MODEL { "" } else { crate::DEFAULT_MODEL }
228}
229
230/// Outcome of the most recent conformance run (feedback 02).
231#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
232#[serde(rename_all = "snake_case")]
233pub enum VerificationStatus {
234	/// No conformance data yet.
235	Untested,
236	/// Latest run met all acceptance criteria.
237	Passing,
238	/// Latest run passed some, but not all, acceptance criteria.
239	Degraded,
240	/// Latest run failed acceptance criteria outright.
241	Failing,
242	/// Data is older than the freshness window - treat as untested.
243	Expired,
244}
245
246/// How much evidence backs a verification verdict (feedback 02 - the
247/// selector prefers verified capability over provider claims).
248#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
249#[serde(rename_all = "snake_case")]
250pub enum VerificationConfidence {
251	/// No evidence at all.
252	None,
253	/// Only catalog/schema metadata, never executed.
254	StaticMetadataOnly,
255	/// A cheap smoke suite (completion, stream, one tool call) passed.
256	SmokeTested,
257	/// The full conformance suite passed at least once.
258	ConformanceTested,
259	/// Passing across repeated runs with regression coverage.
260	RegressionTested,
261}
262
263/// Canonical verification record - the *Verified* statement for one model
264/// (feedback 02). Consumed by the picker, policy, and health reporting.
265#[derive(Debug, Clone, Serialize, Deserialize)]
266pub struct ModelVerification {
267	pub model_id: String,
268	pub latest_run_at: Option<DateTime<Utc>>,
269	pub expires_at: Option<DateTime<Utc>>,
270	pub suite_version: String,
271	pub runner_version: String,
272	pub status: VerificationStatus,
273	pub agent_eligible: bool,
274	pub confidence: VerificationConfidence,
275	pub total_runs: u32,
276	pub successful_runs: u32,
277	pub text_completion_success_rate: Option<f64>,
278	pub stream_completion_success_rate: Option<f64>,
279	pub single_tool_success_rate: Option<f64>,
280	pub multi_turn_tool_success_rate: Option<f64>,
281	pub structured_output_success_rate: Option<f64>,
282	pub median_latency_ms: Option<u64>,
283	pub p95_latency_ms: Option<u64>,
284	pub total_failures: u32,
285	pub timeout_failures: u32,
286	pub transport_failures: u32,
287	pub provider_5xx_failures: u32,
288	pub malformed_response_failures: u32,
289	pub malformed_tool_call_failures: u32,
290	pub tool_loop_failures: u32,
291	pub last_failure: Option<FailureEvidence>,
292}
293
294impl ModelVerification {
295	/// Overall delivery rate (`successful_runs / total_runs`), `None` for
296	/// zero runs.
297	pub fn delivery_success_rate(&self) -> Option<f64> {
298		if self.total_runs == 0 {
299			return None;
300		}
301		Some((self.successful_runs as f64 / self.total_runs as f64).min(1.0))
302	}
303
304	/// Feedback-01 acceptance gate for recommended models:
305	///
306	/// - at least 100 completed runs;
307	/// - transport completion >= 98% (transport + timeout + provider-5xx
308	///   failures <= 2% of runs);
309	/// - single-tool success rate >= 95%;
310	/// - multi-turn tool-loop success rate >= 93%.
311	///
312	/// Missing rate evidence fails the gate - unverified is not passing.
313	pub fn passes_acceptance_gate(&self) -> bool {
314		if self.total_runs < MIN_ACCEPTANCE_RUNS {
315			return false;
316		}
317		let transport_failures = self.transport_failures + self.timeout_failures + self.provider_5xx_failures;
318		let transport_fraction = transport_failures as f64 / self.total_runs as f64;
319		if transport_fraction > MAX_TRANSPORT_FAILURE_FRACTION {
320			return false;
321		}
322		match (self.single_tool_success_rate, self.multi_turn_tool_success_rate) {
323			(Some(single), Some(multi)) => {
324				single >= MIN_SINGLE_TOOL_SUCCESS_RATE && multi >= MIN_MULTI_TURN_TOOL_SUCCESS_RATE
325			},
326			_ => false,
327		}
328	}
329}
330
331#[cfg(test)]
332mod tests {
333	use super::*;
334
335	fn timestamp() -> DateTime<Utc> {
336		DateTime::from_timestamp(1_788_800_000, 0).expect("valid timestamp")
337	}
338
339	fn health_record(
340		requests: u64,
341		successful: u64,
342		transport: u64,
343		timeout: u64,
344		fivexx: u64,
345		tool_calls: u64,
346	) -> ModelHealthRecord {
347		ModelHealthRecord {
348			model_id: "@cf/deepseek-ai/deepseek-v4-flash-0731".to_string(),
349			window_started_at: timestamp(),
350			request_count: requests,
351			successful_responses: successful,
352			transport_failures: transport,
353			upstream_5xx_failures: fivexx,
354			timeout_failures: timeout,
355			malformed_tool_calls: tool_calls,
356			median_latency_ms: None,
357			last_failure_at: None,
358			last_failure_class: None,
359		}
360	}
361
362	fn verification_record(
363		total: u32,
364		successful: u32,
365		transport: u32,
366		timeout: u32,
367		fivexx: u32,
368		single: Option<f64>,
369		multi: Option<f64>,
370	) -> ModelVerification {
371		ModelVerification {
372			model_id: "@cf/deepseek-ai/deepseek-v4-flash-0731".to_string(),
373			latest_run_at: Some(timestamp()),
374			expires_at: None,
375			suite_version: CONFORMANCE_SUITE_VERSION.to_string(),
376			runner_version: "test".to_string(),
377			status: VerificationStatus::Untested,
378			agent_eligible: true,
379			confidence: VerificationConfidence::ConformanceTested,
380			total_runs: total,
381			successful_runs: successful,
382			text_completion_success_rate: None,
383			stream_completion_success_rate: None,
384			single_tool_success_rate: single,
385			multi_turn_tool_success_rate: multi,
386			structured_output_success_rate: None,
387			median_latency_ms: None,
388			p95_latency_ms: None,
389			total_failures: total - successful,
390			timeout_failures: timeout,
391			transport_failures: transport,
392			provider_5xx_failures: fivexx,
393			malformed_response_failures: 0,
394			malformed_tool_call_failures: 0,
395			tool_loop_failures: 0,
396			last_failure: None,
397		}
398	}
399
400	#[test]
401	fn success_rate_math() {
402		// GLM-5.3 Flash evidence from feedback 01: 22 successful of 31.
403		let rate = health_record(31, 22, 9, 0, 0, 0).delivery_success_rate().expect("rate present");
404		let expected = 22.0 / 31.0;
405		assert!((rate - expected).abs() < 1e-9, "rate {rate} != {expected}");
406		assert_eq!(health_record(100, 100, 0, 0, 0, 0).delivery_success_rate(), Some(1.0));
407		assert_eq!(health_record(0, 0, 0, 0, 0, 0).delivery_success_rate(), None);
408	}
409
410	#[test]
411	fn degraded_threshold() {
412		// 70.9% delivery is below the 0.9 threshold -> degraded.
413		assert!(health_record(31, 22, 9, 0, 0, 0).is_degraded());
414		// Any failure in the window degrades even a high-rate record.
415		assert!(health_record(100, 99, 0, 1, 0, 0).is_degraded());
416		assert!(health_record(100, 99, 0, 0, 1, 0).is_degraded());
417		assert!(health_record(100, 99, 0, 0, 0, 1).is_degraded());
418		// Clean window at 100% is not degraded.
419		assert!(!health_record(100, 100, 0, 0, 0, 0).is_degraded());
420		// Empty window: no evidence, not degraded.
421		assert!(!health_record(0, 0, 0, 0, 0, 0).is_degraded());
422	}
423
424	#[test]
425	fn acceptance_gate_passes_for_conformant_record() {
426		let record = verification_record(100, 100, 0, 0, 0, Some(0.96), Some(0.94));
427		assert!(record.passes_acceptance_gate());
428	}
429
430	#[test]
431	fn acceptance_gate_allows_boundary_transport_failures() {
432		// Exactly 2% transport failures is still >= 98% transport completion.
433		let record = verification_record(100, 98, 2, 0, 0, Some(0.95), Some(0.93));
434		assert!(record.passes_acceptance_gate());
435	}
436
437	#[test]
438	fn acceptance_gate_rejects_glm_style_record() {
439		// GLM-5.3 Flash evidence: 31 runs, 22 successful, 9 transport failures.
440		let record = verification_record(31, 22, 9, 0, 0, Some(0.709), Some(0.709));
441		assert!(!record.passes_acceptance_gate());
442	}
443
444	#[test]
445	fn acceptance_gate_rejects_missing_rates() {
446		assert!(!verification_record(100, 100, 0, 0, 0, None, None).passes_acceptance_gate());
447		assert!(!verification_record(100, 100, 0, 0, 0, Some(0.96), None).passes_acceptance_gate());
448	}
449
450	#[test]
451	fn acceptance_gate_rejects_low_tool_rates() {
452		assert!(!verification_record(100, 100, 0, 0, 0, Some(0.90), Some(0.94)).passes_acceptance_gate());
453		assert!(!verification_record(100, 100, 0, 0, 0, Some(0.96), Some(0.90)).passes_acceptance_gate());
454	}
455
456	#[test]
457	fn failure_evidence_truncates_long_excerpts() {
458		let evidence = FailureEvidence::new(
459			FailureClass::StreamIdleTimeout,
460			Some(200),
461			Some("ray-1".to_string()),
462			Some(120_000),
463			"@cf/zai-org/glm-5.3-flash",
464			Some("uuid".to_string()),
465			Some("e".repeat(600)),
466		);
467		let excerpt = evidence.response_excerpt.expect("excerpt present");
468		assert_eq!(excerpt.chars().count(), MAX_EXCERPT_CHARS);
469		assert_eq!(excerpt, "e".repeat(MAX_EXCERPT_CHARS));
470	}
471
472	#[test]
473	fn failure_evidence_truncation_is_char_safe() {
474		// Multi-byte scalars must not be split mid-sequence.
475		let sanitized = FailureEvidence::sanitize_excerpt(Some(&"€".repeat(600))).expect("sanitized");
476		assert_eq!(sanitized.chars().count(), MAX_EXCERPT_CHARS);
477		assert_eq!(sanitized, "€".repeat(MAX_EXCERPT_CHARS));
478	}
479
480	#[test]
481	fn failure_evidence_keeps_short_excerpts() {
482		assert_eq!(FailureEvidence::sanitize_excerpt(Some("ok")), Some("ok".to_string()));
483		assert_eq!(FailureEvidence::sanitize_excerpt(None), None);
484	}
485
486	#[test]
487	fn failure_evidence_has_no_token_field() {
488		let evidence = FailureEvidence::new(
489			FailureClass::AuthRejected,
490			Some(401),
491			None,
492			Some(100),
493			"@cf/deepseek-ai/deepseek-v4-flash-0731",
494			None,
495			Some("unauthorized".to_string()),
496		);
497		let json = serde_json::to_string(&evidence).expect("serializes");
498		let lowered = json.to_lowercase();
499		for forbidden in ["token", "authorization", "secret", "api_key", "bearer"] {
500			assert!(!lowered.contains(forbidden), "JSON must not contain {forbidden}: {json}");
501		}
502	}
503
504	#[test]
505	fn failure_evidence_rejects_unknown_fields() {
506		// A leaked credential field must not round-trip through serde.
507		let json = serde_json::json!({
508			"failure_class": "auth_missing",
509			"http_status": null,
510			"cloudflare_ray_id": null,
511			"elapsed_ms": null,
512			"model_id": "@cf/deepseek-ai/deepseek-v4-flash-0731",
513			"request_id": null,
514			"response_excerpt": null,
515			"token": "leaked",
516		});
517		let result = serde_json::from_value::<FailureEvidence>(json);
518		assert!(result.is_err(), "unknown field must be rejected: {result:?}");
519	}
520
521	#[test]
522	fn failure_class_serde_snake_case() {
523		let cases = [
524			(FailureClass::AuthMissing, "\"auth_missing\""),
525			(FailureClass::ToolLoopDidNotConverge, "\"tool_loop_did_not_converge\""),
526			(FailureClass::StreamIdleTimeout, "\"stream_idle_timeout\""),
527			(FailureClass::InvalidChatCompletionShape, "\"invalid_chat_completion_shape\""),
528			(FailureClass::ProviderServerError, "\"provider_server_error\""),
529		];
530		for (class, name) in cases {
531			let json = serde_json::to_string(&class).expect("ser");
532			assert_eq!(json, name);
533			assert_eq!(serde_json::from_str::<FailureClass>(&json).expect("de"), class);
534		}
535		let back: FailureClass = serde_json::from_str("\"cloudflare_edge_error\"").expect("de");
536		assert_eq!(back, FailureClass::CloudflareEdgeError);
537	}
538
539	#[test]
540	fn status_and_confidence_serde_snake_case() {
541		let statuses = [
542			(VerificationStatus::Untested, "untested"),
543			(VerificationStatus::Passing, "passing"),
544			(VerificationStatus::Degraded, "degraded"),
545			(VerificationStatus::Failing, "failing"),
546			(VerificationStatus::Expired, "expired"),
547		];
548		for (status, name) in statuses {
549			let json = serde_json::to_string(&status).expect("ser");
550			assert_eq!(json, format!("\"{name}\""));
551			assert_eq!(serde_json::from_str::<VerificationStatus>(&json).expect("de"), status);
552		}
553		let confidences = [
554			(VerificationConfidence::None, "none"),
555			(VerificationConfidence::StaticMetadataOnly, "static_metadata_only"),
556			(VerificationConfidence::SmokeTested, "smoke_tested"),
557			(VerificationConfidence::ConformanceTested, "conformance_tested"),
558			(VerificationConfidence::RegressionTested, "regression_tested"),
559		];
560		for (confidence, name) in confidences {
561			let json = serde_json::to_string(&confidence).expect("ser");
562			assert_eq!(json, format!("\"{name}\""));
563			assert_eq!(serde_json::from_str::<VerificationConfidence>(&json).expect("de"), confidence);
564		}
565	}
566
567	#[test]
568	fn health_record_serde_roundtrip() {
569		let record = health_record(100, 98, 2, 0, 0, 1);
570		let json = serde_json::to_string(&record).expect("ser");
571		let back: ModelHealthRecord = serde_json::from_str(&json).expect("de");
572		assert_eq!(back, record);
573	}
574
575	#[test]
576	fn verification_serde_roundtrip_with_evidence() {
577		let mut record = verification_record(100, 98, 2, 0, 0, Some(0.96), Some(0.94));
578		record.status = VerificationStatus::Passing;
579		record.last_failure = Some(FailureEvidence::new(
580			FailureClass::RateLimited,
581			Some(429),
582			Some("ray-abc".to_string()),
583			Some(1_200),
584			"@cf/deepseek-ai/deepseek-v4-flash-0731",
585			Some("req-7".to_string()),
586			Some("rate limited".to_string()),
587		));
588		let json = serde_json::to_string(&record).expect("ser");
589		let back: ModelVerification = serde_json::from_str(&json).expect("de");
590		assert_eq!(back.model_id, record.model_id);
591		assert_eq!(back.status, VerificationStatus::Passing);
592		assert_eq!(back.total_runs, 100);
593		assert_eq!(back.single_tool_success_rate, Some(0.96));
594		let evidence = back.last_failure.expect("evidence present");
595		assert_eq!(evidence.failure_class, FailureClass::RateLimited);
596		assert_eq!(evidence.http_status, Some(429));
597		assert_eq!(evidence.cloudflare_ray_id.as_deref(), Some("ray-abc"));
598	}
599
600	#[test]
601	fn health_warning_renders_for_degraded_record() {
602		let record = health_record(100, 89, 11, 0, 0, 0);
603		let warning = health_warning(&record, crate::DEFAULT_MODEL).expect("warning present");
604		assert!(warning.contains("89.0"), "warning must contain the rate: {warning}");
605		assert!(
606			warning.contains(crate::DEFAULT_MODEL),
607			"warning must contain the alternative: {warning}"
608		);
609		assert!(warning.contains("100"), "warning must contain the request count: {warning}");
610		assert!(
611			warning.contains(&record.model_id),
612			"warning must contain the model id: {warning}"
613		);
614	}
615
616	#[test]
617	fn health_warning_none_for_clean_record() {
618		assert_eq!(health_warning(&health_record(100, 100, 0, 0, 0, 0), "alt"), None);
619	}
620
621	#[test]
622	fn health_warning_none_for_empty_record() {
623		assert_eq!(health_warning(&health_record(0, 0, 0, 0, 0, 0), "alt"), None);
624	}
625
626	#[test]
627	fn recommended_stable_alternative_empty_sentinel_for_default() {
628		assert_eq!(recommended_stable_alternative(crate::DEFAULT_MODEL), "");
629	}
630
631	#[test]
632	fn recommended_stable_alternative_returns_default_for_glm() {
633		assert_eq!(
634			recommended_stable_alternative("@cf/zai-org/glm-5.3-flash"),
635			crate::DEFAULT_MODEL
636		);
637	}
638
639	#[test]
640	fn health_warning_is_token_free() {
641		let record = health_record(31, 22, 9, 0, 0, 0);
642		let warning = health_warning(&record, crate::DEFAULT_MODEL).expect("warning present");
643		let lowered = warning.to_lowercase();
644		for forbidden in ["token", "secret", "bearer"] {
645			assert!(!lowered.contains(forbidden), "warning must not contain {forbidden}: {warning}");
646		}
647	}
648}