car_inference/schema.rs
1//! Model schema — declarative metadata for models, analogous to ToolSchema for tools.
2//!
3//! Every model (local GGUF, remote API, Ollama) is described by a `ModelSchema`
4//! that declares identity, capabilities, constraints, cost, and source.
5//! The router uses this schema for initial routing; observed outcomes refine it.
6
7use serde::{Deserialize, Serialize};
8
9/// What a model can do.
10#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, schemars::JsonSchema)]
11#[serde(rename_all = "snake_case")]
12pub enum ModelCapability {
13 /// Text completion / chat generation
14 Generate,
15 /// Vector embeddings
16 Embed,
17 /// Cross-encoder relevance scoring (query + document → relevance
18 /// score). Qwen3-Reranker is the canonical local implementation.
19 Rerank,
20 /// Label assignment / classification
21 Classify,
22 /// Code generation, repair, refactoring
23 Code,
24 /// Chain-of-thought, planning, analysis
25 Reasoning,
26 /// Text condensation
27 Summarize,
28 /// Function/tool calling
29 ToolUse,
30 /// Multiple tool calls in a single response (parallel tool execution)
31 MultiToolCall,
32 /// Vision / image understanding
33 Vision,
34 /// Video understanding (multi-frame sampling + temporal tokens).
35 /// Distinct from `Vision` so routing can prefer video-trained
36 /// models when the caller attaches a video content block.
37 VideoUnderstanding,
38 /// Audio understanding (speech + non-speech audio as an input to
39 /// a chat/reasoning model). Distinct from `SpeechToText` which is
40 /// the transcription-only task. Gemma 4 E2B/E4B and Gemini do
41 /// this; Qwen2.5-VL does not.
42 AudioUnderstanding,
43 /// Visual grounding — structured object-localization output
44 /// (bounding boxes keyed to object labels) in addition to text.
45 Grounding,
46 /// Speech recognition / transcription
47 SpeechToText,
48 /// Speech synthesis / text-to-speech
49 TextToSpeech,
50 /// Image generation
51 ImageGeneration,
52 /// Video generation
53 VideoGeneration,
54}
55
56/// How much the project vouches for a model. Gates automatic upgrades and
57/// is surfaced in recommendation rationale. Closed enum — a new tier is a
58/// deliberate FFI-visible change, never a silent string fallback.
59#[derive(
60 Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default, schemars::JsonSchema,
61)]
62#[serde(rename_all = "snake_case")]
63pub enum TrustTier {
64 /// Vetted by the project — the built-in catalog and verified upgrades.
65 /// Eligible for background auto-apply when the user opts in.
66 #[default]
67 Curated,
68 /// User-registered or upstream-discovered, not project-vetted. Always
69 /// notify-only; never auto-applied regardless of update policy.
70 Community,
71}
72
73/// How to access the model.
74#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
75#[serde(tag = "type", rename_all = "snake_case")]
76pub enum ModelSource {
77 /// Local GGUF file via Candle backend.
78 Local {
79 hf_repo: String,
80 hf_filename: String,
81 tokenizer_repo: String,
82 },
83 /// Remote API endpoint (OpenAI-compatible, Anthropic, etc.)
84 RemoteApi {
85 endpoint: String,
86 /// Environment variable name containing the API key (never the key itself).
87 /// The env var value may contain comma-separated keys for load balancing.
88 api_key_env: String,
89 /// Additional environment variable names for load balancing across multiple keys.
90 /// Each env var may also contain comma-separated keys.
91 #[serde(default)]
92 api_key_envs: Vec<String>,
93 #[serde(default)]
94 api_version: Option<String>,
95 protocol: ApiProtocol,
96 },
97 /// One text-generation turn through the locally installed OpenAI Codex CLI.
98 ///
99 /// Codex remains the sole holder of its ChatGPT-subscription credential:
100 /// CAR neither reads that credential nor accepts an API key for this source.
101 /// An optional reasoning suffix is part of the model string (for example,
102 /// `gpt-5.6-sol:high`) and maps to Codex's `model_reasoning_effort` config.
103 CodexCli { model: String },
104 /// Ollama local server.
105 Ollama {
106 model_tag: String,
107 #[serde(default = "default_ollama_host")]
108 host: String,
109 },
110 /// Local MLX model via mlx-rs backend (Apple Silicon, safetensors format).
111 /// Models from mlx-community on HuggingFace.
112 Mlx {
113 /// HuggingFace repo (e.g., "mlx-community/Qwen3-4B-4bit").
114 hf_repo: String,
115 /// Optional specific weight filename. If None, auto-discovers safetensors files.
116 #[serde(default)]
117 hf_weight_file: Option<String>,
118 },
119 /// Local whisper.cpp speech-to-text model — a ggml `.bin` from the
120 /// `ggerganov/whisper.cpp` HF repo, run in-process via the shared
121 /// `car-whisper` crate. Cross-platform (Windows/Linux/macOS): this is the
122 /// on-device STT path where MLX isn't available. Cached at
123 /// `~/.tokhn/whisper/ggml-<model>.bin`.
124 WhisperCpp {
125 /// whisper.cpp model id — the suffix of `ggml-<model>.bin`
126 /// (e.g. `"large-v3-turbo-q5_0"`).
127 model: String,
128 },
129 /// Windows OS text-to-speech via `Windows.Media.SpeechSynthesis` (WinRT),
130 /// run in-process. The catalog-side analog of the `car-voice`
131 /// `TtsProvider::WindowsSpeech` live path and the parity counterpart of
132 /// Apple's OS synthesizer — free, on-device, no model download, no MLX.
133 /// Windows-only; availability is `false` on every other target (like
134 /// `AppleFoundationModels`).
135 WindowsSpeech {},
136 /// Local vLLM-MLX server (Apple Silicon, OpenAI-compatible API).
137 /// Routes through RemoteBackend with OpenAI protocol handler.
138 VllmMlx {
139 /// Server endpoint (e.g., "http://localhost:8000").
140 endpoint: String,
141 /// The model name as known to vLLM-MLX (e.g., "mlx-community/Qwen3-4B-4bit").
142 model_name: String,
143 },
144 /// CAR-owned supervised vLLM-MLX process backed by a managed HuggingFace
145 /// artifact. Unlike `VllmMlx`, CAR downloads, admits, spawns, reaps, and
146 /// accounts this allocation. Dispatch rewrites a clone to `VllmMlx` only
147 /// after the child reports healthy.
148 ManagedVllmMlx {
149 hf_repo: String,
150 #[serde(default)]
151 hf_weight_file: Option<String>,
152 },
153 /// Apple's on-device system model via the FoundationModels framework
154 /// (macOS 26+, Apple Silicon). Inference happens in-process through a
155 /// Swift shim — there is no HTTP, no API key, and no model file: the
156 /// OS owns the weights. Availability is checked at runtime via
157 /// `@available(macOS 26.0, *)`; on older macOS or non-Apple-Silicon
158 /// hosts the backend reports `UnsupportedMode` and the router falls
159 /// through to the next candidate.
160 AppleFoundationModels {
161 /// Optional Apple use-case hint passed through to
162 /// `LanguageModelSession`. Apple's framework tunes its prompt and
163 /// safety scaffolding per use case (e.g. "general", "summarize").
164 /// `None` uses the default.
165 #[serde(default)]
166 use_case: Option<String>,
167 },
168 /// Proprietary provider with custom auth and protocol.
169 ///
170 /// For vendor-specific APIs that aren't generic OpenAI-compatible endpoints.
171 /// Parslee is the first proprietary provider — custom auth (OAuth2),
172 /// custom response format, multi-provider routing built into the API.
173 Proprietary {
174 /// Provider identifier (e.g., "parslee").
175 provider: String,
176 /// Base URL for the API.
177 endpoint: String,
178 /// Auth configuration.
179 auth: ProprietaryAuth,
180 /// Custom protocol details.
181 protocol: ProprietaryProtocol,
182 },
183 /// Inference is delegated to a host-registered runner. CAR does
184 /// not own the wire format — the runner (typically a JS / Python
185 /// host) translates the `GenerateRequest` to its provider's API,
186 /// streams chunks back through the runner's event callback, and
187 /// returns the final aggregated result.
188 ///
189 /// Closes Parslee-ai/car-releases#24. Use this when the host
190 /// already has an SDK relationship with a provider (Anthropic,
191 /// OpenAI, GitHub Models, Vercel AI SDK) and wants CAR to sit in
192 /// the lifecycle / policy / replay path without learning every
193 /// provider's wire format.
194 ///
195 /// Routing requires that a runner has been registered via
196 /// [`crate::set_inference_runner`] (or its FFI equivalent —
197 /// `registerInferenceRunner` on JS, `register_inference_runner`
198 /// on Python, the `InferenceRunner` foreign trait on UniFFI,
199 /// `inference.register_runner` on the WebSocket protocol).
200 /// Without a runner, dispatch fails with `InferenceFailed`.
201 Delegated {
202 /// Opaque hint passed through to the runner — typically the
203 /// provider id (`"anthropic"`, `"openai"`, `"vercel-ai-sdk"`)
204 /// so a multi-provider runner can dispatch internally. CAR
205 /// does not interpret this string.
206 #[serde(default)]
207 hint: Option<String>,
208 },
209}
210
211impl ModelSource {
212 /// The source's serialized `type` tag (`codex_cli`, `remote_api`, ...):
213 /// where and how the model runs. Two rows of one model on different
214 /// sources are different routes, with their own credentials and billing.
215 pub fn kind(&self) -> &'static str {
216 match self {
217 ModelSource::Local { .. } => "local",
218 ModelSource::RemoteApi { .. } => "remote_api",
219 ModelSource::CodexCli { .. } => "codex_cli",
220 ModelSource::Ollama { .. } => "ollama",
221 ModelSource::Mlx { .. } => "mlx",
222 ModelSource::WhisperCpp { .. } => "whisper_cpp",
223 ModelSource::WindowsSpeech {} => "windows_speech",
224 ModelSource::VllmMlx { .. } => "vllm_mlx",
225 ModelSource::ManagedVllmMlx { .. } => "managed_vllm_mlx",
226 ModelSource::AppleFoundationModels { .. } => "apple_foundation_models",
227 ModelSource::Proprietary { .. } => "proprietary",
228 ModelSource::Delegated { .. } => "delegated",
229 }
230 }
231}
232
233/// Provider id for ChatGPT subscription inference backed by the Codex credential record.
234pub const OPENAI_CODEX_PROVIDER: &str = "openai-codex";
235
236/// What a signed-out openai-codex row tells a person to do.
237pub const OPENAI_CODEX_SIGN_IN_HINT: &str =
238 "sign in to ChatGPT in Settings → AI accounts, or run `car auth login --provider openai-codex`";
239
240/// Authentication method for proprietary providers.
241#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
242#[serde(tag = "type", rename_all = "snake_case")]
243pub enum ProprietaryAuth {
244 /// OAuth2 PKCE flow (e.g., Azure AD for Parslee).
245 #[serde(rename = "oauth2_pkce", alias = "o_auth2_pkce")]
246 OAuth2Pkce {
247 authority: String,
248 client_id: String,
249 scopes: Vec<String>,
250 },
251 /// Static API key from environment variable.
252 ApiKeyEnv { env_var: String },
253 /// Bearer token from environment variable.
254 BearerTokenEnv { env_var: String },
255 /// ChatGPT subscription OAuth read from the car-auth Codex record; admitted only for provider
256 /// `openai-codex`.
257 #[serde(rename = "chatgpt_subscription")]
258 ChatGptSubscription {},
259}
260
261/// Protocol configuration for proprietary providers.
262#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
263pub struct ProprietaryProtocol {
264 /// Chat/completion endpoint path (appended to base URL).
265 #[serde(default = "default_chat_path")]
266 pub chat_path: String,
267 /// Content type for requests.
268 #[serde(default = "default_content_type")]
269 pub content_type: String,
270 /// Whether the API streams responses via SSE.
271 #[serde(default)]
272 pub streaming: bool,
273 /// Custom headers to include in every request.
274 #[serde(default)]
275 pub extra_headers: std::collections::HashMap<String, String>,
276 /// The wire format the endpoint speaks. A row names it rather than CAR
277 /// inferring it from the path, so the model decides the backend.
278 #[serde(default)]
279 pub wire: ProprietaryWire,
280}
281
282/// What a proprietary endpoint speaks.
283#[derive(
284 Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema,
285)]
286#[serde(rename_all = "snake_case")]
287pub enum ProprietaryWire {
288 /// OpenAI Responses-style chat (Parslee's `/inference/responses`).
289 #[default]
290 Responses,
291 /// TypeSafe System One (Parslee's `/inference/typesafe/v1/systemone`):
292 /// typed decisions with a probability per option, no text generation.
293 /// Such a row serves `classify` only.
294 SystemOne,
295}
296
297impl Default for ProprietaryProtocol {
298 fn default() -> Self {
299 Self {
300 chat_path: default_chat_path(),
301 content_type: default_content_type(),
302 streaming: false,
303 extra_headers: std::collections::HashMap::new(),
304 wire: ProprietaryWire::default(),
305 }
306 }
307}
308
309fn default_chat_path() -> String {
310 "/chat".to_string()
311}
312
313fn default_content_type() -> String {
314 "application/json".to_string()
315}
316
317fn default_ollama_host() -> String {
318 "http://localhost:11434".to_string()
319}
320
321#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
322#[serde(rename_all = "snake_case")]
323pub enum ApiProtocol {
324 OpenAiCompat,
325 /// OpenRouter's OpenAI-compatible Chat Completions surface. Distinct so
326 /// credential precedence and error translation stay provider-specific.
327 OpenRouter,
328 /// OpenAI Responses API (/v1/responses) — works with all OpenAI models including codex.
329 OpenAiResponses,
330 Anthropic,
331 Google,
332 /// Azure OpenAI — uses api-key header and deployment-based URLs.
333 /// Endpoint format: {base}/openai/deployments/{model}/chat/completions?api-version={version}
334 AzureOpenAi,
335 /// Google Vertex AI — the enterprise Gemini surface. Same request/response
336 /// shape as the AI-Studio `Google` protocol, but a project/location URL and
337 /// OAuth Bearer auth (a GCP access token from `gcloud auth print-access-token`
338 /// or a service account) instead of an `?key=` query param. Endpoint format:
339 /// `{base}/publishers/google/models/{model}:generateContent`, where `base`
340 /// is `https://{loc}-aiplatform.googleapis.com/v1/projects/{proj}/locations/{loc}`.
341 VertexAi,
342 /// AWS Bedrock — the **Converse** API (`bedrock-runtime`), a unified
343 /// messages surface across Bedrock-hosted models (Claude, Llama, Mistral,
344 /// Titan, …). Auth is **SigV4** request signing (not a bearer token), with
345 /// credentials from the standard AWS env vars; the model's `endpoint` is the
346 /// region (e.g. `us-east-1`) and `name` is the Bedrock model id. Non-stream
347 /// only for now (Converse streaming uses a separate binary event-stream).
348 Bedrock,
349}
350
351impl ApiProtocol {
352 /// Prompt-cache economics for this provider, relative to its base input
353 /// rate — used by the cost scoreboard to price cached tokens correctly.
354 /// Anthropic uses explicit breakpoints (deep read discount + write
355 /// premium); OpenAI/Azure cache automatically (~0.5× read, no write
356 /// charge). Providers whose cache tokens CAR does not parse (Google/Vertex/
357 /// Bedrock) report zero cache tokens, so their rates are inert.
358 pub fn cache_rates(&self) -> crate::outcome::CacheRates {
359 use crate::outcome::CacheRates;
360 match self {
361 ApiProtocol::Anthropic => CacheRates::ANTHROPIC,
362 ApiProtocol::OpenAiCompat | ApiProtocol::OpenAiResponses | ApiProtocol::AzureOpenAi => {
363 CacheRates::OPENAI
364 }
365 // OpenRouter rates differ per upstream model and are carried by
366 // ModelSchema::cost. A blanket OpenAI-shaped discount is false.
367 ApiProtocol::OpenRouter => CacheRates::NONE,
368 ApiProtocol::Google | ApiProtocol::VertexAi | ApiProtocol::Bedrock => CacheRates::NONE,
369 }
370 }
371
372 /// This protocol as an OpenTelemetry GenAI `gen_ai.provider.name` value.
373 ///
374 /// The protocol is the right source for this attribute and
375 /// [`ModelSchema::provider`] is not: that field carries the model's
376 /// *publisher* (`qwen`, `mlx-community`, `apple`, `ggerganov`), while
377 /// `gen_ai.provider.name` names the API surface the request is spoken to.
378 /// A Qwen checkpoint served over Bedrock is `aws.bedrock`, not `qwen`.
379 ///
380 /// Both OpenAI protocols map to `openai` on purpose. The convention
381 /// defines this attribute as a discriminator for the *telemetry format
382 /// flavor*, not the company serving the weights — so every
383 /// OpenAI-REST-shaped surface reports `openai`, and Chat Completions vs
384 /// Responses is not a distinction it draws. `openrouter` has no
385 /// well-known value; the convention's fallback is a provider-specific
386 /// string, which is what this returns.
387 ///
388 /// Exhaustive by intent: a new protocol must decide its own value rather
389 /// than inherit a wildcard's guess.
390 pub fn genai_provider_name(&self) -> &'static str {
391 match self {
392 ApiProtocol::OpenAiCompat | ApiProtocol::OpenAiResponses => "openai",
393 ApiProtocol::OpenRouter => "openrouter",
394 ApiProtocol::Anthropic => "anthropic",
395 ApiProtocol::Google => "gcp.gemini",
396 ApiProtocol::VertexAi => "gcp.vertex_ai",
397 ApiProtocol::AzureOpenAi => "azure.ai.openai",
398 ApiProtocol::Bedrock => "aws.bedrock",
399 }
400 }
401}
402
403/// The numeric format a checkpoint's weights are stored in.
404///
405/// Deliberately says nothing about *which engine* serves the file —
406/// [`ModelSource`] already carries that, and keying this on the container
407/// instead of the format produces falsehoods: whisper.cpp's `q5_0` ggml
408/// checkpoints use the same round-to-nearest block format as llama.cpp's, and
409/// a `Gguf*` variant would assert a GGUF text path that cannot load them.
410///
411/// What it does express is the axis neither the bit width nor the source can:
412/// an MLX 4-bit affine checkpoint and a `Q4_K_M` are both "4-bit", were
413/// produced by different algorithms, and do not have the same quality.
414#[derive(
415 Debug, Clone, Copy, PartialEq, Eq, Hash, Default, Serialize, Deserialize, schemars::JsonSchema,
416)]
417#[serde(rename_all = "snake_case")]
418pub enum QuantScheme {
419 /// Integer weights with a scale and bias shared across `group_size`
420 /// elements (`4bit`, `6bit`). MLX's default and only affine format.
421 AffineGroupInt,
422 /// Block-scaled float — MLX's `mxfp4`, `mxfp8` and NVIDIA's `nvfp4`. A
423 /// genuinely different loader path from [`QuantScheme::AffineGroupInt`] at
424 /// the same nominal width, which is why the width alone cannot classify a
425 /// checkpoint; see `backend::local::quantization_is_decodable`.
426 BlockScaledFloat,
427 /// Mixed per-tensor bit allocation, optionally importance-weighted
428 /// (`Q4_K_M`, `Q5_K_S`, `IQ4_XS`, `TQ1_0`).
429 KQuantMixed,
430 /// Uniform round-to-nearest blocks, no per-tensor mixing and no importance
431 /// weighting (`Q8_0`, `q5_0`, `Q4_0_4_4`).
432 RtnBlock,
433 /// Full-precision weights (`bf16`, `f16`, `f32`). Distinct from `None` at
434 /// the [`ModelSchema::quantization`] level: a positive claim that the
435 /// checkpoint is unquantized, not an absence of information.
436 Unquantized,
437 /// A format this parser could not identify. The label is still preserved
438 /// verbatim; only the classification is missing.
439 #[default]
440 Unknown,
441}
442
443/// Structured quantization descriptor.
444///
445/// This replaces a free-text string that mixed two vocabularies — MLX's
446/// `4bit`/`6bit` and GGUF's `Q4_K_M`/`Q8_0` — under one field, so nothing could
447/// tell a group-quantized integer checkpoint from a mixed k-quant one. It also
448/// threw away what the ingest paths already had in hand: MLX `config.json`
449/// declares `{bits, group_size, mode}` and only `bits` survived.
450///
451/// Deserializes from **either** the legacy bare string or the structured
452/// object, so catalogs, user `models.json` files, and `models.register`
453/// payloads written before this change keep loading unchanged. The object form
454/// requires `label` and rejects unknown fields: a misspelled key is a producer
455/// bug, and accepting it silently is how the field it replaced lost
456/// information in the first place.
457///
458/// **Writes the bare label back whenever that is lossless**, and the object
459/// only when it carries something parsing cannot recover — a `group_size`, or
460/// a `scheme` that disambiguates a label the parser reads as `Unknown`. Two
461/// reasons, both about blast radius rather than taste. This value is inside
462/// `catalog_identity::row_digest`, so an unconditional shape change would move
463/// the digest of every quantized row and hard-fail any client that pinned
464/// `expected_catalog_revision`. And `registry::load_user_config` fails a
465/// `models.json` **whole**, with its only production caller discarding the
466/// error — so an older daemon meeting an object it cannot parse boots with
467/// zero registered models and says nothing. Emitting the object only where it
468/// adds information keeps both costs proportional to what actually changed.
469#[allow(dead_code)]
470#[derive(schemars::JsonSchema)]
471#[serde(untagged)]
472enum QuantizationWireSchema {
473 Label(String),
474 Structured(QuantizationObjectWireSchema),
475}
476
477#[allow(dead_code)]
478#[derive(schemars::JsonSchema)]
479#[serde(deny_unknown_fields)]
480struct QuantizationObjectWireSchema {
481 bits: Option<u8>,
482 scheme: QuantScheme,
483 group_size: Option<u32>,
484 label: String,
485}
486
487#[derive(Debug, Clone, PartialEq, Eq)]
488pub struct Quantization {
489 /// Weight bit width. `None` when the label names none.
490 pub bits: Option<u8>,
491 /// Which quantization family this is.
492 pub scheme: QuantScheme,
493 /// Elements sharing one scale/zero point (MLX: 32, 64, 128). `None` when
494 /// the scheme has no group concept, or the source declared none.
495 pub group_size: Option<u32>,
496 /// The label exactly as published. Never synthesized: it is what the user
497 /// sees, what Hugging Face repos are named after, and the only thing that
498 /// survives a scheme CAR does not recognize yet.
499 pub label: String,
500}
501
502/// Widths affine group quantization actually uses. A `16bit` or `32bit`
503/// label is full precision that happens to be spelled like a quant.
504const AFFINE_GROUP_WIDTHS: std::ops::RangeInclusive<u8> = 2..=8;
505
506impl Quantization {
507 /// Best-effort structure from a published label.
508 ///
509 /// Never fails and never invents: an unrecognized label yields
510 /// [`QuantScheme::Unknown`] with the label intact.
511 pub fn parse(label: &str) -> Self {
512 let label = label.trim();
513 let lower = label.to_ascii_lowercase();
514 let build = |bits: Option<u8>, scheme: QuantScheme| Self {
515 bits,
516 scheme,
517 group_size: None,
518 label: label.to_string(),
519 };
520
521 // Full precision. `fp8` is deliberately NOT here — an 8-bit float is a
522 // quantized weight format, just not one this parser can attribute.
523 match lower.as_str() {
524 "bf16" | "f16" | "fp16" | "float16" | "half" => {
525 return build(Some(16), QuantScheme::Unquantized)
526 }
527 "f32" | "fp32" | "float32" | "full" => {
528 return build(Some(32), QuantScheme::Unquantized)
529 }
530 "none" => return build(None, QuantScheme::Unquantized),
531 "f8" | "fp8" | "float8" => return build(Some(8), QuantScheme::Unknown),
532 // Empty means nobody said, which is not a claim of full precision.
533 "" => return build(None, QuantScheme::Unknown),
534 _ => {}
535 }
536
537 // MLX block-scaled floats: `mxfp4`, `mxfp8`. A bare `mxfp` names no
538 // width, so it identifies nothing.
539 if let Some(rest) = lower.strip_prefix("mxfp") {
540 return match leading_number(rest) {
541 Some((bits, _)) => build(Some(bits), QuantScheme::BlockScaledFloat),
542 None => build(None, QuantScheme::Unknown),
543 };
544 }
545
546 // MLX affine group quant: `4bit`, `4-bit`, `6bit`.
547 if let Some(width) = lower.strip_suffix("bit").map(|w| w.trim_end_matches('-')) {
548 if let Some((bits, consumed)) = leading_number(width) {
549 if consumed == width.len() && AFFINE_GROUP_WIDTHS.contains(&bits) {
550 return build(Some(bits), QuantScheme::AffineGroupInt);
551 }
552 // `16bit`/`32bit` are full precision spelled as a width.
553 if consumed == width.len() && (bits == 16 || bits == 32) {
554 return build(Some(bits), QuantScheme::Unquantized);
555 }
556 }
557 }
558
559 // GGUF. `iq`/`tq` are k-quant families; a bare `q` needs its suffix
560 // read to tell k-quant from legacy round-to-nearest.
561 let gguf = lower
562 .strip_prefix("iq")
563 .or_else(|| lower.strip_prefix("tq"))
564 .map(|rest| (rest, true))
565 .or_else(|| lower.strip_prefix('q').map(|rest| (rest, false)));
566 if let Some((rest, k_family)) = gguf {
567 if let Some((bits, consumed)) = leading_number(rest) {
568 if bits == 0 {
569 return build(None, QuantScheme::Unknown);
570 }
571 // Slice by digits consumed, not by the width's decimal length —
572 // `q08_0` has a two-character prefix for a one-character number.
573 let suffix = &rest[consumed..];
574 let scheme = if k_family || suffix.contains("_k") {
575 QuantScheme::KQuantMixed
576 } else if suffix.starts_with("_0") || suffix.starts_with("_1") {
577 // `starts_with`, not equality: the aarch64 repack quants
578 // are `Q4_0_4_4`, `Q4_0_4_8`, `Q4_0_8_8`.
579 QuantScheme::RtnBlock
580 } else {
581 // A bare `Q4` names a width and no producer — llama.cpp
582 // has no such format, so this came from somewhere else.
583 QuantScheme::Unknown
584 };
585 return build(Some(bits), scheme);
586 }
587 }
588
589 build(None, QuantScheme::Unknown)
590 }
591
592 /// Recover the quantization a GGUF file names in its own filename
593 /// (`Qwen3-8B-Q4_K_M.gguf`, `ggml-large-v3-turbo-q5_0.gguf`).
594 ///
595 /// Scans the hyphen-separated segments from the right and takes the first
596 /// that classifies, so a model whose *name* contains something quant-shaped
597 /// does not outrank the real suffix. Returns `None` rather than guessing
598 /// when nothing in the name is recognizable.
599 pub fn from_gguf_filename(filename: &str) -> Option<Self> {
600 let stem = filename
601 .rsplit_once('.')
602 .map(|(stem, _)| stem)
603 .unwrap_or(filename);
604 stem.rsplit('-')
605 .map(Self::parse)
606 .find(|q| q.scheme != QuantScheme::Unknown)
607 }
608
609 /// Descriptor for an MLX checkpoint, from the `quantization` block of its
610 /// `config.json`. `mode` is MLX's own name for the format (`affine`,
611 /// `mxfp4`, `mxfp8`); absent means affine, which is MLX's default.
612 ///
613 /// Returns `None` when the block carried nothing usable — declaring
614 /// `AffineGroupInt` on the strength of an empty object would assert exactly the
615 /// thing this type exists to establish.
616 pub fn from_mlx_config(
617 bits: Option<u8>,
618 group_size: Option<u32>,
619 mode: Option<&str>,
620 ) -> Option<Self> {
621 if bits.is_none() && group_size.is_none() && mode.is_none() {
622 return None;
623 }
624 let normalized = mode.map(str::to_ascii_lowercase);
625 let scheme = match normalized.as_deref() {
626 // MLX's `QuantizationMode` is exactly these four. `nvfp4` used to
627 // fall through to `Unknown`, which was a refusal dressed as a parse
628 // failure — it is block-scaled float like the MX pair and the
629 // linked loader reads it.
630 Some(m) if m.starts_with("mxfp") || m == "nvfp4" => QuantScheme::BlockScaledFloat,
631 Some("affine") | None => QuantScheme::AffineGroupInt,
632 Some(_) => QuantScheme::Unknown,
633 };
634 let label = match (normalized.as_deref(), bits) {
635 (Some(m), _) if m != "affine" => m.to_string(),
636 (_, Some(b)) => format!("{b}bit"),
637 // No width and no distinguishing mode: MLX said "affine" and
638 // nothing else. Say that rather than inventing a width.
639 (_, None) => "affine".to_string(),
640 };
641 Some(Self {
642 bits,
643 scheme,
644 group_size,
645 label,
646 })
647 }
648}
649
650/// Leading run of ASCII digits as a bit width, with the number of bytes it
651/// occupied. The byte count is returned because it is not recoverable from the
652/// value — `08` and `8` parse the same and slice differently.
653fn leading_number(s: &str) -> Option<(u8, usize)> {
654 let digits: String = s.chars().take_while(char::is_ascii_digit).collect();
655 if digits.is_empty() {
656 return None;
657 }
658 // Overflow (`Q256_K`) is a parse failure, not a silent truncation.
659 digits.parse().ok().map(|n| (n, digits.len()))
660}
661
662impl std::fmt::Display for Quantization {
663 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
664 f.write_str(&self.label)
665 }
666}
667
668impl Serialize for Quantization {
669 fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
670 use serde::ser::SerializeStruct;
671
672 // Round-tripping through `parse` is the exact test for "the label
673 // carries everything": if it reproduces this value, the object form
674 // would be a longer spelling of the same information.
675 if Self::parse(&self.label) == *self {
676 return serializer.serialize_str(&self.label);
677 }
678
679 let len = 2 + usize::from(self.bits.is_some()) + usize::from(self.group_size.is_some());
680 let mut row = serializer.serialize_struct("Quantization", len)?;
681 if let Some(bits) = self.bits {
682 row.serialize_field("bits", &bits)?;
683 }
684 row.serialize_field("scheme", &self.scheme)?;
685 if let Some(group_size) = self.group_size {
686 row.serialize_field("group_size", &group_size)?;
687 }
688 row.serialize_field("label", &self.label)?;
689 row.end()
690 }
691}
692
693impl<'de> Deserialize<'de> for Quantization {
694 fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
695 /// The object form. `label` is required and unknown fields are
696 /// rejected, so a typo'd or double-nested payload is an error rather
697 /// than a row that silently claims `Unknown`.
698 #[derive(Deserialize)]
699 #[serde(deny_unknown_fields)]
700 struct Structured {
701 #[serde(default)]
702 bits: Option<u8>,
703 #[serde(default)]
704 scheme: Option<QuantScheme>,
705 #[serde(default)]
706 group_size: Option<u32>,
707 label: String,
708 }
709
710 struct QuantizationVisitor;
711
712 impl<'de> serde::de::Visitor<'de> for QuantizationVisitor {
713 type Value = Quantization;
714
715 fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
716 f.write_str("a quantization label string, or an object with a `label` field")
717 }
718
719 fn visit_str<E: serde::de::Error>(self, label: &str) -> Result<Self::Value, E> {
720 Ok(Quantization::parse(label))
721 }
722
723 fn visit_map<M: serde::de::MapAccess<'de>>(
724 self,
725 map: M,
726 ) -> Result<Self::Value, M::Error> {
727 // Dispatched by hand rather than via `#[serde(untagged)]`,
728 // which buffers the input and reports only "data did not match
729 // any variant" — for a `models.json` holding dozens of models
730 // that error names neither the field nor the row.
731 let s = Structured::deserialize(serde::de::value::MapAccessDeserializer::new(map))?;
732 // A row that omits `scheme` or `bits` recovers what its label
733 // implies; an explicit value always wins.
734 let inferred = Quantization::parse(&s.label);
735 Ok(Quantization {
736 bits: s.bits.or(inferred.bits),
737 scheme: s.scheme.unwrap_or(inferred.scheme),
738 group_size: s.group_size,
739 label: s.label,
740 })
741 }
742 }
743
744 d.deserialize_any(QuantizationVisitor)
745 }
746}
747
748/// Declared performance expectations. Overridden by observed data once available.
749#[derive(Debug, Clone, Default, Serialize, Deserialize, schemars::JsonSchema)]
750pub struct PerformanceEnvelope {
751 /// Median latency in milliseconds (declared/estimated).
752 #[serde(default)]
753 pub latency_p50_ms: Option<u64>,
754 /// 99th percentile latency in milliseconds.
755 #[serde(default)]
756 pub latency_p99_ms: Option<u64>,
757 /// Tokens per second throughput.
758 #[serde(default)]
759 pub tokens_per_second: Option<f64>,
760}
761
762/// Cost model for routing optimization.
763/// Generation parameters that a model may or may not support.
764/// Models declare which params they accept. The inference layer
765/// strips unsupported params before sending to the API.
766#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, schemars::JsonSchema)]
767#[serde(rename_all = "snake_case")]
768pub enum GenerateParam {
769 Temperature,
770 TopP,
771 TopK,
772 MaxTokens,
773 StopSequences,
774 FrequencyPenalty,
775 PresencePenalty,
776 Seed,
777 ResponseFormat,
778 /// Extended thinking / internal reasoning before responding.
779 ExtendedThinking,
780}
781
782/// Standard parameter set for most models.
783pub fn standard_params() -> Vec<GenerateParam> {
784 vec![
785 GenerateParam::Temperature,
786 GenerateParam::TopP,
787 GenerateParam::MaxTokens,
788 GenerateParam::StopSequences,
789 GenerateParam::FrequencyPenalty,
790 GenerateParam::PresencePenalty,
791 GenerateParam::Seed,
792 ]
793}
794
795/// Parameter set for reasoning models (no temperature, no top_p).
796pub fn reasoning_params() -> Vec<GenerateParam> {
797 vec![GenerateParam::MaxTokens, GenerateParam::StopSequences]
798}
799
800#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
801pub struct TokenPrices {
802 /// USD per 1M uncached input tokens.
803 #[serde(default)]
804 pub input_per_mtok: Option<f64>,
805 /// USD per 1M output tokens.
806 #[serde(default)]
807 pub output_per_mtok: Option<f64>,
808 /// USD per 1M cache-read input tokens.
809 #[serde(default)]
810 pub cache_read_input_per_mtok: Option<f64>,
811 /// USD per 1M cache-write input tokens.
812 #[serde(default)]
813 pub cache_write_input_per_mtok: Option<f64>,
814}
815
816#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
817pub struct TokenPricingTier {
818 /// Inclusive prompt-token threshold at which this tier applies.
819 pub min_prompt_tokens: usize,
820 #[serde(flatten)]
821 pub prices: TokenPrices,
822}
823
824#[derive(Debug, Clone, Default, Serialize, Deserialize, schemars::JsonSchema)]
825pub struct CostModel {
826 /// USD per 1M input tokens (remote models).
827 #[serde(default)]
828 pub input_per_mtok: Option<f64>,
829 /// USD per 1M output tokens (remote models).
830 #[serde(default)]
831 pub output_per_mtok: Option<f64>,
832 /// USD per 1M cache-read input tokens. Unlike protocol-wide cache
833 /// multipliers, this is model-specific and comes from the provider's
834 /// published catalog.
835 #[serde(default)]
836 pub cache_read_input_per_mtok: Option<f64>,
837 /// USD per 1M cache-write input tokens, when the provider charges one.
838 #[serde(default)]
839 pub cache_write_input_per_mtok: Option<f64>,
840 /// Prompt-size pricing overrides, sorted by increasing threshold.
841 /// The highest threshold not greater than the prompt size wins.
842 #[serde(default)]
843 pub pricing_tiers: Vec<TokenPricingTier>,
844 /// On-disk size in MB (local models).
845 #[serde(default)]
846 pub size_mb: Option<u64>,
847 /// RAM required during inference in MB.
848 #[serde(default)]
849 pub ram_mb: Option<u64>,
850}
851
852impl CostModel {
853 pub fn prices_for(&self, prompt_tokens: usize) -> TokenPrices {
854 let mut prices = TokenPrices {
855 input_per_mtok: self.input_per_mtok,
856 output_per_mtok: self.output_per_mtok,
857 cache_read_input_per_mtok: self.cache_read_input_per_mtok,
858 cache_write_input_per_mtok: self.cache_write_input_per_mtok,
859 };
860 for tier in self
861 .pricing_tiers
862 .iter()
863 .filter(|tier| tier.min_prompt_tokens <= prompt_tokens)
864 {
865 if tier.prices.input_per_mtok.is_some() {
866 prices.input_per_mtok = tier.prices.input_per_mtok;
867 }
868 if tier.prices.output_per_mtok.is_some() {
869 prices.output_per_mtok = tier.prices.output_per_mtok;
870 }
871 if tier.prices.cache_read_input_per_mtok.is_some() {
872 prices.cache_read_input_per_mtok = tier.prices.cache_read_input_per_mtok;
873 }
874 if tier.prices.cache_write_input_per_mtok.is_some() {
875 prices.cache_write_input_per_mtok = tier.prices.cache_write_input_per_mtok;
876 }
877 }
878 prices
879 }
880
881 /// Estimated request cost from the provider's declared token prices.
882 /// Unknown price components contribute zero; callers that need to
883 /// distinguish unknown pricing should inspect `prices_for` first.
884 ///
885 /// **This is the routing-score input, not a display or billing figure.**
886 /// The zero-fill is load-bearing here — `adaptive_router` normalizes the
887 /// result into a 0..1 cost score, and it separately neutralizes models
888 /// with no pricing at all, so changing the fill would change routing.
889 /// For anything a human reads, use
890 /// [`estimated_usd_bounded`](Self::estimated_usd_bounded), which refuses
891 /// to bill an unrated bucket at zero and says which way it can be wrong.
892 pub fn estimated_usd(
893 &self,
894 prompt_tokens: usize,
895 output_tokens: usize,
896 cache_read_tokens: usize,
897 cache_write_tokens: usize,
898 ) -> f64 {
899 let prices = self.prices_for(prompt_tokens);
900 let uncached_input = prompt_tokens
901 .saturating_sub(cache_read_tokens)
902 .saturating_sub(cache_write_tokens);
903 (uncached_input as f64 * prices.input_per_mtok.unwrap_or(0.0)
904 + output_tokens as f64 * prices.output_per_mtok.unwrap_or(0.0)
905 + cache_read_tokens as f64 * prices.cache_read_input_per_mtok.unwrap_or(0.0)
906 + cache_write_tokens as f64 * prices.cache_write_input_per_mtok.unwrap_or(0.0))
907 / 1_000_000.0
908 }
909
910 /// Effective per-bucket rates under one resolved price sheet, in the
911 /// bucket order `[uncached input, output, cache read, cache write]`, each
912 /// paired with which way a *substituted* rate can be wrong:
913 /// `(rate, may_overstate, may_understate)`.
914 ///
915 /// The two cache buckets substitute the uncached-input rate but are **not
916 /// governed by one rule**, because the providers in this catalog do not
917 /// price them the same way relative to input:
918 ///
919 /// | bucket | observed vs input, curated table |
920 /// |---|---|
921 /// | cache read | `0.10x`–`0.64x` — always a discount |
922 /// | cache write | `0.1875x` (Google) … `1.25x` (Anthropic) — **both sides** |
923 ///
924 /// So substituting input for a missing **cache-read** rate can only be too
925 /// high (`may_overstate`), while for a missing **cache-write** rate it can
926 /// land either side and gets both flags, rendering `~` rather than a `≤`
927 /// the figure cannot honour. Treating the two alike is how `claude-opus-4.8`
928 /// — `input 5.0`, `cache_write 6.25` — would have worn a `≤$5.00` ceiling
929 /// over a true cost of `$6.25`, or `$10.00` at OpenRouter's 1h-TTL rate.
930 ///
931 /// **Output** substitutes nothing and refuses instead: output runs
932 /// `1.5x`–`8x` input across this same table, which is not a ballpark
933 /// estimate in either direction, merely a wrong number wearing a marker.
934 /// The line is whether a substitute is *within range and merely of unknown
935 /// sign* (estimate, flag it) or *out of range entirely* (refuse) — see
936 /// [`estimated_usd_bounded`](Self::estimated_usd_bounded).
937 fn bucket_rates(prices: &TokenPrices) -> [(Option<f64>, bool, bool); 4] {
938 // A cached READ is always discounted relative to uncached input — the
939 // entire point of the cache — so the input rate is a true ceiling.
940 let cache_read = match prices.cache_read_input_per_mtok {
941 Some(rate) => (Some(rate), false, false),
942 None => {
943 let substituted = prices.input_per_mtok.is_some();
944 (prices.input_per_mtok, substituted, false)
945 }
946 };
947 // A cache WRITE may be a surcharge (Anthropic 1.25x, OpenRouter's 1h
948 // TTL 2x) or a discount (Google 0.1875x). Unknown sign, so no bound.
949 let cache_write = match prices.cache_write_input_per_mtok {
950 Some(rate) => (Some(rate), false, false),
951 None => {
952 let substituted = prices.input_per_mtok.is_some();
953 (prices.input_per_mtok, substituted, substituted)
954 }
955 };
956 [
957 (prices.input_per_mtok, false, false),
958 (prices.output_per_mtok, false, false),
959 cache_read,
960 cache_write,
961 ]
962 }
963
964 /// Cost estimate for a figure a person will read, carrying which way it
965 /// can be wrong.
966 ///
967 /// Differs from [`estimated_usd`](Self::estimated_usd) in refusing to
968 /// invent numbers. A token bucket the provider charges for but whose rate
969 /// this catalog does not declare is **never billed at zero**, and a figure
970 /// that might be wrong never presents itself as exact.
971 ///
972 /// `tier_prompt_tokens` selects the prompt-size pricing tier and is the
973 /// parameter callers most often get wrong:
974 ///
975 /// - `Some(n)` — the prompt size of **one request**. Tiers resolve exactly.
976 /// - `None` — the caller cannot say (a lifetime accumulator has summed
977 /// many requests and lost their boundaries). Base rates are used and the
978 /// result is flagged in whichever direction the model's own tiers run.
979 ///
980 /// Passing a *summed* token count as `Some` is the bug this signature
981 /// exists to prevent: thirty 10K-token requests sum to 300K, which crosses
982 /// a 272K threshold that no individual request came near, and every token
983 /// ever sent gets priced at the high-context rate — roughly double, stated
984 /// with total confidence.
985 ///
986 /// Two rejected alternatives, for the next person who wants tiers on an
987 /// aggregate. **Pricing lifetime totals at the tier their sum lands in** is
988 /// the bug above. **Pricing everything at the highest declared tier** does
989 /// yield a true ceiling, but a useless one — it doubles the figure for a
990 /// user whose prompts never approached the threshold, which is the same
991 /// confident wrongness in the other direction. Resolving tiers properly
992 /// needs per-request prompt sizes, which means [`crate::ModelProfile`]
993 /// would have to accumulate per-tier token buckets at record time; that is
994 /// a real feature with a persisted-schema change, not something to fake
995 /// here from data that has already been summed away.
996 ///
997 /// Returns `None` when no defensible number exists — either the model
998 /// declares no rate card at all, or a bucket carrying tokens has no rate
999 /// and no usable substitute. Output deliberately has no input-rate
1000 /// fallback: it runs 1.5x–8x input across this catalog, far enough out of
1001 /// range that no marker could rescue the number. Which buckets substitute,
1002 /// and which way each substitution can be wrong, is decided in
1003 /// [`bucket_rates`](Self::bucket_rates) from observed provider pricing —
1004 /// notably cache *reads* and cache *writes* do not share a direction.
1005 /// `None` means unpriced, and a caller must render it as such, not as free.
1006 pub fn estimated_usd_bounded(
1007 &self,
1008 tier_prompt_tokens: Option<usize>,
1009 uncached_input_tokens: usize,
1010 output_tokens: usize,
1011 cache_read_tokens: usize,
1012 cache_write_tokens: usize,
1013 ) -> Option<ApproxCost> {
1014 // `prices_for(0)` is the base sheet: no tier threshold is <= 0 in a
1015 // catalog whose thresholds are positive, so nothing overrides.
1016 let prices = self.prices_for(tier_prompt_tokens.unwrap_or(0));
1017 if prices.input_per_mtok.is_none() && prices.output_per_mtok.is_none() {
1018 // No rate card. Not free — unknown.
1019 return None;
1020 }
1021
1022 let tokens = [
1023 uncached_input_tokens,
1024 output_tokens,
1025 cache_read_tokens,
1026 cache_write_tokens,
1027 ];
1028 let rates = Self::bucket_rates(&prices);
1029 let mut usd = 0.0;
1030 let mut may_overstate = false;
1031 let mut may_understate = false;
1032 for (count, (rate, substitute_high, substitute_low)) in tokens.into_iter().zip(rates) {
1033 if count == 0 {
1034 continue;
1035 }
1036 // Charged, rate unknown, no usable substitute — admit we can't.
1037 let rate = rate?;
1038 // Direction comes from the bucket, not from one blanket rule: a
1039 // substituted cache-READ rate can only be high, a substituted
1040 // cache-WRITE rate can land either side.
1041 may_overstate |= substitute_high;
1042 may_understate |= substitute_low;
1043 usd += count as f64 * rate;
1044 }
1045
1046 // Unresolvable tiers: say which way the base sheet can be wrong rather
1047 // than assuming tiers always cost more. Compare the rate each bucket
1048 // would actually pay at every declared threshold against the base.
1049 if tier_prompt_tokens.is_none() {
1050 for tier in &self.pricing_tiers {
1051 let at_tier = Self::bucket_rates(&self.prices_for(tier.min_prompt_tokens));
1052 for (count, ((base_rate, _, _), (tier_rate, _, _))) in
1053 tokens.into_iter().zip(rates.iter().zip(at_tier))
1054 {
1055 if count == 0 {
1056 continue;
1057 }
1058 if let (Some(base), Some(tiered)) = (base_rate, tier_rate) {
1059 may_understate |= tiered > *base;
1060 may_overstate |= tiered < *base;
1061 }
1062 }
1063 }
1064 }
1065
1066 Some(ApproxCost {
1067 usd: usd / 1_000_000.0,
1068 may_overstate,
1069 may_understate,
1070 })
1071 }
1072}
1073
1074/// A cost figure plus which way it can be wrong.
1075///
1076/// Presenting an estimate as an exact price is the same failure as billing an
1077/// unrated bucket at zero, one step later — so the direction travels with the
1078/// number instead of being re-derived (or forgotten) at each display site.
1079#[derive(Debug, Clone, Copy, PartialEq)]
1080pub struct ApproxCost {
1081 /// USD.
1082 pub usd: f64,
1083 /// The true cost may be **lower** — a bucket was priced at a substitute
1084 /// rate that can only be too high (a cache read at the uncached-input
1085 /// rate), or a pricing tier is cheaper than the base sheet used.
1086 pub may_overstate: bool,
1087 /// The true cost may be **higher** — a pricing tier dearer than the base
1088 /// sheet could not be resolved from the tokens the caller had, or a cache
1089 /// *write* was priced at the uncached-input rate and the provider charges
1090 /// a surcharge for it (Anthropic 1.25x, OpenRouter's 1h TTL 2x).
1091 pub may_understate: bool,
1092}
1093
1094impl ApproxCost {
1095 /// Every rate applied exactly; the figure is the price.
1096 pub fn is_exact(&self) -> bool {
1097 !self.may_overstate && !self.may_understate
1098 }
1099
1100 /// Prefix for the figure: `≤` a ceiling, `≥` a floor, `~` neither bound
1101 /// holds, empty when exact. Rendering the number without this is the
1102 /// defect the type exists to prevent.
1103 pub fn marker(&self) -> &'static str {
1104 match (self.may_overstate, self.may_understate) {
1105 (false, false) => "",
1106 (true, false) => "≤",
1107 (false, true) => "≥",
1108 (true, true) => "~",
1109 }
1110 }
1111}
1112
1113/// A score on a public benchmark from a published source (model card,
1114/// paper, leaderboard). The schema is deliberately permissive — no enum
1115/// of benchmark names — so the catalog can carry whichever benchmarks
1116/// the upstream provider chose to publish, and new ones can be added
1117/// without a code change. Scores are stored on a 0.0–1.0 scale (e.g.
1118/// 73.5% accuracy → 0.735) so they compare cleanly across benchmarks
1119/// and so `routing_ext::apply_benchmark_priors` can consume them
1120/// directly when wired in later.
1121#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
1122pub struct BenchmarkScore {
1123 /// Benchmark name as published (e.g., "MMLU-Pro", "GPQA-Diamond",
1124 /// "SWE-bench-Verified", "HumanEval", "MATH").
1125 pub name: String,
1126 /// Score on a 0.0–1.0 scale.
1127 pub score: f64,
1128 /// Evaluation harness or setup label (e.g., "5-shot", "0-shot CoT",
1129 /// "agentic", "pass@1"). Optional but strongly recommended — the
1130 /// same benchmark name can mean different things under different
1131 /// harnesses.
1132 #[serde(default)]
1133 pub harness: Option<String>,
1134 /// Where the score came from (model card URL, paper, leaderboard
1135 /// snapshot). Empty when the source is the upstream provider's
1136 /// announcement and a stable URL is not yet known.
1137 #[serde(default)]
1138 pub source_url: Option<String>,
1139 /// ISO 8601 date of the score snapshot (e.g., "2025-08-12"). Lets
1140 /// downstream code judge how stale a number is.
1141 #[serde(default)]
1142 pub measured_at: Option<String>,
1143 /// How many independent runs the score pools, when CAR measured it.
1144 /// `None` for a published score, whose sample is not ours to know.
1145 #[serde(default, skip_serializing_if = "Option::is_none")]
1146 pub runs: Option<u32>,
1147 /// Highest minus lowest score across those runs; present only with two
1148 /// or more. A difference smaller than this is not a measured difference.
1149 #[serde(default, skip_serializing_if = "Option::is_none")]
1150 pub spread: Option<f64>,
1151}
1152
1153/// The full declarative schema for a model.
1154///
1155/// Analogous to `ToolSchema` — describes what a model is, what it can do,
1156/// and how to access it. The router uses this for constraint-based filtering
1157/// and cold-start scoring before observed performance data is available.
1158#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
1159pub struct ModelSchema {
1160 /// Unique identifier: "provider/model-name:variant" (e.g., "qwen/qwen3-4b:q4_k_m").
1161 pub id: String,
1162 /// Human-readable display name.
1163 pub name: String,
1164 /// Provider (qwen, openai, anthropic, google, meta, ollama, custom).
1165 pub provider: String,
1166 /// Model family for grouping (qwen3, gpt-4, claude-4, llama-3).
1167 pub family: String,
1168 /// Semantic version or checkpoint label.
1169 #[serde(default)]
1170 pub version: String,
1171 /// What this model can do — ordered by primary capability first.
1172 pub capabilities: Vec<ModelCapability>,
1173 /// Context window in tokens.
1174 pub context_length: usize,
1175 /// Per-model maximum OUTPUT tokens the provider will return in one
1176 /// response. None = unknown; callers fall back to
1177 /// effective_max_output() which derives a fraction of context_length.
1178 #[serde(default)]
1179 pub max_output_tokens: Option<usize>,
1180 /// Parameter count as human-readable string (e.g., "4B", "30B (3B active)").
1181 #[serde(default)]
1182 pub param_count: String,
1183 /// How the weights are quantized, if at all. `None` for remote models and
1184 /// for local ones whose source declared nothing. See [`Quantization`] —
1185 /// this accepts the legacy bare-string form on the wire.
1186 #[serde(default)]
1187 #[schemars(with = "Option<QuantizationWireSchema>")]
1188 pub quantization: Option<Quantization>,
1189 /// Declared performance envelope (initial estimate, overridden by observed data).
1190 #[serde(default)]
1191 pub performance: PerformanceEnvelope,
1192 /// Cost structure.
1193 #[serde(default)]
1194 pub cost: CostModel,
1195 /// How to access this model.
1196 pub source: ModelSource,
1197 /// Free-form tags for filtering (e.g., "fast", "multilingual", "moe").
1198 #[serde(default)]
1199 pub tags: Vec<String>,
1200 /// Supported generation parameters. The inference layer strips any parameter
1201 /// not in this set before sending to the API. Empty = all supported.
1202 #[serde(default)]
1203 pub supported_params: Vec<GenerateParam>,
1204 /// Public benchmark scores as published by the model provider or
1205 /// reproduced on a public leaderboard (MMLU-Pro, GPQA-Diamond,
1206 /// SWE-bench, HumanEval, etc.). The built-in catalog ships this
1207 /// empty — population is a curation step, not a code change. See
1208 /// `BenchmarkScore` for the field shape and the 0.0–1.0 scoring
1209 /// convention.
1210 #[serde(default)]
1211 pub public_benchmarks: Vec<BenchmarkScore>,
1212 /// How much the project vouches for this model. The built-in catalog is
1213 /// `Curated`. Deserialization retains the legacy `Curated` default, so
1214 /// every user-controlled ingestion boundary must call
1215 /// [`Self::mark_user_registered`] before persistence or registration.
1216 /// Gates auto-apply (task #8) and this is surfaced in recommendation
1217 /// rationale.
1218 #[serde(default)]
1219 pub trust_tier: TrustTier,
1220 /// Superseded models stay listed if installed but are excluded from
1221 /// fresh recommendations. `#[serde(default)]` → not deprecated.
1222 #[serde(default)]
1223 pub deprecated: bool,
1224 /// Whether this model is currently available (downloaded / reachable).
1225 /// Not serialized — computed at runtime.
1226 #[serde(skip)]
1227 pub available: bool,
1228 /// Whether this model can be used **right now, without a download**.
1229 ///
1230 /// Deliberately narrower than [`Self::available`], which for a local MLX
1231 /// model is true as soon as an `hf_repo` is declared — `ensure_local()`
1232 /// lazy-downloads on first use, so a declared repo is "functionally
1233 /// available" (see #164). That is the right default for open-ended work and
1234 /// wrong for work on a deadline: a step with a bounded budget that picks a
1235 /// model it must first fetch spends the whole budget downloading and fails.
1236 /// That is exactly how `car code`'s 120s contract derivation became
1237 /// unusable on a machine with no local weights (Parslee-ai/car#638).
1238 ///
1239 /// Callers express the requirement with [`crate::IntentHint::require_ready`];
1240 /// this is the per-candidate fact that hint filters on. Recomputed on every
1241 /// registration, so a cached schema can't carry a stale value.
1242 #[serde(skip)]
1243 pub weights_ready: bool,
1244}
1245
1246impl ModelSchema {
1247 /// Mark a schema as user-controlled rather than project-vetted.
1248 ///
1249 /// This is intentionally separate from serde's legacy default: old built-in
1250 /// and test fixtures omit `trust_tier` and must continue to deserialize,
1251 /// while `models.json`, CLI imports, and daemon `models.register` must never
1252 /// inherit `Curated` merely because a caller omitted the field or supplied
1253 /// a forged value.
1254 pub fn mark_user_registered(&mut self) {
1255 self.trust_tier = TrustTier::Community;
1256 }
1257
1258 /// Check if this model has a given capability.
1259 pub fn has_capability(&self, cap: ModelCapability) -> bool {
1260 self.capabilities.contains(&cap)
1261 }
1262
1263 /// Whether usage is covered by the person's ChatGPT subscription.
1264 /// Callers must render `subscription` and never a dollar figure.
1265 pub fn is_subscription_billed(&self) -> bool {
1266 matches!(
1267 &self.source,
1268 ModelSource::Proprietary {
1269 auth: ProprietaryAuth::ChatGptSubscription {},
1270 ..
1271 }
1272 )
1273 }
1274
1275 /// Live availability for credential-backed providers. The catalog field is
1276 /// a startup snapshot; Settings/OAuth changes must affect the next list and
1277 /// route without a daemon restart.
1278 /// The credential this row is waiting on, when a missing credential is the
1279 /// only thing standing between the user and using it.
1280 ///
1281 /// Deliberately narrow. It answers `Some` only for a plain remote API row
1282 /// that declares one credential variable and is currently unavailable —
1283 /// the case where "add this key" is complete and correct advice. Every
1284 /// other shape answers `None`, because for them the advice would be wrong:
1285 ///
1286 /// - available rows need nothing;
1287 /// - local rows have no credential;
1288 /// - `OpenRouter` rows resolve through a credential *source* (pasted key
1289 /// or OAuth) rather than a named variable a person can type;
1290 /// - `Proprietary` OAuth rows (`parslee/*`) are a sign-in, not a key, and
1291 /// their availability additionally depends on the gateway having an
1292 /// upstream — credential presence alone has already been shipped as a
1293 /// false "available" for those (car#786);
1294 /// - `CodexCli` owns its own auth;
1295 /// - a deprecated row will not become usable by adding anything.
1296 ///
1297 /// A row declaring several alternative variables answers `None` as well:
1298 /// telling a person to set one of four names is not actionable, and
1299 /// picking one for them would be a guess about which account they have.
1300 pub fn credential_required(&self) -> Option<String> {
1301 if self.available_now() || self.deprecated || self.is_local() {
1302 return None;
1303 }
1304 match &self.source {
1305 ModelSource::RemoteApi {
1306 protocol: ApiProtocol::OpenRouter,
1307 ..
1308 } => None,
1309 ModelSource::RemoteApi {
1310 api_key_env,
1311 api_key_envs,
1312 ..
1313 } if api_key_envs.is_empty() && !api_key_env.trim().is_empty() => {
1314 Some(api_key_env.clone())
1315 }
1316 _ => None,
1317 }
1318 }
1319
1320 pub fn available_now(&self) -> bool {
1321 match &self.source {
1322 ModelSource::RemoteApi {
1323 protocol: ApiProtocol::OpenRouter,
1324 ..
1325 } => self.available && crate::openrouter::credential_source().is_some(),
1326 ModelSource::CodexCli { .. } => crate::backend::codex_cli::is_available(),
1327 // Live (cached a few seconds): Apple Intelligence can be switched
1328 // on or off, or finish provisioning, while CAR runs, and the
1329 // catalog snapshot would not see it until the next refresh.
1330 ModelSource::AppleFoundationModels { .. } => {
1331 #[cfg(any(
1332 all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)),
1333 all(target_os = "ios", target_arch = "aarch64")
1334 ))]
1335 {
1336 crate::backend::foundation_models::is_available()
1337 }
1338 #[cfg(not(any(
1339 all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)),
1340 all(target_os = "ios", target_arch = "aarch64")
1341 )))]
1342 {
1343 false
1344 }
1345 }
1346 _ => self.available,
1347 }
1348 }
1349
1350 /// Prompt-cache economics for this model, derived from its remote
1351 /// protocol. Local / non-remote models have no remote prompt cache, so
1352 /// their cache rates are inert ([`CacheRates::NONE`](crate::outcome::CacheRates::NONE)).
1353 pub fn cache_rates(&self) -> crate::outcome::CacheRates {
1354 match &self.source {
1355 ModelSource::RemoteApi {
1356 protocol: ApiProtocol::OpenRouter,
1357 ..
1358 } => {
1359 let input = self.cost.input_per_mtok.unwrap_or(0.0);
1360 if input > 0.0 {
1361 crate::outcome::CacheRates {
1362 read_mult: self.cost.cache_read_input_per_mtok.unwrap_or(0.0) / input,
1363 write_mult: self.cost.cache_write_input_per_mtok.unwrap_or(0.0) / input,
1364 }
1365 } else {
1366 crate::outcome::CacheRates::NONE
1367 }
1368 }
1369 ModelSource::RemoteApi { protocol, .. } => protocol.cache_rates(),
1370 _ => crate::outcome::CacheRates::NONE,
1371 }
1372 }
1373
1374 /// The organization whose model this is, when that can be honestly known.
1375 ///
1376 /// Answers one question — could two models be expected to fail the same
1377 /// way? — so it is deliberately conservative. `None` means UNKNOWABLE, not
1378 /// "none", and callers must treat it as "cannot tell" rather than folding
1379 /// it into a count of distinct vendors.
1380 ///
1381 /// NOT [`Self::provider`], which means four different things depending on
1382 /// which path built the schema:
1383 ///
1384 /// * curated remote rows — the real vendor (`openai`, `anthropic`);
1385 /// * OpenRouter and the Parslee gateway — the AGGREGATOR, so three vendors
1386 /// behind one gateway all report `openrouter`/`parslee` and one vendor
1387 /// reached two ways reports as two;
1388 /// * a HuggingFace-derived row — the repo ORG that uploaded it, so
1389 /// `unsloth/Qwen3` and `mlx-community/Qwen3` are the same weights under
1390 /// two "vendors";
1391 /// * a discovered local-server row — a guess from the model name.
1392 ///
1393 /// Only the first is a vendor, so only the first is reported. NOT `family`
1394 /// either — that is the model line, so `claude-4.6` and `claude-4.8` read as
1395 /// different and are both Anthropic.
1396 pub fn vendor(&self) -> Option<&str> {
1397 // The aggregator case: the curated table knows which upstream a gateway
1398 // alias resolves to, which is the only place that survives an id
1399 // carrying no trace of its vendor.
1400 if self.provider.eq_ignore_ascii_case("openrouter")
1401 || self.provider.eq_ignore_ascii_case("parslee")
1402 {
1403 return crate::openrouter::curated_vendor(&self.id);
1404 }
1405 // A local model is served by the operator's own machine. Whoever
1406 // uploaded the weights is not an organization that could fail
1407 // independently of the process running beside it.
1408 if self.is_local() {
1409 return None;
1410 }
1411 // Only a project-vetted row's `provider` was assigned deliberately. On a
1412 // community row it is the uploader or a guess, and claiming it here is
1413 // how two repacks of one checkpoint would pass as two vendors.
1414 if self.trust_tier != TrustTier::Curated {
1415 return None;
1416 }
1417 (!self.provider.is_empty()).then_some(self.provider.as_str())
1418 }
1419
1420 /// Check if this model is local (runs on-device).
1421 pub fn is_local(&self) -> bool {
1422 matches!(
1423 self.source,
1424 ModelSource::Local { .. }
1425 | ModelSource::Mlx { .. }
1426 | ModelSource::WhisperCpp { .. }
1427 | ModelSource::WindowsSpeech { .. }
1428 | ModelSource::ManagedVllmMlx { .. }
1429 | ModelSource::AppleFoundationModels { .. }
1430 )
1431 }
1432
1433 /// Whether this model has weights CAR fetches to disk before it can be
1434 /// used — i.e. whether "is it installed?" is a question with an answer.
1435 ///
1436 /// Three predicates in this area are easy to conflate, and conflating them
1437 /// is what Parslee-ai/car#894 was about:
1438 ///
1439 /// - [`is_local`](Self::is_local) — *owned on this machine*. True for
1440 /// `WindowsSpeech` (the OS owns the voices), `AppleFoundationModels`
1441 /// (the OS owns the weights), and CAR-managed sources. An external
1442 /// `VllmMlx` endpoint is remote even when its URL happens to be loopback.
1443 /// - [`weights_ready`](Self::weights_ready) — *the weights are on disk
1444 /// now*. Only meaningful when this predicate is true; for everything
1445 /// else the registry sets it to `true` as a "nothing blocks an attempt"
1446 /// sentinel, which reads as "installed" if taken literally.
1447 /// - `downloads_weights` (this one) — *there is something to install at
1448 /// all*. Use it to decide whether an install/download status should be
1449 /// reported, then use `weights_ready` for the status itself.
1450 ///
1451 /// Rendering `weights_ready` without this gate is what made
1452 /// `windows/speech-synthesis:os` claim `INSTALLED yes` while `car doctor`
1453 /// said `Models: none installed`, and made `apple/foundation:default` and
1454 /// the `vllm-mlx/*` rows claim `INSTALLED no` for models that install
1455 /// nothing.
1456 ///
1457 /// Written as an exhaustive `match` rather than `matches!` so that adding
1458 /// a `ModelSource` variant is a compile error here instead of a silently
1459 /// wrong answer in the CLI.
1460 pub fn downloads_weights(&self) -> bool {
1461 match self.source {
1462 // CAR fetches these to disk itself: a GGUF file, an MLX
1463 // safetensors repo, a whisper.cpp ggml `.bin`.
1464 ModelSource::Local { .. }
1465 | ModelSource::Mlx { .. }
1466 | ModelSource::WhisperCpp { .. }
1467 | ModelSource::ManagedVllmMlx { .. } => true,
1468 // The OS owns the voices / the weights — nothing to download.
1469 ModelSource::WindowsSpeech {} | ModelSource::AppleFoundationModels { .. } => false,
1470 // Someone else holds the weights: a local server (vLLM-MLX,
1471 // Ollama), a remote API, or a host-registered runner.
1472 ModelSource::VllmMlx { .. }
1473 | ModelSource::Ollama { .. }
1474 | ModelSource::RemoteApi { .. }
1475 | ModelSource::CodexCli { .. }
1476 | ModelSource::Proprietary { .. }
1477 | ModelSource::Delegated { .. } => false,
1478 }
1479 }
1480
1481 /// Whether a CAR-downloadable artifact is physically present now.
1482 /// General runtime availability and OS/server-owned weights are not an
1483 /// installation claim.
1484 pub fn has_installed_weights(&self) -> bool {
1485 self.downloads_weights() && self.weights_ready
1486 }
1487
1488 /// Whether CAR decodes this model **in its own process**, token by token,
1489 /// through the shared decode loop.
1490 ///
1491 /// Narrower than [`is_local`](Self::is_local) on purpose. `is_local` also
1492 /// covers CAR-managed vLLM-MLX and the speech backends. Those do not spend
1493 /// an output-token budget as this process's wall clock, so they must keep
1494 /// the remote treatment. Getting that distinction wrong takes the
1495 /// anti-truncation budget away from vLLM-MLX, which is the documented way
1496 /// to get structured tool calls out of a local model. (car#851)
1497 pub fn decodes_in_process(&self) -> bool {
1498 matches!(
1499 self.source,
1500 ModelSource::Local { .. } | ModelSource::Mlx { .. }
1501 )
1502 }
1503
1504 /// Check if this model delegates inference to a host-registered
1505 /// runner (closes Parslee-ai/car-releases#24).
1506 pub fn is_delegated(&self) -> bool {
1507 matches!(self.source, ModelSource::Delegated { .. })
1508 }
1509
1510 /// Check if this model runs one tool-free turn through `codex exec`.
1511 pub fn is_codex_cli(&self) -> bool {
1512 matches!(self.source, ModelSource::CodexCli { .. })
1513 }
1514
1515 /// Check if this model uses the MLX backend.
1516 pub fn is_mlx(&self) -> bool {
1517 matches!(self.source, ModelSource::Mlx { .. })
1518 }
1519
1520 /// Check if this model routes to Apple's on-device FoundationModels
1521 /// framework. True only for `ModelSource::AppleFoundationModels`;
1522 /// callers must still verify runtime availability before dispatch
1523 /// (the schema can describe the model on any host, but execution
1524 /// requires macOS 26+ on Apple Silicon).
1525 pub fn is_foundation_models(&self) -> bool {
1526 matches!(self.source, ModelSource::AppleFoundationModels { .. })
1527 }
1528
1529 /// Whether the operating system owns the implementation and its memory.
1530 /// These rows have no CAR-downloadable artifact and no local-model memory
1531 /// figure to compare with the admission budget.
1532 pub fn is_os_provided(&self) -> bool {
1533 matches!(
1534 self.source,
1535 ModelSource::WindowsSpeech {} | ModelSource::AppleFoundationModels { .. }
1536 )
1537 }
1538
1539 /// Check if this model uses vLLM-MLX backend.
1540 pub fn is_vllm_mlx(&self) -> bool {
1541 matches!(
1542 self.source,
1543 ModelSource::VllmMlx { .. } | ModelSource::ManagedVllmMlx { .. }
1544 )
1545 }
1546
1547 /// Whether CAR, rather than an independently managed endpoint, owns the
1548 /// vLLM-MLX child process and its physical weight allocation. The existing
1549 /// HTTP `VllmMlx` contract remains external/server-owned. A supervised
1550 /// source must opt in explicitly and provide an already-installed local
1551 /// model path in `model_name`; it is then replaced with a loopback HTTP
1552 /// endpoint only after admission and a successful readiness ACK.
1553 pub fn is_car_managed_vllm_mlx(&self) -> bool {
1554 matches!(self.source, ModelSource::ManagedVllmMlx { .. })
1555 }
1556
1557 /// Whether a CAR/OS-owned model can only run on Apple Silicon (Metal).
1558 /// External vLLM-MLX endpoints own their hardware and are not constrained
1559 /// by the client machine's accelerator.
1560 pub fn requires_apple_silicon(&self) -> bool {
1561 self.is_mlx() || self.is_car_managed_vllm_mlx() || self.is_foundation_models()
1562 }
1563
1564 /// Check if this model is remote (requires API call).
1565 pub fn is_remote(&self) -> bool {
1566 matches!(
1567 self.source,
1568 ModelSource::RemoteApi { .. }
1569 | ModelSource::CodexCli { .. }
1570 | ModelSource::Proprietary { .. }
1571 | ModelSource::VllmMlx { .. }
1572 )
1573 }
1574
1575 /// Collect all API key env var names for this model (primary + extras).
1576 /// Returns empty vec for non-remote models.
1577 pub fn all_api_key_envs(&self) -> Vec<String> {
1578 match &self.source {
1579 ModelSource::RemoteApi {
1580 api_key_env,
1581 api_key_envs,
1582 ..
1583 } => {
1584 let mut all = vec![api_key_env.clone()];
1585 all.extend(api_key_envs.iter().cloned());
1586 all
1587 }
1588 ModelSource::Proprietary {
1589 auth: ProprietaryAuth::ApiKeyEnv { env_var },
1590 ..
1591 }
1592 | ModelSource::Proprietary {
1593 auth: ProprietaryAuth::BearerTokenEnv { env_var },
1594 ..
1595 } => vec![env_var.clone()],
1596 ModelSource::Proprietary {
1597 auth: ProprietaryAuth::OAuth2Pkce { .. } | ProprietaryAuth::ChatGptSubscription {},
1598 ..
1599 } => vec![],
1600 _ => vec![],
1601 }
1602 }
1603
1604 /// Get the size in MB (from cost model or 0 if unknown).
1605 pub fn size_mb(&self) -> u64 {
1606 self.cost.size_mb.unwrap_or(0)
1607 }
1608
1609 /// Get the RAM requirement in MB (from cost model, falls back to size_mb).
1610 pub fn ram_mb(&self) -> u64 {
1611 self.cost.ram_mb.unwrap_or_else(|| self.size_mb())
1612 }
1613
1614 /// Estimated cost per 1K output tokens in USD. Returns 0.0 for local models.
1615 pub fn cost_per_1k_output(&self) -> f64 {
1616 self.cost.output_per_mtok.map(|c| c / 1000.0).unwrap_or(0.0)
1617 }
1618
1619 /// The per-turn output-token ceiling to use when the caller didn't
1620 /// specify one. Prefers the registry-declared `max_output_tokens`;
1621 /// otherwise derives a quarter of the context window, clamped to a
1622 /// sane [4096, 32768] band so a 1M-context model doesn't request a
1623 /// 250K-token response the API rejects and a tiny 8K model doesn't
1624 /// get an absurdly small ceiling. (Registry value first, computed
1625 /// fallback second — mirrors a provider lookup with a derived default.)
1626 pub fn effective_max_output(&self) -> usize {
1627 self.max_output_tokens
1628 .unwrap_or_else(|| (self.context_length / 4).clamp(4096, 32_768))
1629 }
1630}
1631
1632#[cfg(test)]
1633mod tests {
1634 use super::*;
1635
1636 fn sample_local() -> ModelSchema {
1637 ModelSchema {
1638 id: "qwen/qwen3-4b:q4_k_m".into(),
1639 name: "Qwen3-4B".into(),
1640 provider: "qwen".into(),
1641 family: "qwen3".into(),
1642 version: "1.0".into(),
1643 capabilities: vec![ModelCapability::Generate, ModelCapability::Code],
1644 context_length: 32768,
1645 max_output_tokens: None,
1646 param_count: "4B".into(),
1647 quantization: Some(Quantization::parse("Q4_K_M")),
1648 performance: PerformanceEnvelope {
1649 tokens_per_second: Some(45.0),
1650 ..Default::default()
1651 },
1652 cost: CostModel {
1653 size_mb: Some(2500),
1654 ram_mb: Some(2500),
1655 ..Default::default()
1656 },
1657 source: ModelSource::Local {
1658 hf_repo: "Qwen/Qwen3-4B-GGUF".into(),
1659 hf_filename: "Qwen3-4B-Q4_K_M.gguf".into(),
1660 tokenizer_repo: "Qwen/Qwen3-4B".into(),
1661 },
1662 tags: vec!["code".into(), "fast".into()],
1663 supported_params: vec![],
1664 public_benchmarks: vec![],
1665 trust_tier: TrustTier::Curated,
1666 deprecated: false,
1667 available: false,
1668 weights_ready: false,
1669 }
1670 }
1671
1672 fn sample_remote() -> ModelSchema {
1673 ModelSchema {
1674 id: "anthropic/claude-sonnet-4-6:latest".into(),
1675 name: "Claude Sonnet 4.6".into(),
1676 provider: "anthropic".into(),
1677 family: "claude-4".into(),
1678 version: "latest".into(),
1679 capabilities: vec![
1680 ModelCapability::Generate,
1681 ModelCapability::Code,
1682 ModelCapability::Reasoning,
1683 ModelCapability::ToolUse,
1684 ModelCapability::Vision,
1685 ],
1686 context_length: 200000,
1687 max_output_tokens: None,
1688 param_count: String::new(),
1689 quantization: None,
1690 performance: PerformanceEnvelope {
1691 latency_p50_ms: Some(2000),
1692 latency_p99_ms: Some(8000),
1693 tokens_per_second: Some(80.0),
1694 },
1695 cost: CostModel {
1696 input_per_mtok: Some(3.0),
1697 output_per_mtok: Some(15.0),
1698 ..Default::default()
1699 },
1700 source: ModelSource::RemoteApi {
1701 endpoint: "https://api.anthropic.com/v1/messages".into(),
1702 api_key_env: "ANTHROPIC_API_KEY".into(),
1703 api_key_envs: vec![],
1704 api_version: Some("2023-06-01".into()),
1705 protocol: ApiProtocol::Anthropic,
1706 },
1707 tags: vec!["reasoning".into(), "tool_use".into()],
1708 supported_params: vec![],
1709 public_benchmarks: vec![],
1710 trust_tier: TrustTier::Curated,
1711 deprecated: false,
1712 available: false,
1713 weights_ready: false,
1714 }
1715 }
1716
1717 #[test]
1718 fn capabilities() {
1719 let m = sample_local();
1720 assert!(m.has_capability(ModelCapability::Code));
1721 assert!(!m.has_capability(ModelCapability::Vision));
1722 }
1723
1724 #[test]
1725 fn local_vs_remote() {
1726 assert!(sample_local().is_local());
1727 assert!(!sample_local().is_remote());
1728 assert!(sample_remote().is_remote());
1729 assert!(!sample_remote().is_local());
1730 let codex = ModelSchema {
1731 source: ModelSource::CodexCli {
1732 model: "gpt-5.6-sol:high".into(),
1733 },
1734 ..sample_local()
1735 };
1736 assert!(codex.is_remote());
1737 assert!(!codex.is_local());
1738 }
1739
1740 #[test]
1741 fn vllm_ownership_drives_local_remote_and_apple_predicates() {
1742 let external = ModelSchema {
1743 source: ModelSource::VllmMlx {
1744 endpoint: "https://gpu-owner.example/v1".into(),
1745 model_name: "owner/runtime-model".into(),
1746 },
1747 ..sample_local()
1748 };
1749 assert!(!external.is_local());
1750 assert!(external.is_remote());
1751 assert!(!external.requires_apple_silicon());
1752
1753 let managed = ModelSchema {
1754 source: ModelSource::ManagedVllmMlx {
1755 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
1756 hf_weight_file: None,
1757 },
1758 ..sample_local()
1759 };
1760 assert!(managed.is_local());
1761 assert!(!managed.is_remote());
1762 assert!(managed.requires_apple_silicon());
1763 }
1764
1765 #[test]
1766 fn cost() {
1767 let local = sample_local();
1768 assert_eq!(local.cost_per_1k_output(), 0.0);
1769
1770 let remote = sample_remote();
1771 assert!(remote.cost_per_1k_output() > 0.0);
1772 }
1773
1774 #[test]
1775 fn serde_roundtrip() {
1776 let local = sample_local();
1777 let json = serde_json::to_string(&local).unwrap();
1778 let parsed: ModelSchema = serde_json::from_str(&json).unwrap();
1779 assert_eq!(parsed.id, local.id);
1780 assert_eq!(parsed.capabilities, local.capabilities);
1781
1782 let remote = sample_remote();
1783 let json = serde_json::to_string(&remote).unwrap();
1784 let parsed: ModelSchema = serde_json::from_str(&json).unwrap();
1785 assert_eq!(parsed.id, remote.id);
1786 // available is skip-serialized, defaults to false
1787 assert!(!parsed.available);
1788 }
1789
1790 #[test]
1791 fn managed_vllm_source_is_versioned_without_reinterpreting_legacy_vllm_json() {
1792 let legacy: ModelSource = serde_json::from_str(
1793 r#"{"type":"vllm_mlx","endpoint":"http://localhost:8000","model_name":"legacy"}"#,
1794 )
1795 .unwrap();
1796 assert!(matches!(
1797 legacy,
1798 ModelSource::VllmMlx {
1799 endpoint,
1800 model_name
1801 } if endpoint == "http://localhost:8000" && model_name == "legacy"
1802 ));
1803
1804 let managed = ModelSource::ManagedVllmMlx {
1805 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
1806 hf_weight_file: Some("model.safetensors".into()),
1807 };
1808 let encoded = serde_json::to_string(&managed).unwrap();
1809 assert!(encoded.contains(r#""type":"managed_vllm_mlx""#));
1810 assert!(matches!(
1811 serde_json::from_str::<ModelSource>(&encoded).unwrap(),
1812 ModelSource::ManagedVllmMlx { .. }
1813 ));
1814 }
1815
1816 #[test]
1817 fn vendor_refuses_to_claim_a_repackager_or_a_guess() {
1818 // A HuggingFace-derived row's `provider` is the repo ORG that uploaded
1819 // it, so `unsloth/Qwen3` and `mlx-community/Qwen3` — one checkpoint,
1820 // two repacks — would otherwise read as two independent vendors. That
1821 // is a false independence claim produced silently, which is the exact
1822 // failure a vendor check exists to prevent.
1823 let mut m = sample_remote();
1824 m.provider = "mlx-community".into();
1825 m.trust_tier = TrustTier::Community;
1826 assert_eq!(m.vendor(), None);
1827
1828 // A project-vetted remote row IS authoritative.
1829 m.provider = "openai".into();
1830 m.trust_tier = TrustTier::Curated;
1831 assert_eq!(m.vendor(), Some("openai"));
1832
1833 // A local model is served by the operator's own machine; whoever
1834 // uploaded the weights is not an independently-failing organization.
1835 assert_eq!(sample_local().vendor(), None);
1836 }
1837
1838 #[test]
1839 fn trust_tier_and_deprecated_default_when_absent() {
1840 // Pre-existing ~/.car/models.json configs omit the new fields.
1841 // They must deserialize to Curated / not-deprecated, not error.
1842 let json = serde_json::to_string(&sample_local()).unwrap();
1843 let stripped = json
1844 .replace(",\"trust_tier\":\"curated\"", "")
1845 .replace(",\"deprecated\":false", "");
1846 let parsed: ModelSchema = serde_json::from_str(&stripped).unwrap();
1847 assert_eq!(parsed.trust_tier, TrustTier::Curated);
1848 assert!(!parsed.deprecated);
1849 }
1850
1851 #[test]
1852 fn trust_tier_serializes_snake_case() {
1853 assert_eq!(
1854 serde_json::to_string(&TrustTier::Community).unwrap(),
1855 "\"community\""
1856 );
1857 assert_eq!(TrustTier::default(), TrustTier::Curated);
1858 }
1859
1860 #[test]
1861 fn requires_apple_silicon_only_for_metal_backends() {
1862 // GGUF/Candle local and remote models run anywhere CAR builds for.
1863 assert!(!sample_local().requires_apple_silicon());
1864 assert!(!sample_remote().requires_apple_silicon());
1865
1866 let mlx = ModelSchema {
1867 source: ModelSource::Mlx {
1868 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
1869 hf_weight_file: None,
1870 },
1871 ..sample_local()
1872 };
1873 assert!(mlx.requires_apple_silicon());
1874
1875 // Only CAR-managed vLLM-MLX and Apple FoundationModels are Metal-bound.
1876 // A raw endpoint is external and its owner's hardware is opaque to CAR.
1877 let managed_vllm = ModelSchema {
1878 source: ModelSource::ManagedVllmMlx {
1879 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
1880 hf_weight_file: None,
1881 },
1882 ..sample_local()
1883 };
1884 assert!(managed_vllm.requires_apple_silicon());
1885
1886 let external_vllm = ModelSchema {
1887 source: ModelSource::VllmMlx {
1888 endpoint: "https://gpu-owner.example/v1".into(),
1889 model_name: "mlx-community/Qwen3-4B-4bit".into(),
1890 },
1891 ..sample_local()
1892 };
1893 assert!(!external_vllm.requires_apple_silicon());
1894
1895 let foundation = ModelSchema {
1896 source: ModelSource::AppleFoundationModels { use_case: None },
1897 ..sample_local()
1898 };
1899 assert!(foundation.requires_apple_silicon());
1900 }
1901
1902 fn priced(
1903 input: Option<f64>,
1904 output: Option<f64>,
1905 cache_read: Option<f64>,
1906 cache_write: Option<f64>,
1907 ) -> CostModel {
1908 CostModel {
1909 input_per_mtok: input,
1910 output_per_mtok: output,
1911 cache_read_input_per_mtok: cache_read,
1912 cache_write_input_per_mtok: cache_write,
1913 ..Default::default()
1914 }
1915 }
1916
1917 #[test]
1918 fn an_undeclared_cache_rate_is_bounded_by_the_input_rate_not_billed_at_zero() {
1919 // The real shape this exists for: qwen3.5-plus declares input and
1920 // output but no cache-read rate. 1M cache-read tokens must not be free.
1921 let cost = priced(Some(0.26), Some(1.56), None, None);
1922 let bounded = cost
1923 .estimated_usd_bounded(Some(1_000_000), 0, 0, 1_000_000, 0)
1924 .expect("a model with input+output rates is priceable");
1925 assert!(bounded.may_overstate, "substituted rate can only be high");
1926 assert!(!bounded.may_understate);
1927 assert_eq!(bounded.marker(), "≤");
1928 assert!(
1929 (bounded.usd - 0.26).abs() < 1e-9,
1930 "cache reads fall back to the $0.26/MTok input rate, got {}",
1931 bounded.usd
1932 );
1933
1934 // The routing-score path still zero-fills, deliberately and untouched.
1935 assert_eq!(cost.estimated_usd(1_000_000, 0, 1_000_000, 0), 0.0);
1936 }
1937
1938 #[test]
1939 fn an_undeclared_cache_write_rate_claims_no_bound_it_cannot_keep() {
1940 // The shipped `claude-opus-4.8` shape with its cache-write rate
1941 // omitted. Anthropic's cache write is a 1.25x SURCHARGE (6.25 against
1942 // 5.0 input), and OpenRouter's 1h-TTL rate is 2x — so pricing the
1943 // bucket at the input rate is a FLOOR, and the `≤` a blanket
1944 // "cache is always cheaper" rule would have produced is a false
1945 // ceiling over a true cost of $6.25, or $10.00 at the 1h rate.
1946 let cost = priced(Some(5.0), Some(25.0), Some(0.5), None);
1947 let bounded = cost
1948 .estimated_usd_bounded(Some(1_000_000), 0, 0, 0, 1_000_000)
1949 .expect("input and output rates are published");
1950 assert!((bounded.usd - 5.0).abs() < 1e-9, "got {}", bounded.usd);
1951 assert!(
1952 bounded.may_understate,
1953 "a cache-write surcharge can exceed the input rate"
1954 );
1955 assert_ne!(bounded.marker(), "≤", "must not claim a ceiling it fails");
1956 assert_eq!(bounded.marker(), "~", "sign is unknown, so neither bound");
1957 // Both real cache-write rates this shape could carry sit ABOVE the
1958 // substituted figure — which is exactly why `≤` would have been a
1959 // false ceiling. Assert the tighter of the two; it implies the looser,
1960 // and spelling out both comparisons is a `redundant_comparisons` lint.
1961 let at_anthropics_1_25x: f64 = 1_000_000.0 * 6.25 / 1e6;
1962 let at_openrouters_1h_2x: f64 = 1_000_000.0 * 10.0 / 1e6;
1963 assert!(bounded.usd < at_anthropics_1_25x.min(at_openrouters_1h_2x));
1964
1965 // Declaring the rate makes it exact — the flag tracks substitution,
1966 // not the mere presence of cache-write tokens.
1967 let declared = priced(Some(5.0), Some(25.0), Some(0.5), Some(6.25))
1968 .estimated_usd_bounded(Some(1_000_000), 0, 0, 0, 1_000_000)
1969 .unwrap();
1970 assert!(declared.is_exact());
1971 assert!((declared.usd - 6.25).abs() < 1e-9);
1972
1973 // The same omission on the cache READ side DOES keep its ceiling —
1974 // the two directions are not shared. Note the rate must be missing for
1975 // a substitution to happen at all.
1976 let read = priced(Some(5.0), Some(25.0), None, Some(6.25))
1977 .estimated_usd_bounded(Some(1_000_000), 0, 0, 1_000_000, 0)
1978 .unwrap();
1979 assert_eq!(read.marker(), "≤");
1980 assert!((read.usd - 5.0).abs() < 1e-9);
1981 // A real cache-read rate is a discount, so the ceiling holds.
1982 assert!(read.usd > 1_000_000.0 * 0.5 / 1e6);
1983 }
1984
1985 #[test]
1986 fn a_declared_cache_rate_is_exact_and_never_flagged() {
1987 let cost = priced(Some(5.0), Some(25.0), Some(0.5), Some(6.25));
1988 let bounded = cost
1989 .estimated_usd_bounded(Some(1_600_000), 1_000_000, 200_000, 500_000, 100_000)
1990 .expect("fully rated");
1991 assert!(bounded.is_exact(), "nothing was substituted or unresolved");
1992 assert_eq!(bounded.marker(), "");
1993 // 1M uncached x 5 + 200k x 25 + 500k x 0.5 + 100k x 6.25, per MTok.
1994 assert!((bounded.usd - 10.875).abs() < 1e-9, "got {}", bounded.usd);
1995 // Identical to the router's figure when every rate is published.
1996 assert!(
1997 (bounded.usd - cost.estimated_usd(1_600_000, 200_000, 500_000, 100_000)).abs() < 1e-9
1998 );
1999 }
2000
2001 #[test]
2002 fn an_unbounded_bucket_refuses_rather_than_understating() {
2003 // Output has no safe substitute — it is normally the dearer side, so
2004 // pricing it at the input rate would UNDER-state. Refuse instead.
2005 let cost = priced(Some(1.0), None, None, None);
2006 assert_eq!(
2007 cost.estimated_usd_bounded(Some(1_000), 1_000, 1_000, 0, 0),
2008 None
2009 );
2010 // With no output tokens the same model is priceable and exact.
2011 let bounded = cost
2012 .estimated_usd_bounded(Some(1_000), 1_000, 0, 0, 0)
2013 .unwrap();
2014 assert!(bounded.is_exact());
2015 }
2016
2017 #[test]
2018 fn no_rate_card_is_unpriced_rather_than_free() {
2019 let cost = CostModel::default();
2020 assert_eq!(
2021 cost.estimated_usd_bounded(Some(1_000), 1_000, 1_000, 0, 0),
2022 None
2023 );
2024 // And a zero-usage priced model is genuinely free, not unpriced.
2025 let free = priced(Some(0.0), Some(0.0), None, None)
2026 .estimated_usd_bounded(Some(0), 0, 0, 0, 0)
2027 .expect("a declared zero rate card is priced");
2028 assert_eq!(free.usd, 0.0);
2029 assert!(free.is_exact());
2030 }
2031
2032 fn tiered() -> CostModel {
2033 CostModel {
2034 input_per_mtok: Some(2.5),
2035 output_per_mtok: Some(15.0),
2036 cache_read_input_per_mtok: Some(0.25),
2037 pricing_tiers: vec![TokenPricingTier {
2038 min_prompt_tokens: 272_000,
2039 prices: TokenPrices {
2040 input_per_mtok: Some(5.0),
2041 output_per_mtok: Some(22.5),
2042 cache_read_input_per_mtok: Some(0.5),
2043 cache_write_input_per_mtok: None,
2044 },
2045 }],
2046 ..Default::default()
2047 }
2048 }
2049
2050 #[test]
2051 fn a_known_prompt_size_resolves_the_tier_exactly() {
2052 let cost = tiered();
2053 let below = cost
2054 .estimated_usd_bounded(Some(271_999), 271_999, 0, 0, 0)
2055 .unwrap();
2056 assert!(below.is_exact());
2057 assert!((below.usd - 271_999.0 * 2.5 / 1e6).abs() < 1e-9);
2058
2059 let above = cost
2060 .estimated_usd_bounded(Some(272_000), 272_000, 0, 0, 0)
2061 .unwrap();
2062 assert!(above.is_exact());
2063 assert!((above.usd - 272_000.0 * 5.0 / 1e6).abs() < 1e-9);
2064 }
2065
2066 #[test]
2067 fn a_lifetime_aggregate_uses_base_rates_and_admits_it_may_be_low() {
2068 // Thirty 10K-token requests. Their SUM crosses the 272K threshold that
2069 // no single request came near — the double-charging bug. `None` says
2070 // "boundaries lost", so base rates apply and the figure is marked.
2071 let cost = tiered();
2072 let aggregate = cost.estimated_usd_bounded(None, 300_000, 0, 0, 0).unwrap();
2073 assert!(
2074 (aggregate.usd - 300_000.0 * 2.5 / 1e6).abs() < 1e-9,
2075 "must use the $2.50 base rate, got {}",
2076 aggregate.usd
2077 );
2078 assert!(aggregate.may_understate, "a dearer tier may apply");
2079 assert!(!aggregate.may_overstate);
2080 assert_eq!(aggregate.marker(), "≥");
2081
2082 // What the bug looked like: summed tokens passed as a real prompt size
2083 // price at the high-context rate — exactly double, and unmarked.
2084 let bug = cost
2085 .estimated_usd_bounded(Some(300_000), 300_000, 0, 0, 0)
2086 .unwrap();
2087 assert!((bug.usd - 2.0 * aggregate.usd).abs() < 1e-9);
2088 assert!(bug.is_exact(), "and it would have claimed to be exact");
2089 }
2090
2091 #[test]
2092 fn a_cheaper_tier_flags_the_aggregate_as_possibly_high_instead() {
2093 // Direction is derived from the tiers, not assumed. A volume DISCOUNT
2094 // makes the base-rate figure too high, not too low.
2095 let cost = CostModel {
2096 input_per_mtok: Some(2.0),
2097 output_per_mtok: Some(10.0),
2098 pricing_tiers: vec![TokenPricingTier {
2099 min_prompt_tokens: 100_000,
2100 prices: TokenPrices {
2101 input_per_mtok: Some(1.0),
2102 ..Default::default()
2103 },
2104 }],
2105 ..Default::default()
2106 };
2107 let aggregate = cost.estimated_usd_bounded(None, 500_000, 0, 0, 0).unwrap();
2108 assert!(aggregate.may_overstate);
2109 assert!(!aggregate.may_understate);
2110 assert_eq!(aggregate.marker(), "≤");
2111 }
2112
2113 #[test]
2114 fn both_directions_at_once_claims_neither_bound() {
2115 // Unrated cache bucket (can be high) plus an unresolved dearer tier
2116 // (can be low). Neither bound survives, so the figure is just an
2117 // estimate and must not wear a `≤` it cannot honour.
2118 let cost = CostModel {
2119 input_per_mtok: Some(2.0),
2120 output_per_mtok: Some(10.0),
2121 pricing_tiers: vec![TokenPricingTier {
2122 min_prompt_tokens: 100_000,
2123 prices: TokenPrices {
2124 input_per_mtok: Some(4.0),
2125 ..Default::default()
2126 },
2127 }],
2128 ..Default::default()
2129 };
2130 let aggregate = cost.estimated_usd_bounded(None, 0, 0, 500_000, 0).unwrap();
2131 assert!(aggregate.may_overstate && aggregate.may_understate);
2132 assert_eq!(aggregate.marker(), "~");
2133 }
2134
2135 /// Parslee-ai/car#894 (follow-up): "runs here" and "has weights to fetch"
2136 /// are different questions, and `is_local` answers only the first. Three
2137 /// `is_local` sources download nothing — the CLI must not offer an
2138 /// install status for them.
2139 #[test]
2140 fn downloads_weights_is_true_only_for_sources_car_fetches() {
2141 let mut schema = sample_local();
2142
2143 // CAR downloads these itself.
2144 schema.source = ModelSource::Local {
2145 hf_repo: "Qwen/Qwen3-4B-GGUF".into(),
2146 hf_filename: "Qwen3-4B-Q4_K_M.gguf".into(),
2147 tokenizer_repo: "Qwen/Qwen3-4B".into(),
2148 };
2149 assert!(schema.downloads_weights(), "GGUF weights are downloaded");
2150
2151 schema.source = ModelSource::Mlx {
2152 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
2153 hf_weight_file: None,
2154 };
2155 assert!(schema.downloads_weights(), "MLX weights are downloaded");
2156
2157 schema.source = ModelSource::WhisperCpp {
2158 model: "large-v3-turbo-q5_0".into(),
2159 };
2160 assert!(
2161 schema.downloads_weights(),
2162 "the whisper.cpp ggml bin is downloaded"
2163 );
2164
2165 // The OS owns these — there is nothing to install. `WindowsSpeech` is
2166 // the row that rendered `INSTALLED yes` against `car doctor`'s
2167 // `Models: none installed`.
2168 schema.source = ModelSource::WindowsSpeech {};
2169 assert!(
2170 !schema.downloads_weights(),
2171 "WinRT speech synthesis has no weights to download"
2172 );
2173
2174 schema.source = ModelSource::AppleFoundationModels { use_case: None };
2175 assert!(
2176 !schema.downloads_weights(),
2177 "Apple FoundationModels weights belong to the OS"
2178 );
2179
2180 // Someone else holds the weights.
2181 schema.source = ModelSource::VllmMlx {
2182 endpoint: "http://localhost:8000".into(),
2183 model_name: "mlx-community/Qwen3-4B-4bit".into(),
2184 };
2185 assert!(
2186 !schema.downloads_weights(),
2187 "the vLLM-MLX server owns its own weights"
2188 );
2189
2190 schema.source = ModelSource::ManagedVllmMlx {
2191 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
2192 hf_weight_file: None,
2193 };
2194 assert!(
2195 schema.downloads_weights(),
2196 "the explicitly managed vLLM-MLX contract is CAR-owned"
2197 );
2198
2199 schema.source = ModelSource::Ollama {
2200 model_tag: "qwen3:4b".into(),
2201 host: default_ollama_host(),
2202 };
2203 assert!(!schema.downloads_weights(), "Ollama owns its own weights");
2204
2205 schema.source = sample_remote().source;
2206 assert!(
2207 !schema.downloads_weights(),
2208 "a remote API has no weights on this machine"
2209 );
2210
2211 schema.source = ModelSource::Delegated { hint: None };
2212 assert!(
2213 !schema.downloads_weights(),
2214 "a host-registered runner owns its own weights"
2215 );
2216 }
2217
2218 /// `is_local` is the predicate the CLI used to reach for. These OS-owned
2219 /// sources are where the two answers diverge, which is why a
2220 /// separate predicate exists rather than a reuse of `is_local`.
2221 #[test]
2222 fn downloads_weights_differs_from_is_local_on_os_owned_sources() {
2223 let mut schema = sample_local();
2224 for source in [
2225 ModelSource::WindowsSpeech {},
2226 ModelSource::AppleFoundationModels { use_case: None },
2227 ] {
2228 schema.source = source;
2229 assert!(
2230 schema.is_local(),
2231 "this source runs on-device: {:?}",
2232 schema.source
2233 );
2234 assert!(
2235 !schema.downloads_weights(),
2236 "…but CAR downloads nothing for it: {:?}",
2237 schema.source
2238 );
2239 }
2240 }
2241}
2242
2243#[cfg(test)]
2244mod quantization_tests {
2245 use super::*;
2246
2247 /// Every distinct label the built-in catalog ships, the hyphenated form
2248 /// `registry.rs` writes into auto-discovered entries, and the formats that
2249 /// review found misclassified.
2250 #[test]
2251 fn classifies_known_labels() {
2252 let cases: &[(&str, Option<u8>, QuantScheme)] = &[
2253 ("4bit", Some(4), QuantScheme::AffineGroupInt),
2254 ("6bit", Some(6), QuantScheme::AffineGroupInt),
2255 ("3bit", Some(3), QuantScheme::AffineGroupInt),
2256 ("5bit", Some(5), QuantScheme::AffineGroupInt),
2257 // registry.rs emits the hyphen; users already have it on disk.
2258 ("4-bit", Some(4), QuantScheme::AffineGroupInt),
2259 ("mxfp8", Some(8), QuantScheme::BlockScaledFloat),
2260 ("mxfp4", Some(4), QuantScheme::BlockScaledFloat),
2261 ("Q4_K_M", Some(4), QuantScheme::KQuantMixed),
2262 ("Q5_K_S", Some(5), QuantScheme::KQuantMixed),
2263 ("IQ4_XS", Some(4), QuantScheme::KQuantMixed),
2264 ("TQ1_0", Some(1), QuantScheme::KQuantMixed),
2265 ("Q8_0", Some(8), QuantScheme::RtnBlock),
2266 ("q5_0", Some(5), QuantScheme::RtnBlock),
2267 // aarch64 repack quants: the suffix is a prefix match, not equality.
2268 ("Q4_0_4_4", Some(4), QuantScheme::RtnBlock),
2269 ("Q4_0_8_8", Some(4), QuantScheme::RtnBlock),
2270 // A zero-padded width must slice by digits consumed, not by the
2271 // decimal length of the parsed number.
2272 ("q08_0", Some(8), QuantScheme::RtnBlock),
2273 ("bf16", Some(16), QuantScheme::Unquantized),
2274 ("F16", Some(16), QuantScheme::Unquantized),
2275 ("f32", Some(32), QuantScheme::Unquantized),
2276 // Spelled like a quant, but affine group quant has no such width.
2277 ("16bit", Some(16), QuantScheme::Unquantized),
2278 ("32bit", Some(32), QuantScheme::Unquantized),
2279 // An 8-bit float IS quantized — just not attributable from this
2280 // label alone. Calling it full precision was the bug.
2281 ("fp8", Some(8), QuantScheme::Unknown),
2282 // Names a width and no producer.
2283 ("Q4", Some(4), QuantScheme::Unknown),
2284 ];
2285 for (label, bits, scheme) in cases {
2286 let q = Quantization::parse(label);
2287 assert_eq!(q.bits, *bits, "bits for {label}");
2288 assert_eq!(q.scheme, *scheme, "scheme for {label}");
2289 assert_eq!(q.label, *label, "label must survive verbatim");
2290 }
2291 }
2292
2293 /// Labels that identify nothing must say so rather than guess.
2294 #[test]
2295 fn refuses_to_guess() {
2296 for label in [
2297 "", // nobody said; not a claim of full precision
2298 "mxfp", // MX family, no width
2299 "Q256_K", // width overflows u8
2300 "q0_0", // a zero-bit quantization is not a thing
2301 "awq-marlin-w4a16", // real format, unknown to this parser
2302 ] {
2303 let q = Quantization::parse(label);
2304 assert_eq!(q.scheme, QuantScheme::Unknown, "scheme for {label:?}");
2305 assert_eq!(q.bits, None, "bits for {label:?}");
2306 assert_eq!(q.label, label, "label for {label:?}");
2307 }
2308 }
2309
2310 /// The distinction the free-text field could not express: same width,
2311 /// different algorithm, different loader.
2312 #[test]
2313 fn same_width_different_scheme() {
2314 let affine = Quantization::parse("4bit");
2315 let kquant = Quantization::parse("Q4_K_M");
2316 let mx = Quantization::parse("mxfp4");
2317 assert_eq!(affine.bits, kquant.bits);
2318 assert_eq!(affine.bits, mx.bits);
2319 assert_ne!(affine.scheme, kquant.scheme);
2320 assert_ne!(affine.scheme, mx.scheme);
2321 // And within GGUF, k-quant is not round-to-nearest.
2322 assert_ne!(
2323 Quantization::parse("Q5_K_S").scheme,
2324 Quantization::parse("q5_0").scheme
2325 );
2326 }
2327
2328 /// The scheme names a numeric format, never a container or an engine.
2329 /// whisper.cpp ships `q5_0` ggml checkpoints that no GGUF text path can
2330 /// load; a container-named variant would assert otherwise.
2331 #[test]
2332 fn scheme_does_not_imply_an_engine() {
2333 let whisper = Quantization::parse("q5_0");
2334 let llama = Quantization::parse("Q5_0");
2335 assert_eq!(whisper.scheme, llama.scheme);
2336 assert_eq!(whisper.scheme, QuantScheme::RtnBlock);
2337 }
2338
2339 #[test]
2340 fn mlx_config_block_survives_ingest() {
2341 let q = Quantization::from_mlx_config(Some(8), Some(32), Some("mxfp8")).unwrap();
2342 assert_eq!(q.bits, Some(8));
2343 assert_eq!(q.group_size, Some(32));
2344 assert_eq!(q.scheme, QuantScheme::BlockScaledFloat);
2345
2346 // Absent `mode` means affine — MLX's default, not unknown.
2347 let affine = Quantization::from_mlx_config(Some(4), Some(64), None).unwrap();
2348 assert_eq!(affine.scheme, QuantScheme::AffineGroupInt);
2349 assert_eq!(affine.group_size, Some(64));
2350 assert_eq!(affine.label, "4bit");
2351 }
2352
2353 /// An empty or absent block must not become a claim of affine group quant.
2354 /// The field it replaced returned nothing here, and asserting a scheme is
2355 /// exactly the error a router acting on `scheme` would inherit.
2356 #[test]
2357 fn empty_mlx_block_asserts_nothing() {
2358 assert!(Quantization::from_mlx_config(None, None, None).is_none());
2359 }
2360
2361 #[test]
2362 fn deserializes_legacy_bare_string() {
2363 let q: Quantization = serde_json::from_str(r#""Q4_K_M""#).unwrap();
2364 assert_eq!(q.scheme, QuantScheme::KQuantMixed);
2365 assert_eq!(q.bits, Some(4));
2366 assert_eq!(q.label, "Q4_K_M");
2367 }
2368
2369 #[test]
2370 fn deserializes_structured_object() {
2371 let q: Quantization = serde_json::from_str(
2372 r#"{"bits":4,"scheme":"affine_group_int","group_size":64,"label":"4bit"}"#,
2373 )
2374 .unwrap();
2375 assert_eq!(q.group_size, Some(64));
2376 assert_eq!(q.scheme, QuantScheme::AffineGroupInt);
2377 }
2378
2379 /// A partial object recovers from its label rather than defaulting to
2380 /// Unknown — the same reason the bare string is still accepted.
2381 #[test]
2382 fn partial_object_recovers_from_label() {
2383 let q: Quantization = serde_json::from_str(r#"{"label":"Q8_0"}"#).unwrap();
2384 assert_eq!(q.scheme, QuantScheme::RtnBlock);
2385 assert_eq!(q.bits, Some(8));
2386 }
2387
2388 /// A malformed object must be an error, not a row that silently claims
2389 /// `Unknown`. Under `#[serde(untagged)]` every one of these deserialized
2390 /// successfully into a fabricated descriptor.
2391 #[test]
2392 fn malformed_objects_are_rejected() {
2393 for bad in [
2394 r#"{"btis":4,"scehme":"k_quant_mixed","labl":"Q4_K_M"}"#, // typos
2395 r#"{"quantization":{"bits":4}}"#, // double-nested
2396 r#"{}"#, // nothing at all
2397 r#"{"group_size":64}"#, // no label
2398 r#"{"bits":4,"scheme":"k_quant_mixed"}"#, // no label
2399 ] {
2400 let parsed: Result<Quantization, _> = serde_json::from_str(bad);
2401 assert!(parsed.is_err(), "should have rejected {bad}");
2402 }
2403 }
2404
2405 /// The error must name what was wrong. `#[serde(untagged)]` reported only
2406 /// "data did not match any variant", for a `models.json` that fails whole.
2407 #[test]
2408 fn rejection_names_the_offending_field() {
2409 let err = serde_json::from_str::<Quantization>(
2410 r#"{"bits":4,"scheme":"affine_grp_int","label":"4bit"}"#,
2411 )
2412 .unwrap_err()
2413 .to_string();
2414 assert!(
2415 err.contains("affine_grp_int") || err.contains("scheme"),
2416 "unhelpful error: {err}"
2417 );
2418 }
2419
2420 #[test]
2421 fn round_trips_through_the_object_form() {
2422 for label in ["Q4_K_M", "4bit", "mxfp8", "bf16", "weird-vendor-format"] {
2423 let q = Quantization::parse(label);
2424 let round: Quantization =
2425 serde_json::from_str(&serde_json::to_string(&q).unwrap()).unwrap();
2426 assert_eq!(q, round, "round trip for {label}");
2427 }
2428 }
2429
2430 /// The label alone is the wire form whenever it is lossless. This is what
2431 /// keeps `row_digest` stable for rows that gained no new information.
2432 #[test]
2433 fn serializes_as_a_bare_label_when_lossless() {
2434 for label in [
2435 "Q4_K_M",
2436 "4bit",
2437 "mxfp8",
2438 "bf16",
2439 "q5_0",
2440 "weird-vendor-format",
2441 ] {
2442 let json = serde_json::to_string(&Quantization::parse(label)).unwrap();
2443 assert_eq!(json, format!("\"{label}\""), "should stay a bare string");
2444 }
2445 }
2446
2447 /// ...and the object form appears exactly where the label would lose
2448 /// something: a group size, or a scheme the label cannot express.
2449 #[test]
2450 fn serializes_as_an_object_only_when_it_adds_information() {
2451 let with_group = Quantization::from_mlx_config(Some(4), Some(64), None).unwrap();
2452 let json = serde_json::to_value(&with_group).unwrap();
2453 assert_eq!(json["group_size"], 64, "group_size must survive the write");
2454 assert_eq!(json["label"], "4bit");
2455
2456 // A bare `Q4` parses as Unknown, so an explicit scheme is real
2457 // information and has to be written out.
2458 let disambiguated = Quantization {
2459 bits: Some(4),
2460 scheme: QuantScheme::AffineGroupInt,
2461 group_size: None,
2462 label: "Q4".into(),
2463 };
2464 let json = serde_json::to_value(&disambiguated).unwrap();
2465 assert_eq!(json["scheme"], "affine_group_int");
2466 assert!(json.get("group_size").is_none(), "no null padding");
2467 }
2468
2469 /// Both write forms must read back identically, or the minimal-write rule
2470 /// would trade a digest change for silent data loss.
2471 #[test]
2472 fn every_write_form_round_trips() {
2473 let cases = [
2474 Quantization::parse("Q4_K_M"),
2475 Quantization::parse("bf16"),
2476 Quantization::parse("unattributable-format"),
2477 Quantization::from_mlx_config(Some(8), Some(32), Some("mxfp8")).unwrap(),
2478 Quantization::from_mlx_config(Some(4), Some(64), None).unwrap(),
2479 Quantization {
2480 bits: Some(4),
2481 scheme: QuantScheme::AffineGroupInt,
2482 group_size: None,
2483 label: "Q4".into(),
2484 },
2485 ];
2486 for q in cases {
2487 let round: Quantization =
2488 serde_json::from_str(&serde_json::to_string(&q).unwrap()).unwrap();
2489 assert_eq!(q, round, "round trip for {q:?}");
2490 }
2491 }
2492
2493 /// The digest guard. `catalog_identity::row_digest` is a SHA-256 over the
2494 /// serialized schema and clients pin it through
2495 /// `expected_catalog_revision`, so a row whose quantization gained no new
2496 /// information must re-serialize to the byte-identical JSON it was read
2497 /// from. Only rows that genuinely changed may move.
2498 #[test]
2499 fn catalog_quantizations_reserialize_unchanged() {
2500 let raw: Vec<serde_json::Value> =
2501 serde_json::from_str(include_str!("builtin_catalog.json")).unwrap();
2502 let parsed: Vec<ModelSchema> =
2503 serde_json::from_str(include_str!("builtin_catalog.json")).unwrap();
2504 let mut moved = Vec::new();
2505 for (raw_row, model) in raw.iter().zip(&parsed) {
2506 let before = raw_row
2507 .get("quantization")
2508 .cloned()
2509 .unwrap_or(serde_json::Value::Null);
2510 let after = serde_json::to_value(&model.quantization).unwrap();
2511 if before != after {
2512 moved.push(format!("{}: {before} -> {after}", model.id));
2513 }
2514 }
2515 assert!(
2516 moved.is_empty(),
2517 "these rows would change catalog digest: {moved:#?}"
2518 );
2519 }
2520
2521 /// Every quantization the built-in catalog ships must classify. An entry
2522 /// may carry an explicit `scheme` only where its label is genuinely
2523 /// ambiguous — the guard is that an explicit scheme never *contradicts* a
2524 /// label the parser can already read, which is how catalog data stops
2525 /// being a place to make a failing test pass.
2526 #[test]
2527 fn builtin_catalog_quantizations_are_coherent() {
2528 let catalog: Vec<ModelSchema> =
2529 serde_json::from_str(include_str!("builtin_catalog.json")).unwrap();
2530 let mut problems = Vec::new();
2531 for model in &catalog {
2532 let Some(q) = &model.quantization else {
2533 continue;
2534 };
2535 if q.scheme == QuantScheme::Unknown {
2536 problems.push(format!("{}: unclassified label {:?}", model.id, q.label));
2537 continue;
2538 }
2539 let from_label = Quantization::parse(&q.label).scheme;
2540 if from_label != QuantScheme::Unknown && from_label != q.scheme {
2541 problems.push(format!(
2542 "{}: label {:?} parses as {:?} but the row claims {:?}",
2543 model.id, q.label, from_label, q.scheme
2544 ));
2545 }
2546 }
2547 assert!(
2548 problems.is_empty(),
2549 "incoherent catalog rows: {problems:#?}"
2550 );
2551 }
2552}