car_inference/schema.rs
1//! Model schema — declarative metadata for models, analogous to ToolSchema for tools.
2//!
3//! Every model (local GGUF, remote API, Ollama) is described by a `ModelSchema`
4//! that declares identity, capabilities, constraints, cost, and source.
5//! The router uses this schema for initial routing; observed outcomes refine it.
6
7use serde::{Deserialize, Serialize};
8
9/// What a model can do.
10#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, schemars::JsonSchema)]
11#[serde(rename_all = "snake_case")]
12pub enum ModelCapability {
13 /// Text completion / chat generation
14 Generate,
15 /// Vector embeddings
16 Embed,
17 /// Cross-encoder relevance scoring (query + document → relevance
18 /// score). Qwen3-Reranker is the canonical local implementation.
19 Rerank,
20 /// Label assignment / classification
21 Classify,
22 /// Code generation, repair, refactoring
23 Code,
24 /// Chain-of-thought, planning, analysis
25 Reasoning,
26 /// Text condensation
27 Summarize,
28 /// Function/tool calling
29 ToolUse,
30 /// Multiple tool calls in a single response (parallel tool execution)
31 MultiToolCall,
32 /// Vision / image understanding
33 Vision,
34 /// Video understanding (multi-frame sampling + temporal tokens).
35 /// Distinct from `Vision` so routing can prefer video-trained
36 /// models when the caller attaches a video content block.
37 VideoUnderstanding,
38 /// Audio understanding (speech + non-speech audio as an input to
39 /// a chat/reasoning model). Distinct from `SpeechToText` which is
40 /// the transcription-only task. Gemma 4 E2B/E4B and Gemini do
41 /// this; Qwen2.5-VL does not.
42 AudioUnderstanding,
43 /// Visual grounding — structured object-localization output
44 /// (bounding boxes keyed to object labels) in addition to text.
45 Grounding,
46 /// Speech recognition / transcription
47 SpeechToText,
48 /// Speech synthesis / text-to-speech
49 TextToSpeech,
50 /// Image generation
51 ImageGeneration,
52 /// Video generation
53 VideoGeneration,
54}
55
56/// How much the project vouches for a model. Gates automatic upgrades and
57/// is surfaced in recommendation rationale. Closed enum — a new tier is a
58/// deliberate FFI-visible change, never a silent string fallback.
59#[derive(
60 Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default, schemars::JsonSchema,
61)]
62#[serde(rename_all = "snake_case")]
63pub enum TrustTier {
64 /// Vetted by the project — the built-in catalog and verified upgrades.
65 /// Eligible for background auto-apply when the user opts in.
66 #[default]
67 Curated,
68 /// User-registered or upstream-discovered, not project-vetted. Always
69 /// notify-only; never auto-applied regardless of update policy.
70 Community,
71}
72
73/// How to access the model.
74#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
75#[serde(tag = "type", rename_all = "snake_case")]
76pub enum ModelSource {
77 /// Local GGUF file via Candle backend.
78 Local {
79 hf_repo: String,
80 hf_filename: String,
81 tokenizer_repo: String,
82 },
83 /// Remote API endpoint (OpenAI-compatible, Anthropic, etc.)
84 RemoteApi {
85 endpoint: String,
86 /// Environment variable name containing the API key (never the key itself).
87 /// The env var value may contain comma-separated keys for load balancing.
88 api_key_env: String,
89 /// Additional environment variable names for load balancing across multiple keys.
90 /// Each env var may also contain comma-separated keys.
91 #[serde(default)]
92 api_key_envs: Vec<String>,
93 #[serde(default)]
94 api_version: Option<String>,
95 protocol: ApiProtocol,
96 },
97 /// One text-generation turn through the locally installed OpenAI Codex CLI.
98 ///
99 /// Codex remains the sole holder of its ChatGPT-subscription credential:
100 /// CAR neither reads that credential nor accepts an API key for this source.
101 /// An optional reasoning suffix is part of the model string (for example,
102 /// `gpt-5.6-sol:high`) and maps to Codex's `model_reasoning_effort` config.
103 CodexCli { model: String },
104 /// Ollama local server.
105 Ollama {
106 model_tag: String,
107 #[serde(default = "default_ollama_host")]
108 host: String,
109 },
110 /// Local MLX model via mlx-rs backend (Apple Silicon, safetensors format).
111 /// Models from mlx-community on HuggingFace.
112 Mlx {
113 /// HuggingFace repo (e.g., "mlx-community/Qwen3-4B-4bit").
114 hf_repo: String,
115 /// Optional specific weight filename. If None, auto-discovers safetensors files.
116 #[serde(default)]
117 hf_weight_file: Option<String>,
118 },
119 /// Local whisper.cpp speech-to-text model — a ggml `.bin` from the
120 /// `ggerganov/whisper.cpp` HF repo, run in-process via the shared
121 /// `car-whisper` crate. Cross-platform (Windows/Linux/macOS): this is the
122 /// on-device STT path where MLX isn't available. Cached at
123 /// `~/.tokhn/whisper/ggml-<model>.bin`.
124 WhisperCpp {
125 /// whisper.cpp model id — the suffix of `ggml-<model>.bin`
126 /// (e.g. `"large-v3-turbo-q5_0"`).
127 model: String,
128 },
129 /// Windows OS text-to-speech via `Windows.Media.SpeechSynthesis` (WinRT),
130 /// run in-process. The catalog-side analog of the `car-voice`
131 /// `TtsProvider::WindowsSpeech` live path and the parity counterpart of
132 /// Apple's OS synthesizer — free, on-device, no model download, no MLX.
133 /// Windows-only; availability is `false` on every other target (like
134 /// `AppleFoundationModels`).
135 WindowsSpeech {},
136 /// Local vLLM-MLX server (Apple Silicon, OpenAI-compatible API).
137 /// Routes through RemoteBackend with OpenAI protocol handler.
138 VllmMlx {
139 /// Server endpoint (e.g., "http://localhost:8000").
140 endpoint: String,
141 /// The model name as known to vLLM-MLX (e.g., "mlx-community/Qwen3-4B-4bit").
142 model_name: String,
143 },
144 /// CAR-owned supervised vLLM-MLX process backed by a managed HuggingFace
145 /// artifact. Unlike `VllmMlx`, CAR downloads, admits, spawns, reaps, and
146 /// accounts this allocation. Dispatch rewrites a clone to `VllmMlx` only
147 /// after the child reports healthy.
148 ManagedVllmMlx {
149 hf_repo: String,
150 #[serde(default)]
151 hf_weight_file: Option<String>,
152 /// A speculative-decoding drafter for this checkpoint: a separate HF
153 /// repo holding only MTP drafter weights (`model_type` like
154 /// `qwen3_5_mtp`), which the target supplies embeddings and head for.
155 /// CAR fetches it beside the model and serves with it attached; decode
156 /// on a dense model is memory-bandwidth-bound, and the drafter roughly
157 /// doubles it there. Omitted from serialization when absent so rows
158 /// without one keep their canonical form and `row_digest`.
159 #[serde(default, skip_serializing_if = "Option::is_none")]
160 draft_hf_repo: Option<String>,
161 },
162 /// Apple's on-device system model via the FoundationModels framework
163 /// (macOS 26+, Apple Silicon). Inference happens in-process through a
164 /// Swift shim — there is no HTTP, no API key, and no model file: the
165 /// OS owns the weights. Availability is checked at runtime via
166 /// `@available(macOS 26.0, *)`; on older macOS or non-Apple-Silicon
167 /// hosts the backend reports `UnsupportedMode` and the router falls
168 /// through to the next candidate.
169 AppleFoundationModels {
170 /// Optional Apple use-case hint passed through to
171 /// `LanguageModelSession`. Apple's framework tunes its prompt and
172 /// safety scaffolding per use case (e.g. "general", "summarize").
173 /// `None` uses the default.
174 #[serde(default)]
175 use_case: Option<String>,
176 },
177 /// Proprietary provider with custom auth and protocol.
178 ///
179 /// For vendor-specific APIs that aren't generic OpenAI-compatible endpoints.
180 /// Parslee is the first proprietary provider — custom auth (OAuth2),
181 /// custom response format, multi-provider routing built into the API.
182 Proprietary {
183 /// Provider identifier (e.g., "parslee").
184 provider: String,
185 /// Base URL for the API.
186 endpoint: String,
187 /// Auth configuration.
188 auth: ProprietaryAuth,
189 /// Custom protocol details.
190 protocol: ProprietaryProtocol,
191 },
192 /// Inference is delegated to a host-registered runner. CAR does
193 /// not own the wire format — the runner (typically a JS / Python
194 /// host) translates the `GenerateRequest` to its provider's API,
195 /// streams chunks back through the runner's event callback, and
196 /// returns the final aggregated result.
197 ///
198 /// Closes Parslee-ai/car-releases#24. Use this when the host
199 /// already has an SDK relationship with a provider (Anthropic,
200 /// OpenAI, GitHub Models, Vercel AI SDK) and wants CAR to sit in
201 /// the lifecycle / policy / replay path without learning every
202 /// provider's wire format.
203 ///
204 /// Routing requires that a runner has been registered via
205 /// [`crate::set_inference_runner`] (or its FFI equivalent —
206 /// `registerInferenceRunner` on JS, `register_inference_runner`
207 /// on Python, the `InferenceRunner` foreign trait on UniFFI,
208 /// `inference.register_runner` on the WebSocket protocol).
209 /// Without a runner, dispatch fails with `InferenceFailed`.
210 Delegated {
211 /// Opaque hint passed through to the runner — typically the
212 /// provider id (`"anthropic"`, `"openai"`, `"vercel-ai-sdk"`)
213 /// so a multi-provider runner can dispatch internally. CAR
214 /// does not interpret this string.
215 #[serde(default)]
216 hint: Option<String>,
217 },
218}
219
220impl ModelSource {
221 /// The source's serialized `type` tag (`codex_cli`, `remote_api`, ...):
222 /// where and how the model runs. Two rows of one model on different
223 /// sources are different routes, with their own credentials and billing.
224 pub fn kind(&self) -> &'static str {
225 match self {
226 ModelSource::Local { .. } => "local",
227 ModelSource::RemoteApi { .. } => "remote_api",
228 ModelSource::CodexCli { .. } => "codex_cli",
229 ModelSource::Ollama { .. } => "ollama",
230 ModelSource::Mlx { .. } => "mlx",
231 ModelSource::WhisperCpp { .. } => "whisper_cpp",
232 ModelSource::WindowsSpeech {} => "windows_speech",
233 ModelSource::VllmMlx { .. } => "vllm_mlx",
234 ModelSource::ManagedVllmMlx { .. } => "managed_vllm_mlx",
235 ModelSource::AppleFoundationModels { .. } => "apple_foundation_models",
236 ModelSource::Proprietary { .. } => "proprietary",
237 ModelSource::Delegated { .. } => "delegated",
238 }
239 }
240}
241
242/// Provider id for ChatGPT subscription inference backed by the Codex credential record.
243pub const OPENAI_CODEX_PROVIDER: &str = "openai-codex";
244
245/// What a signed-out openai-codex row tells a person to do.
246pub const OPENAI_CODEX_SIGN_IN_HINT: &str =
247 "sign in to ChatGPT in Settings → AI accounts, or run `car auth login --provider openai-codex`";
248
249/// Authentication method for proprietary providers.
250#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
251#[serde(tag = "type", rename_all = "snake_case")]
252pub enum ProprietaryAuth {
253 /// OAuth2 PKCE flow (e.g., Azure AD for Parslee).
254 #[serde(rename = "oauth2_pkce", alias = "o_auth2_pkce")]
255 OAuth2Pkce {
256 authority: String,
257 client_id: String,
258 scopes: Vec<String>,
259 },
260 /// Static API key from environment variable.
261 ApiKeyEnv { env_var: String },
262 /// Bearer token from environment variable.
263 BearerTokenEnv { env_var: String },
264 /// ChatGPT subscription OAuth read from the car-auth Codex record; admitted only for provider
265 /// `openai-codex`.
266 #[serde(rename = "chatgpt_subscription")]
267 ChatGptSubscription {},
268}
269
270/// Protocol configuration for proprietary providers.
271#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
272pub struct ProprietaryProtocol {
273 /// Chat/completion endpoint path (appended to base URL).
274 #[serde(default = "default_chat_path")]
275 pub chat_path: String,
276 /// Content type for requests.
277 #[serde(default = "default_content_type")]
278 pub content_type: String,
279 /// Whether the API streams responses via SSE.
280 #[serde(default)]
281 pub streaming: bool,
282 /// Custom headers to include in every request.
283 #[serde(default)]
284 pub extra_headers: std::collections::HashMap<String, String>,
285 /// The wire format the endpoint speaks. A row names it rather than CAR
286 /// inferring it from the path, so the model decides the backend.
287 #[serde(default)]
288 pub wire: ProprietaryWire,
289}
290
291/// What a proprietary endpoint speaks.
292#[derive(
293 Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema,
294)]
295#[serde(rename_all = "snake_case")]
296pub enum ProprietaryWire {
297 /// OpenAI Responses-style chat (Parslee's `/inference/responses`).
298 #[default]
299 Responses,
300 /// TypeSafe System One (Parslee's `/inference/typesafe/v1/systemone`):
301 /// typed decisions with a probability per option, no text generation.
302 /// Such a row serves `classify` only.
303 SystemOne,
304}
305
306impl Default for ProprietaryProtocol {
307 fn default() -> Self {
308 Self {
309 chat_path: default_chat_path(),
310 content_type: default_content_type(),
311 streaming: false,
312 extra_headers: std::collections::HashMap::new(),
313 wire: ProprietaryWire::default(),
314 }
315 }
316}
317
318fn default_chat_path() -> String {
319 "/chat".to_string()
320}
321
322fn default_content_type() -> String {
323 "application/json".to_string()
324}
325
326fn default_ollama_host() -> String {
327 "http://localhost:11434".to_string()
328}
329
330#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
331#[serde(rename_all = "snake_case")]
332pub enum ApiProtocol {
333 OpenAiCompat,
334 /// OpenRouter's OpenAI-compatible Chat Completions surface. Distinct so
335 /// credential precedence and error translation stay provider-specific.
336 OpenRouter,
337 /// OpenAI Responses API (/v1/responses) — works with all OpenAI models including codex.
338 OpenAiResponses,
339 Anthropic,
340 Google,
341 /// Azure OpenAI — uses api-key header and deployment-based URLs.
342 /// Endpoint format: {base}/openai/deployments/{model}/chat/completions?api-version={version}
343 AzureOpenAi,
344 /// Google Vertex AI — the enterprise Gemini surface. Same request/response
345 /// shape as the AI-Studio `Google` protocol, but a project/location URL and
346 /// OAuth Bearer auth (a GCP access token from `gcloud auth print-access-token`
347 /// or a service account) instead of an `?key=` query param. Endpoint format:
348 /// `{base}/publishers/google/models/{model}:generateContent`, where `base`
349 /// is `https://{loc}-aiplatform.googleapis.com/v1/projects/{proj}/locations/{loc}`.
350 VertexAi,
351 /// AWS Bedrock — the **Converse** API (`bedrock-runtime`), a unified
352 /// messages surface across Bedrock-hosted models (Claude, Llama, Mistral,
353 /// Titan, …). Auth is **SigV4** request signing (not a bearer token), with
354 /// credentials from the standard AWS env vars; the model's `endpoint` is the
355 /// region (e.g. `us-east-1`) and `name` is the Bedrock model id. Non-stream
356 /// only for now (Converse streaming uses a separate binary event-stream).
357 Bedrock,
358}
359
360impl ApiProtocol {
361 /// Prompt-cache economics for this provider, relative to its base input
362 /// rate — used by the cost scoreboard to price cached tokens correctly.
363 /// Anthropic uses explicit breakpoints (deep read discount + write
364 /// premium); OpenAI/Azure cache automatically (~0.5× read, no write
365 /// charge). Providers whose cache tokens CAR does not parse (Google/Vertex/
366 /// Bedrock) report zero cache tokens, so their rates are inert.
367 pub fn cache_rates(&self) -> crate::outcome::CacheRates {
368 use crate::outcome::CacheRates;
369 match self {
370 ApiProtocol::Anthropic => CacheRates::ANTHROPIC,
371 ApiProtocol::OpenAiCompat | ApiProtocol::OpenAiResponses | ApiProtocol::AzureOpenAi => {
372 CacheRates::OPENAI
373 }
374 // OpenRouter rates differ per upstream model and are carried by
375 // ModelSchema::cost. A blanket OpenAI-shaped discount is false.
376 ApiProtocol::OpenRouter => CacheRates::NONE,
377 ApiProtocol::Google | ApiProtocol::VertexAi | ApiProtocol::Bedrock => CacheRates::NONE,
378 }
379 }
380
381 /// This protocol as an OpenTelemetry GenAI `gen_ai.provider.name` value.
382 ///
383 /// The protocol is the right source for this attribute and
384 /// [`ModelSchema::provider`] is not: that field carries the model's
385 /// *publisher* (`qwen`, `mlx-community`, `apple`, `ggerganov`), while
386 /// `gen_ai.provider.name` names the API surface the request is spoken to.
387 /// A Qwen checkpoint served over Bedrock is `aws.bedrock`, not `qwen`.
388 ///
389 /// Both OpenAI protocols map to `openai` on purpose. The convention
390 /// defines this attribute as a discriminator for the *telemetry format
391 /// flavor*, not the company serving the weights — so every
392 /// OpenAI-REST-shaped surface reports `openai`, and Chat Completions vs
393 /// Responses is not a distinction it draws. `openrouter` has no
394 /// well-known value; the convention's fallback is a provider-specific
395 /// string, which is what this returns.
396 ///
397 /// Exhaustive by intent: a new protocol must decide its own value rather
398 /// than inherit a wildcard's guess.
399 pub fn genai_provider_name(&self) -> &'static str {
400 match self {
401 ApiProtocol::OpenAiCompat | ApiProtocol::OpenAiResponses => "openai",
402 ApiProtocol::OpenRouter => "openrouter",
403 ApiProtocol::Anthropic => "anthropic",
404 ApiProtocol::Google => "gcp.gemini",
405 ApiProtocol::VertexAi => "gcp.vertex_ai",
406 ApiProtocol::AzureOpenAi => "azure.ai.openai",
407 ApiProtocol::Bedrock => "aws.bedrock",
408 }
409 }
410}
411
412/// The numeric format a checkpoint's weights are stored in.
413///
414/// Deliberately says nothing about *which engine* serves the file —
415/// [`ModelSource`] already carries that, and keying this on the container
416/// instead of the format produces falsehoods: whisper.cpp's `q5_0` ggml
417/// checkpoints use the same round-to-nearest block format as llama.cpp's, and
418/// a `Gguf*` variant would assert a GGUF text path that cannot load them.
419///
420/// What it does express is the axis neither the bit width nor the source can:
421/// an MLX 4-bit affine checkpoint and a `Q4_K_M` are both "4-bit", were
422/// produced by different algorithms, and do not have the same quality.
423#[derive(
424 Debug, Clone, Copy, PartialEq, Eq, Hash, Default, Serialize, Deserialize, schemars::JsonSchema,
425)]
426#[serde(rename_all = "snake_case")]
427pub enum QuantScheme {
428 /// Integer weights with a scale and bias shared across `group_size`
429 /// elements (`4bit`, `6bit`). MLX's default and only affine format.
430 AffineGroupInt,
431 /// Block-scaled float — MLX's `mxfp4`, `mxfp8` and NVIDIA's `nvfp4`. A
432 /// genuinely different loader path from [`QuantScheme::AffineGroupInt`] at
433 /// the same nominal width, which is why the width alone cannot classify a
434 /// checkpoint; see `backend::local::quantization_is_decodable`.
435 BlockScaledFloat,
436 /// Mixed per-tensor bit allocation, optionally importance-weighted
437 /// (`Q4_K_M`, `Q5_K_S`, `IQ4_XS`, `TQ1_0`).
438 KQuantMixed,
439 /// Uniform round-to-nearest blocks, no per-tensor mixing and no importance
440 /// weighting (`Q8_0`, `q5_0`, `Q4_0_4_4`).
441 RtnBlock,
442 /// Full-precision weights (`bf16`, `f16`, `f32`). Distinct from `None` at
443 /// the [`ModelSchema::quantization`] level: a positive claim that the
444 /// checkpoint is unquantized, not an absence of information.
445 Unquantized,
446 /// A format this parser could not identify. The label is still preserved
447 /// verbatim; only the classification is missing.
448 #[default]
449 Unknown,
450}
451
452/// Structured quantization descriptor.
453///
454/// This replaces a free-text string that mixed two vocabularies — MLX's
455/// `4bit`/`6bit` and GGUF's `Q4_K_M`/`Q8_0` — under one field, so nothing could
456/// tell a group-quantized integer checkpoint from a mixed k-quant one. It also
457/// threw away what the ingest paths already had in hand: MLX `config.json`
458/// declares `{bits, group_size, mode}` and only `bits` survived.
459///
460/// Deserializes from **either** the legacy bare string or the structured
461/// object, so catalogs, user `models.json` files, and `models.register`
462/// payloads written before this change keep loading unchanged. The object form
463/// requires `label` and rejects unknown fields: a misspelled key is a producer
464/// bug, and accepting it silently is how the field it replaced lost
465/// information in the first place.
466///
467/// **Writes the bare label back whenever that is lossless**, and the object
468/// only when it carries something parsing cannot recover — a `group_size`, or
469/// a `scheme` that disambiguates a label the parser reads as `Unknown`. Two
470/// reasons, both about blast radius rather than taste. This value is inside
471/// `catalog_identity::row_digest`, so an unconditional shape change would move
472/// the digest of every quantized row and hard-fail any client that pinned
473/// `expected_catalog_revision`. And `registry::load_user_config` fails a
474/// `models.json` **whole**, with its only production caller discarding the
475/// error — so an older daemon meeting an object it cannot parse boots with
476/// zero registered models and says nothing. Emitting the object only where it
477/// adds information keeps both costs proportional to what actually changed.
478#[allow(dead_code)]
479#[derive(schemars::JsonSchema)]
480#[serde(untagged)]
481enum QuantizationWireSchema {
482 Label(String),
483 Structured(QuantizationObjectWireSchema),
484}
485
486#[allow(dead_code)]
487#[derive(schemars::JsonSchema)]
488#[serde(deny_unknown_fields)]
489struct QuantizationObjectWireSchema {
490 bits: Option<u8>,
491 scheme: QuantScheme,
492 group_size: Option<u32>,
493 label: String,
494}
495
496#[derive(Debug, Clone, PartialEq, Eq)]
497pub struct Quantization {
498 /// Weight bit width. `None` when the label names none.
499 pub bits: Option<u8>,
500 /// Which quantization family this is.
501 pub scheme: QuantScheme,
502 /// Elements sharing one scale/zero point (MLX: 32, 64, 128). `None` when
503 /// the scheme has no group concept, or the source declared none.
504 pub group_size: Option<u32>,
505 /// The label exactly as published. Never synthesized: it is what the user
506 /// sees, what Hugging Face repos are named after, and the only thing that
507 /// survives a scheme CAR does not recognize yet.
508 pub label: String,
509}
510
511/// Widths affine group quantization actually uses. A `16bit` or `32bit`
512/// label is full precision that happens to be spelled like a quant.
513const AFFINE_GROUP_WIDTHS: std::ops::RangeInclusive<u8> = 2..=8;
514
515impl Quantization {
516 /// Best-effort structure from a published label.
517 ///
518 /// Never fails and never invents: an unrecognized label yields
519 /// [`QuantScheme::Unknown`] with the label intact.
520 pub fn parse(label: &str) -> Self {
521 let label = label.trim();
522 let lower = label.to_ascii_lowercase();
523 let build = |bits: Option<u8>, scheme: QuantScheme| Self {
524 bits,
525 scheme,
526 group_size: None,
527 label: label.to_string(),
528 };
529
530 // Full precision. `fp8` is deliberately NOT here — an 8-bit float is a
531 // quantized weight format, just not one this parser can attribute.
532 match lower.as_str() {
533 "bf16" | "f16" | "fp16" | "float16" | "half" => {
534 return build(Some(16), QuantScheme::Unquantized)
535 }
536 "f32" | "fp32" | "float32" | "full" => {
537 return build(Some(32), QuantScheme::Unquantized)
538 }
539 "none" => return build(None, QuantScheme::Unquantized),
540 "f8" | "fp8" | "float8" => return build(Some(8), QuantScheme::Unknown),
541 // Empty means nobody said, which is not a claim of full precision.
542 "" => return build(None, QuantScheme::Unknown),
543 _ => {}
544 }
545
546 // MLX block-scaled floats: `mxfp4`, `mxfp8`. A bare `mxfp` names no
547 // width, so it identifies nothing.
548 if let Some(rest) = lower.strip_prefix("mxfp") {
549 return match leading_number(rest) {
550 Some((bits, _)) => build(Some(bits), QuantScheme::BlockScaledFloat),
551 None => build(None, QuantScheme::Unknown),
552 };
553 }
554
555 // MLX affine group quant: `4bit`, `4-bit`, `6bit`.
556 if let Some(width) = lower.strip_suffix("bit").map(|w| w.trim_end_matches('-')) {
557 if let Some((bits, consumed)) = leading_number(width) {
558 if consumed == width.len() && AFFINE_GROUP_WIDTHS.contains(&bits) {
559 return build(Some(bits), QuantScheme::AffineGroupInt);
560 }
561 // `16bit`/`32bit` are full precision spelled as a width.
562 if consumed == width.len() && (bits == 16 || bits == 32) {
563 return build(Some(bits), QuantScheme::Unquantized);
564 }
565 }
566 }
567
568 // GGUF. `iq`/`tq` are k-quant families; a bare `q` needs its suffix
569 // read to tell k-quant from legacy round-to-nearest.
570 let gguf = lower
571 .strip_prefix("iq")
572 .or_else(|| lower.strip_prefix("tq"))
573 .map(|rest| (rest, true))
574 .or_else(|| lower.strip_prefix('q').map(|rest| (rest, false)));
575 if let Some((rest, k_family)) = gguf {
576 if let Some((bits, consumed)) = leading_number(rest) {
577 if bits == 0 {
578 return build(None, QuantScheme::Unknown);
579 }
580 // Slice by digits consumed, not by the width's decimal length —
581 // `q08_0` has a two-character prefix for a one-character number.
582 let suffix = &rest[consumed..];
583 let scheme = if k_family || suffix.contains("_k") {
584 QuantScheme::KQuantMixed
585 } else if suffix.starts_with("_0") || suffix.starts_with("_1") {
586 // `starts_with`, not equality: the aarch64 repack quants
587 // are `Q4_0_4_4`, `Q4_0_4_8`, `Q4_0_8_8`.
588 QuantScheme::RtnBlock
589 } else {
590 // A bare `Q4` names a width and no producer — llama.cpp
591 // has no such format, so this came from somewhere else.
592 QuantScheme::Unknown
593 };
594 return build(Some(bits), scheme);
595 }
596 }
597
598 build(None, QuantScheme::Unknown)
599 }
600
601 /// Recover the quantization a GGUF file names in its own filename
602 /// (`Qwen3-8B-Q4_K_M.gguf`, `ggml-large-v3-turbo-q5_0.gguf`).
603 ///
604 /// Scans the hyphen-separated segments from the right and takes the first
605 /// that classifies, so a model whose *name* contains something quant-shaped
606 /// does not outrank the real suffix. Returns `None` rather than guessing
607 /// when nothing in the name is recognizable.
608 pub fn from_gguf_filename(filename: &str) -> Option<Self> {
609 let stem = filename
610 .rsplit_once('.')
611 .map(|(stem, _)| stem)
612 .unwrap_or(filename);
613 stem.rsplit('-')
614 .map(Self::parse)
615 .find(|q| q.scheme != QuantScheme::Unknown)
616 }
617
618 /// Descriptor for an MLX checkpoint, from the `quantization` block of its
619 /// `config.json`. `mode` is MLX's own name for the format (`affine`,
620 /// `mxfp4`, `mxfp8`); absent means affine, which is MLX's default.
621 ///
622 /// Returns `None` when the block carried nothing usable — declaring
623 /// `AffineGroupInt` on the strength of an empty object would assert exactly the
624 /// thing this type exists to establish.
625 pub fn from_mlx_config(
626 bits: Option<u8>,
627 group_size: Option<u32>,
628 mode: Option<&str>,
629 ) -> Option<Self> {
630 if bits.is_none() && group_size.is_none() && mode.is_none() {
631 return None;
632 }
633 let normalized = mode.map(str::to_ascii_lowercase);
634 let scheme = match normalized.as_deref() {
635 // MLX's `QuantizationMode` is exactly these four. `nvfp4` used to
636 // fall through to `Unknown`, which was a refusal dressed as a parse
637 // failure — it is block-scaled float like the MX pair and the
638 // linked loader reads it.
639 Some(m) if m.starts_with("mxfp") || m == "nvfp4" => QuantScheme::BlockScaledFloat,
640 Some("affine") | None => QuantScheme::AffineGroupInt,
641 Some(_) => QuantScheme::Unknown,
642 };
643 let label = match (normalized.as_deref(), bits) {
644 (Some(m), _) if m != "affine" => m.to_string(),
645 (_, Some(b)) => format!("{b}bit"),
646 // No width and no distinguishing mode: MLX said "affine" and
647 // nothing else. Say that rather than inventing a width.
648 (_, None) => "affine".to_string(),
649 };
650 Some(Self {
651 bits,
652 scheme,
653 group_size,
654 label,
655 })
656 }
657}
658
659/// Leading run of ASCII digits as a bit width, with the number of bytes it
660/// occupied. The byte count is returned because it is not recoverable from the
661/// value — `08` and `8` parse the same and slice differently.
662fn leading_number(s: &str) -> Option<(u8, usize)> {
663 let digits: String = s.chars().take_while(char::is_ascii_digit).collect();
664 if digits.is_empty() {
665 return None;
666 }
667 // Overflow (`Q256_K`) is a parse failure, not a silent truncation.
668 digits.parse().ok().map(|n| (n, digits.len()))
669}
670
671impl std::fmt::Display for Quantization {
672 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
673 f.write_str(&self.label)
674 }
675}
676
677impl Serialize for Quantization {
678 fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
679 use serde::ser::SerializeStruct;
680
681 // Round-tripping through `parse` is the exact test for "the label
682 // carries everything": if it reproduces this value, the object form
683 // would be a longer spelling of the same information.
684 if Self::parse(&self.label) == *self {
685 return serializer.serialize_str(&self.label);
686 }
687
688 let len = 2 + usize::from(self.bits.is_some()) + usize::from(self.group_size.is_some());
689 let mut row = serializer.serialize_struct("Quantization", len)?;
690 if let Some(bits) = self.bits {
691 row.serialize_field("bits", &bits)?;
692 }
693 row.serialize_field("scheme", &self.scheme)?;
694 if let Some(group_size) = self.group_size {
695 row.serialize_field("group_size", &group_size)?;
696 }
697 row.serialize_field("label", &self.label)?;
698 row.end()
699 }
700}
701
702impl<'de> Deserialize<'de> for Quantization {
703 fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
704 /// The object form. `label` is required and unknown fields are
705 /// rejected, so a typo'd or double-nested payload is an error rather
706 /// than a row that silently claims `Unknown`.
707 #[derive(Deserialize)]
708 #[serde(deny_unknown_fields)]
709 struct Structured {
710 #[serde(default)]
711 bits: Option<u8>,
712 #[serde(default)]
713 scheme: Option<QuantScheme>,
714 #[serde(default)]
715 group_size: Option<u32>,
716 label: String,
717 }
718
719 struct QuantizationVisitor;
720
721 impl<'de> serde::de::Visitor<'de> for QuantizationVisitor {
722 type Value = Quantization;
723
724 fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
725 f.write_str("a quantization label string, or an object with a `label` field")
726 }
727
728 fn visit_str<E: serde::de::Error>(self, label: &str) -> Result<Self::Value, E> {
729 Ok(Quantization::parse(label))
730 }
731
732 fn visit_map<M: serde::de::MapAccess<'de>>(
733 self,
734 map: M,
735 ) -> Result<Self::Value, M::Error> {
736 // Dispatched by hand rather than via `#[serde(untagged)]`,
737 // which buffers the input and reports only "data did not match
738 // any variant" — for a `models.json` holding dozens of models
739 // that error names neither the field nor the row.
740 let s = Structured::deserialize(serde::de::value::MapAccessDeserializer::new(map))?;
741 // A row that omits `scheme` or `bits` recovers what its label
742 // implies; an explicit value always wins.
743 let inferred = Quantization::parse(&s.label);
744 Ok(Quantization {
745 bits: s.bits.or(inferred.bits),
746 scheme: s.scheme.unwrap_or(inferred.scheme),
747 group_size: s.group_size,
748 label: s.label,
749 })
750 }
751 }
752
753 d.deserialize_any(QuantizationVisitor)
754 }
755}
756
757/// Declared performance expectations. Overridden by observed data once available.
758#[derive(Debug, Clone, Default, Serialize, Deserialize, schemars::JsonSchema)]
759pub struct PerformanceEnvelope {
760 /// Median latency in milliseconds (declared/estimated).
761 #[serde(default)]
762 pub latency_p50_ms: Option<u64>,
763 /// 99th percentile latency in milliseconds.
764 #[serde(default)]
765 pub latency_p99_ms: Option<u64>,
766 /// Tokens per second throughput.
767 #[serde(default)]
768 pub tokens_per_second: Option<f64>,
769}
770
771/// Cost model for routing optimization.
772/// Generation parameters that a model may or may not support.
773/// Models declare which params they accept. The inference layer
774/// strips unsupported params before sending to the API.
775#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, schemars::JsonSchema)]
776#[serde(rename_all = "snake_case")]
777pub enum GenerateParam {
778 Temperature,
779 TopP,
780 TopK,
781 MaxTokens,
782 StopSequences,
783 FrequencyPenalty,
784 PresencePenalty,
785 Seed,
786 ResponseFormat,
787 /// Extended thinking / internal reasoning before responding.
788 ExtendedThinking,
789}
790
791/// Standard parameter set for most models.
792pub fn standard_params() -> Vec<GenerateParam> {
793 vec![
794 GenerateParam::Temperature,
795 GenerateParam::TopP,
796 GenerateParam::MaxTokens,
797 GenerateParam::StopSequences,
798 GenerateParam::FrequencyPenalty,
799 GenerateParam::PresencePenalty,
800 GenerateParam::Seed,
801 ]
802}
803
804/// Parameter set for reasoning models (no temperature, no top_p).
805pub fn reasoning_params() -> Vec<GenerateParam> {
806 vec![GenerateParam::MaxTokens, GenerateParam::StopSequences]
807}
808
809#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
810pub struct TokenPrices {
811 /// USD per 1M uncached input tokens.
812 #[serde(default)]
813 pub input_per_mtok: Option<f64>,
814 /// USD per 1M output tokens.
815 #[serde(default)]
816 pub output_per_mtok: Option<f64>,
817 /// USD per 1M cache-read input tokens.
818 #[serde(default)]
819 pub cache_read_input_per_mtok: Option<f64>,
820 /// USD per 1M cache-write input tokens.
821 #[serde(default)]
822 pub cache_write_input_per_mtok: Option<f64>,
823}
824
825#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
826pub struct TokenPricingTier {
827 /// Inclusive prompt-token threshold at which this tier applies.
828 pub min_prompt_tokens: usize,
829 #[serde(flatten)]
830 pub prices: TokenPrices,
831}
832
833#[derive(Debug, Clone, Default, Serialize, Deserialize, schemars::JsonSchema)]
834pub struct CostModel {
835 /// USD per 1M input tokens (remote models).
836 #[serde(default)]
837 pub input_per_mtok: Option<f64>,
838 /// USD per 1M output tokens (remote models).
839 #[serde(default)]
840 pub output_per_mtok: Option<f64>,
841 /// USD per 1M cache-read input tokens. Unlike protocol-wide cache
842 /// multipliers, this is model-specific and comes from the provider's
843 /// published catalog.
844 #[serde(default)]
845 pub cache_read_input_per_mtok: Option<f64>,
846 /// USD per 1M cache-write input tokens, when the provider charges one.
847 #[serde(default)]
848 pub cache_write_input_per_mtok: Option<f64>,
849 /// Prompt-size pricing overrides, sorted by increasing threshold.
850 /// The highest threshold not greater than the prompt size wins.
851 #[serde(default)]
852 pub pricing_tiers: Vec<TokenPricingTier>,
853 /// USD per minute of audio, for a model priced by the length of the audio
854 /// it hears (speech to text). Omitted when the catalog has no such price.
855 #[serde(default, skip_serializing_if = "Option::is_none")]
856 pub audio_per_minute: Option<f64>,
857 /// USD per 1,000 characters of text, for a model priced by the text it
858 /// speaks (text to speech). Omitted when the catalog has no such price.
859 #[serde(default, skip_serializing_if = "Option::is_none")]
860 pub per_1k_characters: Option<f64>,
861 /// On-disk size in MB (local models).
862 #[serde(default)]
863 pub size_mb: Option<u64>,
864 /// RAM required during inference in MB.
865 #[serde(default)]
866 pub ram_mb: Option<u64>,
867}
868
869impl CostModel {
870 pub fn prices_for(&self, prompt_tokens: usize) -> TokenPrices {
871 let mut prices = TokenPrices {
872 input_per_mtok: self.input_per_mtok,
873 output_per_mtok: self.output_per_mtok,
874 cache_read_input_per_mtok: self.cache_read_input_per_mtok,
875 cache_write_input_per_mtok: self.cache_write_input_per_mtok,
876 };
877 for tier in self
878 .pricing_tiers
879 .iter()
880 .filter(|tier| tier.min_prompt_tokens <= prompt_tokens)
881 {
882 if tier.prices.input_per_mtok.is_some() {
883 prices.input_per_mtok = tier.prices.input_per_mtok;
884 }
885 if tier.prices.output_per_mtok.is_some() {
886 prices.output_per_mtok = tier.prices.output_per_mtok;
887 }
888 if tier.prices.cache_read_input_per_mtok.is_some() {
889 prices.cache_read_input_per_mtok = tier.prices.cache_read_input_per_mtok;
890 }
891 if tier.prices.cache_write_input_per_mtok.is_some() {
892 prices.cache_write_input_per_mtok = tier.prices.cache_write_input_per_mtok;
893 }
894 }
895 prices
896 }
897
898 /// Estimated request cost from the provider's declared token prices.
899 /// Unknown price components contribute zero; callers that need to
900 /// distinguish unknown pricing should inspect `prices_for` first.
901 ///
902 /// **This is the routing-score input, not a display or billing figure.**
903 /// The zero-fill is load-bearing here — `adaptive_router` normalizes the
904 /// result into a 0..1 cost score, and it separately neutralizes models
905 /// with no pricing at all, so changing the fill would change routing.
906 /// For anything a human reads, use
907 /// [`estimated_usd_bounded`](Self::estimated_usd_bounded), which refuses
908 /// to bill an unrated bucket at zero and says which way it can be wrong.
909 pub fn estimated_usd(
910 &self,
911 prompt_tokens: usize,
912 output_tokens: usize,
913 cache_read_tokens: usize,
914 cache_write_tokens: usize,
915 ) -> f64 {
916 let prices = self.prices_for(prompt_tokens);
917 let uncached_input = prompt_tokens
918 .saturating_sub(cache_read_tokens)
919 .saturating_sub(cache_write_tokens);
920 (uncached_input as f64 * prices.input_per_mtok.unwrap_or(0.0)
921 + output_tokens as f64 * prices.output_per_mtok.unwrap_or(0.0)
922 + cache_read_tokens as f64 * prices.cache_read_input_per_mtok.unwrap_or(0.0)
923 + cache_write_tokens as f64 * prices.cache_write_input_per_mtok.unwrap_or(0.0))
924 / 1_000_000.0
925 }
926
927 /// Effective per-bucket rates under one resolved price sheet, in the
928 /// bucket order `[uncached input, output, cache read, cache write]`, each
929 /// paired with which way a *substituted* rate can be wrong:
930 /// `(rate, may_overstate, may_understate)`.
931 ///
932 /// The two cache buckets substitute the uncached-input rate but are **not
933 /// governed by one rule**, because the providers in this catalog do not
934 /// price them the same way relative to input:
935 ///
936 /// | bucket | observed vs input, curated table |
937 /// |---|---|
938 /// | cache read | `0.10x`–`0.64x` — always a discount |
939 /// | cache write | `0.1875x` (Google) … `1.25x` (Anthropic) — **both sides** |
940 ///
941 /// So substituting input for a missing **cache-read** rate can only be too
942 /// high (`may_overstate`), while for a missing **cache-write** rate it can
943 /// land either side and gets both flags, rendering `~` rather than a `≤`
944 /// the figure cannot honour. Treating the two alike is how `claude-opus-4.8`
945 /// — `input 5.0`, `cache_write 6.25` — would have worn a `≤$5.00` ceiling
946 /// over a true cost of `$6.25`, or `$10.00` at OpenRouter's 1h-TTL rate.
947 ///
948 /// **Output** substitutes nothing and refuses instead: output runs
949 /// `1.5x`–`8x` input across this same table, which is not a ballpark
950 /// estimate in either direction, merely a wrong number wearing a marker.
951 /// The line is whether a substitute is *within range and merely of unknown
952 /// sign* (estimate, flag it) or *out of range entirely* (refuse) — see
953 /// [`estimated_usd_bounded`](Self::estimated_usd_bounded).
954 fn bucket_rates(prices: &TokenPrices) -> [(Option<f64>, bool, bool); 4] {
955 // A cached READ is always discounted relative to uncached input — the
956 // entire point of the cache — so the input rate is a true ceiling.
957 let cache_read = match prices.cache_read_input_per_mtok {
958 Some(rate) => (Some(rate), false, false),
959 None => {
960 let substituted = prices.input_per_mtok.is_some();
961 (prices.input_per_mtok, substituted, false)
962 }
963 };
964 // A cache WRITE may be a surcharge (Anthropic 1.25x, OpenRouter's 1h
965 // TTL 2x) or a discount (Google 0.1875x). Unknown sign, so no bound.
966 let cache_write = match prices.cache_write_input_per_mtok {
967 Some(rate) => (Some(rate), false, false),
968 None => {
969 let substituted = prices.input_per_mtok.is_some();
970 (prices.input_per_mtok, substituted, substituted)
971 }
972 };
973 [
974 (prices.input_per_mtok, false, false),
975 (prices.output_per_mtok, false, false),
976 cache_read,
977 cache_write,
978 ]
979 }
980
981 /// Cost estimate for a figure a person will read, carrying which way it
982 /// can be wrong.
983 ///
984 /// Differs from [`estimated_usd`](Self::estimated_usd) in refusing to
985 /// invent numbers. A token bucket the provider charges for but whose rate
986 /// this catalog does not declare is **never billed at zero**, and a figure
987 /// that might be wrong never presents itself as exact.
988 ///
989 /// `tier_prompt_tokens` selects the prompt-size pricing tier and is the
990 /// parameter callers most often get wrong:
991 ///
992 /// - `Some(n)` — the prompt size of **one request**. Tiers resolve exactly.
993 /// - `None` — the caller cannot say (a lifetime accumulator has summed
994 /// many requests and lost their boundaries). Base rates are used and the
995 /// result is flagged in whichever direction the model's own tiers run.
996 ///
997 /// Passing a *summed* token count as `Some` is the bug this signature
998 /// exists to prevent: thirty 10K-token requests sum to 300K, which crosses
999 /// a 272K threshold that no individual request came near, and every token
1000 /// ever sent gets priced at the high-context rate — roughly double, stated
1001 /// with total confidence.
1002 ///
1003 /// Two rejected alternatives, for the next person who wants tiers on an
1004 /// aggregate. **Pricing lifetime totals at the tier their sum lands in** is
1005 /// the bug above. **Pricing everything at the highest declared tier** does
1006 /// yield a true ceiling, but a useless one — it doubles the figure for a
1007 /// user whose prompts never approached the threshold, which is the same
1008 /// confident wrongness in the other direction. Resolving tiers properly
1009 /// needs per-request prompt sizes, which means [`crate::ModelProfile`]
1010 /// would have to accumulate per-tier token buckets at record time; that is
1011 /// a real feature with a persisted-schema change, not something to fake
1012 /// here from data that has already been summed away.
1013 ///
1014 /// Returns `None` when no defensible number exists — either the model
1015 /// declares no rate card at all, or a bucket carrying tokens has no rate
1016 /// and no usable substitute. Output deliberately has no input-rate
1017 /// fallback: it runs 1.5x–8x input across this catalog, far enough out of
1018 /// range that no marker could rescue the number. Which buckets substitute,
1019 /// and which way each substitution can be wrong, is decided in
1020 /// [`bucket_rates`](Self::bucket_rates) from observed provider pricing —
1021 /// notably cache *reads* and cache *writes* do not share a direction.
1022 /// `None` means unpriced, and a caller must render it as such, not as free.
1023 pub fn estimated_usd_bounded(
1024 &self,
1025 tier_prompt_tokens: Option<usize>,
1026 uncached_input_tokens: usize,
1027 output_tokens: usize,
1028 cache_read_tokens: usize,
1029 cache_write_tokens: usize,
1030 ) -> Option<ApproxCost> {
1031 // `prices_for(0)` is the base sheet: no tier threshold is <= 0 in a
1032 // catalog whose thresholds are positive, so nothing overrides.
1033 let prices = self.prices_for(tier_prompt_tokens.unwrap_or(0));
1034 if prices.input_per_mtok.is_none() && prices.output_per_mtok.is_none() {
1035 // No rate card. Not free — unknown.
1036 return None;
1037 }
1038
1039 let tokens = [
1040 uncached_input_tokens,
1041 output_tokens,
1042 cache_read_tokens,
1043 cache_write_tokens,
1044 ];
1045 let rates = Self::bucket_rates(&prices);
1046 let mut usd = 0.0;
1047 let mut may_overstate = false;
1048 let mut may_understate = false;
1049 for (count, (rate, substitute_high, substitute_low)) in tokens.into_iter().zip(rates) {
1050 if count == 0 {
1051 continue;
1052 }
1053 // Charged, rate unknown, no usable substitute — admit we can't.
1054 let rate = rate?;
1055 // Direction comes from the bucket, not from one blanket rule: a
1056 // substituted cache-READ rate can only be high, a substituted
1057 // cache-WRITE rate can land either side.
1058 may_overstate |= substitute_high;
1059 may_understate |= substitute_low;
1060 usd += count as f64 * rate;
1061 }
1062
1063 // Unresolvable tiers: say which way the base sheet can be wrong rather
1064 // than assuming tiers always cost more. Compare the rate each bucket
1065 // would actually pay at every declared threshold against the base.
1066 if tier_prompt_tokens.is_none() {
1067 for tier in &self.pricing_tiers {
1068 let at_tier = Self::bucket_rates(&self.prices_for(tier.min_prompt_tokens));
1069 for (count, ((base_rate, _, _), (tier_rate, _, _))) in
1070 tokens.into_iter().zip(rates.iter().zip(at_tier))
1071 {
1072 if count == 0 {
1073 continue;
1074 }
1075 if let (Some(base), Some(tiered)) = (base_rate, tier_rate) {
1076 may_understate |= tiered > *base;
1077 may_overstate |= tiered < *base;
1078 }
1079 }
1080 }
1081 }
1082
1083 Some(ApproxCost {
1084 usd: usd / 1_000_000.0,
1085 may_overstate,
1086 may_understate,
1087 })
1088 }
1089}
1090
1091/// A cost figure plus which way it can be wrong.
1092///
1093/// Presenting an estimate as an exact price is the same failure as billing an
1094/// unrated bucket at zero, one step later — so the direction travels with the
1095/// number instead of being re-derived (or forgotten) at each display site.
1096#[derive(Debug, Clone, Copy, PartialEq)]
1097pub struct ApproxCost {
1098 /// USD.
1099 pub usd: f64,
1100 /// The true cost may be **lower** — a bucket was priced at a substitute
1101 /// rate that can only be too high (a cache read at the uncached-input
1102 /// rate), or a pricing tier is cheaper than the base sheet used.
1103 pub may_overstate: bool,
1104 /// The true cost may be **higher** — a pricing tier dearer than the base
1105 /// sheet could not be resolved from the tokens the caller had, or a cache
1106 /// *write* was priced at the uncached-input rate and the provider charges
1107 /// a surcharge for it (Anthropic 1.25x, OpenRouter's 1h TTL 2x).
1108 pub may_understate: bool,
1109}
1110
1111impl ApproxCost {
1112 /// Every rate applied exactly; the figure is the price.
1113 pub fn is_exact(&self) -> bool {
1114 !self.may_overstate && !self.may_understate
1115 }
1116
1117 /// Prefix for the figure: `≤` a ceiling, `≥` a floor, `~` neither bound
1118 /// holds, empty when exact. Rendering the number without this is the
1119 /// defect the type exists to prevent.
1120 pub fn marker(&self) -> &'static str {
1121 match (self.may_overstate, self.may_understate) {
1122 (false, false) => "",
1123 (true, false) => "≤",
1124 (false, true) => "≥",
1125 (true, true) => "~",
1126 }
1127 }
1128}
1129
1130/// A score on a public benchmark from a published source (model card,
1131/// paper, leaderboard). The schema is deliberately permissive — no enum
1132/// of benchmark names — so the catalog can carry whichever benchmarks
1133/// the upstream provider chose to publish, and new ones can be added
1134/// without a code change. Scores are stored on a 0.0–1.0 scale (e.g.
1135/// 73.5% accuracy → 0.735) so they compare cleanly across benchmarks
1136/// and so `routing_ext::apply_benchmark_priors` can consume them
1137/// directly when wired in later.
1138#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
1139pub struct BenchmarkScore {
1140 /// Benchmark name as published (e.g., "MMLU-Pro", "GPQA-Diamond",
1141 /// "SWE-bench-Verified", "HumanEval", "MATH").
1142 pub name: String,
1143 /// Score on a 0.0–1.0 scale.
1144 pub score: f64,
1145 /// Evaluation harness or setup label (e.g., "5-shot", "0-shot CoT",
1146 /// "agentic", "pass@1"). Optional but strongly recommended — the
1147 /// same benchmark name can mean different things under different
1148 /// harnesses.
1149 #[serde(default)]
1150 pub harness: Option<String>,
1151 /// Where the score came from (model card URL, paper, leaderboard
1152 /// snapshot). Empty when the source is the upstream provider's
1153 /// announcement and a stable URL is not yet known.
1154 #[serde(default)]
1155 pub source_url: Option<String>,
1156 /// ISO 8601 date of the score snapshot (e.g., "2025-08-12"). Lets
1157 /// downstream code judge how stale a number is.
1158 #[serde(default)]
1159 pub measured_at: Option<String>,
1160 /// How many independent runs the score pools, when CAR measured it.
1161 /// `None` for a published score, whose sample is not ours to know.
1162 #[serde(default, skip_serializing_if = "Option::is_none")]
1163 pub runs: Option<u32>,
1164 /// Highest minus lowest score across those runs; present only with two
1165 /// or more. A difference smaller than this is not a measured difference.
1166 #[serde(default, skip_serializing_if = "Option::is_none")]
1167 pub spread: Option<f64>,
1168}
1169
1170/// The full declarative schema for a model.
1171///
1172/// Analogous to `ToolSchema` — describes what a model is, what it can do,
1173/// and how to access it. The router uses this for constraint-based filtering
1174/// and cold-start scoring before observed performance data is available.
1175#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
1176pub struct ModelSchema {
1177 /// Unique identifier: "provider/model-name:variant" (e.g., "qwen/qwen3-4b:q4_k_m").
1178 pub id: String,
1179 /// Human-readable display name.
1180 pub name: String,
1181 /// Provider (qwen, openai, anthropic, google, meta, ollama, custom).
1182 pub provider: String,
1183 /// Model family for grouping (qwen3, gpt-4, claude-4, llama-3).
1184 pub family: String,
1185 /// Semantic version or checkpoint label.
1186 #[serde(default)]
1187 pub version: String,
1188 /// What this model can do — ordered by primary capability first.
1189 pub capabilities: Vec<ModelCapability>,
1190 /// Context window in tokens.
1191 pub context_length: usize,
1192 /// Per-model maximum OUTPUT tokens the provider will return in one
1193 /// response. None = unknown; callers fall back to
1194 /// effective_max_output() which derives a fraction of context_length.
1195 #[serde(default)]
1196 pub max_output_tokens: Option<usize>,
1197 /// Parameter count as human-readable string (e.g., "4B", "30B (3B active)").
1198 #[serde(default)]
1199 pub param_count: String,
1200 /// How the weights are quantized, if at all. `None` for remote models and
1201 /// for local ones whose source declared nothing. See [`Quantization`] —
1202 /// this accepts the legacy bare-string form on the wire.
1203 #[serde(default)]
1204 #[schemars(with = "Option<QuantizationWireSchema>")]
1205 pub quantization: Option<Quantization>,
1206 /// Declared performance envelope (initial estimate, overridden by observed data).
1207 #[serde(default)]
1208 pub performance: PerformanceEnvelope,
1209 /// Cost structure.
1210 #[serde(default)]
1211 pub cost: CostModel,
1212 /// How to access this model.
1213 pub source: ModelSource,
1214 /// Free-form tags for filtering (e.g., "fast", "multilingual", "moe").
1215 #[serde(default)]
1216 pub tags: Vec<String>,
1217 /// Supported generation parameters. The inference layer strips any parameter
1218 /// not in this set before sending to the API. Empty = all supported.
1219 #[serde(default)]
1220 pub supported_params: Vec<GenerateParam>,
1221 /// Public benchmark scores as published by the model provider or
1222 /// reproduced on a public leaderboard (MMLU-Pro, GPQA-Diamond,
1223 /// SWE-bench, HumanEval, etc.). The built-in catalog ships this
1224 /// empty — population is a curation step, not a code change. See
1225 /// `BenchmarkScore` for the field shape and the 0.0–1.0 scoring
1226 /// convention.
1227 #[serde(default)]
1228 pub public_benchmarks: Vec<BenchmarkScore>,
1229 /// How much the project vouches for this model. The built-in catalog is
1230 /// `Curated`. Deserialization retains the legacy `Curated` default, so
1231 /// every user-controlled ingestion boundary must call
1232 /// [`Self::mark_user_registered`] before persistence or registration.
1233 /// Gates auto-apply (task #8) and this is surfaced in recommendation
1234 /// rationale.
1235 #[serde(default)]
1236 pub trust_tier: TrustTier,
1237 /// Superseded models stay listed if installed but are excluded from
1238 /// fresh recommendations. `#[serde(default)]` → not deprecated.
1239 #[serde(default)]
1240 pub deprecated: bool,
1241 /// Whether this model is currently available (downloaded / reachable).
1242 /// Not serialized — computed at runtime.
1243 #[serde(skip)]
1244 pub available: bool,
1245 /// Whether this model can be used **right now, without a download**.
1246 ///
1247 /// Deliberately narrower than [`Self::available`], which for a local MLX
1248 /// model is true as soon as an `hf_repo` is declared — `ensure_local()`
1249 /// lazy-downloads on first use, so a declared repo is "functionally
1250 /// available" (see #164). That is the right default for open-ended work and
1251 /// wrong for work on a deadline: a step with a bounded budget that picks a
1252 /// model it must first fetch spends the whole budget downloading and fails.
1253 /// That is exactly how `car code`'s 120s contract derivation became
1254 /// unusable on a machine with no local weights (Parslee-ai/car#638).
1255 ///
1256 /// Callers express the requirement with [`crate::IntentHint::require_ready`];
1257 /// this is the per-candidate fact that hint filters on. Recomputed on every
1258 /// registration, so a cached schema can't carry a stale value.
1259 #[serde(skip)]
1260 pub weights_ready: bool,
1261}
1262
1263impl ModelSchema {
1264 /// Mark a schema as user-controlled rather than project-vetted.
1265 ///
1266 /// This is intentionally separate from serde's legacy default: old built-in
1267 /// and test fixtures omit `trust_tier` and must continue to deserialize,
1268 /// while `models.json`, CLI imports, and daemon `models.register` must never
1269 /// inherit `Curated` merely because a caller omitted the field or supplied
1270 /// a forged value.
1271 pub fn mark_user_registered(&mut self) {
1272 self.trust_tier = TrustTier::Community;
1273 }
1274
1275 /// Check if this model has a given capability.
1276 pub fn has_capability(&self, cap: ModelCapability) -> bool {
1277 self.capabilities.contains(&cap)
1278 }
1279
1280 /// Whether usage is covered by the person's ChatGPT subscription.
1281 /// Callers must render `subscription` and never a dollar figure.
1282 pub fn is_subscription_billed(&self) -> bool {
1283 matches!(
1284 &self.source,
1285 ModelSource::Proprietary {
1286 auth: ProprietaryAuth::ChatGptSubscription {},
1287 ..
1288 }
1289 )
1290 }
1291
1292 /// Live availability for credential-backed providers. The catalog field is
1293 /// a startup snapshot; Settings/OAuth changes must affect the next list and
1294 /// route without a daemon restart.
1295 /// The credential this row is waiting on, when a missing credential is the
1296 /// only thing standing between the user and using it.
1297 ///
1298 /// Deliberately narrow. It answers `Some` only for a plain remote API row
1299 /// that declares one credential variable and is currently unavailable —
1300 /// the case where "add this key" is complete and correct advice. Every
1301 /// other shape answers `None`, because for them the advice would be wrong:
1302 ///
1303 /// - available rows need nothing;
1304 /// - local rows have no credential;
1305 /// - `OpenRouter` rows resolve through a credential *source* (pasted key
1306 /// or OAuth) rather than a named variable a person can type;
1307 /// - `Proprietary` OAuth rows (`parslee/*`) are a sign-in, not a key, and
1308 /// their availability additionally depends on the gateway having an
1309 /// upstream — credential presence alone has already been shipped as a
1310 /// false "available" for those (car#786);
1311 /// - `CodexCli` owns its own auth;
1312 /// - a deprecated row will not become usable by adding anything.
1313 ///
1314 /// A row declaring several alternative variables answers `None` as well:
1315 /// telling a person to set one of four names is not actionable, and
1316 /// picking one for them would be a guess about which account they have.
1317 pub fn credential_required(&self) -> Option<String> {
1318 if self.available_now() || self.deprecated || self.is_local() {
1319 return None;
1320 }
1321 match &self.source {
1322 ModelSource::RemoteApi {
1323 protocol: ApiProtocol::OpenRouter,
1324 ..
1325 } => None,
1326 ModelSource::RemoteApi {
1327 api_key_env,
1328 api_key_envs,
1329 ..
1330 } if api_key_envs.is_empty() && !api_key_env.trim().is_empty() => {
1331 Some(api_key_env.clone())
1332 }
1333 _ => None,
1334 }
1335 }
1336
1337 pub fn available_now(&self) -> bool {
1338 match &self.source {
1339 ModelSource::RemoteApi {
1340 protocol: ApiProtocol::OpenRouter,
1341 ..
1342 } => self.available && crate::openrouter::credential_source().is_some(),
1343 ModelSource::CodexCli { .. } => crate::backend::codex_cli::is_available(),
1344 // Live (cached a few seconds): Apple Intelligence can be switched
1345 // on or off, or finish provisioning, while CAR runs, and the
1346 // catalog snapshot would not see it until the next refresh.
1347 ModelSource::AppleFoundationModels { .. } => {
1348 #[cfg(any(
1349 all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)),
1350 all(target_os = "ios", target_arch = "aarch64")
1351 ))]
1352 {
1353 crate::backend::foundation_models::is_available()
1354 }
1355 #[cfg(not(any(
1356 all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)),
1357 all(target_os = "ios", target_arch = "aarch64")
1358 )))]
1359 {
1360 false
1361 }
1362 }
1363 _ => self.available,
1364 }
1365 }
1366
1367 /// Prompt-cache economics for this model, derived from its remote
1368 /// protocol. Local / non-remote models have no remote prompt cache, so
1369 /// their cache rates are inert ([`CacheRates::NONE`](crate::outcome::CacheRates::NONE)).
1370 pub fn cache_rates(&self) -> crate::outcome::CacheRates {
1371 self.cache_rates_at(&TokenPrices {
1372 input_per_mtok: self.cost.input_per_mtok,
1373 output_per_mtok: self.cost.output_per_mtok,
1374 cache_read_input_per_mtok: self.cost.cache_read_input_per_mtok,
1375 cache_write_input_per_mtok: self.cost.cache_write_input_per_mtok,
1376 })
1377 }
1378
1379 /// [`Self::cache_rates`] under one resolved price sheet, such as a
1380 /// prompt-size tier from [`CostModel::prices_for`]: an OpenRouter row's
1381 /// cache multipliers are its cache rates over its input rate, so a tier
1382 /// that changes them changes the multipliers too.
1383 pub fn cache_rates_at(&self, prices: &TokenPrices) -> crate::outcome::CacheRates {
1384 match &self.source {
1385 ModelSource::RemoteApi {
1386 protocol: ApiProtocol::OpenRouter,
1387 ..
1388 } => {
1389 let input = prices.input_per_mtok.unwrap_or(0.0);
1390 if input > 0.0 {
1391 crate::outcome::CacheRates {
1392 read_mult: prices.cache_read_input_per_mtok.unwrap_or(0.0) / input,
1393 write_mult: prices.cache_write_input_per_mtok.unwrap_or(0.0) / input,
1394 }
1395 } else {
1396 crate::outcome::CacheRates::NONE
1397 }
1398 }
1399 ModelSource::RemoteApi { protocol, .. } => protocol.cache_rates(),
1400 _ => crate::outcome::CacheRates::NONE,
1401 }
1402 }
1403
1404 /// The organization whose model this is, when that can be honestly known.
1405 ///
1406 /// Answers one question — could two models be expected to fail the same
1407 /// way? — so it is deliberately conservative. `None` means UNKNOWABLE, not
1408 /// "none", and callers must treat it as "cannot tell" rather than folding
1409 /// it into a count of distinct vendors.
1410 ///
1411 /// NOT [`Self::provider`], which means four different things depending on
1412 /// which path built the schema:
1413 ///
1414 /// * curated remote rows — the real vendor (`openai`, `anthropic`);
1415 /// * OpenRouter and the Parslee gateway — the AGGREGATOR, so three vendors
1416 /// behind one gateway all report `openrouter`/`parslee` and one vendor
1417 /// reached two ways reports as two;
1418 /// * a HuggingFace-derived row — the repo ORG that uploaded it, so
1419 /// `unsloth/Qwen3` and `mlx-community/Qwen3` are the same weights under
1420 /// two "vendors";
1421 /// * a discovered local-server row — a guess from the model name.
1422 ///
1423 /// Only the first is a vendor, so only the first is reported. NOT `family`
1424 /// either — that is the model line, so `claude-4.6` and `claude-4.8` read as
1425 /// different and are both Anthropic.
1426 pub fn vendor(&self) -> Option<&str> {
1427 // The aggregator case: the curated table knows which upstream a gateway
1428 // alias resolves to, which is the only place that survives an id
1429 // carrying no trace of its vendor.
1430 if self.provider.eq_ignore_ascii_case("openrouter")
1431 || self.provider.eq_ignore_ascii_case("parslee")
1432 {
1433 return crate::openrouter::curated_vendor(&self.id);
1434 }
1435 // A local model is served by the operator's own machine. Whoever
1436 // uploaded the weights is not an organization that could fail
1437 // independently of the process running beside it.
1438 if self.is_local() {
1439 return None;
1440 }
1441 // Only a project-vetted row's `provider` was assigned deliberately. On a
1442 // community row it is the uploader or a guess, and claiming it here is
1443 // how two repacks of one checkpoint would pass as two vendors.
1444 if self.trust_tier != TrustTier::Curated {
1445 return None;
1446 }
1447 (!self.provider.is_empty()).then_some(self.provider.as_str())
1448 }
1449
1450 /// Check if this model is local (runs on-device).
1451 pub fn is_local(&self) -> bool {
1452 matches!(
1453 self.source,
1454 ModelSource::Local { .. }
1455 | ModelSource::Mlx { .. }
1456 | ModelSource::WhisperCpp { .. }
1457 | ModelSource::WindowsSpeech { .. }
1458 | ModelSource::ManagedVllmMlx { .. }
1459 | ModelSource::AppleFoundationModels { .. }
1460 )
1461 }
1462
1463 /// Whether this model has weights CAR fetches to disk before it can be
1464 /// used — i.e. whether "is it installed?" is a question with an answer.
1465 ///
1466 /// Three predicates in this area are easy to conflate, and conflating them
1467 /// is what Parslee-ai/car#894 was about:
1468 ///
1469 /// - [`is_local`](Self::is_local) — *owned on this machine*. True for
1470 /// `WindowsSpeech` (the OS owns the voices), `AppleFoundationModels`
1471 /// (the OS owns the weights), and CAR-managed sources. An external
1472 /// `VllmMlx` endpoint is remote even when its URL happens to be loopback.
1473 /// - [`weights_ready`](Self::weights_ready) — *the weights are on disk
1474 /// now*. Only meaningful when this predicate is true; for everything
1475 /// else the registry sets it to `true` as a "nothing blocks an attempt"
1476 /// sentinel, which reads as "installed" if taken literally.
1477 /// - `downloads_weights` (this one) — *there is something to install at
1478 /// all*. Use it to decide whether an install/download status should be
1479 /// reported, then use `weights_ready` for the status itself.
1480 ///
1481 /// Rendering `weights_ready` without this gate is what made
1482 /// `windows/speech-synthesis:os` claim `INSTALLED yes` while `car doctor`
1483 /// said `Models: none installed`, and made `apple/foundation:default` and
1484 /// the `vllm-mlx/*` rows claim `INSTALLED no` for models that install
1485 /// nothing.
1486 ///
1487 /// Written as an exhaustive `match` rather than `matches!` so that adding
1488 /// a `ModelSource` variant is a compile error here instead of a silently
1489 /// wrong answer in the CLI.
1490 pub fn downloads_weights(&self) -> bool {
1491 match self.source {
1492 // CAR fetches these to disk itself: a GGUF file, an MLX
1493 // safetensors repo, a whisper.cpp ggml `.bin`.
1494 ModelSource::Local { .. }
1495 | ModelSource::Mlx { .. }
1496 | ModelSource::WhisperCpp { .. }
1497 | ModelSource::ManagedVllmMlx { .. } => true,
1498 // The OS owns the voices / the weights — nothing to download.
1499 ModelSource::WindowsSpeech {} | ModelSource::AppleFoundationModels { .. } => false,
1500 // Someone else holds the weights: a local server (vLLM-MLX,
1501 // Ollama), a remote API, or a host-registered runner.
1502 ModelSource::VllmMlx { .. }
1503 | ModelSource::Ollama { .. }
1504 | ModelSource::RemoteApi { .. }
1505 | ModelSource::CodexCli { .. }
1506 | ModelSource::Proprietary { .. }
1507 | ModelSource::Delegated { .. } => false,
1508 }
1509 }
1510
1511 /// Whether a CAR-downloadable artifact is physically present now.
1512 /// General runtime availability and OS/server-owned weights are not an
1513 /// installation claim.
1514 pub fn has_installed_weights(&self) -> bool {
1515 self.downloads_weights() && self.weights_ready
1516 }
1517
1518 /// Whether CAR decodes this model **in its own process**, token by token,
1519 /// through the shared decode loop.
1520 ///
1521 /// Narrower than [`is_local`](Self::is_local) on purpose. `is_local` also
1522 /// covers CAR-managed vLLM-MLX and the speech backends. Those do not spend
1523 /// an output-token budget as this process's wall clock, so they must keep
1524 /// the remote treatment. Getting that distinction wrong takes the
1525 /// anti-truncation budget away from vLLM-MLX, which is the documented way
1526 /// to get structured tool calls out of a local model. (car#851)
1527 pub fn decodes_in_process(&self) -> bool {
1528 matches!(
1529 self.source,
1530 ModelSource::Local { .. } | ModelSource::Mlx { .. }
1531 )
1532 }
1533
1534 /// Check if this model delegates inference to a host-registered
1535 /// runner (closes Parslee-ai/car-releases#24).
1536 pub fn is_delegated(&self) -> bool {
1537 matches!(self.source, ModelSource::Delegated { .. })
1538 }
1539
1540 /// Check if this model runs one tool-free turn through `codex exec`.
1541 pub fn is_codex_cli(&self) -> bool {
1542 matches!(self.source, ModelSource::CodexCli { .. })
1543 }
1544
1545 /// Check if this model uses the MLX backend.
1546 pub fn is_mlx(&self) -> bool {
1547 matches!(self.source, ModelSource::Mlx { .. })
1548 }
1549
1550 /// Check if this model routes to Apple's on-device FoundationModels
1551 /// framework. True only for `ModelSource::AppleFoundationModels`;
1552 /// callers must still verify runtime availability before dispatch
1553 /// (the schema can describe the model on any host, but execution
1554 /// requires macOS 26+ on Apple Silicon).
1555 pub fn is_foundation_models(&self) -> bool {
1556 matches!(self.source, ModelSource::AppleFoundationModels { .. })
1557 }
1558
1559 /// Whether the operating system owns the implementation and its memory.
1560 /// These rows have no CAR-downloadable artifact and no local-model memory
1561 /// figure to compare with the admission budget.
1562 pub fn is_os_provided(&self) -> bool {
1563 matches!(
1564 self.source,
1565 ModelSource::WindowsSpeech {} | ModelSource::AppleFoundationModels { .. }
1566 )
1567 }
1568
1569 /// Check if this model uses vLLM-MLX backend.
1570 pub fn is_vllm_mlx(&self) -> bool {
1571 matches!(
1572 self.source,
1573 ModelSource::VllmMlx { .. } | ModelSource::ManagedVllmMlx { .. }
1574 )
1575 }
1576
1577 /// Whether CAR, rather than an independently managed endpoint, owns the
1578 /// vLLM-MLX child process and its physical weight allocation. The existing
1579 /// HTTP `VllmMlx` contract remains external/server-owned. A supervised
1580 /// source must opt in explicitly and provide an already-installed local
1581 /// model path in `model_name`; it is then replaced with a loopback HTTP
1582 /// endpoint only after admission and a successful readiness ACK.
1583 pub fn is_car_managed_vllm_mlx(&self) -> bool {
1584 matches!(self.source, ModelSource::ManagedVllmMlx { .. })
1585 }
1586
1587 /// Whether a CAR/OS-owned model can only run on Apple Silicon (Metal).
1588 /// External vLLM-MLX endpoints own their hardware and are not constrained
1589 /// by the client machine's accelerator.
1590 pub fn requires_apple_silicon(&self) -> bool {
1591 self.is_mlx() || self.is_car_managed_vllm_mlx() || self.is_foundation_models()
1592 }
1593
1594 /// Check if this model is remote (requires API call).
1595 pub fn is_remote(&self) -> bool {
1596 matches!(
1597 self.source,
1598 ModelSource::RemoteApi { .. }
1599 | ModelSource::CodexCli { .. }
1600 | ModelSource::Proprietary { .. }
1601 | ModelSource::VllmMlx { .. }
1602 )
1603 }
1604
1605 /// Collect all API key env var names for this model (primary + extras).
1606 /// Returns empty vec for non-remote models.
1607 pub fn all_api_key_envs(&self) -> Vec<String> {
1608 match &self.source {
1609 ModelSource::RemoteApi {
1610 api_key_env,
1611 api_key_envs,
1612 ..
1613 } => {
1614 let mut all = vec![api_key_env.clone()];
1615 all.extend(api_key_envs.iter().cloned());
1616 all
1617 }
1618 ModelSource::Proprietary {
1619 auth: ProprietaryAuth::ApiKeyEnv { env_var },
1620 ..
1621 }
1622 | ModelSource::Proprietary {
1623 auth: ProprietaryAuth::BearerTokenEnv { env_var },
1624 ..
1625 } => vec![env_var.clone()],
1626 ModelSource::Proprietary {
1627 auth: ProprietaryAuth::OAuth2Pkce { .. } | ProprietaryAuth::ChatGptSubscription {},
1628 ..
1629 } => vec![],
1630 _ => vec![],
1631 }
1632 }
1633
1634 /// Get the size in MB (from cost model or 0 if unknown).
1635 pub fn size_mb(&self) -> u64 {
1636 self.cost.size_mb.unwrap_or(0)
1637 }
1638
1639 /// Get the RAM requirement in MB (from cost model, falls back to size_mb).
1640 pub fn ram_mb(&self) -> u64 {
1641 self.cost.ram_mb.unwrap_or_else(|| self.size_mb())
1642 }
1643
1644 /// Estimated cost per 1K output tokens in USD. Returns 0.0 for local models.
1645 pub fn cost_per_1k_output(&self) -> f64 {
1646 self.cost.output_per_mtok.map(|c| c / 1000.0).unwrap_or(0.0)
1647 }
1648
1649 /// The per-turn output-token ceiling to use when the caller didn't
1650 /// specify one. Prefers the registry-declared `max_output_tokens`;
1651 /// otherwise derives a quarter of the context window, clamped to a
1652 /// sane [4096, 32768] band so a 1M-context model doesn't request a
1653 /// 250K-token response the API rejects and a tiny 8K model doesn't
1654 /// get an absurdly small ceiling. (Registry value first, computed
1655 /// fallback second — mirrors a provider lookup with a derived default.)
1656 pub fn effective_max_output(&self) -> usize {
1657 self.max_output_tokens
1658 .unwrap_or_else(|| (self.context_length / 4).clamp(4096, 32_768))
1659 }
1660}
1661
1662#[cfg(test)]
1663mod tests {
1664 use super::*;
1665
1666 fn sample_local() -> ModelSchema {
1667 ModelSchema {
1668 id: "qwen/qwen3-4b:q4_k_m".into(),
1669 name: "Qwen3-4B".into(),
1670 provider: "qwen".into(),
1671 family: "qwen3".into(),
1672 version: "1.0".into(),
1673 capabilities: vec![ModelCapability::Generate, ModelCapability::Code],
1674 context_length: 32768,
1675 max_output_tokens: None,
1676 param_count: "4B".into(),
1677 quantization: Some(Quantization::parse("Q4_K_M")),
1678 performance: PerformanceEnvelope {
1679 tokens_per_second: Some(45.0),
1680 ..Default::default()
1681 },
1682 cost: CostModel {
1683 size_mb: Some(2500),
1684 ram_mb: Some(2500),
1685 ..Default::default()
1686 },
1687 source: ModelSource::Local {
1688 hf_repo: "Qwen/Qwen3-4B-GGUF".into(),
1689 hf_filename: "Qwen3-4B-Q4_K_M.gguf".into(),
1690 tokenizer_repo: "Qwen/Qwen3-4B".into(),
1691 },
1692 tags: vec!["code".into(), "fast".into()],
1693 supported_params: vec![],
1694 public_benchmarks: vec![],
1695 trust_tier: TrustTier::Curated,
1696 deprecated: false,
1697 available: false,
1698 weights_ready: false,
1699 }
1700 }
1701
1702 fn sample_remote() -> ModelSchema {
1703 ModelSchema {
1704 id: "anthropic/claude-sonnet-4-6:latest".into(),
1705 name: "Claude Sonnet 4.6".into(),
1706 provider: "anthropic".into(),
1707 family: "claude-4".into(),
1708 version: "latest".into(),
1709 capabilities: vec![
1710 ModelCapability::Generate,
1711 ModelCapability::Code,
1712 ModelCapability::Reasoning,
1713 ModelCapability::ToolUse,
1714 ModelCapability::Vision,
1715 ],
1716 context_length: 200000,
1717 max_output_tokens: None,
1718 param_count: String::new(),
1719 quantization: None,
1720 performance: PerformanceEnvelope {
1721 latency_p50_ms: Some(2000),
1722 latency_p99_ms: Some(8000),
1723 tokens_per_second: Some(80.0),
1724 },
1725 cost: CostModel {
1726 input_per_mtok: Some(3.0),
1727 output_per_mtok: Some(15.0),
1728 ..Default::default()
1729 },
1730 source: ModelSource::RemoteApi {
1731 endpoint: "https://api.anthropic.com/v1/messages".into(),
1732 api_key_env: "ANTHROPIC_API_KEY".into(),
1733 api_key_envs: vec![],
1734 api_version: Some("2023-06-01".into()),
1735 protocol: ApiProtocol::Anthropic,
1736 },
1737 tags: vec!["reasoning".into(), "tool_use".into()],
1738 supported_params: vec![],
1739 public_benchmarks: vec![],
1740 trust_tier: TrustTier::Curated,
1741 deprecated: false,
1742 available: false,
1743 weights_ready: false,
1744 }
1745 }
1746
1747 #[test]
1748 fn capabilities() {
1749 let m = sample_local();
1750 assert!(m.has_capability(ModelCapability::Code));
1751 assert!(!m.has_capability(ModelCapability::Vision));
1752 }
1753
1754 #[test]
1755 fn local_vs_remote() {
1756 assert!(sample_local().is_local());
1757 assert!(!sample_local().is_remote());
1758 assert!(sample_remote().is_remote());
1759 assert!(!sample_remote().is_local());
1760 let codex = ModelSchema {
1761 source: ModelSource::CodexCli {
1762 model: "gpt-5.6-sol:high".into(),
1763 },
1764 ..sample_local()
1765 };
1766 assert!(codex.is_remote());
1767 assert!(!codex.is_local());
1768 }
1769
1770 #[test]
1771 fn vllm_ownership_drives_local_remote_and_apple_predicates() {
1772 let external = ModelSchema {
1773 source: ModelSource::VllmMlx {
1774 endpoint: "https://gpu-owner.example/v1".into(),
1775 model_name: "owner/runtime-model".into(),
1776 },
1777 ..sample_local()
1778 };
1779 assert!(!external.is_local());
1780 assert!(external.is_remote());
1781 assert!(!external.requires_apple_silicon());
1782
1783 let managed = ModelSchema {
1784 source: ModelSource::ManagedVllmMlx {
1785 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
1786 hf_weight_file: None,
1787 draft_hf_repo: None,
1788 },
1789 ..sample_local()
1790 };
1791 assert!(managed.is_local());
1792 assert!(!managed.is_remote());
1793 assert!(managed.requires_apple_silicon());
1794 }
1795
1796 #[test]
1797 fn cost() {
1798 let local = sample_local();
1799 assert_eq!(local.cost_per_1k_output(), 0.0);
1800
1801 let remote = sample_remote();
1802 assert!(remote.cost_per_1k_output() > 0.0);
1803 }
1804
1805 #[test]
1806 fn serde_roundtrip() {
1807 let local = sample_local();
1808 let json = serde_json::to_string(&local).unwrap();
1809 let parsed: ModelSchema = serde_json::from_str(&json).unwrap();
1810 assert_eq!(parsed.id, local.id);
1811 assert_eq!(parsed.capabilities, local.capabilities);
1812
1813 let remote = sample_remote();
1814 let json = serde_json::to_string(&remote).unwrap();
1815 let parsed: ModelSchema = serde_json::from_str(&json).unwrap();
1816 assert_eq!(parsed.id, remote.id);
1817 // available is skip-serialized, defaults to false
1818 assert!(!parsed.available);
1819 }
1820
1821 #[test]
1822 fn managed_vllm_source_is_versioned_without_reinterpreting_legacy_vllm_json() {
1823 let legacy: ModelSource = serde_json::from_str(
1824 r#"{"type":"vllm_mlx","endpoint":"http://localhost:8000","model_name":"legacy"}"#,
1825 )
1826 .unwrap();
1827 assert!(matches!(
1828 legacy,
1829 ModelSource::VllmMlx {
1830 endpoint,
1831 model_name
1832 } if endpoint == "http://localhost:8000" && model_name == "legacy"
1833 ));
1834
1835 let managed = ModelSource::ManagedVllmMlx {
1836 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
1837 hf_weight_file: Some("model.safetensors".into()),
1838 draft_hf_repo: None,
1839 };
1840 let encoded = serde_json::to_string(&managed).unwrap();
1841 assert!(encoded.contains(r#""type":"managed_vllm_mlx""#));
1842 assert!(matches!(
1843 serde_json::from_str::<ModelSource>(&encoded).unwrap(),
1844 ModelSource::ManagedVllmMlx { .. }
1845 ));
1846 }
1847
1848 /// Every catalog row is content-addressed (`row_digest`), so the drafter
1849 /// field must vanish from rows that don't declare one; otherwise adding it
1850 /// would have re-hashed every managed vLLM-MLX row in every catalog.
1851 #[test]
1852 fn draft_repo_is_absent_from_rows_without_one() {
1853 let plain = ModelSource::ManagedVllmMlx {
1854 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
1855 hf_weight_file: None,
1856 draft_hf_repo: None,
1857 };
1858 let encoded = serde_json::to_value(&plain).unwrap();
1859 assert!(encoded.get("draft_hf_repo").is_none(), "{encoded}");
1860
1861 let declared: ModelSource = serde_json::from_str(
1862 r#"{"type":"managed_vllm_mlx","hf_repo":"a/b","draft_hf_repo":"a/b-MTP"}"#,
1863 )
1864 .unwrap();
1865 assert!(matches!(
1866 declared,
1867 ModelSource::ManagedVllmMlx { draft_hf_repo: Some(ref d), .. } if d == "a/b-MTP"
1868 ));
1869 }
1870
1871 #[test]
1872 fn vendor_refuses_to_claim_a_repackager_or_a_guess() {
1873 // A HuggingFace-derived row's `provider` is the repo ORG that uploaded
1874 // it, so `unsloth/Qwen3` and `mlx-community/Qwen3` — one checkpoint,
1875 // two repacks — would otherwise read as two independent vendors. That
1876 // is a false independence claim produced silently, which is the exact
1877 // failure a vendor check exists to prevent.
1878 let mut m = sample_remote();
1879 m.provider = "mlx-community".into();
1880 m.trust_tier = TrustTier::Community;
1881 assert_eq!(m.vendor(), None);
1882
1883 // A project-vetted remote row IS authoritative.
1884 m.provider = "openai".into();
1885 m.trust_tier = TrustTier::Curated;
1886 assert_eq!(m.vendor(), Some("openai"));
1887
1888 // A local model is served by the operator's own machine; whoever
1889 // uploaded the weights is not an independently-failing organization.
1890 assert_eq!(sample_local().vendor(), None);
1891 }
1892
1893 #[test]
1894 fn trust_tier_and_deprecated_default_when_absent() {
1895 // Pre-existing ~/.car/models.json configs omit the new fields.
1896 // They must deserialize to Curated / not-deprecated, not error.
1897 let json = serde_json::to_string(&sample_local()).unwrap();
1898 let stripped = json
1899 .replace(",\"trust_tier\":\"curated\"", "")
1900 .replace(",\"deprecated\":false", "");
1901 let parsed: ModelSchema = serde_json::from_str(&stripped).unwrap();
1902 assert_eq!(parsed.trust_tier, TrustTier::Curated);
1903 assert!(!parsed.deprecated);
1904 }
1905
1906 #[test]
1907 fn trust_tier_serializes_snake_case() {
1908 assert_eq!(
1909 serde_json::to_string(&TrustTier::Community).unwrap(),
1910 "\"community\""
1911 );
1912 assert_eq!(TrustTier::default(), TrustTier::Curated);
1913 }
1914
1915 #[test]
1916 fn requires_apple_silicon_only_for_metal_backends() {
1917 // GGUF/Candle local and remote models run anywhere CAR builds for.
1918 assert!(!sample_local().requires_apple_silicon());
1919 assert!(!sample_remote().requires_apple_silicon());
1920
1921 let mlx = ModelSchema {
1922 source: ModelSource::Mlx {
1923 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
1924 hf_weight_file: None,
1925 },
1926 ..sample_local()
1927 };
1928 assert!(mlx.requires_apple_silicon());
1929
1930 // Only CAR-managed vLLM-MLX and Apple FoundationModels are Metal-bound.
1931 // A raw endpoint is external and its owner's hardware is opaque to CAR.
1932 let managed_vllm = ModelSchema {
1933 source: ModelSource::ManagedVllmMlx {
1934 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
1935 hf_weight_file: None,
1936 draft_hf_repo: None,
1937 },
1938 ..sample_local()
1939 };
1940 assert!(managed_vllm.requires_apple_silicon());
1941
1942 let external_vllm = ModelSchema {
1943 source: ModelSource::VllmMlx {
1944 endpoint: "https://gpu-owner.example/v1".into(),
1945 model_name: "mlx-community/Qwen3-4B-4bit".into(),
1946 },
1947 ..sample_local()
1948 };
1949 assert!(!external_vllm.requires_apple_silicon());
1950
1951 let foundation = ModelSchema {
1952 source: ModelSource::AppleFoundationModels { use_case: None },
1953 ..sample_local()
1954 };
1955 assert!(foundation.requires_apple_silicon());
1956 }
1957
1958 fn priced(
1959 input: Option<f64>,
1960 output: Option<f64>,
1961 cache_read: Option<f64>,
1962 cache_write: Option<f64>,
1963 ) -> CostModel {
1964 CostModel {
1965 input_per_mtok: input,
1966 output_per_mtok: output,
1967 cache_read_input_per_mtok: cache_read,
1968 cache_write_input_per_mtok: cache_write,
1969 ..Default::default()
1970 }
1971 }
1972
1973 #[test]
1974 fn an_undeclared_cache_rate_is_bounded_by_the_input_rate_not_billed_at_zero() {
1975 // The real shape this exists for: qwen3.5-plus declares input and
1976 // output but no cache-read rate. 1M cache-read tokens must not be free.
1977 let cost = priced(Some(0.26), Some(1.56), None, None);
1978 let bounded = cost
1979 .estimated_usd_bounded(Some(1_000_000), 0, 0, 1_000_000, 0)
1980 .expect("a model with input+output rates is priceable");
1981 assert!(bounded.may_overstate, "substituted rate can only be high");
1982 assert!(!bounded.may_understate);
1983 assert_eq!(bounded.marker(), "≤");
1984 assert!(
1985 (bounded.usd - 0.26).abs() < 1e-9,
1986 "cache reads fall back to the $0.26/MTok input rate, got {}",
1987 bounded.usd
1988 );
1989
1990 // The routing-score path still zero-fills, deliberately and untouched.
1991 assert_eq!(cost.estimated_usd(1_000_000, 0, 1_000_000, 0), 0.0);
1992 }
1993
1994 #[test]
1995 fn an_undeclared_cache_write_rate_claims_no_bound_it_cannot_keep() {
1996 // The shipped `claude-opus-4.8` shape with its cache-write rate
1997 // omitted. Anthropic's cache write is a 1.25x SURCHARGE (6.25 against
1998 // 5.0 input), and OpenRouter's 1h-TTL rate is 2x — so pricing the
1999 // bucket at the input rate is a FLOOR, and the `≤` a blanket
2000 // "cache is always cheaper" rule would have produced is a false
2001 // ceiling over a true cost of $6.25, or $10.00 at the 1h rate.
2002 let cost = priced(Some(5.0), Some(25.0), Some(0.5), None);
2003 let bounded = cost
2004 .estimated_usd_bounded(Some(1_000_000), 0, 0, 0, 1_000_000)
2005 .expect("input and output rates are published");
2006 assert!((bounded.usd - 5.0).abs() < 1e-9, "got {}", bounded.usd);
2007 assert!(
2008 bounded.may_understate,
2009 "a cache-write surcharge can exceed the input rate"
2010 );
2011 assert_ne!(bounded.marker(), "≤", "must not claim a ceiling it fails");
2012 assert_eq!(bounded.marker(), "~", "sign is unknown, so neither bound");
2013 // Both real cache-write rates this shape could carry sit ABOVE the
2014 // substituted figure — which is exactly why `≤` would have been a
2015 // false ceiling. Assert the tighter of the two; it implies the looser,
2016 // and spelling out both comparisons is a `redundant_comparisons` lint.
2017 let at_anthropics_1_25x: f64 = 1_000_000.0 * 6.25 / 1e6;
2018 let at_openrouters_1h_2x: f64 = 1_000_000.0 * 10.0 / 1e6;
2019 assert!(bounded.usd < at_anthropics_1_25x.min(at_openrouters_1h_2x));
2020
2021 // Declaring the rate makes it exact — the flag tracks substitution,
2022 // not the mere presence of cache-write tokens.
2023 let declared = priced(Some(5.0), Some(25.0), Some(0.5), Some(6.25))
2024 .estimated_usd_bounded(Some(1_000_000), 0, 0, 0, 1_000_000)
2025 .unwrap();
2026 assert!(declared.is_exact());
2027 assert!((declared.usd - 6.25).abs() < 1e-9);
2028
2029 // The same omission on the cache READ side DOES keep its ceiling —
2030 // the two directions are not shared. Note the rate must be missing for
2031 // a substitution to happen at all.
2032 let read = priced(Some(5.0), Some(25.0), None, Some(6.25))
2033 .estimated_usd_bounded(Some(1_000_000), 0, 0, 1_000_000, 0)
2034 .unwrap();
2035 assert_eq!(read.marker(), "≤");
2036 assert!((read.usd - 5.0).abs() < 1e-9);
2037 // A real cache-read rate is a discount, so the ceiling holds.
2038 assert!(read.usd > 1_000_000.0 * 0.5 / 1e6);
2039 }
2040
2041 #[test]
2042 fn a_declared_cache_rate_is_exact_and_never_flagged() {
2043 let cost = priced(Some(5.0), Some(25.0), Some(0.5), Some(6.25));
2044 let bounded = cost
2045 .estimated_usd_bounded(Some(1_600_000), 1_000_000, 200_000, 500_000, 100_000)
2046 .expect("fully rated");
2047 assert!(bounded.is_exact(), "nothing was substituted or unresolved");
2048 assert_eq!(bounded.marker(), "");
2049 // 1M uncached x 5 + 200k x 25 + 500k x 0.5 + 100k x 6.25, per MTok.
2050 assert!((bounded.usd - 10.875).abs() < 1e-9, "got {}", bounded.usd);
2051 // Identical to the router's figure when every rate is published.
2052 assert!(
2053 (bounded.usd - cost.estimated_usd(1_600_000, 200_000, 500_000, 100_000)).abs() < 1e-9
2054 );
2055 }
2056
2057 #[test]
2058 fn an_unbounded_bucket_refuses_rather_than_understating() {
2059 // Output has no safe substitute — it is normally the dearer side, so
2060 // pricing it at the input rate would UNDER-state. Refuse instead.
2061 let cost = priced(Some(1.0), None, None, None);
2062 assert_eq!(
2063 cost.estimated_usd_bounded(Some(1_000), 1_000, 1_000, 0, 0),
2064 None
2065 );
2066 // With no output tokens the same model is priceable and exact.
2067 let bounded = cost
2068 .estimated_usd_bounded(Some(1_000), 1_000, 0, 0, 0)
2069 .unwrap();
2070 assert!(bounded.is_exact());
2071 }
2072
2073 #[test]
2074 fn no_rate_card_is_unpriced_rather_than_free() {
2075 let cost = CostModel::default();
2076 assert_eq!(
2077 cost.estimated_usd_bounded(Some(1_000), 1_000, 1_000, 0, 0),
2078 None
2079 );
2080 // And a zero-usage priced model is genuinely free, not unpriced.
2081 let free = priced(Some(0.0), Some(0.0), None, None)
2082 .estimated_usd_bounded(Some(0), 0, 0, 0, 0)
2083 .expect("a declared zero rate card is priced");
2084 assert_eq!(free.usd, 0.0);
2085 assert!(free.is_exact());
2086 }
2087
2088 fn tiered() -> CostModel {
2089 CostModel {
2090 input_per_mtok: Some(2.5),
2091 output_per_mtok: Some(15.0),
2092 cache_read_input_per_mtok: Some(0.25),
2093 pricing_tiers: vec![TokenPricingTier {
2094 min_prompt_tokens: 272_000,
2095 prices: TokenPrices {
2096 input_per_mtok: Some(5.0),
2097 output_per_mtok: Some(22.5),
2098 cache_read_input_per_mtok: Some(0.5),
2099 cache_write_input_per_mtok: None,
2100 },
2101 }],
2102 ..Default::default()
2103 }
2104 }
2105
2106 #[test]
2107 fn a_known_prompt_size_resolves_the_tier_exactly() {
2108 let cost = tiered();
2109 let below = cost
2110 .estimated_usd_bounded(Some(271_999), 271_999, 0, 0, 0)
2111 .unwrap();
2112 assert!(below.is_exact());
2113 assert!((below.usd - 271_999.0 * 2.5 / 1e6).abs() < 1e-9);
2114
2115 let above = cost
2116 .estimated_usd_bounded(Some(272_000), 272_000, 0, 0, 0)
2117 .unwrap();
2118 assert!(above.is_exact());
2119 assert!((above.usd - 272_000.0 * 5.0 / 1e6).abs() < 1e-9);
2120 }
2121
2122 #[test]
2123 fn a_lifetime_aggregate_uses_base_rates_and_admits_it_may_be_low() {
2124 // Thirty 10K-token requests. Their SUM crosses the 272K threshold that
2125 // no single request came near — the double-charging bug. `None` says
2126 // "boundaries lost", so base rates apply and the figure is marked.
2127 let cost = tiered();
2128 let aggregate = cost.estimated_usd_bounded(None, 300_000, 0, 0, 0).unwrap();
2129 assert!(
2130 (aggregate.usd - 300_000.0 * 2.5 / 1e6).abs() < 1e-9,
2131 "must use the $2.50 base rate, got {}",
2132 aggregate.usd
2133 );
2134 assert!(aggregate.may_understate, "a dearer tier may apply");
2135 assert!(!aggregate.may_overstate);
2136 assert_eq!(aggregate.marker(), "≥");
2137
2138 // What the bug looked like: summed tokens passed as a real prompt size
2139 // price at the high-context rate — exactly double, and unmarked.
2140 let bug = cost
2141 .estimated_usd_bounded(Some(300_000), 300_000, 0, 0, 0)
2142 .unwrap();
2143 assert!((bug.usd - 2.0 * aggregate.usd).abs() < 1e-9);
2144 assert!(bug.is_exact(), "and it would have claimed to be exact");
2145 }
2146
2147 #[test]
2148 fn a_cheaper_tier_flags_the_aggregate_as_possibly_high_instead() {
2149 // Direction is derived from the tiers, not assumed. A volume DISCOUNT
2150 // makes the base-rate figure too high, not too low.
2151 let cost = CostModel {
2152 input_per_mtok: Some(2.0),
2153 output_per_mtok: Some(10.0),
2154 pricing_tiers: vec![TokenPricingTier {
2155 min_prompt_tokens: 100_000,
2156 prices: TokenPrices {
2157 input_per_mtok: Some(1.0),
2158 ..Default::default()
2159 },
2160 }],
2161 ..Default::default()
2162 };
2163 let aggregate = cost.estimated_usd_bounded(None, 500_000, 0, 0, 0).unwrap();
2164 assert!(aggregate.may_overstate);
2165 assert!(!aggregate.may_understate);
2166 assert_eq!(aggregate.marker(), "≤");
2167 }
2168
2169 #[test]
2170 fn both_directions_at_once_claims_neither_bound() {
2171 // Unrated cache bucket (can be high) plus an unresolved dearer tier
2172 // (can be low). Neither bound survives, so the figure is just an
2173 // estimate and must not wear a `≤` it cannot honour.
2174 let cost = CostModel {
2175 input_per_mtok: Some(2.0),
2176 output_per_mtok: Some(10.0),
2177 pricing_tiers: vec![TokenPricingTier {
2178 min_prompt_tokens: 100_000,
2179 prices: TokenPrices {
2180 input_per_mtok: Some(4.0),
2181 ..Default::default()
2182 },
2183 }],
2184 ..Default::default()
2185 };
2186 let aggregate = cost.estimated_usd_bounded(None, 0, 0, 500_000, 0).unwrap();
2187 assert!(aggregate.may_overstate && aggregate.may_understate);
2188 assert_eq!(aggregate.marker(), "~");
2189 }
2190
2191 /// Parslee-ai/car#894 (follow-up): "runs here" and "has weights to fetch"
2192 /// are different questions, and `is_local` answers only the first. Three
2193 /// `is_local` sources download nothing — the CLI must not offer an
2194 /// install status for them.
2195 #[test]
2196 fn downloads_weights_is_true_only_for_sources_car_fetches() {
2197 let mut schema = sample_local();
2198
2199 // CAR downloads these itself.
2200 schema.source = ModelSource::Local {
2201 hf_repo: "Qwen/Qwen3-4B-GGUF".into(),
2202 hf_filename: "Qwen3-4B-Q4_K_M.gguf".into(),
2203 tokenizer_repo: "Qwen/Qwen3-4B".into(),
2204 };
2205 assert!(schema.downloads_weights(), "GGUF weights are downloaded");
2206
2207 schema.source = ModelSource::Mlx {
2208 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
2209 hf_weight_file: None,
2210 };
2211 assert!(schema.downloads_weights(), "MLX weights are downloaded");
2212
2213 schema.source = ModelSource::WhisperCpp {
2214 model: "large-v3-turbo-q5_0".into(),
2215 };
2216 assert!(
2217 schema.downloads_weights(),
2218 "the whisper.cpp ggml bin is downloaded"
2219 );
2220
2221 // The OS owns these — there is nothing to install. `WindowsSpeech` is
2222 // the row that rendered `INSTALLED yes` against `car doctor`'s
2223 // `Models: none installed`.
2224 schema.source = ModelSource::WindowsSpeech {};
2225 assert!(
2226 !schema.downloads_weights(),
2227 "WinRT speech synthesis has no weights to download"
2228 );
2229
2230 schema.source = ModelSource::AppleFoundationModels { use_case: None };
2231 assert!(
2232 !schema.downloads_weights(),
2233 "Apple FoundationModels weights belong to the OS"
2234 );
2235
2236 // Someone else holds the weights.
2237 schema.source = ModelSource::VllmMlx {
2238 endpoint: "http://localhost:8000".into(),
2239 model_name: "mlx-community/Qwen3-4B-4bit".into(),
2240 };
2241 assert!(
2242 !schema.downloads_weights(),
2243 "the vLLM-MLX server owns its own weights"
2244 );
2245
2246 schema.source = ModelSource::ManagedVllmMlx {
2247 hf_repo: "mlx-community/Qwen3-4B-4bit".into(),
2248 hf_weight_file: None,
2249 draft_hf_repo: None,
2250 };
2251 assert!(
2252 schema.downloads_weights(),
2253 "the explicitly managed vLLM-MLX contract is CAR-owned"
2254 );
2255
2256 schema.source = ModelSource::Ollama {
2257 model_tag: "qwen3:4b".into(),
2258 host: default_ollama_host(),
2259 };
2260 assert!(!schema.downloads_weights(), "Ollama owns its own weights");
2261
2262 schema.source = sample_remote().source;
2263 assert!(
2264 !schema.downloads_weights(),
2265 "a remote API has no weights on this machine"
2266 );
2267
2268 schema.source = ModelSource::Delegated { hint: None };
2269 assert!(
2270 !schema.downloads_weights(),
2271 "a host-registered runner owns its own weights"
2272 );
2273 }
2274
2275 /// `is_local` is the predicate the CLI used to reach for. These OS-owned
2276 /// sources are where the two answers diverge, which is why a
2277 /// separate predicate exists rather than a reuse of `is_local`.
2278 #[test]
2279 fn downloads_weights_differs_from_is_local_on_os_owned_sources() {
2280 let mut schema = sample_local();
2281 for source in [
2282 ModelSource::WindowsSpeech {},
2283 ModelSource::AppleFoundationModels { use_case: None },
2284 ] {
2285 schema.source = source;
2286 assert!(
2287 schema.is_local(),
2288 "this source runs on-device: {:?}",
2289 schema.source
2290 );
2291 assert!(
2292 !schema.downloads_weights(),
2293 "…but CAR downloads nothing for it: {:?}",
2294 schema.source
2295 );
2296 }
2297 }
2298}
2299
2300#[cfg(test)]
2301mod quantization_tests {
2302 use super::*;
2303
2304 /// Every distinct label the built-in catalog ships, the hyphenated form
2305 /// `registry.rs` writes into auto-discovered entries, and the formats that
2306 /// review found misclassified.
2307 #[test]
2308 fn classifies_known_labels() {
2309 let cases: &[(&str, Option<u8>, QuantScheme)] = &[
2310 ("4bit", Some(4), QuantScheme::AffineGroupInt),
2311 ("6bit", Some(6), QuantScheme::AffineGroupInt),
2312 ("3bit", Some(3), QuantScheme::AffineGroupInt),
2313 ("5bit", Some(5), QuantScheme::AffineGroupInt),
2314 // registry.rs emits the hyphen; users already have it on disk.
2315 ("4-bit", Some(4), QuantScheme::AffineGroupInt),
2316 ("mxfp8", Some(8), QuantScheme::BlockScaledFloat),
2317 ("mxfp4", Some(4), QuantScheme::BlockScaledFloat),
2318 ("Q4_K_M", Some(4), QuantScheme::KQuantMixed),
2319 ("Q5_K_S", Some(5), QuantScheme::KQuantMixed),
2320 ("IQ4_XS", Some(4), QuantScheme::KQuantMixed),
2321 ("TQ1_0", Some(1), QuantScheme::KQuantMixed),
2322 ("Q8_0", Some(8), QuantScheme::RtnBlock),
2323 ("q5_0", Some(5), QuantScheme::RtnBlock),
2324 // aarch64 repack quants: the suffix is a prefix match, not equality.
2325 ("Q4_0_4_4", Some(4), QuantScheme::RtnBlock),
2326 ("Q4_0_8_8", Some(4), QuantScheme::RtnBlock),
2327 // A zero-padded width must slice by digits consumed, not by the
2328 // decimal length of the parsed number.
2329 ("q08_0", Some(8), QuantScheme::RtnBlock),
2330 ("bf16", Some(16), QuantScheme::Unquantized),
2331 ("F16", Some(16), QuantScheme::Unquantized),
2332 ("f32", Some(32), QuantScheme::Unquantized),
2333 // Spelled like a quant, but affine group quant has no such width.
2334 ("16bit", Some(16), QuantScheme::Unquantized),
2335 ("32bit", Some(32), QuantScheme::Unquantized),
2336 // An 8-bit float IS quantized — just not attributable from this
2337 // label alone. Calling it full precision was the bug.
2338 ("fp8", Some(8), QuantScheme::Unknown),
2339 // Names a width and no producer.
2340 ("Q4", Some(4), QuantScheme::Unknown),
2341 ];
2342 for (label, bits, scheme) in cases {
2343 let q = Quantization::parse(label);
2344 assert_eq!(q.bits, *bits, "bits for {label}");
2345 assert_eq!(q.scheme, *scheme, "scheme for {label}");
2346 assert_eq!(q.label, *label, "label must survive verbatim");
2347 }
2348 }
2349
2350 /// Labels that identify nothing must say so rather than guess.
2351 #[test]
2352 fn refuses_to_guess() {
2353 for label in [
2354 "", // nobody said; not a claim of full precision
2355 "mxfp", // MX family, no width
2356 "Q256_K", // width overflows u8
2357 "q0_0", // a zero-bit quantization is not a thing
2358 "awq-marlin-w4a16", // real format, unknown to this parser
2359 ] {
2360 let q = Quantization::parse(label);
2361 assert_eq!(q.scheme, QuantScheme::Unknown, "scheme for {label:?}");
2362 assert_eq!(q.bits, None, "bits for {label:?}");
2363 assert_eq!(q.label, label, "label for {label:?}");
2364 }
2365 }
2366
2367 /// The distinction the free-text field could not express: same width,
2368 /// different algorithm, different loader.
2369 #[test]
2370 fn same_width_different_scheme() {
2371 let affine = Quantization::parse("4bit");
2372 let kquant = Quantization::parse("Q4_K_M");
2373 let mx = Quantization::parse("mxfp4");
2374 assert_eq!(affine.bits, kquant.bits);
2375 assert_eq!(affine.bits, mx.bits);
2376 assert_ne!(affine.scheme, kquant.scheme);
2377 assert_ne!(affine.scheme, mx.scheme);
2378 // And within GGUF, k-quant is not round-to-nearest.
2379 assert_ne!(
2380 Quantization::parse("Q5_K_S").scheme,
2381 Quantization::parse("q5_0").scheme
2382 );
2383 }
2384
2385 /// The scheme names a numeric format, never a container or an engine.
2386 /// whisper.cpp ships `q5_0` ggml checkpoints that no GGUF text path can
2387 /// load; a container-named variant would assert otherwise.
2388 #[test]
2389 fn scheme_does_not_imply_an_engine() {
2390 let whisper = Quantization::parse("q5_0");
2391 let llama = Quantization::parse("Q5_0");
2392 assert_eq!(whisper.scheme, llama.scheme);
2393 assert_eq!(whisper.scheme, QuantScheme::RtnBlock);
2394 }
2395
2396 #[test]
2397 fn mlx_config_block_survives_ingest() {
2398 let q = Quantization::from_mlx_config(Some(8), Some(32), Some("mxfp8")).unwrap();
2399 assert_eq!(q.bits, Some(8));
2400 assert_eq!(q.group_size, Some(32));
2401 assert_eq!(q.scheme, QuantScheme::BlockScaledFloat);
2402
2403 // Absent `mode` means affine — MLX's default, not unknown.
2404 let affine = Quantization::from_mlx_config(Some(4), Some(64), None).unwrap();
2405 assert_eq!(affine.scheme, QuantScheme::AffineGroupInt);
2406 assert_eq!(affine.group_size, Some(64));
2407 assert_eq!(affine.label, "4bit");
2408 }
2409
2410 /// An empty or absent block must not become a claim of affine group quant.
2411 /// The field it replaced returned nothing here, and asserting a scheme is
2412 /// exactly the error a router acting on `scheme` would inherit.
2413 #[test]
2414 fn empty_mlx_block_asserts_nothing() {
2415 assert!(Quantization::from_mlx_config(None, None, None).is_none());
2416 }
2417
2418 #[test]
2419 fn deserializes_legacy_bare_string() {
2420 let q: Quantization = serde_json::from_str(r#""Q4_K_M""#).unwrap();
2421 assert_eq!(q.scheme, QuantScheme::KQuantMixed);
2422 assert_eq!(q.bits, Some(4));
2423 assert_eq!(q.label, "Q4_K_M");
2424 }
2425
2426 #[test]
2427 fn deserializes_structured_object() {
2428 let q: Quantization = serde_json::from_str(
2429 r#"{"bits":4,"scheme":"affine_group_int","group_size":64,"label":"4bit"}"#,
2430 )
2431 .unwrap();
2432 assert_eq!(q.group_size, Some(64));
2433 assert_eq!(q.scheme, QuantScheme::AffineGroupInt);
2434 }
2435
2436 /// A partial object recovers from its label rather than defaulting to
2437 /// Unknown — the same reason the bare string is still accepted.
2438 #[test]
2439 fn partial_object_recovers_from_label() {
2440 let q: Quantization = serde_json::from_str(r#"{"label":"Q8_0"}"#).unwrap();
2441 assert_eq!(q.scheme, QuantScheme::RtnBlock);
2442 assert_eq!(q.bits, Some(8));
2443 }
2444
2445 /// A malformed object must be an error, not a row that silently claims
2446 /// `Unknown`. Under `#[serde(untagged)]` every one of these deserialized
2447 /// successfully into a fabricated descriptor.
2448 #[test]
2449 fn malformed_objects_are_rejected() {
2450 for bad in [
2451 r#"{"btis":4,"scehme":"k_quant_mixed","labl":"Q4_K_M"}"#, // typos
2452 r#"{"quantization":{"bits":4}}"#, // double-nested
2453 r#"{}"#, // nothing at all
2454 r#"{"group_size":64}"#, // no label
2455 r#"{"bits":4,"scheme":"k_quant_mixed"}"#, // no label
2456 ] {
2457 let parsed: Result<Quantization, _> = serde_json::from_str(bad);
2458 assert!(parsed.is_err(), "should have rejected {bad}");
2459 }
2460 }
2461
2462 /// The error must name what was wrong. `#[serde(untagged)]` reported only
2463 /// "data did not match any variant", for a `models.json` that fails whole.
2464 #[test]
2465 fn rejection_names_the_offending_field() {
2466 let err = serde_json::from_str::<Quantization>(
2467 r#"{"bits":4,"scheme":"affine_grp_int","label":"4bit"}"#,
2468 )
2469 .unwrap_err()
2470 .to_string();
2471 assert!(
2472 err.contains("affine_grp_int") || err.contains("scheme"),
2473 "unhelpful error: {err}"
2474 );
2475 }
2476
2477 #[test]
2478 fn round_trips_through_the_object_form() {
2479 for label in ["Q4_K_M", "4bit", "mxfp8", "bf16", "weird-vendor-format"] {
2480 let q = Quantization::parse(label);
2481 let round: Quantization =
2482 serde_json::from_str(&serde_json::to_string(&q).unwrap()).unwrap();
2483 assert_eq!(q, round, "round trip for {label}");
2484 }
2485 }
2486
2487 /// The label alone is the wire form whenever it is lossless. This is what
2488 /// keeps `row_digest` stable for rows that gained no new information.
2489 #[test]
2490 fn serializes_as_a_bare_label_when_lossless() {
2491 for label in [
2492 "Q4_K_M",
2493 "4bit",
2494 "mxfp8",
2495 "bf16",
2496 "q5_0",
2497 "weird-vendor-format",
2498 ] {
2499 let json = serde_json::to_string(&Quantization::parse(label)).unwrap();
2500 assert_eq!(json, format!("\"{label}\""), "should stay a bare string");
2501 }
2502 }
2503
2504 /// ...and the object form appears exactly where the label would lose
2505 /// something: a group size, or a scheme the label cannot express.
2506 #[test]
2507 fn serializes_as_an_object_only_when_it_adds_information() {
2508 let with_group = Quantization::from_mlx_config(Some(4), Some(64), None).unwrap();
2509 let json = serde_json::to_value(&with_group).unwrap();
2510 assert_eq!(json["group_size"], 64, "group_size must survive the write");
2511 assert_eq!(json["label"], "4bit");
2512
2513 // A bare `Q4` parses as Unknown, so an explicit scheme is real
2514 // information and has to be written out.
2515 let disambiguated = Quantization {
2516 bits: Some(4),
2517 scheme: QuantScheme::AffineGroupInt,
2518 group_size: None,
2519 label: "Q4".into(),
2520 };
2521 let json = serde_json::to_value(&disambiguated).unwrap();
2522 assert_eq!(json["scheme"], "affine_group_int");
2523 assert!(json.get("group_size").is_none(), "no null padding");
2524 }
2525
2526 /// Both write forms must read back identically, or the minimal-write rule
2527 /// would trade a digest change for silent data loss.
2528 #[test]
2529 fn every_write_form_round_trips() {
2530 let cases = [
2531 Quantization::parse("Q4_K_M"),
2532 Quantization::parse("bf16"),
2533 Quantization::parse("unattributable-format"),
2534 Quantization::from_mlx_config(Some(8), Some(32), Some("mxfp8")).unwrap(),
2535 Quantization::from_mlx_config(Some(4), Some(64), None).unwrap(),
2536 Quantization {
2537 bits: Some(4),
2538 scheme: QuantScheme::AffineGroupInt,
2539 group_size: None,
2540 label: "Q4".into(),
2541 },
2542 ];
2543 for q in cases {
2544 let round: Quantization =
2545 serde_json::from_str(&serde_json::to_string(&q).unwrap()).unwrap();
2546 assert_eq!(q, round, "round trip for {q:?}");
2547 }
2548 }
2549
2550 /// The digest guard. `catalog_identity::row_digest` is a SHA-256 over the
2551 /// serialized schema and clients pin it through
2552 /// `expected_catalog_revision`, so a row whose quantization gained no new
2553 /// information must re-serialize to the byte-identical JSON it was read
2554 /// from. Only rows that genuinely changed may move.
2555 #[test]
2556 fn catalog_quantizations_reserialize_unchanged() {
2557 let raw: Vec<serde_json::Value> =
2558 serde_json::from_str(include_str!("builtin_catalog.json")).unwrap();
2559 let parsed: Vec<ModelSchema> =
2560 serde_json::from_str(include_str!("builtin_catalog.json")).unwrap();
2561 let mut moved = Vec::new();
2562 for (raw_row, model) in raw.iter().zip(&parsed) {
2563 let before = raw_row
2564 .get("quantization")
2565 .cloned()
2566 .unwrap_or(serde_json::Value::Null);
2567 let after = serde_json::to_value(&model.quantization).unwrap();
2568 if before != after {
2569 moved.push(format!("{}: {before} -> {after}", model.id));
2570 }
2571 }
2572 assert!(
2573 moved.is_empty(),
2574 "these rows would change catalog digest: {moved:#?}"
2575 );
2576 }
2577
2578 /// Every quantization the built-in catalog ships must classify. An entry
2579 /// may carry an explicit `scheme` only where its label is genuinely
2580 /// ambiguous — the guard is that an explicit scheme never *contradicts* a
2581 /// label the parser can already read, which is how catalog data stops
2582 /// being a place to make a failing test pass.
2583 #[test]
2584 fn builtin_catalog_quantizations_are_coherent() {
2585 let catalog: Vec<ModelSchema> =
2586 serde_json::from_str(include_str!("builtin_catalog.json")).unwrap();
2587 let mut problems = Vec::new();
2588 for model in &catalog {
2589 let Some(q) = &model.quantization else {
2590 continue;
2591 };
2592 if q.scheme == QuantScheme::Unknown {
2593 problems.push(format!("{}: unclassified label {:?}", model.id, q.label));
2594 continue;
2595 }
2596 let from_label = Quantization::parse(&q.label).scheme;
2597 if from_label != QuantScheme::Unknown && from_label != q.scheme {
2598 problems.push(format!(
2599 "{}: label {:?} parses as {:?} but the row claims {:?}",
2600 model.id, q.label, from_label, q.scheme
2601 ));
2602 }
2603 }
2604 assert!(
2605 problems.is_empty(),
2606 "incoherent catalog rows: {problems:#?}"
2607 );
2608 }
2609}