car-inference 0.49.0

//! Derive a [`ModelSchema`] from a HuggingFace repo, so adopting a new model
//! does not require shipping a CAR release.
//!
//! The built-in catalog is compiled in. That made *learning a model exists* a
//! release-gated operation, which is why it once went seven weeks without a new
//! text entry while the open-weight world moved through three Qwen generations.
//! Hand-authoring a `ModelSchema` JSON for `car models register` was the only
//! escape hatch, and it asked the user to know CAR's internal enum spellings.
//!
//! This module reads what the repo already declares — `config.json` plus the
//! file listing — and produces a registerable schema. The load-bearing decision
//! is the one the user cannot be expected to make: whether `model_type` has an
//! in-process Rust backend ([`ModelSource::Mlx`]) or has to go through the
//! supervised external runtime ([`ModelSource::VllmMlx`]). That answer lives in
//! [`backend::local::has_native_backend`](crate::backend::local::has_native_backend)
//! and is applied here, so `qwen3_5`, `glm4_moe_lite`, and every family that
//! lands next month route correctly without anyone editing Rust.
//!
//! Derived schemas are always [`TrustTier::Community`]: they are user-directed,
//! not project-vetted, and must never be eligible for background auto-apply.

use serde_json::Value;

use crate::schema::{
    CostModel, ModelCapability, ModelSchema, ModelSource, PerformanceEnvelope, TrustTier,
};
use crate::InferenceError;

/// A schema derived from a repo, plus the facts that decided its shape.
#[derive(Debug, Clone)]
pub struct DerivedModel {
    pub schema: ModelSchema,
    /// The repo's declared `config.json` `model_type`.
    pub model_type: String,
    /// True when an in-process Rust backend services `model_type` on this
    /// build; false means the schema points at the external runtime.
    pub native: bool,
}

/// Build a registerable [`ModelSchema`] for `repo` (e.g.
/// `mlx-community/Qwen3.8-27B-4bit`) from what HuggingFace already publishes.
///
/// Two public endpoints are read: `config.json` for the architecture, and the
/// repo file listing for on-disk size. Nothing is downloaded.
pub async fn derive_from_hf_repo(repo: &str) -> Result<DerivedModel, InferenceError> {
    let repo = repo.trim().trim_matches('/');
    if repo.split('/').count() != 2 || repo.split('/').any(str::is_empty) {
        return Err(InferenceError::InferenceFailed(format!(
            "`{repo}` is not a HuggingFace repo id — expected `org/name`"
        )));
    }

    let config = fetch_config(repo).await?;
    let listing = fetch_repo_listing(repo).await.unwrap_or_default();
    let size_bytes = listing.total_bytes;

    // A repo that carries a config but no weights derives into a perfectly
    // well-formed schema that can never load. Catching it here costs one
    // already-fetched listing; catching it at inference time costs a confusing
    // runtime error, minutes later, inside whatever job was running.
    if listing.observed && !listing.has_weights {
        return Err(InferenceError::InferenceFailed(format!(
            "{repo}: no model weights found (no .safetensors, .bin, or .gguf files). \
             If this is a base repo that only holds a config, use one of its \
             quantized conversions instead."
        )));
    }

    let model_type = config
        .get("model_type")
        .and_then(Value::as_str)
        .unwrap_or_default()
        .to_ascii_lowercase();
    if model_type.is_empty() {
        return Err(InferenceError::InferenceFailed(format!(
            "{repo}: config.json declares no `model_type`, so CAR cannot tell \
             which backend should serve it"
        )));
    }

    let native = crate::backend::local::has_native_backend(&model_type);
    let bits = quantization_bits(&config);
    let size_mb = size_bytes / 1_000_000;

    let schema = ModelSchema {
        id: derive_id(repo, bits),
        name: basename(repo).to_string(),
        provider: org(repo).to_ascii_lowercase(),
        family: model_type.clone(),
        version: String::new(),
        capabilities: capabilities(&config),
        context_length: context_length(&config),
        max_output_tokens: None,
        param_count: String::new(),
        quantization: bits.map(|b| format!("{b}bit")),
        performance: PerformanceEnvelope::default(),
        cost: CostModel {
            size_mb: (size_mb > 0).then_some(size_mb),
            // Weights plus working set. Deliberately generous: under-declaring
            // makes the router pick a model the machine cannot hold.
            ram_mb: (size_mb > 0).then(|| size_mb + size_mb / 4),
            ..Default::default()
        },
        source: if native {
            ModelSource::Mlx {
                hf_repo: repo.to_string(),
                hf_weight_file: None,
            }
        } else {
            ModelSource::VllmMlx {
                endpoint: "http://localhost:8000".to_string(),
                model_name: repo.to_string(),
            }
        },
        tags: {
            let mut t = vec![
                "derived".to_string(),
                "local".to_string(),
                model_type.clone(),
            ];
            t.push(if native { "native-mlx" } else { "vllm-mlx" }.to_string());
            if is_moe(&config) {
                t.push("moe".to_string());
            }
            t
        },
        supported_params: Vec::new(),
        public_benchmarks: Vec::new(),
        trust_tier: TrustTier::Community,
        deprecated: false,
        available: false,
        weights_ready: false,
    };

    Ok(DerivedModel {
        schema,
        model_type,
        native,
    })
}

async fn fetch_config(repo: &str) -> Result<Value, InferenceError> {
    let url = format!("https://huggingface.co/{repo}/resolve/main/config.json");
    // `model_download_client`, not `Client::new()`: the latter panics when the
    // OS trust store yields zero valid certificates. huggingface.co is public,
    // so the public-CA rung serves this fully.
    crate::tls_client::model_download_client()
        .get(&url)
        .send()
        .await
        .map_err(|e| InferenceError::InferenceFailed(format!("fetch {repo} config.json: {e}")))?
        .error_for_status()
        .map_err(|e| {
            InferenceError::InferenceFailed(format!(
                "{repo}: no readable config.json ({e}) — check the repo id"
            ))
        })?
        .json()
        .await
        .map_err(|e| InferenceError::InferenceFailed(format!("parse {repo} config.json: {e}")))
}

/// What the repo's file listing tells us: total size, and whether any of it is
/// actually model weights.
#[derive(Debug, Default)]
struct RepoListing {
    total_bytes: u64,
    has_weights: bool,
    /// False when the listing could not be fetched at all, so callers do not
    /// mistake "we could not look" for "there is nothing there".
    observed: bool,
}

/// Fetch the repo's file listing. Best-effort for sizing: a failure degrades
/// the schema rather than failing the call, and leaves `observed` false so the
/// weights check does not fire on a network hiccup.
async fn fetch_repo_listing(repo: &str) -> Result<RepoListing, InferenceError> {
    let url = format!("https://huggingface.co/api/models/{repo}?blobs=true");
    let info: Value = crate::tls_client::model_download_client()
        .get(&url)
        .send()
        .await
        .map_err(|e| InferenceError::InferenceFailed(e.to_string()))?
        .error_for_status()
        .map_err(|e| InferenceError::InferenceFailed(e.to_string()))?
        .json()
        .await
        .map_err(|e| InferenceError::InferenceFailed(e.to_string()))?;

    let files = info.get("siblings").and_then(Value::as_array);
    let total_bytes = files
        .map(|f| {
            f.iter()
                .filter_map(|f| f.get("size").and_then(Value::as_u64))
                .sum()
        })
        .unwrap_or(0);
    let has_weights = files
        .map(|f| {
            f.iter()
                .filter_map(|f| f.get("rfilename").and_then(Value::as_str))
                .any(is_weight_file)
        })
        .unwrap_or(false);
    Ok(RepoListing {
        total_bytes,
        has_weights,
        observed: true,
    })
}

/// Whether `name` is a model weight file in any format CAR's backends load.
fn is_weight_file(name: &str) -> bool {
    let lower = name.to_ascii_lowercase();
    [".safetensors", ".gguf", ".bin", ".npz"]
        .iter()
        .any(|ext| lower.ends_with(ext))
}

/// `config.json` may nest the language-model fields under `text_config` on
/// multimodal repos. Look in both places before giving up.
fn nested<'a>(config: &'a Value, key: &str) -> Option<&'a Value> {
    config.get(key).or_else(|| {
        config
            .get("text_config")
            .and_then(|t| t.get(key))
            .filter(|v| !v.is_null())
    })
}

fn context_length(config: &Value) -> usize {
    nested(config, "max_position_embeddings")
        .and_then(Value::as_u64)
        .unwrap_or(32_768) as usize
}

fn quantization_bits(config: &Value) -> Option<u64> {
    config
        .get("quantization")
        .and_then(|q| q.get("bits"))
        .and_then(Value::as_u64)
}

fn is_moe(config: &Value) -> bool {
    nested(config, "num_experts")
        .and_then(Value::as_u64)
        .is_some_and(|n| n > 1)
}

/// Capabilities a text LLM is assumed to have, plus vision when the repo
/// declares a vision tower.
///
/// Deliberately conservative about vision — claiming it wrongly makes the
/// router send images to a model that cannot see them — and deliberately
/// generous about the text set, which every instruct-tuned LLM services.
fn capabilities(config: &Value) -> Vec<ModelCapability> {
    let mut caps = vec![
        ModelCapability::Generate,
        ModelCapability::Code,
        ModelCapability::Reasoning,
        ModelCapability::Summarize,
        ModelCapability::ToolUse,
        ModelCapability::MultiToolCall,
    ];
    // `language_model_only` is authoritative when present: the Qwen3.5+ line
    // unified vision into the base config, so a text-only *conversion* still
    // carries `vision_config` and the image token ids from its parent. Trusting
    // the tower's presence alone would claim vision for a checkpoint whose
    // vision weights were stripped.
    let text_only = config
        .get("language_model_only")
        .and_then(Value::as_bool)
        .unwrap_or(false);
    let has_vision_tower = config.get("vision_config").is_some()
        || config.get("image_token_id").is_some()
        || config.get("image_token_index").is_some();
    if has_vision_tower && !text_only {
        caps.push(ModelCapability::Vision);
    }
    caps
}

fn org(repo: &str) -> &str {
    repo.split('/').next().unwrap_or("custom")
}

fn basename(repo: &str) -> &str {
    repo.rsplit('/').next().unwrap_or(repo)
}

/// `custom/<repo-basename>` lowercased, with the quantization as the variant
/// tag so two quantizations of one model do not collide.
fn derive_id(repo: &str, bits: Option<u64>) -> String {
    let base = basename(repo).to_ascii_lowercase();
    match bits {
        Some(b) => format!("custom/{base}:{b}bit"),
        None => format!("custom/{base}"),
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json::json;

    #[test]
    fn rejects_a_non_repo_id() {
        let err = tokio::runtime::Runtime::new()
            .unwrap()
            .block_on(derive_from_hf_repo("Qwen3.8-27B"))
            .unwrap_err();
        assert!(err.to_string().contains("org/name"), "got {err}");
    }

    #[test]
    fn context_falls_back_to_text_config() {
        let c = json!({ "text_config": { "max_position_embeddings": 262144 } });
        assert_eq!(context_length(&c), 262_144);
    }

    #[test]
    fn vision_is_claimed_only_when_declared() {
        let text_only = json!({ "model_type": "qwen3" });
        assert!(!capabilities(&text_only).contains(&ModelCapability::Vision));

        let vlm = json!({ "model_type": "qwen3_vl", "vision_config": {} });
        assert!(capabilities(&vlm).contains(&ModelCapability::Vision));

        // A text-only conversion of a multimodal base keeps the parent's
        // `vision_config`; `language_model_only` is what distinguishes it.
        let stripped = json!({
            "model_type": "qwen3_5",
            "vision_config": { "depth": 27 },
            "image_token_id": 248056,
            "language_model_only": true
        });
        assert!(
            !capabilities(&stripped).contains(&ModelCapability::Vision),
            "a language_model_only conversion must not claim vision"
        );
    }

    #[test]
    fn id_keeps_quantizations_distinct() {
        assert_eq!(
            derive_id("mlx-community/Qwen3.8-27B-4bit", Some(4)),
            "custom/qwen3.8-27b-4bit:4bit"
        );
        assert_ne!(
            derive_id("mlx-community/Qwen3.8-27B-4bit", Some(4)),
            derive_id("mlx-community/Qwen3.8-27B-8bit", Some(8))
        );
    }

    #[test]
    fn weight_files_are_recognized_across_formats() {
        for name in [
            "model-00001-of-00003.safetensors",
            "model.gguf",
            "pytorch_model.bin",
            "weights.npz",
            "MODEL.SAFETENSORS",
        ] {
            assert!(is_weight_file(name), "{name} should count as weights");
        }
        for name in [
            "config.json",
            "tokenizer.json",
            "README.md",
            "chat_template.jinja",
            ".gitattributes",
        ] {
            assert!(!is_weight_file(name), "{name} is not weights");
        }
    }

    /// A repo that carries a config but no weights derives into a well-formed
    /// schema that can never load. "Could not look" must not be confused with
    /// "there is nothing there", so the check only fires on an observed listing.
    #[test]
    fn an_unobserved_listing_does_not_trigger_the_weights_check() {
        let unobserved = RepoListing::default();
        assert!(!unobserved.observed);
        assert!(!unobserved.has_weights);
        // The guard is `observed && !has_weights`; with `observed` false a
        // network hiccup must not be reported as an empty repo.
        assert!(
            !(unobserved.observed && !unobserved.has_weights),
            "a failed listing must not be reported as a weightless repo"
        );
    }

    #[test]
    fn moe_needs_more_than_one_expert() {
        assert!(is_moe(&json!({ "num_experts": 128 })));
        assert!(!is_moe(&json!({ "num_experts": 1 })));
        assert!(!is_moe(&json!({})));
    }

    /// The decision this module exists to make: an architecture with no Rust
    /// backend must route to the external runtime rather than being rejected.
    #[test]
    fn unknown_architectures_are_not_native() {
        assert!(!crate::backend::local::has_native_backend("qwen3_5"));
        assert!(!crate::backend::local::has_native_backend("qwen3_5_moe"));
        assert!(!crate::backend::local::has_native_backend("glm4_moe_lite"));
    }

    #[test]
    fn known_architectures_are_native_on_apple_silicon() {
        let expected = cfg!(all(
            target_os = "macos",
            target_arch = "aarch64",
            not(car_skip_mlx)
        ));
        assert_eq!(
            crate::backend::local::has_native_backend("qwen3_moe"),
            expected
        );
        assert_eq!(
            crate::backend::local::has_native_backend("gemma4_unified"),
            expected
        );
    }
}