car-inference 0.56.1

//! Upstream-aware upgrade detection.
//!
//! The registry's `available_upgrades()` only fires on hand-authored rules in
//! `model-upgrades.json` — the *curated*, verified tier. This module unifies
//! that with *upstream* discovery: for an installed model it can ask the Hub
//! whether a newer revision exists. Findings are tagged by trust tier and
//! source so the UI (and auto-apply policy) can treat verified curated
//! upgrades differently from unverified upstream ones.
//!
//! Properties required by the design:
//! - **Channel-aware**: upstream probing runs only on the `Latest` channel;
//!   `Stable` is curated-only.
//! - **Offline-safe**: any probe error (no network, Hub down, rate limit)
//!   degrades silently to curated-only — never an error to the caller.
//! - **Cached / rate-limited**: upstream results are cached with a TTL so we
//!   don't hit the Hub on every check.
//!
//! The probe is a trait so the orchestration is unit-testable without a
//! network (inject a fake), and the real Hub implementation stays thin.

use std::future::Future;
use std::path::{Path, PathBuf};

use serde::{Deserialize, Serialize};

use crate::registry::ModelUpgrade;
use crate::schema::{ModelSchema, ModelSource, TrustTier};
use crate::update_prefs::{UpdateChannel, UpdatePreferences};

/// Where an upgrade finding came from.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum UpgradeSource {
    /// A vetted rule in `model-upgrades.json`.
    Curated,
    /// A newer revision discovered upstream on the Hub. Unverified.
    Upstream,
    /// A model that scores clearly better on CAR's own benchmark at a similar
    /// size (see [`benchmark_findings`]). Measured by CAR, so trusted like a
    /// curated rule.
    Benchmark,
}

/// A single "something newer is available" result, unifying curated rules and
/// upstream discoveries.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct UpgradeFinding {
    pub from_id: String,
    pub from_name: String,
    pub to_id: String,
    pub to_name: String,
    /// Plain-language reason to show the user.
    pub reason: String,
    /// `Curated` (verified) or `Community` (upstream, unverified).
    pub trust_tier: TrustTier,
    pub source: UpgradeSource,
    /// Whether CAR can pull the target directly (local/MLX) vs needs setup.
    pub target_pullable: bool,
}

impl UpgradeFinding {
    fn from_curated(u: ModelUpgrade) -> Self {
        UpgradeFinding {
            from_id: u.from_id,
            from_name: u.from_name,
            to_id: u.to_id,
            to_name: u.to_name,
            reason: u.reason,
            trust_tier: TrustTier::Curated,
            source: UpgradeSource::Curated,
            target_pullable: u.target_pullable,
        }
    }
}

/// Asks whether a newer upstream revision exists for an installed model.
/// Implementations must be offline-safe: return `None` on any failure rather
/// than erroring.
pub trait UpstreamProbe {
    /// `Some(reason)` if a newer revision exists upstream; `None` if not, or
    /// if it can't be determined (offline, uncached, error).
    fn newer_revision(&self, schema: &ModelSchema) -> impl Future<Output = Option<String>> + Send;
}

/// Cached upstream findings with a freshness timestamp, persisted so repeated
/// checks within the TTL don't hit the Hub.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct UpgradeCache {
    /// Unix seconds of the last successful upstream check.
    #[serde(default)]
    pub checked_at_secs: u64,
    /// Fingerprint of the installed-model set the cache was built for. If the
    /// user installs/removes a model the fingerprint changes, invalidating the
    /// cache so the new model gets probed before the TTL expires.
    #[serde(default)]
    pub models_fingerprint: String,
    #[serde(default)]
    pub upstream: Vec<UpgradeFinding>,
}

impl UpgradeCache {
    /// Default path: `upgrade-cache.json` under the CAR state root —
    /// `~/.car/upgrade-cache.json` unless `CAR_HOME` moves the root, in which
    /// case the cache moves with it. The cache is keyed to the installed-model
    /// set *this* daemon sees, so it follows the daemon's own state rather than
    /// being shared with an unrelated install.
    pub fn default_path() -> PathBuf {
        car_home::root_or_relative().join("upgrade-cache.json")
    }

    pub fn load_from(path: &Path) -> Self {
        std::fs::read_to_string(path)
            .ok()
            .and_then(|s| serde_json::from_str(&s).ok())
            .unwrap_or_default()
    }

    pub fn save_to(&self, path: &Path) -> Result<(), String> {
        if let Some(parent) = path.parent() {
            std::fs::create_dir_all(parent).map_err(|e| e.to_string())?;
        }
        let json = serde_json::to_string_pretty(self).map_err(|e| e.to_string())?;
        std::fs::write(path, json).map_err(|e| e.to_string())
    }

    /// Fresh if the last check was within `ttl_secs` of `now_secs`.
    pub fn is_fresh(&self, now_secs: u64, ttl_secs: u64) -> bool {
        self.checked_at_secs != 0 && now_secs.saturating_sub(self.checked_at_secs) < ttl_secs
    }
}

/// Default cache TTL: re-probe the Hub at most once a day.
pub const DEFAULT_TTL_SECS: u64 = 24 * 60 * 60;

/// Detect upgrades for `installed` models, combining curated rules with
/// upstream discovery. Pure-ish: caller supplies the curated rules, the probe,
/// the cache path, and `now_secs`, so it's fully testable offline.
///
/// - Curated findings are always included (the trusted tier).
/// - Upstream probing runs only when `prefs.channel == Latest`, is cached for
///   `ttl_secs`, and degrades to the cached/empty set on any probe failure.
pub async fn detect_upgrades<P: UpstreamProbe>(
    curated: Vec<ModelUpgrade>,
    installed: &[&ModelSchema],
    prefs: &UpdatePreferences,
    probe: &P,
    cache_path: &Path,
    now_secs: u64,
    ttl_secs: u64,
) -> Vec<UpgradeFinding> {
    let mut findings: Vec<UpgradeFinding> = curated
        .into_iter()
        .map(UpgradeFinding::from_curated)
        .collect();

    if prefs.channel == UpdateChannel::Latest && prefs.checks_enabled() {
        let upstream = upstream_findings(installed, probe, cache_path, now_secs, ttl_secs).await;
        // Dedup: a curated rule for the same from_id wins (it's verified).
        for f in upstream {
            if !findings.iter().any(|c| c.from_id == f.from_id) {
                findings.push(f);
            }
        }
    }

    findings.sort_by(|a, b| a.from_id.cmp(&b.from_id).then(a.to_id.cmp(&b.to_id)));
    findings.dedup_by(|a, b| a.from_id == b.from_id && a.to_id == b.to_id);
    findings
}

/// Upstream findings, served from cache when fresh, else re-probed and cached.
async fn upstream_findings<P: UpstreamProbe>(
    installed: &[&ModelSchema],
    probe: &P,
    cache_path: &Path,
    now_secs: u64,
    ttl_secs: u64,
) -> Vec<UpgradeFinding> {
    let fingerprint = installed_fingerprint(installed);
    let cache = UpgradeCache::load_from(cache_path);
    // Serve the cache only when it's both fresh AND built for the same
    // installed set — so installing a new model re-probes immediately.
    if cache.is_fresh(now_secs, ttl_secs) && cache.models_fingerprint == fingerprint {
        return cache.upstream;
    }

    // Probes run sequentially (one Hub request at a time) and only on a
    // cache miss (≤ once per TTL window), so the Hub request rate is inherently
    // bounded by the installed-model count, not the call rate.
    let mut found = Vec::new();
    for schema in installed {
        // Only locally-installed models with a Hub repo can have an upstream.
        if !schema.has_installed_weights() || repo_of(schema).is_none() {
            continue;
        }
        if let Some(reason) = probe.newer_revision(schema).await {
            found.push(UpgradeFinding {
                from_id: schema.id.clone(),
                from_name: schema.name.clone(),
                // Upstream = same model line, newer revision; target is the
                // same id (re-pull refreshes the cache to the new revision).
                to_id: schema.id.clone(),
                to_name: schema.name.clone(),
                reason,
                trust_tier: TrustTier::Community,
                source: UpgradeSource::Upstream,
                target_pullable: matches!(
                    schema.source,
                    ModelSource::Local { .. } | ModelSource::Mlx { .. }
                ),
            });
        }
    }

    // Persist (best-effort; a write failure must not break detection).
    // Empty results are cached too — a fresh empty cache suppresses re-probing.
    let _ = UpgradeCache {
        checked_at_secs: now_secs,
        models_fingerprint: fingerprint,
        upstream: found.clone(),
    }
    .save_to(cache_path);
    found
}

/// Stable fingerprint of the physically installed downloadable-model set.
/// Runtime availability is deliberately not installation evidence.
fn installed_fingerprint(installed: &[&ModelSchema]) -> String {
    use std::collections::hash_map::DefaultHasher;
    use std::hash::{Hash, Hasher};
    let mut ids: Vec<&str> = installed
        .iter()
        .filter(|m| m.has_installed_weights())
        .map(|m| m.id.as_str())
        .collect();
    ids.sort_unstable();
    let mut h = DefaultHasher::new();
    ids.hash(&mut h);
    format!("{:x}", h.finish())
}

/// The Hub repo for a model, if it has one.
fn repo_of(schema: &ModelSchema) -> Option<&str> {
    match &schema.source {
        ModelSource::Local { hf_repo, .. } | ModelSource::Mlx { hf_repo, .. } => Some(hf_repo),
        _ => None,
    }
}

// --- real Hub probe --------------------------------------------------------

/// Probes the HuggingFace Hub for a newer commit than the one cached locally.
/// Compares the locally cached `refs/main` sha against the repo's current sha
/// from the Hub model-info API. Fully offline-safe: any error → `None`.
pub struct HuggingFaceProbe {
    client: reqwest::Client,
}

impl Default for HuggingFaceProbe {
    fn default() -> Self {
        Self::new()
    }
}

impl HuggingFaceProbe {
    /// Infallible, and offline-safe by construction. If the OS trust store
    /// cannot be loaded the client degrades (with a warning, once) rather than
    /// panicking, and the `Option`-returning probe methods below then report
    /// "unknown" — the same answer they already give when the machine is
    /// offline.
    pub fn new() -> Self {
        let (client, _degradation) = crate::tls_client::build_client_with_degradation(
            &crate::tls_client::HUGGINGFACE_PROBE,
            || reqwest::Client::builder().timeout(std::time::Duration::from_secs(8)),
        );
        HuggingFaceProbe { client }
    }

    async fn remote_sha(&self, repo: &str) -> Option<String> {
        let url = format!("https://huggingface.co/api/models/{repo}");
        let resp = self.client.get(&url).send().await.ok()?;
        if !resp.status().is_success() {
            return None;
        }
        let json: serde_json::Value = resp.json().await.ok()?;
        json.get("sha")?.as_str().map(|s| s.to_string())
    }
}

impl UpstreamProbe for HuggingFaceProbe {
    async fn newer_revision(&self, schema: &ModelSchema) -> Option<String> {
        let repo = repo_of(schema)?;
        let local_sha = local_main_sha(repo)?; // not cached ⇒ can't compare
        let remote_sha = self.remote_sha(repo).await?; // offline ⇒ None
        if remote_sha != local_sha {
            Some(format!(
                "A newer revision of {repo} is available on Hugging Face."
            ))
        } else {
            None
        }
    }
}

/// Read the locally cached `refs/main` sha for a Hub repo, if present.
fn local_main_sha(repo: &str) -> Option<String> {
    let ref_path = crate::hf_cache::repo_dir(repo).join("refs").join("main");
    std::fs::read_to_string(ref_path)
        .ok()
        .map(|s| s.trim().to_string())
        .filter(|s| !s.is_empty())
}

/// The benchmark CAR publishes per family and track (`bench-consolidate.py`).
pub const CAR_BENCHMARK: &str = "car-judged";
/// How much better a replacement must score to be worth mentioning: more
/// than one task of the seven-task suite (1/7 ≈ 0.143). A smaller gap is
/// one judge's call on one task, not a better model.
pub const BENCHMARK_MARGIN: f64 = 0.15;
/// How much larger a replacement may be. A better model at a much larger size
/// is a different trade-off (memory, download), not an upgrade.
pub const BENCHMARK_SIZE_RATIO: f64 = 1.25;
/// The slowest a replacement may decode, as a fraction of the installed
/// model's measured speed. A better answer that arrives four times later is
/// a different trade-off too.
pub const BENCHMARK_MIN_SPEED_RATIO: f64 = 0.75;
/// Runs each score must pool before CAR acts on the difference by itself.
pub const BENCHMARK_CONFIRMING_RUNS: u32 = 2;

/// A row's entry for CAR's own benchmark, if it has one.
pub fn car_benchmark(schema: &ModelSchema) -> Option<&crate::schema::BenchmarkScore> {
    schema
        .public_benchmarks
        .iter()
        .find(|b| b.name == CAR_BENCHMARK)
}

/// A row's score on CAR's own benchmark, if it has one.
pub fn car_benchmark_score(schema: &ModelSchema) -> Option<f64> {
    car_benchmark(schema).map(|b| b.score)
}

/// What a replacement must also be able to do.
const KEPT_CAPABILITIES: [crate::schema::ModelCapability; 5] = [
    crate::schema::ModelCapability::Generate,
    crate::schema::ModelCapability::ToolUse,
    crate::schema::ModelCapability::Code,
    crate::schema::ModelCapability::Reasoning,
    crate::schema::ModelCapability::Vision,
];

/// Whether the measurements behind a difference are enough for CAR to act on
/// it without asking: both scores pool [`BENCHMARK_CONFIRMING_RUNS`] or more
/// runs, the difference exceeds the margin and twice both scores' run-to-run
/// spread, and both models' decode speed was measured.
fn confirmed(
    from: &crate::schema::BenchmarkScore,
    to: &crate::schema::BenchmarkScore,
    speeds_known: bool,
) -> bool {
    let repeated =
        |b: &crate::schema::BenchmarkScore| b.runs.is_some_and(|n| n >= BENCHMARK_CONFIRMING_RUNS);
    // Two runs estimate their own noise poorly, so the gap must clear twice
    // the larger spread, not just equal it.
    let noise = BENCHMARK_MARGIN.max(
        2.0 * from
            .spread
            .unwrap_or(f64::INFINITY)
            .max(to.spread.unwrap_or(f64::INFINITY)),
    );
    speeds_known && repeated(from) && repeated(to) && to.score - from.score > noise
}

/// For each installed model, the best replacement CAR's own benchmark knows
/// of: a local row that scores more than [`BENCHMARK_MARGIN`] higher, is at
/// most [`BENCHMARK_SIZE_RATIO`] times its size, holds at least as much
/// context, decodes no slower than [`BENCHMARK_MIN_SPEED_RATIO`] of it where
/// both speeds are known, does everything it does, and that this machine can
/// run: `fit` is `None` where the platform cannot run it at all, else its fit
/// in the active memory budget, and `TooBig` is refused. Both rows need a
/// score: no score is no opinion, never "worse". Pure over its inputs.
///
/// A finding is `Curated` — CAR may apply it by itself — only when the
/// difference is [`confirmed`] and the replacement plainly `Fits`; otherwise
/// it is `Community`: worth telling the user, not worth switching their model
/// over. Confirmed candidates rank ahead of unconfirmed ones, so a single
/// noisy high score cannot hide one CAR could act on.
pub fn benchmark_findings(
    installed: &[&ModelSchema],
    candidates: &[&ModelSchema],
    fit: &dyn Fn(&ModelSchema) -> Option<crate::recommend::ModelFitStatus>,
) -> Vec<UpgradeFinding> {
    use crate::recommend::ModelFitStatus;
    let mut out = Vec::new();
    for from in installed {
        let (Some(from_bench), from_size) = (car_benchmark(from), from.size_mb()) else {
            continue;
        };
        if from_size == 0 {
            continue;
        }
        let from_speed = from.performance.tokens_per_second.filter(|s| *s > 0.0);
        let needed: Vec<_> = KEPT_CAPABILITIES
            .iter()
            .filter(|c| from.capabilities.contains(c))
            .collect();
        let best = candidates
            .iter()
            .filter(|c| c.id != from.id && c.is_local() && c.downloads_weights())
            .filter(|c| needed.iter().all(|cap| c.capabilities.contains(cap)))
            .filter(|c| c.context_length >= from.context_length)
            .filter_map(|c| car_benchmark(c).map(|bench| (*c, bench)))
            .filter(|(c, bench)| {
                bench.score > from_bench.score + BENCHMARK_MARGIN
                    && c.size_mb() > 0
                    && c.size_mb() as f64 <= from_size as f64 * BENCHMARK_SIZE_RATIO
            })
            .filter(|(c, _)| {
                match (
                    from_speed,
                    c.performance.tokens_per_second.filter(|s| *s > 0.0),
                ) {
                    (Some(was), Some(is)) => is >= was * BENCHMARK_MIN_SPEED_RATIO,
                    _ => true,
                }
            })
            .filter_map(|(c, bench)| match fit(c) {
                None | Some(ModelFitStatus::TooBig) => None,
                Some(status) => {
                    let speeds_known = from_speed.is_some()
                        && c.performance.tokens_per_second.is_some_and(|s| s > 0.0);
                    let sure = status == ModelFitStatus::Fits
                        && confirmed(from_bench, bench, speeds_known);
                    Some((c, bench, sure))
                }
            })
            .max_by(|(a, ba, sa), (b, bb, sb)| {
                sa.cmp(sb)
                    .then_with(|| ba.score.total_cmp(&bb.score))
                    .then_with(|| b.size_mb().cmp(&a.size_mb()))
                    .then_with(|| b.id.cmp(&a.id))
            });
        if let Some((to, to_bench, sure)) = best {
            out.push(UpgradeFinding {
                from_id: from.id.clone(),
                from_name: from.name.clone(),
                to_id: to.id.clone(),
                to_name: to.name.clone(),
                reason: format!(
                    "{} scores {:.2} on CAR's benchmark against {:.2} for {}, at a similar size{}",
                    to.name,
                    to_bench.score,
                    from_bench.score,
                    from.name,
                    if sure {
                        ""
                    } else {
                        " (not yet measured often enough for CAR to switch by itself)"
                    }
                ),
                trust_tier: if sure {
                    TrustTier::Curated
                } else {
                    TrustTier::Community
                },
                source: UpgradeSource::Benchmark,
                target_pullable: matches!(
                    to.source,
                    ModelSource::Local { .. } | ModelSource::Mlx { .. }
                ),
            });
        }
    }
    out
}

/// A replacement only the online catalog names is suggested, never applied
/// by CAR on its own: one signing key would otherwise reach every machine's
/// lane defaults. Builtin rows — including ones a signed catalog re-measured
/// — keep their trust.
pub fn suggest_only_unless_builtin(
    findings: Vec<UpgradeFinding>,
    is_builtin: &dyn Fn(&str) -> bool,
) -> Vec<UpgradeFinding> {
    findings
        .into_iter()
        .map(|mut finding| {
            if finding.trust_tier == TrustTier::Curated && !is_builtin(&finding.to_id) {
                finding.trust_tier = TrustTier::Community;
                finding
                    .reason
                    .push_str(" (from CAR's online catalog, so CAR suggests rather than switches)");
            }
            finding
        })
        .collect()
}

#[cfg(test)]
mod tests {
    use super::*;

    mod benchmark {
        use super::super::*;
        use crate::schema::ModelCapability;

        fn row(id: &str) -> ModelSchema {
            crate::registry::builtin_catalog()
                .into_iter()
                .find(|m| m.id == id)
                .unwrap_or_else(|| panic!("catalog row {id}"))
        }

        /// The row with its CAR score replaced (it must already carry one).
        fn scored(mut m: ModelSchema, score: f64) -> ModelSchema {
            let entry = m
                .public_benchmarks
                .iter_mut()
                .find(|b| b.name == CAR_BENCHMARK)
                .expect("a CAR score to replace");
            entry.score = score;
            m
        }

        fn fits(_: &ModelSchema) -> Option<crate::recommend::ModelFitStatus> {
            Some(crate::recommend::ModelFitStatus::Fits)
        }

        /// The row with its CAR score pooling `runs` runs `spread` apart.
        fn repeated(mut m: ModelSchema, runs: u32, spread: f64) -> ModelSchema {
            let entry = m
                .public_benchmarks
                .iter_mut()
                .find(|b| b.name == CAR_BENCHMARK)
                .expect("a CAR score to repeat");
            entry.runs = Some(runs);
            entry.spread = Some(spread);
            m
        }

        /// The row as if measured once: no run count, no spread.
        fn single_run(mut m: ModelSchema) -> ModelSchema {
            if let Some(entry) = m
                .public_benchmarks
                .iter_mut()
                .find(|b| b.name == CAR_BENCHMARK)
            {
                entry.runs = None;
                entry.spread = None;
            }
            m
        }

        /// The real qwen3.6-35b-a3b row with a clear lead over qwen3-30b-a3b's
        /// measured 0.696 (+0.164), so each gate test below has a finding to
        /// lose. The measured catalog's own gap (0.134) is under the margin.
        fn better() -> ModelSchema {
            scored(row("vllm-mlx/qwen3.6-35b-a3b:4bit"), 0.86)
        }

        #[test]
        fn the_measured_catalog_suggests_no_replacement_between_these_two_today() {
            // 0.696 against 0.830: real, repeated, and under the 0.15 margin.
            let from = row("mlx/qwen3-30b-a3b:4bit");
            let to = row("vllm-mlx/qwen3.6-35b-a3b:4bit");
            assert!(benchmark_findings(&[&from], &[&to], &fits).is_empty());
        }

        #[test]
        fn a_clearly_better_model_of_similar_size_is_suggested() {
            let from = row("mlx/qwen3-30b-a3b:4bit");
            // Both catalog scores pool 3 runs, and the lead clears the margin
            // and twice their spreads, so CAR may act on it.
            let confirmed = benchmark_findings(&[&from], &[&better()], &fits);
            assert_eq!(confirmed[0].trust_tier, TrustTier::Curated, "{confirmed:?}");
            let to = single_run(better());
            let found = benchmark_findings(&[&from], &[&from, &to], &fits);
            assert_eq!(found.len(), 1, "{found:?}");
            assert_eq!(found[0].to_id, to.id);
            assert_eq!(found[0].source, UpgradeSource::Benchmark);
            assert!(found[0].reason.contains("0.86"), "{}", found[0].reason);
            // One run each: a suggestion, never a switch CAR makes by itself.
            assert_eq!(found[0].trust_tier, TrustTier::Community);
            assert!(
                found[0].reason.contains("not yet measured"),
                "{}",
                found[0].reason
            );
            // The managed vLLM-MLX row needs setup; it is not one click.
            assert!(!found[0].target_pullable);
        }

        #[test]
        fn only_a_difference_beyond_repeated_measurement_noise_is_confirmed() {
            let from = repeated(row("mlx/qwen3-30b-a3b:4bit"), 2, 0.05);
            let to = repeated(better(), 3, 0.05);
            let found = benchmark_findings(&[&from], &[&to], &fits);
            assert_eq!(found[0].trust_tier, TrustTier::Curated, "{found:?}");
            assert!(!found[0].reason.contains("not yet measured"));
            // The runs disagree with each other by more than the difference.
            let noisy = repeated(to.clone(), 3, 0.2);
            let found = benchmark_findings(&[&from], &[&noisy], &fits);
            assert_eq!(found[0].trust_tier, TrustTier::Community);
            // Fit that cannot be asserted is a suggestion, never a switch.
            let unknown = |_: &ModelSchema| Some(crate::recommend::ModelFitStatus::Unknown);
            let found = benchmark_findings(&[&from], &[&to], &unknown);
            assert_eq!(found[0].trust_tier, TrustTier::Community);
            // A confirmed candidate outranks a higher single-run one.
            let mut outlier = single_run(scored(better(), 0.95));
            outlier.id = "vllm-mlx/outlier".into();
            let found = benchmark_findings(&[&from], &[&outlier, &to], &fits);
            assert_eq!(found[0].to_id, to.id, "{found:?}");
            assert_eq!(found[0].trust_tier, TrustTier::Curated);
            // A speed nobody measured is not confirmed either.
            let mut unmeasured = to.clone();
            unmeasured.performance.tokens_per_second = None;
            let found = benchmark_findings(&[&from], &[&unmeasured], &fits);
            assert_eq!(found[0].trust_tier, TrustTier::Community);
        }

        #[test]
        fn a_confirmed_finding_to_a_model_only_the_online_catalog_names_is_only_suggested() {
            let from = repeated(row("mlx/qwen3-30b-a3b:4bit"), 2, 0.05);
            let to = repeated(better(), 3, 0.05);
            let found = benchmark_findings(&[&from], &[&to], &fits);
            assert_eq!(found[0].trust_tier, TrustTier::Curated);
            let builtin = suggest_only_unless_builtin(found.clone(), &|_| true);
            assert_eq!(builtin[0].trust_tier, TrustTier::Curated);
            let online = suggest_only_unless_builtin(found, &|_| false);
            assert_eq!(online[0].trust_tier, TrustTier::Community);
            assert!(online[0].reason.contains("online catalog"));
        }

        #[test]
        fn less_context_or_a_much_slower_decode_is_not_an_upgrade() {
            let from = row("mlx/qwen3-30b-a3b:4bit");
            let mut to = better();
            to.context_length = from.context_length - 1;
            assert!(benchmark_findings(&[&from], &[&to], &fits).is_empty());
            let mut to = better();
            let was = from.performance.tokens_per_second.expect("measured speed");
            to.performance.tokens_per_second = Some(was * 0.5);
            assert!(benchmark_findings(&[&from], &[&to], &fits).is_empty());
        }

        #[test]
        fn a_much_larger_model_is_a_different_trade_off_not_an_upgrade() {
            let small = row("mlx/qwen3-8b:4bit");
            let big = scored(row("vllm-mlx/qwen3.6-35b-a3b:4bit"), 0.99);
            assert!(benchmark_findings(&[&small], &[&small, &big], &fits).is_empty());
            // Positive control: the same score at a similar size is suggested.
            let mut similar = big.clone();
            similar.cost.size_mb = small.cost.size_mb;
            similar.performance.tokens_per_second = small.performance.tokens_per_second;
            similar.context_length = small.context_length;
            assert_eq!(benchmark_findings(&[&small], &[&similar], &fits).len(), 1);
        }

        #[test]
        fn no_score_is_no_opinion_and_the_margin_and_machine_gate() {
            let from = row("mlx/qwen3-30b-a3b:4bit");
            let to = better();
            let mut unscored_from = from.clone();
            unscored_from.public_benchmarks.clear();
            assert!(benchmark_findings(&[&unscored_from], &[&to], &fits).is_empty());
            let mut unscored_to = to.clone();
            unscored_to.public_benchmarks.clear();
            assert!(benchmark_findings(&[&from], &[&unscored_to], &fits).is_empty());
            // Within one task of the suite: one judge's call, not better.
            let close = scored(to.clone(), 0.82);
            assert!(benchmark_findings(&[&from], &[&close], &fits).is_empty());
            // Better, but this machine cannot run it.
            assert!(benchmark_findings(&[&from], &[&to], &|_| None).is_empty());
            let too_big = |_: &ModelSchema| Some(crate::recommend::ModelFitStatus::TooBig);
            assert!(benchmark_findings(&[&from], &[&to], &too_big).is_empty());
        }

        #[test]
        fn a_replacement_must_do_everything_the_installed_model_does() {
            let from = row("mlx/qwen3-30b-a3b:4bit");
            let mut to = better();
            assert!(from.capabilities.contains(&ModelCapability::Generate));
            to.capabilities.retain(|c| *c != ModelCapability::Generate);
            assert!(benchmark_findings(&[&from], &[&to], &fits).is_empty());
        }
    }
    use crate::schema::{CostModel, ModelCapability, PerformanceEnvelope};

    fn local_schema(id: &str, available: bool) -> ModelSchema {
        ModelSchema {
            id: id.into(),
            name: id.into(),
            provider: "qwen".into(),
            family: "qwen3".into(),
            version: String::new(),
            capabilities: vec![ModelCapability::Generate],
            context_length: 8192,
            max_output_tokens: None,
            param_count: "4B".into(),
            quantization: None,
            performance: PerformanceEnvelope::default(),
            cost: CostModel::default(),
            source: ModelSource::Local {
                hf_repo: format!("org/{id}"),
                hf_filename: "m.gguf".into(),
                tokenizer_repo: format!("org/{id}"),
            },
            tags: vec![],
            supported_params: vec![],
            public_benchmarks: vec![],
            trust_tier: TrustTier::Curated,
            deprecated: false,
            available,
            weights_ready: available,
        }
    }

    struct FakeProbe {
        newer: bool,
    }
    impl UpstreamProbe for FakeProbe {
        async fn newer_revision(&self, _schema: &ModelSchema) -> Option<String> {
            if self.newer {
                Some("newer upstream".into())
            } else {
                None
            }
        }
    }

    /// A probe that panics if called — proves we didn't hit the network.
    struct NeverProbe;
    impl UpstreamProbe for NeverProbe {
        async fn newer_revision(&self, _schema: &ModelSchema) -> Option<String> {
            panic!("probe must not be called");
        }
    }

    fn tmp_cache(tag: &str) -> PathBuf {
        std::env::temp_dir().join(format!("car-upgrade-{tag}-{}.json", std::process::id()))
    }

    #[tokio::test]
    async fn stable_channel_is_curated_only_and_never_probes() {
        let prefs = UpdatePreferences::default(); // Stable
        let installed = local_schema("qwen3-4b", true);
        let cache = tmp_cache("stable");
        let findings = detect_upgrades(
            vec![],
            &[&installed],
            &prefs,
            &NeverProbe, // would panic if probed
            &cache,
            1000,
            DEFAULT_TTL_SECS,
        )
        .await;
        assert!(findings.is_empty());
        let _ = std::fs::remove_file(&cache);
    }

    #[tokio::test]
    async fn latest_channel_adds_upstream_findings() {
        let prefs = UpdatePreferences {
            channel: UpdateChannel::Latest,
            ..Default::default()
        };
        let installed = local_schema("qwen3-4b", true);
        let cache = tmp_cache("latest");
        let _ = std::fs::remove_file(&cache);
        let findings = detect_upgrades(
            vec![],
            &[&installed],
            &prefs,
            &FakeProbe { newer: true },
            &cache,
            1000,
            DEFAULT_TTL_SECS,
        )
        .await;
        assert_eq!(findings.len(), 1);
        assert_eq!(findings[0].source, UpgradeSource::Upstream);
        assert_eq!(findings[0].trust_tier, TrustTier::Community);
        let _ = std::fs::remove_file(&cache);
    }

    #[tokio::test]
    async fn uninstalled_models_are_not_probed() {
        let prefs = UpdatePreferences {
            channel: UpdateChannel::Latest,
            ..Default::default()
        };
        let installed = local_schema("qwen3-4b", false); // not installed
        let cache = tmp_cache("uninstalled");
        let _ = std::fs::remove_file(&cache);
        let findings = detect_upgrades(
            vec![],
            &[&installed],
            &prefs,
            &NeverProbe, // skipped before probe because !available
            &cache,
            1000,
            DEFAULT_TTL_SECS,
        )
        .await;
        assert!(findings.is_empty());
        let _ = std::fs::remove_file(&cache);
    }

    #[tokio::test]
    async fn lazy_available_model_without_weights_is_not_probed_as_installed() {
        let prefs = UpdatePreferences {
            channel: UpdateChannel::Latest,
            ..Default::default()
        };
        let mut model = local_schema("qwen3-4b", false);
        model.available = true;
        model.weights_ready = false;
        let cache = tmp_cache("lazy-available");
        let _ = std::fs::remove_file(&cache);

        let findings = detect_upgrades(
            vec![],
            &[&model],
            &prefs,
            &NeverProbe,
            &cache,
            1000,
            DEFAULT_TTL_SECS,
        )
        .await;

        assert!(findings.is_empty());
        let _ = std::fs::remove_file(&cache);
    }

    #[tokio::test]
    async fn fresh_cache_is_served_without_probing() {
        let prefs = UpdatePreferences {
            channel: UpdateChannel::Latest,
            ..Default::default()
        };
        let installed = local_schema("qwen3-4b", true);
        let cache = tmp_cache("fresh");
        // Seed a fresh cache with a finding, matching the installed-set
        // fingerprint so it isn't invalidated.
        UpgradeCache {
            checked_at_secs: 1000,
            models_fingerprint: installed_fingerprint(&[&installed]),
            upstream: vec![UpgradeFinding {
                from_id: "qwen3-4b".into(),
                from_name: "qwen3-4b".into(),
                to_id: "qwen3-4b".into(),
                to_name: "qwen3-4b".into(),
                reason: "cached".into(),
                trust_tier: TrustTier::Community,
                source: UpgradeSource::Upstream,
                target_pullable: true,
            }],
        }
        .save_to(&cache)
        .unwrap();
        // now within TTL of checked_at ⇒ NeverProbe must not be called.
        let findings = detect_upgrades(
            vec![],
            &[&installed],
            &prefs,
            &NeverProbe,
            &cache,
            1500,
            DEFAULT_TTL_SECS,
        )
        .await;
        assert_eq!(findings.len(), 1);
        assert_eq!(findings[0].reason, "cached");
        let _ = std::fs::remove_file(&cache);
    }

    #[tokio::test]
    async fn fresh_cache_for_a_different_model_set_is_invalidated() {
        // A fresh cache built for a DIFFERENT installed set must not be served;
        // the newly installed model has to be probed.
        let prefs = UpdatePreferences {
            channel: UpdateChannel::Latest,
            ..Default::default()
        };
        let installed = local_schema("qwen3-8b", true); // different model
        let cache = tmp_cache("fingerprint");
        UpgradeCache {
            checked_at_secs: 1000,
            models_fingerprint: "stale-different-set".into(),
            upstream: vec![],
        }
        .save_to(&cache)
        .unwrap();
        let findings = detect_upgrades(
            vec![],
            &[&installed],
            &prefs,
            &FakeProbe { newer: true }, // must be called → fingerprint mismatch
            &cache,
            1500,
            DEFAULT_TTL_SECS,
        )
        .await;
        assert_eq!(findings.len(), 1, "stale-fingerprint cache must re-probe");
        let _ = std::fs::remove_file(&cache);
    }

    #[tokio::test]
    async fn curated_wins_over_upstream_for_same_model() {
        let prefs = UpdatePreferences {
            channel: UpdateChannel::Latest,
            ..Default::default()
        };
        let installed = local_schema("qwen3-4b", true);
        let cache = tmp_cache("dedup");
        let _ = std::fs::remove_file(&cache);
        let curated = vec![ModelUpgrade {
            from_id: "qwen3-4b".into(),
            from_name: "qwen3-4b".into(),
            to_id: "qwen3-8b".into(),
            to_name: "qwen3-8b".into(),
            reason: "curated replacement".into(),
            target_runtime: None,
            target_runtime_requirement: None,
            minimum_runtimes: vec![],
            target_available: true,
            target_pullable: true,
            remove_old_supported: true,
        }];
        let findings = detect_upgrades(
            curated,
            &[&installed],
            &prefs,
            &FakeProbe { newer: true },
            &cache,
            1000,
            DEFAULT_TTL_SECS,
        )
        .await;
        // Only the curated finding for qwen3-4b; upstream for same from_id dropped.
        assert_eq!(findings.len(), 1);
        assert_eq!(findings[0].source, UpgradeSource::Curated);
        let _ = std::fs::remove_file(&cache);
    }

    #[tokio::test]
    async fn probe_construction_degrades_instead_of_panicking() {
        use crate::tls_client::test_seam::{TrustStoreScope, UNPARSEABLE_CERT_PEM};
        use crate::tls_client::{TrustFallback, HUGGINGFACE_PROBE, REMOTE_BACKEND};

        let scope = TrustStoreScope::acquire_async(UNPARSEABLE_CERT_PEM).await;
        scope.assert_breaks_client_construction();
        crate::tls_client::reset_sites_for_test();

        // Construction only. `remote_sha` hardcodes `https://huggingface.co/...`
        // with no endpoint parameter and no mock seam, so any assertion about
        // what the probe *returns* would either hit the real network or assert
        // nothing at all.
        let _probe = HuggingFaceProbe::new();
        let _defaulted = HuggingFaceProbe::default();

        // The record is the assertion that matters: a degraded probe's "None"
        // is indistinguishable from an ordinary offline "None", which is
        // exactly the masking this guards against.
        let record = crate::tls_client::last_degradation(&HUGGINGFACE_PROBE)
            .expect("a degraded probe must leave a readable record");
        assert_eq!(record.fallback, TrustFallback::PublicCaOnly);
        assert!(!record.source.is_empty());
        assert!(
            crate::tls_client::last_degradation(&REMOTE_BACKEND).is_none(),
            "records are per site — this one must not be attributed elsewhere"
        );
    }
}