modelshelf 0.1.0

//! The live model catalog behind recommendations.
//!
//! The catalog is versioned *data*, not code: a small curated JSON list of
//! GGUF models (see `catalog/catalog.json` and `catalog/README.md` in this
//! crate). Model rankings shift irregularly, so the list must be updatable
//! without shipping a new binary. Resolution works in two layers:
//!
//! 1. a **cached** copy at `<shelf home>/catalog.json`, refreshed from
//!    [`DEFAULT_CATALOG_URL`] (a raw file on the repository's `main` branch);
//! 2. the **embedded** snapshot compiled into this crate.
//!
//! Whichever has the higher `catalog_version` wins, so an offline machine or
//! a failed fetch always degrades to the build-time snapshot.

use std::io::Read;
use std::time::{Duration, SystemTime};

use serde::{Deserialize, Serialize};

use crate::paths::ShelfPaths;
use crate::{Error, Result};

/// Environment variable that overrides the catalog download URL
/// (tests, mirrors, air-gapped deployments).
pub const CATALOG_URL_ENV: &str = "MODELSHELF_CATALOG_URL";

/// Where fresh catalogs are published: the canonical file on `main`.
pub const DEFAULT_CATALOG_URL: &str =
    "https://raw.githubusercontent.com/KOIYAL/modelshelf/main/crates/modelshelf/catalog/catalog.json";

/// How old the cache may grow before an opportunistic refresh is attempted.
pub const REFRESH_TTL: Duration = Duration::from_secs(7 * 24 * 60 * 60);

/// Network timeout for catalog fetches. Kept short: a fetch may run
/// opportunistically at the start of user-facing commands.
const FETCH_TIMEOUT: Duration = Duration::from_secs(5);

/// Upper bound on the catalog document size (it is a few KiB in practice).
const MAX_CATALOG_BYTES: u64 = 2 * 1024 * 1024;

const EMBEDDED: &str = include_str!("../catalog/catalog.json");

/// A companion file an entry needs at runtime (a TTS vocoder, a vision
/// projector, …). Provisioning fetches it alongside the main file.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ExtraFile {
    /// Repository holding the file; `None` means the entry's own repo.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub repo: Option<String>,
    /// Exact file within the repository.
    pub filename: String,
    /// Exact download size in bytes.
    pub file_bytes: u64,
}

/// One recommendable model.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct CatalogEntry {
    /// Stable unique key (kebab-case), never renamed or reused.
    pub name: String,
    /// The use case this entry serves (`chat`, `code`, `reasoning`,
    /// `embedding`, `stt`, `tts`, `vision`, …). Free-form so new use cases
    /// can ship as catalog data; see [`crate::recommend::KNOWN_TASKS`].
    #[serde(default = "default_task")]
    pub task: String,
    /// Hugging Face repository id (`org/repo`).
    pub repo: String,
    /// Exact model file within the repository.
    pub filename: String,
    /// File format: `gguf` (default), `ggml` (whisper.cpp `.bin`), `onnx`.
    #[serde(default = "default_format")]
    pub format: String,
    /// Quantization label as spelled in the model metadata (e.g. `Q4_K_M`).
    pub quant: String,
    /// Exact download size of the main file in bytes.
    pub file_bytes: u64,
    /// Total parameters in billions. A matching hint, not a fit input.
    pub params_b: f64,
    /// True when Japanese quality is strong.
    #[serde(default)]
    pub japanese: bool,
    /// One-line description shown to users.
    #[serde(default)]
    pub notes: String,
    /// Superseded entries keep resolving but are never recommended.
    #[serde(default)]
    pub deprecated: bool,
    /// Companion files required at runtime, fetched by provisioning.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub extra_files: Vec<ExtraFile>,
}

fn default_task() -> String {
    "chat".to_owned()
}

fn default_format() -> String {
    "gguf".to_owned()
}

impl CatalogEntry {
    /// Total download size: the main file plus every extra file.
    pub fn total_bytes(&self) -> u64 {
        self.file_bytes + self.extra_files.iter().map(|f| f.file_bytes).sum::<u64>()
    }
}

/// File extensions the catalog may reference (matching the formats current
/// runtimes load directly).
const ALLOWED_EXTENSIONS: [&str; 3] = [".gguf", ".bin", ".onnx"];

fn allowed_extension(filename: &str) -> bool {
    let lower = filename.to_ascii_lowercase();
    ALLOWED_EXTENSIONS.iter().any(|ext| lower.ends_with(ext))
}

/// A parsed model catalog.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Catalog {
    /// Monotonically increasing version; the higher of {cached, embedded}
    /// wins at load time.
    pub catalog_version: u64,
    /// RFC 3339 timestamp of the last catalog edit.
    pub updated: String,
    /// The recommendable models.
    pub entries: Vec<CatalogEntry>,
}

/// Which layer a loaded catalog came from.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CatalogOrigin {
    /// The snapshot compiled into this build.
    Embedded,
    /// A newer copy previously fetched into `<shelf home>/catalog.json`.
    Cached,
}

impl Catalog {
    /// The catalog snapshot embedded at build time.
    ///
    /// # Panics
    /// Only if the crate was built with a broken `catalog/catalog.json`;
    /// unit tests and CI parse the exact embedded bytes.
    pub fn embedded() -> Catalog {
        serde_json::from_str(EMBEDDED).expect("embedded catalog.json must parse")
    }

    /// The entry with this `name`, if any.
    pub fn entry(&self, name: &str) -> Option<&CatalogEntry> {
        self.entries.iter().find(|e| e.name == name)
    }

    /// Check structural invariants. Fetched documents must pass before they
    /// are cached or used.
    pub fn validate(&self) -> Result<()> {
        let fail = |msg: String| Err(Error::InvalidSpec(format!("catalog: {msg}")));
        if self.catalog_version == 0 {
            return fail("catalog_version must be >= 1".into());
        }
        if self.entries.is_empty() {
            return fail("no entries".into());
        }
        let mut names = std::collections::HashSet::new();
        for e in &self.entries {
            if !names.insert(e.name.as_str()) {
                return fail(format!("duplicate entry name {:?}", e.name));
            }
            if e.name.is_empty() {
                return fail("empty entry name".into());
            }
            if e.task.is_empty() {
                return fail(format!("{}: empty task", e.name));
            }
            if e.repo.split('/').filter(|p| !p.is_empty()).count() != 2 {
                return fail(format!(
                    "{}: repo must be org/name, got {:?}",
                    e.name, e.repo
                ));
            }
            if !allowed_extension(&e.filename) {
                return fail(format!(
                    "{}: filename must end in one of {ALLOWED_EXTENSIONS:?}",
                    e.name
                ));
            }
            if e.file_bytes == 0 {
                return fail(format!("{}: file_bytes must be > 0", e.name));
            }
            for extra in &e.extra_files {
                if !allowed_extension(&extra.filename) {
                    return fail(format!(
                        "{}: extra file must end in one of {ALLOWED_EXTENSIONS:?}",
                        e.name
                    ));
                }
                if extra.file_bytes == 0 {
                    return fail(format!("{}: extra file_bytes must be > 0", e.name));
                }
                if let Some(repo) = &extra.repo {
                    if repo.split('/').filter(|p| !p.is_empty()).count() != 2 {
                        return fail(format!("{}: extra repo must be org/name", e.name));
                    }
                }
            }
        }
        Ok(())
    }
}

/// Load the best available catalog: the cached copy when it is valid and
/// strictly newer than the embedded snapshot, the embedded snapshot
/// otherwise. Never fails and never touches the network.
pub fn load(paths: &ShelfPaths) -> (Catalog, CatalogOrigin) {
    let embedded = Catalog::embedded();
    let cached = std::fs::read_to_string(paths.catalog_json())
        .ok()
        .and_then(|s| serde_json::from_str::<Catalog>(&s).ok())
        .filter(|c| c.validate().is_ok());
    match cached {
        Some(c) if c.catalog_version > embedded.catalog_version => (c, CatalogOrigin::Cached),
        _ => (embedded, CatalogOrigin::Embedded),
    }
}

/// Download the catalog and atomically replace the cache with the raw
/// fetched document (preserving fields this build does not know about).
///
/// `url` falls back to [`CATALOG_URL_ENV`], then [`DEFAULT_CATALOG_URL`].
/// The fetched document must parse and [`Catalog::validate`]; a bad document
/// leaves the cache untouched.
pub fn fetch_and_cache(paths: &ShelfPaths, url: Option<&str>) -> Result<Catalog> {
    let url = match url {
        Some(u) => u.to_owned(),
        None => std::env::var(CATALOG_URL_ENV)
            .ok()
            .filter(|s| !s.is_empty())
            .unwrap_or_else(|| DEFAULT_CATALOG_URL.to_owned()),
    };
    let agent = ureq::AgentBuilder::new()
        .user_agent(concat!("modelshelf/", env!("CARGO_PKG_VERSION")))
        .timeout(FETCH_TIMEOUT)
        .build();
    let response = agent
        .get(&url)
        .call()
        .map_err(|e| Error::Network(format!("catalog fetch failed: {e}")))?;
    let mut body = String::new();
    response
        .into_reader()
        .take(MAX_CATALOG_BYTES)
        .read_to_string(&mut body)
        .map_err(|e| Error::Network(format!("catalog fetch failed while reading: {e}")))?;

    let catalog: Catalog = serde_json::from_str(&body)
        .map_err(|e| Error::Network(format!("fetched catalog is not valid JSON: {e}")))?;
    catalog.validate()?;
    crate::registry::lock::write_atomic(&paths.catalog_json(), body.as_bytes())?;
    Ok(catalog)
}

/// True when the cache warrants a refresh attempt: it does not exist, or its
/// mtime (the time of the last refresh attempt) is older than
/// [`REFRESH_TTL`].
pub fn should_refresh(cache_mtime: Option<SystemTime>, now: SystemTime) -> bool {
    match cache_mtime {
        None => true,
        Some(t) => now
            .duration_since(t)
            .map(|age| age > REFRESH_TTL)
            .unwrap_or(false), // mtime in the future: clock skew, don't churn
    }
}

/// Opportunistic refresh: fetch at most once per [`REFRESH_TTL`], swallowing
/// every failure (recommendations must keep working offline).
///
/// The cache file's mtime records the last *attempt*: after a failed fetch
/// the existing cache is re-stamped (or seeded with the embedded snapshot)
/// so an offline machine retries weekly instead of on every command.
pub fn refresh_if_stale(paths: &ShelfPaths, url: Option<&str>) {
    let cache = paths.catalog_json();
    let mtime = std::fs::metadata(&cache)
        .ok()
        .and_then(|m| m.modified().ok());
    if !should_refresh(mtime, SystemTime::now()) {
        return;
    }
    if let Err(e) = fetch_and_cache(paths, url) {
        tracing::debug!("catalog refresh skipped: {e}");
        let stamped = std::fs::File::options()
            .append(true)
            .open(&cache)
            .and_then(|f| f.set_modified(SystemTime::now()));
        if stamped.is_err() {
            // No cache yet: seed it with the embedded snapshot so the mtime
            // exists to rate-limit future attempts.
            let _ = crate::registry::lock::write_atomic(&cache, EMBEDDED.as_bytes());
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn shelf_paths(tmp: &tempfile::TempDir) -> ShelfPaths {
        let p = ShelfPaths::at(tmp.path().join("shelf"));
        p.ensure_layout().unwrap();
        p
    }

    fn catalog_json(version: u64, name: &str) -> String {
        format!(
            r#"{{"catalog_version": {version}, "updated": "2026-07-10T00:00:00Z",
                 "entries": [{{"name": "{name}", "repo": "org/repo",
                               "filename": "m.gguf", "quant": "Q4_K_M",
                               "file_bytes": 1000, "params_b": 1.0}}]}}"#
        )
    }

    #[test]
    fn embedded_catalog_parses_and_validates() {
        let c = Catalog::embedded();
        c.validate().unwrap();
        assert!(c.catalog_version >= 1);
        assert!(c.entries.len() >= 8, "curated ladder shrank unexpectedly");
        // The ladder must include something an 8 GiB machine can run.
        assert!(c
            .entries
            .iter()
            .any(|e| e.file_bytes < 2 * 1024 * 1024 * 1024));
    }

    #[test]
    fn validate_rejects_structural_problems() {
        let mut dup = Catalog::embedded();
        let clone = dup.entries[0].clone();
        dup.entries.push(clone);
        assert!(dup.validate().is_err(), "duplicate names must fail");

        let mut bad_repo = Catalog::embedded();
        bad_repo.entries[0].repo = "no-slash".into();
        assert!(bad_repo.validate().is_err());

        let mut bad_ext = Catalog::embedded();
        bad_ext.entries[0].filename = "weights.safetensors".into();
        assert!(bad_ext.validate().is_err());

        let mut zero = Catalog::embedded();
        zero.entries[0].file_bytes = 0;
        assert!(zero.validate().is_err());

        let mut v0 = Catalog::embedded();
        v0.catalog_version = 0;
        assert!(v0.validate().is_err());
    }

    #[test]
    fn load_prefers_strictly_newer_valid_cache() {
        let tmp = tempfile::tempdir().unwrap();
        let paths = shelf_paths(&tmp);
        let embedded_version = Catalog::embedded().catalog_version;

        // No cache: embedded.
        assert_eq!(load(&paths).1, CatalogOrigin::Embedded);

        // Newer cache wins.
        std::fs::write(
            paths.catalog_json(),
            catalog_json(embedded_version + 5, "newer"),
        )
        .unwrap();
        let (c, origin) = load(&paths);
        assert_eq!(origin, CatalogOrigin::Cached);
        assert_eq!(c.catalog_version, embedded_version + 5);

        // Same or older cache loses.
        std::fs::write(paths.catalog_json(), catalog_json(embedded_version, "same")).unwrap();
        assert_eq!(load(&paths).1, CatalogOrigin::Embedded);

        // Garbage cache is ignored.
        std::fs::write(paths.catalog_json(), "{ not json").unwrap();
        assert_eq!(load(&paths).1, CatalogOrigin::Embedded);
    }

    #[test]
    fn should_refresh_honors_ttl() {
        let now = SystemTime::now();
        assert!(should_refresh(None, now));
        assert!(!should_refresh(Some(now), now));
        assert!(!should_refresh(Some(now - Duration::from_secs(3600)), now));
        assert!(should_refresh(
            Some(now - (REFRESH_TTL + Duration::from_secs(1))),
            now
        ));
        // Future mtime (clock skew) must not cause a refresh loop.
        assert!(!should_refresh(Some(now + Duration::from_secs(3600)), now));
    }

    #[test]
    fn fetch_and_cache_stores_valid_and_rejects_invalid() {
        let tmp = tempfile::tempdir().unwrap();
        let paths = shelf_paths(&tmp);
        let server = httpmock::MockServer::start();

        let good = server.mock(|when, then| {
            when.method(httpmock::Method::GET).path("/good.json");
            then.status(200).body(catalog_json(42, "fresh"));
        });
        let fetched =
            fetch_and_cache(&paths, Some(&format!("{}/good.json", server.base_url()))).unwrap();
        good.assert_hits(1);
        assert_eq!(fetched.catalog_version, 42);
        let (loaded, origin) = load(&paths);
        assert_eq!(origin, CatalogOrigin::Cached);
        assert_eq!(loaded.catalog_version, 42);

        // Invalid JSON: error, cache untouched.
        server.mock(|when, then| {
            when.method(httpmock::Method::GET).path("/bad.json");
            then.status(200).body("<html>not a catalog</html>");
        });
        let err =
            fetch_and_cache(&paths, Some(&format!("{}/bad.json", server.base_url()))).unwrap_err();
        assert_eq!(err.kind(), crate::ErrorKind::Network);
        assert_eq!(
            load(&paths).0.catalog_version,
            42,
            "cache must be untouched"
        );

        // Valid JSON failing validation: error, cache untouched.
        server.mock(|when, then| {
            when.method(httpmock::Method::GET).path("/empty.json");
            then.status(200).body(
                r#"{"catalog_version": 99, "updated": "2026-07-10T00:00:00Z", "entries": []}"#,
            );
        });
        assert!(
            fetch_and_cache(&paths, Some(&format!("{}/empty.json", server.base_url()))).is_err()
        );
        assert_eq!(load(&paths).0.catalog_version, 42);

        // HTTP error: Network error surfaced.
        server.mock(|when, then| {
            when.method(httpmock::Method::GET).path("/missing.json");
            then.status(404);
        });
        let err = fetch_and_cache(&paths, Some(&format!("{}/missing.json", server.base_url())))
            .unwrap_err();
        assert_eq!(err.kind(), crate::ErrorKind::Network);
    }

    #[test]
    fn refresh_if_stale_is_silent_and_rate_limited_on_failure() {
        let tmp = tempfile::tempdir().unwrap();
        let paths = shelf_paths(&tmp);
        let server = httpmock::MockServer::start();
        let failing = server.mock(|when, then| {
            when.method(httpmock::Method::GET).path("/down.json");
            then.status(500);
        });
        let url = format!("{}/down.json", server.base_url());

        // First attempt: fetch fails silently, cache is seeded to rate-limit.
        refresh_if_stale(&paths, Some(&url));
        failing.assert_hits(1);
        assert!(
            paths.catalog_json().is_file(),
            "failure must seed the cache"
        );
        let (c, origin) = load(&paths);
        // The seeded cache is the embedded snapshot: same version, so the
        // embedded copy still wins.
        assert_eq!(origin, CatalogOrigin::Embedded);
        assert_eq!(c.catalog_version, Catalog::embedded().catalog_version);

        // Second call within the TTL: no new network attempt.
        refresh_if_stale(&paths, Some(&url));
        failing.assert_hits(1);
    }
}