coremlit 0.1.2

//! Shared helpers for the granite embedding tests.
//!
//! Two data sources, kept distinct:
//!
//! - **Committed goldens** (`tests/granite/fixtures/goldens/corpus.json`) — the
//!   in-tree transformers-fp32 oracle (token-ids AND unit-normalized embedding
//!   goldens). Read hermetically; no model, no network. This is the parity
//!   ground truth (the embedkit "no ort anywhere" rule), committed the way the
//!   speaker/vad Swift goldens are.
//! - **CoreML artifact** (`granite_97m_512.mlmodelc`) — a gitignored dev-time
//!   download from `FinDIT-Studio/embedkit-coreml` (revision `a61241cb`), under
//!   `Models/embedkit-granite/` (overridable via `EMBEDKIT_TEST_MODELS`).
//!   Model-gated tests are `#[ignore]` by default and run only when present.
//!
//! Both are re-derivable from the upstream Apache-2.0 checkpoint by
//! `coremlit/conversion/granite/` — to the cosine floors that recipe
//! gates on, NOT byte-for-byte (its README records the measured byte-identity
//! result). The hashes pinned in `model_io.rs` are the PUBLISHED bundle's, so a
//! locally re-derived bundle will not satisfy that gate.

// The workspace-root anchor every `models_dir()` below resolves against, and
// the sibling-checkout anchor the oracle gates read. FOUND by searching upward
// for the `[workspace]` manifest, never counted in `../` hops — see its module
// doc for why a count is the wrong shape here. Re-exported so the binaries
// that pull this `common` in share the one resolver.
// The committed per-table file manifest the artifact digests are read from.
// See its module doc for the grammar and for why the gates read a data file
// rather than each holding their own copy.
#[path = "../../support/models_lock_manifest.rs"]
#[allow(dead_code)]
pub mod models_lock_manifest;

#[path = "../../support/workspace_root.rs"]
#[allow(dead_code)]
mod workspace_root;
#[allow(unused_imports)]
pub use workspace_root::{checkout_parent, models_root, workspace_root};

use std::path::{Path, PathBuf};

/// The HF revision (commit SHA) of `FinDIT-Studio/embedkit-coreml` the model
/// artifact and its per-file SHA-256 pins are recorded at — the revision
/// `MODELS_LOCK` pins, so this is what CI actually stages. It supersedes
/// `81852f70` by ADDING the `tokenizer.json` sidecar; the `.mlmodelc` and
/// `.mlpackage` bytes the pins below cover are unchanged between the two, which
/// is why bumping this did not require re-pinning any hash.
#[allow(dead_code)]
pub const EMBEDKIT_REVISION: &str = "a61241cb";

/// The `MODELS_LOCK` `local-dir` this kit stages into, with `Models/` removed,
/// and the FULL revision the lock pins — together the key into
/// `MODELS_LOCK.d/`. [`EMBEDKIT_REVISION`] is the short form used in messages.
#[allow(dead_code)]
pub const VENDOR_DIR: &str = "embedkit-granite";

/// The full pinned revision, as `MODELS_LOCK` and the committed manifest's file
/// name spell it.
#[allow(dead_code)]
pub const EMBEDKIT_LOCK_REVISION: &str = "a61241cb18689de1a8b83c315875807030e2878f";

/// The `.mlmodelc` bundle's path relative to the table's `local-dir`.
#[allow(dead_code)]
pub const BUNDLE_PATH: &str = "granite-97m-multilingual-r2/granite_97m_512.mlmodelc";

/// The artifact-root `tokenizer.json`'s path relative to the table's
/// `local-dir`.
#[allow(dead_code)]
pub const TOKENIZER_PATH: &str = "granite-97m-multilingual-r2/tokenizer.json";

/// Exact per-file SHA-256 of the staged `.mlmodelc`, read from
/// `MODELS_LOCK.d/embedkit-granite@<revision>.sha256` — upstream's own
/// `CHECKSUMS.sha256` at [`EMBEDKIT_LOCK_REVISION`], paths made
/// table-relative.
///
/// It used to be a `const ARTIFACT_SHA256` in `tests/granite/model_io.rs`, with
/// a second copy of the same digests in `tests/model_licences.rs` tied to it by
/// a scanner over Rust source (coremlit #139 retired both).
#[allow(dead_code)]
pub fn artifact_sha256() -> Vec<(String, String)> {
  models_lock_manifest::bundle_manifest(
    &workspace_root::workspace_root(),
    VENDOR_DIR,
    EMBEDKIT_LOCK_REVISION,
    BUNDLE_PATH,
  )
}

/// Directory containing the downloaded granite CoreML artifact tree.
///
/// Overridable via `EMBEDKIT_TEST_MODELS`; otherwise
/// `<workspace>/Models/embedkit-granite` — gitignored, fetched dev-time
/// (mirrors the clapkit `CLAPKIT_TEST_MODELS` / speakerkit `SPEAKERKIT_TEST_MODELS`
/// conventions).
#[allow(dead_code)]
pub fn models_dir() -> PathBuf {
  std::env::var_os("EMBEDKIT_TEST_MODELS").map_or_else(
    || workspace_root::models_root().join("embedkit-granite"),
    PathBuf::from,
  )
}

/// The granite model bundle's parent directory
/// (`.../granite-97m-multilingual-r2`), which holds the `.mlmodelc`,
/// `.mlpackage`, and `CHECKSUMS.sha256`.
#[allow(dead_code)]
pub fn model_root() -> PathBuf {
  models_dir().join("granite-97m-multilingual-r2")
}

/// Path to the compiled granite text encoder, `granite_97m_512.mlmodelc`.
#[allow(dead_code)]
pub fn model_path() -> PathBuf {
  model_root().join("granite_97m_512.mlmodelc")
}

/// Absolute path to a committed fixture under `coremlit/tests/granite/fixtures`.
#[allow(dead_code)]
pub fn fixture_path(relative: &str) -> PathBuf {
  PathBuf::from(env!("CARGO_MANIFEST_DIR"))
    .join("tests")
    .join("granite")
    .join("fixtures")
    .join(relative)
}

/// One committed golden corpus entry: the raw text, its exact token-ids (granite
/// tokenizer, truncated at 512, special tokens included), and the
/// transformers-fp32 **unit-normalized** 384-d embedding.
#[derive(Debug, serde::Deserialize)]
#[allow(dead_code)]
pub struct GoldenEntry {
  /// Stable per-entry id (`en_q`, `zh`, `near512`, …).
  pub id: String,
  /// The raw input string (prompt-free — no task prefix).
  pub text: String,
  /// The golden token-id sequence (truncated at 512, specials included).
  pub token_ids: Vec<u32>,
  /// The golden token count (`== token_ids.len()`).
  pub n_tokens: usize,
  /// The transformers-fp32 unit-normalized 384-d embedding.
  pub embedding: Vec<f32>,
}

#[derive(Debug, serde::Deserialize)]
struct Corpus {
  entries: Vec<GoldenEntry>,
}

/// Load the committed golden corpus (16 entries). Hermetic — reads the in-tree
/// fixture, never `Models/`.
#[allow(dead_code)]
pub fn golden_corpus() -> Vec<GoldenEntry> {
  let path = fixture_path("goldens/corpus.json");
  let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display()));
  let corpus: Corpus =
    serde_json::from_slice(&bytes).unwrap_or_else(|e| panic!("parse {}: {e}", path.display()));
  assert_eq!(
    corpus.entries.len(),
    16,
    "the committed granite golden corpus must have 16 entries"
  );
  corpus.entries
}

/// One recorded per-entry cosine from `goldens/driver_crosscheck.json`.
///
/// The cosine is `f64` because that is the precision the fixture stores AND the
/// precision the trap lives in: several recorded values sit a few ULP ABOVE 1.0
/// (see `driver_crosscheck.rs`), and narrowing them to `f32` would round that
/// excess away and silently hide it.
#[derive(Debug, serde::Deserialize)]
#[allow(dead_code)]
pub struct CrosscheckEntry {
  /// The corpus entry id this cosine belongs to (`en_q`, `zh`, `near512`, …).
  pub id: String,
  /// Cosine of the canonical transformers-fp32 pipeline against the fixed-512
  /// static-mask driver that was traced to CoreML.
  pub cosine_canonical_vs_driver: f64,
  /// Largest per-component difference between the unit-normalized driver vector
  /// and the canonical one. This — not the cosine — is what shows the two sides
  /// were computed independently; see `driver_crosscheck.rs`.
  pub max_abs_component_delta: f64,
}

/// The committed canonical-vs-driver crosscheck record — the measurement the
/// conversion gated on before tracing, published verbatim, kept as a golden so
/// the claim is checkable without the model or the Python toolchain.
#[derive(Debug, serde::Deserialize)]
#[allow(dead_code)]
pub struct DriverCrosscheck {
  /// The minimum of `per_entry`.
  pub worst_cosine_canonical_vs_driver: f64,
  /// The smallest per-entry `max_abs_component_delta` — the tightest the two
  /// computations came anywhere in the corpus.
  pub min_max_abs_component_delta: f64,
  /// The divergence budget the recipe reported against (looser than the floor
  /// the recipe gates on, and looser than the floor this crate asserts).
  pub stop_threshold_divergence: f64,
  /// `AGREE` when the worst cosine stayed inside `stop_threshold_divergence`.
  pub verdict: String,
  /// SHA-256 of the `corpus.json` bytes this record was PUBLISHED alongside —
  /// stamped on at publication, not an input to the measurement. Binds the two
  /// goldens as a pair rather than by id alone. The measurement-INPUT binding is
  /// a separate `corpus_input_sha256` the recipe records while measuring; this
  /// committed fixture predates that field, so it is not deserialized here.
  pub corpus_sha256: String,
  /// One row per corpus entry, same ids as `corpus.json`.
  pub per_entry: Vec<CrosscheckEntry>,
}

/// Load the committed driver crosscheck. Hermetic — reads the in-tree fixture,
/// never `Models/`.
#[allow(dead_code)]
pub fn driver_crosscheck() -> DriverCrosscheck {
  let path = fixture_path("goldens/driver_crosscheck.json");
  let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display()));
  serde_json::from_slice(&bytes).unwrap_or_else(|e| panic!("parse {}: {e}", path.display()))
}

/// SHA-256 of the committed `corpus.json` bytes, for the crosscheck's pair
/// binding. Hashes the file exactly as written — the generator digests the same
/// serialized text it puts on disk.
#[allow(dead_code)]
pub fn corpus_sha256() -> String {
  let path = fixture_path("goldens/corpus.json");
  let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display()));
  sha256_hex(&bytes)
}

/// Lowercase-hex SHA-256 of a byte slice. Backs the `model_io` provenance pins
/// (the #30 enumerate-then-hash manifest pattern hashes the bytes it read).
#[allow(dead_code)]
pub fn sha256_hex(bytes: &[u8]) -> String {
  use sha2::{Digest, Sha256};
  Sha256::digest(bytes)
    .iter()
    .map(|b| format!("{b:02x}"))
    .collect()
}

/// Cosine of two equal-length finite vectors — FAIL-CLOSED (the #30 lesson):
/// panics on a length mismatch or any non-finite component rather than returning
/// a silently-wrong scalar. `Embedding` is fixed-dim, but a golden vector is a
/// `Vec`, so its length and finiteness are checked here.
#[allow(dead_code)]
pub fn cosine_checked(a: &[f32], b: &[f32]) -> f32 {
  assert_eq!(a.len(), b.len(), "cosine operands differ in length");
  assert!(!a.is_empty(), "cosine of empty vectors is undefined");
  let mut dot = 0.0f64;
  let mut na = 0.0f64;
  let mut nb = 0.0f64;
  for (i, (&x, &y)) in a.iter().zip(b.iter()).enumerate() {
    assert!(x.is_finite(), "left operand non-finite at {i}");
    assert!(y.is_finite(), "right operand non-finite at {i}");
    dot += (x as f64) * (y as f64);
    na += (x as f64) * (x as f64);
    nb += (y as f64) * (y as f64);
  }
  assert!(na > 0.0 && nb > 0.0, "cosine of a zero vector is undefined");
  (dot / (na.sqrt() * nb.sqrt())) as f32
}

/// Recursively collects every real FILE under `dir` as a path relative to
/// `root`, with `/` separators, into `out`. Used to enumerate a `.mlmodelc`
/// bundle's actual tree so it can be set-compared against a pinned key manifest
/// (no unpinned extras, none missing) BEFORE hashing — the #30 exact-enumerate
/// pattern.
///
/// OS-generated sidecars are skipped: AppleDouble `._*` files and `.DS_Store`.
/// macOS materializes these inside bundles on non-native filesystems
/// (exFAT/FAT/SMB); CoreML's loader never reads them, so excluding them from
/// discovery cannot mask a functional artifact change — whereas NOT excluding
/// them would false-fail the exact-set gate as a phantom "unpinned extra" even
/// though every pinned byte is untouched.
#[allow(dead_code)]
pub fn collect_files_rel(root: &Path, dir: &Path, out: &mut std::collections::BTreeSet<String>) {
  for entry in std::fs::read_dir(dir).unwrap_or_else(|e| panic!("read_dir {}: {e}", dir.display()))
  {
    let entry = entry.expect("read dir entry");
    // Drop OS-generated sidecars (AppleDouble `._*`, `.DS_Store`) at every
    // depth, before the file/dir split — see the doc comment above.
    let name = entry.file_name();
    let name = name.to_string_lossy();
    if name.starts_with("._") || name == ".DS_Store" {
      continue;
    }
    let path = entry.path();
    if entry.file_type().expect("file type").is_dir() {
      collect_files_rel(root, &path, out);
    } else {
      let rel = path
        .strip_prefix(root)
        .expect("walked path is under root")
        .to_str()
        .expect("utf-8 path")
        .replace('\\', "/");
      out.insert(rel);
    }
  }
}

// ── Model-gate visibility (#61) ─────────────────────────────────────────────
//
// NOT `#[ignore]`d, deliberately. This is the ordinary-run half of the gate
// accounting: an ignored-ONLY run (`-- --ignored`, what every CI gate uses)
// never selects it, and it never appears in an ignored-only `--list`, so the
// anti-vacuum counts those gates take are unchanged. What it adds is the case
// no gate covers — a plain, modelless run — where the skipped gates otherwise
// say nothing but `ignored`. Mechanism, and what it does and does not refuse,
// in the shared module.
#[path = "../../support/model_gate_report.rs"]
mod model_gate_report;

/// Reports how many of this binary's tests are `#[ignore]`d granite model gates
/// that did not run, and whether the models root they read is on disk.
#[test]
fn model_gate_report() {
  model_gate_report::report(&[("EMBEDKIT_TEST_MODELS", models_dir())]);
}