promptforge-tool-picker 0.1.0

//! The embedding model, compiled into this library (crate-private).
//!
//! The build script fetches `BAAI/bge-small-en-v1.5` from the Hugging Face Hub
//! at one pinned commit, verifies each file against a hardcoded SHA-256 digest,
//! downcasts the weights to fp16, and stages the result in `OUT_DIR`. The
//! statics below embed those bytes. None of this is public resolver API: the
//! model payloads, provenance, and dimensions are implementation details behind
//! the opaque [`Model`](crate::Model).

/// The fp16 model weights, as a safetensors blob.
pub(crate) static WEIGHTS_SAFETENSORS: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/model-fp16.safetensors"));

/// The tokenizer, as a Hugging Face `tokenizer.json` document.
pub(crate) static TOKENIZER_JSON: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/tokenizer.json"));

/// The model architecture configuration, as a `config.json` document.
pub(crate) static CONFIG_JSON: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/config.json"));

/// The Hugging Face repository the embedded assets came from.
///
/// Generated by the build script alongside the revision, so the two provenance
/// values share one source of truth and cannot drift. Verified against the
/// embedded weights' safetensors metadata when the model loads.
pub(crate) const SOURCE_REPO: &str = include_str!(concat!(env!("OUT_DIR"), "/repo.txt"));

/// The immutable commit the embedded assets were taken from.
///
/// Verified against the embedded weights' safetensors metadata when the model
/// loads, so a mixed or substituted checkpoint fails rather than re-ranking.
pub(crate) const SOURCE_REVISION: &str = include_str!(concat!(env!("OUT_DIR"), "/revision.txt"));

#[cfg(test)]
mod tests {
    use super::{CONFIG_JSON, SOURCE_REPO, SOURCE_REVISION, TOKENIZER_JSON, WEIGHTS_SAFETENSORS};

    #[test]
    fn weights_are_an_fp16_bert_safetensors_blob() {
        let len = WEIGHTS_SAFETENSORS.len();
        assert!(
            (40_000_000..100_000_000).contains(&len),
            "embedded weights are {len} bytes"
        );
        let header_len = u64::from_le_bytes(
            WEIGHTS_SAFETENSORS[..8]
                .try_into()
                .expect("slice of 8 bytes"),
        );
        let header_len = usize::try_from(header_len).expect("header fits in usize");
        let header: serde_json::Value =
            serde_json::from_slice(&WEIGHTS_SAFETENSORS[8..8 + header_len])
                .expect("safetensors header is JSON");
        assert_eq!(header["embeddings.word_embeddings.weight"]["dtype"], "F16");
        assert_eq!(header["embeddings.position_ids"]["dtype"], "I64");
    }

    #[test]
    fn tokenizer_and_config_parse_as_json() {
        let tokenizer: serde_json::Value =
            serde_json::from_slice(TOKENIZER_JSON).expect("tokenizer.json parses");
        assert!(tokenizer.get("model").is_some());
        let config: serde_json::Value =
            serde_json::from_slice(CONFIG_JSON).expect("config.json parses");
        assert_eq!(config["model_type"], "bert");
        assert_eq!(config["hidden_size"], 384);
    }

    #[test]
    fn provenance_is_a_pinned_commit_from_the_source_repository() {
        assert_eq!(SOURCE_REPO, "BAAI/bge-small-en-v1.5");
        assert_eq!(SOURCE_REVISION.len(), 40);
        assert!(SOURCE_REVISION.chars().all(|c| c.is_ascii_hexdigit()));
    }
}