cora-code 0.12.0

CLI-first AI code review — BYOK, diff/scan/branch, pre-commit hooks
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

//! Bag-of-tokens embedding engine.
//!
//! Provides two backends:
//!
//! 1. **Hashing trick** (`tokens` module) — zero-dependency, 256-dim, fast but
//!    low-quality.  Suitable for dedup / near-duplicate detection.
//!
//! 2. **Pre-trained nomic-embed-code** (`token_vocab` module) — real 768-dim
//!    embeddings distilled from [nomic-ai/nomic-embed-code](https://huggingface.co/nomic-ai/nomic-embed-code),
//!    compiled into the binary via `include_bytes!` / `include_str!`.
//!    Higher quality at the cost of ~30 MB binary size.
//!
//! The [`embed_code_dispatch`] function selects the best available backend at
//! compile time: pretrained-embed (768d) → hashing trick (256d) → FTS5-only.

pub mod tokens;

// Pre-trained nomic-embed-code module — only compiled when vendored data
// exists. Excluded from crates.io package (too large for 10 MB upload limit).
#[cfg(feature = "pretrained-embed")]
pub mod token_vocab;

// Re-export tokenizer — used by both backends.
pub use tokens::EMBEDDING_DIM;

// Re-export pretrained constants when feature is enabled.
#[cfg(feature = "pretrained-embed")]
pub use token_vocab::{PRETRAINED_DIM, embed_code_pretrained};

/// Returns the embedding dimensionality used by the active backend.
///
/// - `pretrained-embed` feature → 768
/// - default (hashing trick) → 256
pub const fn active_dims() -> usize {
    #[cfg(feature = "pretrained-embed")]
    {
        PRETRAINED_DIM
    }
    #[cfg(not(feature = "pretrained-embed"))]
    {
        EMBEDDING_DIM
    }
}

/// Returns a human-readable label for the active embedding provider.
pub const fn active_provider_name() -> &'static str {
    #[cfg(feature = "pretrained-embed")]
    {
        "nomic-embed-code (768d, pretrained)"
    }
    #[cfg(not(feature = "pretrained-embed"))]
    {
        "hashing-trick (256d, static)"
    }
}

/// Embed a code snippet using the best available backend.
///
/// Returns an f32 vector that can be passed directly to usearch.
///
/// - **Pretrained path** (`pretrained-embed` feature): tokenises → looks up
///   each token in the nomic vocabulary → accumulates int8 vectors → L2-normalises.
///   Returns 768-dim vector.
///
/// - **Hashing-trick fallback**: tokenises → hashes each token into a
///   pseudo-random 256-dim vector → accumulates → L2-normalises.
///   Returns 256-dim vector.
///
/// Both paths share the same [`tokenize_code`] tokenizer.
pub fn embed_code_dispatch(code: &str) -> Vec<f32> {
    #[cfg(feature = "pretrained-embed")]
    {
        embed_code_pretrained(code)
    }
    #[cfg(not(feature = "pretrained-embed"))]
    {
        let embedding = tokens::embed_code(code);
        embedding.as_slice().iter().map(|&v| v as f32).collect()
    }
}

/// Whether the pretrained embedding backend is available (compile-time).
#[expect(
    dead_code,
    reason = "used by Phase 3+ features; embed module not yet wired at call sites"
)]
pub const fn has_pretrained() -> bool {
    cfg!(feature = "pretrained-embed")
}