hermes-core 1.8.102

Core async search engine library with WASM support
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120

//! Shared ANN index constants and construction helpers.
//!
//! Constants are available on all platforms (including WASM).
//! Builder/serialization functions are native-only.

/// Index type discriminants stored in the vectors file TOC.
/// Type 2 was IVF-PQ, removed after IVF-TQ superseded it; the loader still
/// recognizes it to fail with an actionable message. Never reuse it.
pub const LEGACY_IVF_PQ_TYPE: u8 = 2;
pub const FLAT_TYPE: u8 = 4;
/// Binary IVF payload backed by an index-level global quantizer.
pub const BINARY_IVF_TYPE: u8 = 6;
/// TurboQuant flat payload; training-free, no global artifacts.
pub const TQ_FLAT_TYPE: u8 = 7;
/// IVF-TQ payload: trained coarse router, TurboQuant residual leaves.
pub const IVF_TQ_TYPE: u8 = 8;

// --- Native-only builder/serialization functions ---

#[cfg(feature = "native")]
use crate::structures::CoarseCentroids;

/// Encode one segment's vectors into IVF-TQ leaves against the trained global
/// cosine-space centroids.
#[cfg(feature = "native")]
pub fn build_ivf_tq(
    dim: usize,
    routing: crate::dsl::IvfRoutingMode,
    centroids: &CoarseCentroids,
    doc_id_ordinals: &[(u32, u16)],
    vectors: &[f32],
) -> crate::Result<Vec<u8>> {
    if !crate::structures::is_ivf_tq_cosine_generation(centroids.version) {
        return Err(crate::Error::Corruption(
            "legacy raw IVF-TQ centroids cannot encode a correct cosine index; \
             rebuild the index with a current Hermes version"
                .into(),
        ));
    }
    let codec = crate::structures::vector::quantization::tq_shared_codec(dim);
    let mut index = crate::structures::IvfTqIndex::new(dim, routing, centroids.version, codec);
    index
        .add_vectors_parallel(centroids, doc_id_ordinals, vectors)
        .map_err(|error| crate::Error::Internal(format!("IVF-TQ encode failed: {error}")))?;
    let mut bytes = Vec::new();
    crate::segment::ann_disk::write_built_ivf_tq(&index, centroids.num_clusters, &mut bytes)
        .map_err(crate::Error::Io)?;
    Ok(bytes)
}

/// Encode one segment's vectors with the training-free TurboQuant codec.
#[cfg(feature = "native")]
pub fn build_tq_flat(
    dim: usize,
    doc_id_ordinals: &[(u32, u16)],
    vectors: &[f32],
) -> crate::Result<Vec<u8>> {
    let codec = crate::structures::vector::quantization::tq_shared_codec(dim);
    let mut builder = crate::structures::TqFlatBuilder::new(codec);
    builder
        .add_batch(doc_id_ordinals, vectors)
        .map_err(|error| crate::Error::Internal(format!("TQ encode failed: {error}")))?;
    builder.finish();
    let mut bytes = Vec::new();
    crate::segment::ann_disk::write_built_tq_flat(&builder, &mut bytes)
        .map_err(crate::Error::Io)?;
    Ok(bytes)
}

#[cfg(all(test, feature = "native"))]
mod tests {
    use super::*;
    use crate::directories::OwnedBytes;
    use crate::dsl::IvfRoutingMode;
    use crate::segment::ann_disk::{AnnDiskIndex, AnnKind};

    #[test]
    fn ivf_tq_header_preserves_cosine_generation_marker() {
        let marked = crate::structures::mark_ivf_tq_cosine_generation(7);
        let centroids = CoarseCentroids {
            num_clusters: 1,
            dim: 2,
            centroids: vec![1.0, 0.0],
            version: marked,
            soar_config: None,
            routing_index: None,
        };
        let bytes = build_ivf_tq(
            2,
            IvfRoutingMode::Flat,
            &centroids,
            &[(0, 0)],
            &[100.0, 0.0],
        )
        .unwrap();
        let disk = AnnDiskIndex::open(OwnedBytes::new(bytes), AnnKind::IvfTq, 1).unwrap();

        assert_eq!(disk.header().quantizer_version, marked);
        assert!(crate::structures::is_ivf_tq_cosine_generation(
            disk.header().quantizer_version
        ));
    }

    #[test]
    fn ivf_tq_build_rejects_legacy_raw_generation() {
        let centroids = CoarseCentroids {
            num_clusters: 1,
            dim: 2,
            centroids: vec![1.0, 0.0],
            version: 7,
            soar_config: None,
            routing_index: None,
        };
        let error = build_ivf_tq(2, IvfRoutingMode::Flat, &centroids, &[(0, 0)], &[1.0, 0.0])
            .expect_err("legacy IVF-TQ generation must not encode new segments")
            .to_string();
        assert!(error.contains("legacy raw IVF-TQ"), "{error}");
        assert!(error.contains("rebuild the index"), "{error}");
    }
}