laurus 0.12.0

Unified search library for lexical, vector, and semantic retrieval
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

//! Block-encoder helpers used by [`super::TermDictionaryBuilder::build`].
//!
//! The build pipeline (driven from `dictionary.rs`):
//!
//! 1. Convert the builder's `BTreeMap<String, TermInfo>` to a sorted
//!    `Vec<(String, TermInfo)>`.
//! 2. Split into 128-term blocks (the last block may be partial).
//! 3. For each block, call [`encode_block_into`] to append the block's
//!    bytes to a growing `BlockSection` buffer.
//! 4. Build an [`fst::Map`] keyed by each block's last term, valued by
//!    the block's start offset within `BlockSection`.

#![allow(dead_code)]

use crate::lexical::index::structures::dictionary::BlockMax;

use super::block_max_data::BlockMaxData;
use super::front_coding::encode_block_terms;
use super::term_info_block::{FixedTermInfo, FixedTermInfoBlock};

/// Append the encoded bytes of a single block (BlockHeader + TermBytes
/// + FixedTermInfoBlock + BlockMaxData) to `out`.
///
/// The block layout matches [`super::block_reader`]'s parser. See the
/// module-level doc on `block_reader.rs` for the exact byte layout.
///
/// # Arguments
///
/// * `out` - target byte buffer; the encoded block is appended at the
///   current end.
/// * `terms` - sorted term byte slices for this block (`1..=128`
///   entries).
/// * `fixed_infos` - parallel-indexed fixed-size `TermInfo` portion
///   for each term.
/// * `block_max_per_term` - parallel-indexed variable-length
///   `BlockMax` array for each term (each may be empty).
///
/// # Panics
///
/// Panics if the three input slices have different lengths or if
/// `terms.len()` is outside `1..=128`.
pub(super) fn encode_block_into(
    out: &mut Vec<u8>,
    terms: &[&[u8]],
    fixed_infos: &[FixedTermInfo],
    block_max_per_term: &[Vec<BlockMax>],
) {
    assert_eq!(
        terms.len(),
        fixed_infos.len(),
        "terms and fixed_infos must have equal length"
    );
    assert_eq!(
        terms.len(),
        block_max_per_term.len(),
        "terms and block_max_per_term must have equal length"
    );
    assert!(
        !terms.is_empty() && terms.len() <= 128,
        "terms.len() must be in 1..=128"
    );

    let term_count = terms.len();

    // BlockHeader: term_count varint.
    write_varint_into(out, term_count as u64);

    // TermBytes: front-coded.
    let fc = encode_block_terms(terms);
    write_varint_into(out, fc.len() as u64);
    out.extend_from_slice(&fc);

    // FixedTermInfoBlock.
    let ftib = FixedTermInfoBlock::encode(fixed_infos);
    out.extend_from_slice(&ftib.block_min_posting_offset.to_le_bytes());
    out.extend_from_slice(&ftib.block_min_posting_length.to_le_bytes());
    out.extend_from_slice(&ftib.block_min_doc_frequency.to_le_bytes());
    out.extend_from_slice(&ftib.block_min_total_frequency.to_le_bytes());
    out.extend_from_slice(&ftib.ref_max_score_factor.to_le_bytes());
    out.push(ftib.posting_offset_nbits);
    out.push(ftib.posting_length_nbits);
    out.push(ftib.doc_frequency_nbits);
    out.push(ftib.total_frequency_nbits);
    out.push(ftib.max_score_factor_size);
    out.extend_from_slice(&(ftib.payload.len() as u32).to_le_bytes());
    out.extend_from_slice(&ftib.payload);

    // BlockMaxData.
    let bmd = BlockMaxData::encode(block_max_per_term);
    for off in &bmd.offsets {
        out.extend_from_slice(&off.to_le_bytes());
    }
    out.extend_from_slice(&(bmd.data.len() as u32).to_le_bytes());
    out.extend_from_slice(&bmd.data);
}

/// Append the unsigned LEB128 (uleb128) encoding of `value` to `buf`.
///
/// Mirrors the reader-side decoder in
/// [`super::block_reader::read_varint`]. Kept private to the builder
/// module so the writer/reader pair stays in lockstep.
fn write_varint_into(buf: &mut Vec<u8>, mut value: u64) {
    while value >= 0x80 {
        buf.push(((value as u8) & 0x7F) | 0x80);
        value >>= 7;
    }
    buf.push(value as u8);
}