uqa-scoring 0.3.6

BM25, Bayesian BM25, WAND, BMW, calibration
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//

//! Query-local corpus statistics and durable scorer identities.

use crate::{BM25Params, ScoringMode};
use std::collections::BTreeSet;
use uqa_core::IndexStats;
use uqa_storage::{InvertedIndex, StorageBackendResult, TokenTermKey, DEFAULT_BLOCK_SIZE};

pub(super) fn search_stats_for_terms(
    index: &dyn InvertedIndex,
    field: &str,
    terms: &[TokenTermKey],
    doc_freqs: &[u64],
) -> StorageBackendResult<IndexStats> {
    // The scalar variant skips the vocabulary-wide doc-freq map; every
    // term this query scores gets its frequency set explicitly below.
    let mut stats = index.field_stats_scalar(field)?;

    let mut seen = BTreeSet::<&TokenTermKey>::new();
    for (term, doc_freq) in terms.iter().zip(doc_freqs) {
        if seen.insert(term) {
            stats.set_doc_freq_utf16(field.to_string(), term.to_term().into_utf16(), *doc_freq);
        }
    }
    Ok(stats)
}

pub(super) fn raw_bm25_params(mode: &ScoringMode) -> BM25Params {
    match mode {
        ScoringMode::BM25(params) => *params,
        ScoringMode::BayesianBM25(params) => params.bm25,
    }
}

/// Stable identity for every value used while materializing raw BM25 block
/// bounds. Term document frequency is already reflected in each stored score;
/// posting mutations atomically invalidate the materialization.
pub(super) fn block_max_scorer_fingerprint(params: BM25Params, stats: &IndexStats) -> String {
    format!(
        "bm25-block-v1:block={DEFAULT_BLOCK_SIZE}:k1={:016x}:b={:016x}:boost={:016x}:docs={}:avgdl={:016x}",
        params.k1.to_bits(),
        params.b.to_bits(),
        params.boost.to_bits(),
        stats.total_docs,
        stats.avg_doc_length.to_bits(),
    )
}