uqa-scoring 0.6.2

BM25, Bayesian BM25, WAND, BMW, calibration
Documentation
//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//

//! Cursor-based exhaustive scoring using authoritative occurrence frequencies.

use super::{search_stats_for_terms, storage_error, TextSearchError};
use crate::{BM25Scorer, BayesianBM25Scorer, Scorer, ScoringMode};
use std::sync::Arc;
use uqa_core::ScoredEntry;
use uqa_storage::clustered_postings::{OwnedPostingReadCursor, PostingReadCursor};
use uqa_storage::{InvertedIndex, TokenTermKey};

#[cfg(test)]
mod tests;

trait ScoreOutput {
    fn borrows_index(&self) -> bool {
        false
    }
    fn reserve_single_term(&mut self, _: usize) {}
    fn push(&mut self, entry: ScoredEntry) -> bool;
}

impl ScoreOutput for Vec<ScoredEntry> {
    fn reserve_single_term(&mut self, capacity: usize) {
        self.reserve_exact(capacity);
    }
    fn push(&mut self, entry: ScoredEntry) -> bool {
        Vec::push(self, entry);
        true
    }
}

struct Visitor<'a>(&'a mut dyn FnMut(ScoredEntry) -> bool);

impl ScoreOutput for Visitor<'_> {
    fn borrows_index(&self) -> bool {
        true
    }
    fn push(&mut self, entry: ScoredEntry) -> bool {
        self.0(entry)
    }
}

pub(super) fn visit_text_terms(
    index: &dyn InvertedIndex,
    field: &str,
    terms: &[TokenTermKey],
    mode: &ScoringMode,
    visitor: &mut dyn FnMut(ScoredEntry) -> bool,
) -> Result<bool, TextSearchError> {
    let output = &mut Visitor(visitor);
    match terms.len() {
        0 => Ok(true),
        1 => visit_single_text_term(index, field, terms, mode, output),
        _ => visit_multiple_text_terms(index, field, terms, mode, output),
    }
}

pub(super) fn build_text_scorer(
    mode: &ScoringMode,
    stats_arc: Arc<uqa_core::IndexStats>,
    query_term_count: usize,
) -> Result<Arc<dyn Scorer>, TextSearchError> {
    Ok(match mode {
        ScoringMode::BM25(p) => {
            p.validate().map_err(TextSearchError::Parameters)?;
            Arc::new(BM25Scorer::new(*p, stats_arc))
        }
        ScoringMode::BayesianBM25(p) => Arc::new(
            BayesianBM25Scorer::new(p.scaled_for_query_terms(query_term_count), stats_arc)
                .map_err(TextSearchError::Parameters)?,
        ),
    })
}

pub(super) fn score_single_text_term(
    index: &dyn InvertedIndex,
    field: &str,
    analyzed_terms: &[TokenTermKey],
    mode: &ScoringMode,
) -> Result<Vec<ScoredEntry>, TextSearchError> {
    let mut entries = Vec::new();
    visit_single_text_term(index, field, analyzed_terms, mode, &mut entries)?;
    Ok(entries)
}

fn visit_single_text_term(
    index: &dyn InvertedIndex,
    field: &str,
    analyzed_terms: &[TokenTermKey],
    mode: &ScoringMode,
    output: &mut impl ScoreOutput,
) -> Result<bool, TextSearchError> {
    let term = &analyzed_terms[0];
    let mut cursor: Box<dyn PostingReadCursor + '_> = if output.borrows_index() {
        index.posting_read_cursor_key(field, term)
    } else {
        index
            .posting_cursor_key(field, term)
            .map(|cursor| Box::new(OwnedPostingReadCursor(cursor)) as Box<dyn PostingReadCursor>)
    }
    .map_err(|error| storage_error("open term score cursor", error))?;
    let document_frequency = cursor.doc_freq();
    let term_doc_freqs = [document_frequency];
    let stats = Arc::new(
        search_stats_for_terms(index, field, analyzed_terms, &term_doc_freqs)
            .map_err(|error| storage_error("read field statistics", error))?,
    );
    let scorer = build_text_scorer(mode, stats, 1)?;
    let idf = scorer.idf(document_frequency);
    let scored_capacity = usize::try_from(document_frequency).map_err(|_| {
        TextSearchError::InvalidIndex("text document frequency exceeds usize".into())
    })?;
    output.reserve_single_term(scored_capacity);
    while let Some(entry) = cursor.current() {
        if !output.push(ScoredEntry {
            doc_id: entry.doc_id,
            score: scorer.finalize_score(&[scorer.term_score_with_idf(
                entry.term_freq,
                entry.doc_length,
                idf,
            )]),
        }) {
            return Ok(false);
        }
        cursor
            .advance()
            .map_err(|error| storage_error("advance term score cursor", error))?;
    }
    Ok(true)
}

pub(super) fn score_multiple_text_terms(
    index: &dyn InvertedIndex,
    field: &str,
    analyzed_terms: &[TokenTermKey],
    mode: &ScoringMode,
) -> Result<Vec<ScoredEntry>, TextSearchError> {
    let mut entries = Vec::new();
    visit_multiple_text_terms(index, field, analyzed_terms, mode, &mut entries)?;
    Ok(entries)
}

fn visit_multiple_text_terms(
    index: &dyn InvertedIndex,
    field: &str,
    analyzed_terms: &[TokenTermKey],
    mode: &ScoringMode,
    output: &mut impl ScoreOutput,
) -> Result<bool, TextSearchError> {
    let mut cursors: Vec<Box<dyn PostingReadCursor + '_>> = if output.borrows_index() {
        analyzed_terms
            .iter()
            .map(|term| index.posting_read_cursor_key(field, term))
            .collect::<Result<_, _>>()
    } else {
        index
            .posting_cursors_keys_bulk(field, analyzed_terms)
            .map(|cursors| {
                cursors
                    .into_iter()
                    .map(|cursor| {
                        Box::new(OwnedPostingReadCursor(cursor)) as Box<dyn PostingReadCursor>
                    })
                    .collect()
            })
    }
    .map_err(|error| storage_error("open term score cursors", error))?;
    let term_doc_freqs = cursors
        .iter()
        .map(|cursor| cursor.doc_freq())
        .collect::<Vec<_>>();
    let stats = Arc::new(
        search_stats_for_terms(index, field, analyzed_terms, &term_doc_freqs)
            .map_err(|error| storage_error("read field statistics", error))?,
    );
    let scorer = build_text_scorer(mode, stats, analyzed_terms.len())?;
    let term_idfs: Vec<f64> = term_doc_freqs
        .iter()
        .map(|doc_freq| scorer.idf(*doc_freq))
        .collect();
    let mut term_freqs = vec![0_u64; analyzed_terms.len()];
    let mut per_term = vec![0.0; analyzed_terms.len()];
    while let Some(doc_id) = cursors
        .iter()
        .filter_map(|cursor| cursor.current().map(|entry| entry.doc_id))
        .min()
    {
        term_freqs.fill(0);
        let mut candidate_length = None;
        for (term_index, cursor) in cursors.iter_mut().enumerate() {
            let Some(entry) = cursor.current().filter(|entry| entry.doc_id == doc_id) else {
                continue;
            };
            let doc_length = entry.doc_length;
            if let Some(previous) = candidate_length {
                if previous != doc_length {
                    return Err(TextSearchError::InvalidIndex(format!(
                        "inconsistent indexed document length for document {doc_id}: {previous} and {doc_length}"
                    )));
                }
            } else {
                candidate_length = Some(doc_length);
            }
            term_freqs[term_index] = entry.term_freq;
            cursor
                .advance()
                .map_err(|error| storage_error("advance term score cursor", error))?;
        }
        let doc_length = candidate_length.ok_or_else(|| {
            TextSearchError::InvalidIndex(format!(
                "text cursor merge found no posting for document {doc_id}"
            ))
        })?;
        for ((term_score, term_freq), idf) in per_term.iter_mut().zip(&term_freqs).zip(&term_idfs) {
            *term_score = scorer.term_score_with_idf(*term_freq, doc_length, *idf);
        }
        if !output.push(ScoredEntry {
            doc_id,
            score: scorer.finalize_score(&per_term),
        }) {
            return Ok(false);
        }
    }
    Ok(true)
}