haematite-cli 0.21.8

//! EMB-E2 (g): search over records embedded as windows beside one-vector
//! records.

use std::num::NonZeroUsize;

use super::{
    FIELD, TestResult, code, create, declare, find, find_within, head, ids, model, put, revision,
};
use crate::serve::access::{Standalone, request};
use crate::serve::storage::Storage;
use crate::serve::storage::embed_job::{Made, MadeWindows, Page, Reach, Window};
use crate::serve::storage::embed_search::HitSpan;

/// Writes each record, then embeds them in one walk to the head: `one` as a
/// vector each, `windowed` as a window for each vector, four bytes long and
/// three apart.
fn embed_windows(
    storage: &mut Storage,
    one: &[(&str, [f32; 2])],
    windowed: &[(&str, &[[f32; 2]])],
) -> TestResult {
    for (id, _vector) in one {
        put(storage, id, "text")?;
    }
    for (id, _vectors) in windowed {
        put(storage, id, "a long text")?;
    }
    let mut made = Vec::new();
    for (id, vector) in one {
        made.push(Made {
            id: (*id).to_owned(),
            made_from: revision(storage, id)?,
            vector: vector.to_vec(),
        });
    }
    let mut covered = Vec::new();
    for (id, vectors) in windowed {
        let windows = (0_u64..)
            .zip(vectors.iter())
            .map(|(index, vector)| Window {
                start: index * 3,
                end: index * 3 + 4,
                vector: vector.to_vec(),
            })
            .collect();
        covered.push(MadeWindows {
            id: (*id).to_owned(),
            made_from: revision(storage, id)?,
            windows,
        });
    }
    let page = Page {
        collection: "notes".to_owned(),
        field: FIELD.to_owned(),
        model: model(),
        made,
        removed: Vec::new(),
        cleared: Vec::new(),
        windowed: covered,
        toward: head(storage)?.version,
        reach: Reach::End,
    };
    storage.embed_commit("main", &page, NonZeroUsize::MIN.saturating_add(63))?;
    Ok(())
}

/// Fails while the tail of a long text is not searched: the question
/// matches only `n-long`'s last window, and it is the hit, at that span.
#[test]
fn a_search_for_words_only_in_the_last_window_finds_the_record() -> TestResult {
    let (_directory, mut storage) = create()?;
    declare(&mut storage)?;
    let one = [("n-a", [0.0, 1.0]), ("n-b", [-1.0, 0.0])];
    let long: &[[f32; 2]] = &[[0.0, 1.0], [0.0, 1.0], [1.0, 0.0]];
    embed_windows(&mut storage, &one, &[("n-long", long)])?;
    let found = find(&storage, &request(&Standalone)?, &[1.0, 0.0], 1)?;
    assert_eq!(ids(&found), ["n-long"]);
    let span = found.hits.first().and_then(|hit| hit.span);
    assert_eq!(span, Some(HitSpan { start: 6, end: 10 }));
    assert_eq!(
        (found.compared, found.stale),
        (5, 0),
        "each window is compared"
    );
    Ok(())
}

/// Fails on a record answered more than once, or at a worse window: three
/// windows, the second exactly the question, give one hit scored 1.
#[test]
fn a_record_is_answered_once_at_its_best_window() -> TestResult {
    let (_directory, mut storage) = create()?;
    declare(&mut storage)?;
    let long: &[[f32; 2]] = &[[1.0, 1.0], [1.0, 0.0], [1.0, 0.5]];
    embed_windows(&mut storage, &[("n-2", [0.0, 1.0])], &[("n-long", long)])?;
    let found = find(&storage, &request(&Standalone)?, &[1.0, 0.0], 5)?;
    assert_eq!(ids(&found), ["n-long", "n-2"]);
    let best = found.hits.first().ok_or("no hit")?;
    assert!((best.score - 1.0).abs() < 1e-9, "{best:?}");
    assert_eq!(best.span, Some(HitSpan { start: 3, end: 7 }));
    assert_eq!(found.hits.get(1).and_then(|hit| hit.span), None);
    Ok(())
}

/// Fails while `query_scan_records` bounds records rather than vectors:
/// two short records and one of three windows are five vectors, refused
/// within four and answered within five.
#[test]
fn the_bound_counts_windows_not_records() -> TestResult {
    let (_directory, mut storage) = create()?;
    declare(&mut storage)?;
    let one = [("n-a", [0.0, 1.0]), ("n-b", [1.0, 0.0])];
    let long: &[[f32; 2]] = &[[0.0, 1.0], [0.0, 1.0], [1.0, 0.0]];
    embed_windows(&mut storage, &one, &[("n-long", long)])?;
    let access = request(&Standalone)?;
    let refused = find_within(&storage, &access, &[1.0, 0.0], 5, 4);
    assert_eq!(code(refused).as_deref(), Some("query_scan_limit"));
    let found = find_within(&storage, &access, &[1.0, 0.0], 5, 5)?;
    assert_eq!(found.compared, 5);
    Ok(())
}

/// Fails while a changed long text's windows are compared as current:
/// each is left out and counted stale.
#[test]
fn a_changed_long_texts_windows_are_stale() -> TestResult {
    let (_directory, mut storage) = create()?;
    declare(&mut storage)?;
    let long: &[[f32; 2]] = &[[0.0, 1.0], [1.0, 0.0]];
    embed_windows(&mut storage, &[("n-a", [1.0, 0.0])], &[("n-long", long)])?;
    put(&mut storage, "n-long", "another long text")?;
    let found = find(&storage, &request(&Standalone)?, &[1.0, 0.0], 5)?;
    assert_eq!(ids(&found), ["n-a"]);
    assert_eq!((found.compared, found.stale), (1, 2));
    Ok(())
}