use core::num::NonZeroUsize;
use sqlx::Sqlite;
use super::corpus::{
SEEDED, Segment, SegmentFilter, SegmentKey, SegmentRow, forget, shorter_than, store,
};
use crate::{
Connected, Field, FulltextRecord, FulltextStore, FulltextSubset, FulltextTable, Identifiable,
Ranked, SlotHandle, SlotName, TantivyDoc, TantivyFulltext, TantivyStore, Transformer,
retrieve::{
EqSeen, FulltextQuery, HashSeen, RetrieveError, Seen, SqlGather, filled, keyed, ranked_keys,
},
};
pub(super) struct Transcript {
key: SegmentKey,
body: String,
}
impl Transcript {
fn of(media: &str, id: i64, body: &str) -> Self {
Self {
key: SegmentKey {
media: media.to_owned(),
id,
},
body: body.to_owned(),
}
}
}
impl Identifiable for Transcript {
type Id = <SegmentKey as Identifiable>::Id;
fn id_ref(&self) -> &i64 {
Identifiable::id_ref(&self.key)
}
}
impl Transformer for Transcript {
type Key = SegmentKey;
type Lineage = <Segment as Transformer>::Lineage;
type Record = ();
type Graphql = ();
fn key(&self) -> SegmentKey {
self.key.clone()
}
}
impl FulltextRecord for Transcript {
type Document = String;
fn document(&self) -> &String {
&self.body
}
}
impl FulltextTable for Transcript {
const SLOT: &'static SlotName = SlotName::new("transcript");
}
fn entry<R>(record: &R) -> TantivyDoc
where
R: FulltextTable + FulltextRecord<Document: AsRef<str>>,
R::Id: Field,
R::Key: Field,
{
TantivyDoc::new(R::SLOT.as_ref())
.with_identity(record.id_ref())
.with_address(&record.key())
.with_body(record.document().as_ref())
}
pub(super) async fn index(
extra: &[Transcript],
) -> (
tempfile::TempDir,
TantivyStore,
SlotHandle<TantivyFulltext, Transcript>,
) {
let dir = tempfile::tempdir().expect("a temporary directory");
let store = TantivyStore::open(dir.path()).expect("the store opens");
let mut face = SlotHandle::<_, Transcript>::mint(store.fulltext("transcript"));
{
let mut writer = store
.writer("transcript", TantivyStore::MIN_WRITER_HEAP_BYTES)
.expect("the slot's writer opens");
for (media, id, transcript, _) in SEEDED {
face
.upsert_document(
Connected::new(&store, &mut writer),
entry(&Transcript::of(media, *id, transcript)),
)
.await
.expect("the document queues");
}
for document in extra {
face
.upsert_document(Connected::new(&store, &mut writer), entry(document))
.await
.expect("the document queues");
}
writer.commit().expect("the batch commits");
}
store.reload("transcript").expect("the reader reloads");
(dir, store, face)
}
pub(super) async fn retrieve_keys<D>(
slot: &mut SlotHandle<TantivyFulltext, Transcript>,
store: &TantivyStore,
query: &FulltextQuery<f32>,
limit: usize,
) -> Vec<Ranked<SegmentKey, f32>>
where
D: Seen<SegmentKey>,
{
let Some(k) = NonZeroUsize::new(limit) else {
return Vec::new();
};
let hits = slot
.search(
Connected::new(store, ()),
FulltextSubset::All,
query.text(),
k,
None,
)
.await
.expect("the search runs");
let hits = keyed::<TantivyFulltext, SegmentKey>(hits).expect("the addresses read back");
ranked_keys::<_, _, D>(hits, query.floor(), limit)
}
async fn retrieve(
slot: &mut SlotHandle<TantivyFulltext, Transcript>,
index: &TantivyStore,
db: &mut sqlx::sqlite::SqliteConnection,
query: &FulltextQuery<f32>,
filter: Option<&SegmentFilter<Sqlite>>,
limit: usize,
) -> Result<
Vec<Ranked<Segment, f32>>,
RetrieveError<crate::TantivyFulltextError, crate::query::sql::SqlQueryError>,
> {
let ranked = retrieve_keys::<HashSeen<_>>(slot, index, query, usize::MAX).await;
let mut gather = SqlGather::<SegmentRow, Sqlite>::new(db);
filled(ranked, &mut gather, filter, limit)
.await
.map_err(RetrieveError::Gather)
}
fn ids(page: &[Ranked<Segment, f32>]) -> Vec<i64> {
page.iter().map(|hit| hit.node_ref().id).collect()
}
fn ranks(page: &[Ranked<Segment, f32>]) -> Vec<u64> {
page.iter().map(Ranked::rank).collect()
}
#[tokio::test]
async fn a_german_row_holding_the_english_term_is_a_hit() {
let (_dir, index, mut engine) = index(&[]).await;
let ranked = retrieve_keys::<HashSeen<_>>(
&mut engine,
&index,
&FulltextQuery::new("sunset", 0.0_f32),
usize::MAX,
)
.await;
let mut hit: Vec<i64> = ranked.iter().map(|hit| hit.node_ref().id).collect();
hit.sort_unstable();
assert_eq!(hit, vec![1, 2, 3, 4, 6]);
assert!(
ranked
.windows(2)
.all(|pair| pair[0].score() >= pair[1].score()),
"{ranked:?}"
);
}
#[tokio::test]
async fn the_floor_shortens_a_live_answer() {
let (_dir, index, mut engine) = index(&[]).await;
let all = retrieve_keys::<HashSeen<_>>(
&mut engine,
&index,
&FulltextQuery::new("sunset", 0.0_f32),
usize::MAX,
)
.await;
assert_eq!(all.len(), 5);
let floor = (all[1].score() + all[2].score()) / 2.0;
let floored = retrieve_keys::<HashSeen<_>>(
&mut engine,
&index,
&FulltextQuery::new("sunset", floor),
usize::MAX,
)
.await;
assert_eq!(floored.len(), 2);
assert_eq!(
floored.iter().map(Ranked::rank).collect::<Vec<_>>(),
vec![1, 2]
);
assert_eq!(
floored
.iter()
.map(|hit| hit.node_ref().id)
.collect::<Vec<_>>(),
all[..2]
.iter()
.map(|hit| hit.node_ref().id)
.collect::<Vec<_>>()
);
}
#[tokio::test]
async fn a_duplicate_key_collapses_and_leaves_its_rank_behind() {
struct Echo(Transcript);
impl Identifiable for Echo {
type Id = i64;
fn id_ref(&self) -> &i64 {
self.0.id_ref()
}
}
impl Transformer for Echo {
type Key = SegmentKey;
type Lineage = <Segment as Transformer>::Lineage;
type Record = ();
type Graphql = ();
fn key(&self) -> SegmentKey {
self.0.key()
}
}
impl FulltextRecord for Echo {
type Document = String;
fn document(&self) -> &String {
self.0.document()
}
}
impl FulltextTable for Echo {
const SLOT: &'static SlotName = SlotName::new("echo");
}
let dir = tempfile::tempdir().expect("a temporary directory");
let index = TantivyStore::open(dir.path()).expect("the store opens");
let mut engine = SlotHandle::<_, Transcript>::mint(index.fulltext("transcript"));
{
let mut writer = index
.writer("transcript", TantivyStore::MIN_WRITER_HEAP_BYTES)
.expect("the slot's writer opens");
for (media, id, transcript, _) in SEEDED {
engine
.upsert_document(
Connected::new(&index, &mut writer),
entry(&Transcript::of(media, *id, transcript)),
)
.await
.expect("the document queues");
}
engine
.upsert_document(
Connected::new(&index, &mut writer),
entry(&Echo(Transcript::of(
"m3",
6,
"sunset sunset sunset sunset",
))),
)
.await
.expect("the document queues");
writer.commit().expect("the batch commits");
}
index.reload("transcript").expect("the reader reloads");
let raw = keyed::<TantivyFulltext, SegmentKey>(
engine
.search(
Connected::new(&index, ()),
FulltextSubset::All,
"sunset",
NonZeroUsize::new(10).expect("10 is not zero"),
None,
)
.await
.expect("the search runs"),
)
.expect("the addresses read back");
assert_eq!(raw.len(), 6, "the slot holds two entries at one key");
let duplicated = raw.iter().filter(|hit| hit.data_ref().id == 6).count();
assert_eq!(duplicated, 2);
let ranked = retrieve_keys::<HashSeen<_>>(
&mut engine,
&index,
&FulltextQuery::new("sunset", 0.0_f32),
usize::MAX,
)
.await;
assert_eq!(ranked.len(), 5, "the duplicate collapsed");
assert_eq!(
ranked.iter().filter(|hit| hit.node_ref().id == 6).count(),
1
);
let worn: Vec<u64> = ranked.iter().map(Ranked::rank).collect();
assert_eq!(worn.len(), 5);
assert!(worn.iter().max().is_some_and(|last| *last == 6), "{worn:?}");
let kept = ranked
.iter()
.find(|hit| hit.node_ref().id == 6)
.expect("the survivor");
let best = raw
.iter()
.filter(|hit| hit.data_ref().id == 6)
.map(|hit| hit.score())
.fold(f32::MIN, f32::max);
assert_eq!(kept.score(), best);
}
#[tokio::test]
async fn tier_b_answers_whole_entities_in_engine_order() {
let (_dir, index, mut engine) = index(&[]).await;
let mut db = store().await;
let keys = retrieve_keys::<HashSeen<_>>(
&mut engine,
&index,
&FulltextQuery::new("sunset", 0.0_f32),
usize::MAX,
)
.await;
let page = retrieve(
&mut engine,
&index,
&mut db,
&FulltextQuery::new("sunset", 0.0_f32),
None,
10,
)
.await
.expect("the retrieve runs");
assert_eq!(page.len(), 5);
assert!(page[0].node_ref().transcript.contains("sunset"));
assert!(page[0].node_ref().duration > 0);
assert_eq!(
ids(&page),
keys.iter().map(|hit| hit.node_ref().id).collect::<Vec<_>>()
);
assert_eq!(
ranks(&page),
keys.iter().map(Ranked::rank).collect::<Vec<_>>()
);
}
#[tokio::test]
async fn a_filter_forces_a_second_key_batch() {
let (_dir, index, mut engine) = index(&[]).await;
let mut db = store().await;
let short = shorter_than(15);
let unfiltered = retrieve(
&mut engine,
&index,
&mut db,
&FulltextQuery::new("sunset", 0.0_f32),
None,
10,
)
.await
.expect("the retrieve runs");
let page = retrieve(
&mut engine,
&index,
&mut db,
&FulltextQuery::new("sunset", 0.0_f32),
Some(&short),
2,
)
.await
.expect("the retrieve runs");
assert_eq!(page.len(), 2);
let mut kept = ids(&page);
kept.sort_unstable();
assert_eq!(kept, vec![2, 4]);
assert!(page.iter().all(|hit| hit.node_ref().duration <= 15));
for hit in &page {
let raw = unfiltered
.iter()
.find(|all| all.node_ref().id == hit.node_ref().id)
.expect("the same hit, unfiltered");
assert_eq!(hit.rank(), raw.rank());
}
}
#[tokio::test]
async fn a_store_missing_row_is_skipped_and_the_page_fills() {
let (_dir, index, mut engine) = index(&[]).await;
let mut db = store().await;
let whole = retrieve(
&mut engine,
&index,
&mut db,
&FulltextQuery::new("sunset", 0.0_f32),
None,
10,
)
.await
.expect("the retrieve runs");
let lost = whole[0].node_ref().id;
forget(&mut db, lost).await;
let page = retrieve(
&mut engine,
&index,
&mut db,
&FulltextQuery::new("sunset", 0.0_f32),
None,
10,
)
.await
.expect("the retrieve runs");
assert_eq!(page.len(), whole.len() - 1);
assert!(!ids(&page).contains(&lost));
for hit in &page {
let before = whole
.iter()
.find(|all| all.node_ref().id == hit.node_ref().id)
.expect("the same hit, before the loss");
assert_eq!(hit.rank(), before.rank());
}
}
#[tokio::test]
async fn a_floored_retrieve_answers_short() {
let (_dir, index, mut engine) = index(&[]).await;
let mut db = store().await;
let all = retrieve_keys::<HashSeen<_>>(
&mut engine,
&index,
&FulltextQuery::new("sunset", 0.0_f32),
usize::MAX,
)
.await;
let floor = (all[0].score() + all[1].score()) / 2.0;
let page = retrieve(
&mut engine,
&index,
&mut db,
&FulltextQuery::new("sunset", floor),
None,
10,
)
.await
.expect("the retrieve runs");
assert_eq!(page.len(), 1, "asked for ten, and one cleared the floor");
assert_eq!(page[0].rank(), 1);
}
#[tokio::test]
async fn the_equality_mechanism_answers_the_same_live_stream() {
let (_dir, index, mut engine) = index(&[]).await;
let query = FulltextQuery::new("sunset", 0.0_f32);
let hashed = retrieve_keys::<HashSeen<_>>(&mut engine, &index, &query, usize::MAX).await;
let listed = retrieve_keys::<EqSeen<_>>(&mut engine, &index, &query, usize::MAX).await;
assert_eq!(hashed, listed);
}