use core::num::NonZeroUsize;
use std::sync::Arc;
use sqlx::Sqlite;
use super::corpus::{Segment, SegmentKey, SegmentRow, forget, shorter_than, store};
use crate::{
AsyncVectorRead, AsyncVectorWrite, Connected, Embedding, Field, HashBests, HashSeen, Hits,
Ranked, Scored, SlotHandle, VectorColumn, VectorEngine, VectorRecord,
retrieve::{
AsyncGather, FoldSemantics, Folded, FoldedQuery, Max, SqlGather, Sum, VectorQuery, owner_best,
ranked_keys, set_scored, vector_keyed,
},
store::column::testkit::{Col, Fallback, FallbackRead, Refusal},
};
const DIM: usize = 4;
const FRAME_EMBEDDINGS: VectorColumn = VectorColumn::new("frame_embeddings");
struct Keyframes {
key: SegmentKey,
frame_embeddings: Vec<Embedding<f32, DIM>>,
}
impl Keyframes {
fn of(media: &str, id: i64, frame_embeddings: Vec<[f32; DIM]>) -> Self {
Self {
key: SegmentKey {
media: media.to_owned(),
id,
},
frame_embeddings: frame_embeddings
.into_iter()
.map(|coordinates| {
Embedding::new(Arc::from(coordinates.to_vec())).expect("the fixture declares DIM")
})
.collect(),
}
}
}
impl VectorRecord for Keyframes {
type Element = f32;
type Filter = ();
fn vectors(&self) -> impl Iterator<Item = (VectorColumn, &[f32])> {
core::iter::empty()
}
fn vector_sets(&self) -> impl Iterator<Item = (VectorColumn, u32, &[f32])> {
(0u32..)
.zip(&self.frame_embeddings)
.map(|(at, member)| (FRAME_EMBEDDINGS, at, member.coordinates()))
}
}
struct Frames(Vec<Row>);
struct Row {
owner: SegmentKey,
ordinal: u32,
coordinates: Vec<f32>,
}
fn addressed(key: &SegmentKey) -> Vec<Col> {
let mut buf = Frames::buffer();
key.write(&mut buf);
Frames::finalize(buf)
}
impl VectorEngine for Frames {
type Options = ();
type SearchOptions = ();
type Error = Refusal;
type Score = f32;
type Element = f32;
type Address = Vec<Col>;
type Connection = ();
type WriteBuffer = Fallback;
type ReadBuffer = FallbackRead;
fn buffer() -> Self::WriteBuffer {
Fallback::default()
}
fn finalize(buf: Self::WriteBuffer) -> Self::Address {
buf.0
}
fn open(address: Self::Address) -> Self::ReadBuffer {
Fallback(address).read()
}
}
impl AsyncVectorWrite for Frames {
type Doc = Entry;
type WriteContext<'a> = ();
fn upsert(
&self,
_ctx: Connected<'_, (), Self::WriteContext<'_>>,
_doc: Self::Doc,
) -> impl Future<Output = Result<(), Self::Error>> + Send {
core::future::ready(Ok(()))
}
}
impl AsyncVectorRead for Frames {
type ReadContext<'a> = ();
fn knn(
&self,
_ctx: Connected<'_, (), Self::ReadContext<'_>>,
column: VectorColumn,
query: &[Self::Element],
_k: ::core::num::NonZeroUsize,
_options: ::core::option::Option<&Self::SearchOptions>,
) -> impl Future<Output = Result<Hits<Self::Address, Self::Score>, Self::Error>> + Send {
let mut hits: Vec<Scored<Vec<Col>, f32>> = self
.0
.iter()
.filter(|_| column == FRAME_EMBEDDINGS)
.map(|row| Scored::new(addressed(&row.owner), cosine(&row.coordinates, query)))
.collect();
hits.sort_by(|a, b| b.score().total_cmp(&a.score()));
core::future::ready(Ok(hits))
}
fn get(
&self,
_ctx: Connected<'_, (), Self::ReadContext<'_>>,
column: VectorColumn,
address: &Self::Address,
) -> impl Future<Output = Result<Vec<Vec<Self::Element>>, Self::Error>> + Send {
let mut held: Vec<&Row> = self
.0
.iter()
.filter(|row| column == FRAME_EMBEDDINGS && *address == addressed(&row.owner))
.collect();
held.sort_by_key(|row| row.ordinal);
core::future::ready(Ok(
held
.into_iter()
.map(|row| row.coordinates.clone())
.collect(),
))
}
}
struct Entry {
owner: SegmentKey,
members: Vec<(VectorColumn, u32, Vec<f32>)>,
}
impl Frames {
fn upsert_entry(&mut self, entry: Entry) {
for (column, ordinal, coordinates) in entry.members {
assert_eq!(
column, FRAME_EMBEDDINGS,
"the declaration elects one column"
);
self
.0
.retain(|row| !(row.owner == entry.owner && row.ordinal == ordinal));
self.0.push(Row {
owner: entry.owner.clone(),
ordinal,
coordinates,
});
}
}
}
fn cosine(a: &[f32], b: &[f32]) -> f32 {
let dot: f32 = a.iter().zip(b).map(|(x, y)| x * y).sum();
let norm = |v: &[f32]| v.iter().map(|x| x * x).sum::<f32>().sqrt();
let scale = norm(a) * norm(b);
if scale == 0.0 { 0.0 } else { dot / scale }
}
fn corpus() -> [Keyframes; 6] {
[
Keyframes::of("m1", 1, vec![[1.0, 0.6, 0.0, 0.0], [0.6, 1.0, 0.0, 0.0]]),
Keyframes::of("m1", 2, vec![[1.0, 0.02, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0]]),
Keyframes::of("m2", 3, vec![[1.0, 0.45, 0.0, 0.0]]),
Keyframes::of("m2", 4, vec![[1.0, 0.80, 0.0, 0.0]]),
Keyframes::of("m3", 5, vec![[1.0, 1.40, 0.0, 0.0]]),
Keyframes::of("m3", 6, vec![]),
]
}
fn write(engine: &mut Frames, record: &Keyframes) {
engine.upsert_entry(Entry {
owner: record.key.clone(),
members: record
.vector_sets()
.map(|(column, ordinal, coordinates)| (column, ordinal, coordinates.to_vec()))
.collect(),
});
}
fn index() -> SlotHandle<Frames, Keyframes> {
let mut engine = Frames(Vec::new());
for record in &corpus() {
write(&mut engine, record);
}
SlotHandle::mint(engine)
}
fn set_query(directions: Vec<[f32; DIM]>, floor: f32) -> FoldedQuery<f32, f32, DIM> {
FoldedQuery::new(
directions
.into_iter()
.map(|coordinates| {
Embedding::new(Arc::from(coordinates.to_vec())).expect("the fixture declares DIM")
})
.collect(),
floor,
)
.expect("the fixture asks with at least one vector")
}
async fn retrieve_keys_with<F: FoldSemantics<f32>>(
slot: &mut SlotHandle<Frames, Keyframes>,
query: &FoldedQuery<f32, f32, DIM>,
fold: &F,
limit: usize,
) -> Vec<Folded<SegmentKey, f32>> {
let Some(k) = NonZeroUsize::new(limit) else {
return Vec::new();
};
let mut bests = Vec::with_capacity(query.inputs().len());
for coordinates in query.coordinates() {
let hits = slot
.knn(
Connected::new(&(), ()),
FRAME_EMBEDDINGS,
coordinates,
k,
None,
)
.await
.expect("the toy index cannot fail");
let hits = vector_keyed::<Frames, SegmentKey>(hits).expect("the addresses read back");
bests.push(owner_best::<_, _, HashSeen<_>>(hits, query.floor()));
}
set_scored::<_, _, HashBests<_, _>, _>(bests, fold, limit)
}
async fn retrieve_keys(
slot: &mut SlotHandle<Frames, Keyframes>,
query: &VectorQuery<f32, f32, DIM>,
limit: usize,
) -> Vec<Ranked<SegmentKey, f32>> {
let Some(k) = NonZeroUsize::new(limit) else {
return Vec::new();
};
let hits = slot
.knn(
Connected::new(&(), ()),
FRAME_EMBEDDINGS,
query.coordinates(),
k,
None,
)
.await
.expect("the toy index cannot fail");
let hits = vector_keyed::<Frames, SegmentKey>(hits).expect("the addresses read back");
ranked_keys::<_, _, HashSeen<_>>(hits, query.floor(), limit)
}
fn ids<S>(page: &[Folded<SegmentKey, S>]) -> Vec<i64> {
page.iter().map(|hit| hit.node_ref().id).collect()
}
#[tokio::test]
async fn an_entity_with_several_members_is_one_result() {
let mut engine = index();
let page = retrieve_keys_with(
&mut engine,
&set_query(vec![[1.0, 0.0, 0.0, 0.0]], 0.5),
&Sum::new(),
usize::MAX,
)
.await;
let mut owners = ids(&page);
owners.sort_unstable();
owners.dedup();
assert_eq!(owners.len(), ids(&page).len(), "{page:?}");
}
#[tokio::test]
async fn re_writing_a_set_replaces_its_members_rather_than_collapsing_it() {
let mut engine = Frames(Vec::new());
let record = Keyframes::of("m1", 1, vec![[1.0, 0.6, 0.0, 0.0], [0.6, 1.0, 0.0, 0.0]]);
write(&mut engine, &record);
write(&mut engine, &record);
assert_eq!(engine.0.len(), 2, "one row per member, written twice");
let mut ordinals: Vec<u32> = engine.0.iter().map(|row| row.ordinal).collect();
ordinals.sort_unstable();
assert_eq!(ordinals, vec![0, 1]);
}
#[tokio::test]
async fn sum_ranks_the_entity_that_answered_both_directions_first() {
let mut engine = index();
let page = retrieve_keys_with(
&mut engine,
&set_query(vec![[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0]], 0.5),
&Sum::new(),
10,
)
.await;
assert_eq!(ids(&page)[0], 1, "{page:?}");
assert!(ids(&page).contains(&2), "{page:?}");
}
#[tokio::test]
async fn max_ranks_the_single_best_pair_first() {
let mut engine = index();
let page = retrieve_keys_with(
&mut engine,
&set_query(vec![[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0]], 0.5),
&Max::new(),
10,
)
.await;
assert_eq!(ids(&page)[0], 2, "{page:?}");
}
#[tokio::test]
async fn an_entity_with_an_empty_set_is_never_matched() {
let mut engine = index();
let page = retrieve_keys_with(
&mut engine,
&set_query(vec![[1.0, 0.0, 0.0, 0.0]], 0.0),
&Sum::new(),
usize::MAX,
)
.await;
assert!(!ids(&page).contains(&6), "{page:?}");
}
#[tokio::test]
async fn one_query_vector_agrees_with_the_single_vector_road() {
let mut engine = index();
let direction = [1.0, 0.0, 0.0, 0.0];
let floor = 0.5;
let single = retrieve_keys(
&mut engine,
&VectorQuery::new(
Embedding::new(Arc::from(direction.to_vec())).expect("the fixture declares DIM"),
floor,
),
usize::MAX,
)
.await;
async fn agrees<F: FoldSemantics<f32>>(
slot: &mut SlotHandle<Frames, Keyframes>,
query: &FoldedQuery<f32, f32, DIM>,
fold: &F,
single: &[Ranked<SegmentKey, f32>],
) {
let set = retrieve_keys_with(slot, query, fold, usize::MAX).await;
assert_eq!(set.len(), single.len());
for (folded, ranked) in set.iter().zip(single) {
assert_eq!(folded.node_ref(), ranked.node_ref());
assert!((folded.score() - ranked.score()).abs() < f32::EPSILON);
let [Some(only)] = folded.streams() else {
panic!("one stream, and its own slot: {:?}", folded.streams())
};
assert_eq!(
only.ordinal(),
ranked.rank(),
"one stream, and its own ordinal carried whole"
);
assert!(
(only.score() - ranked.score()).abs() < f32::EPSILON,
"the pre-fold score the stream measured is the single-vector road's own",
);
}
}
let query = set_query(vec![direction], floor);
agrees(&mut engine, &query, &Sum::new(), &single).await;
agrees(&mut engine, &query, &Max::new(), &single).await;
}
#[tokio::test]
async fn the_folded_keys_hydrate_from_the_live_store() {
let mut engine = index();
let mut db = store().await;
let page = retrieve_keys_with(
&mut engine,
&set_query(vec![[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0]], 0.5),
&Sum::new(),
10,
)
.await;
let keys: Vec<SegmentKey> = page.iter().map(|hit| hit.node_ref().clone()).collect();
let mut gather = SqlGather::<SegmentRow, Sqlite>::new(&mut db);
let rows: Vec<Option<Segment>> = AsyncGather::gather(&mut gather, &keys, None)
.await
.expect("the live store answers");
assert_eq!(rows.len(), keys.len());
let first = rows[0].as_ref().expect("the store holds segment 1");
assert_eq!(first.id, 1);
assert_eq!(first.transcript, "a long sunset over the harbour");
}
#[tokio::test]
async fn the_gather_prunes_and_forgets_the_same_way_under_a_set_score() {
let mut engine = index();
let mut db = store().await;
forget(&mut db, 2).await;
let page = retrieve_keys_with(
&mut engine,
&set_query(vec![[1.0, 0.0, 0.0, 0.0]], 0.5),
&Sum::new(),
usize::MAX,
)
.await;
let keys: Vec<SegmentKey> = page.iter().map(|hit| hit.node_ref().clone()).collect();
let short = shorter_than(15);
let mut gather = SqlGather::<SegmentRow, Sqlite>::new(&mut db);
let rows: Vec<Option<Segment>> = AsyncGather::gather(&mut gather, &keys, Some(&short))
.await
.expect("the live store answers");
assert_eq!(rows.len(), keys.len());
for (key, row) in keys.iter().zip(&rows) {
if key.id == 2 {
assert!(row.is_none(), "a forgotten row leaves a hole");
}
}
assert!(rows.iter().any(Option::is_some), "{rows:?}");
}