use core::num::NonZeroUsize;
use std::sync::Arc;
use sqlx::Sqlite;
use super::corpus::{Segment, SegmentFilter, SegmentKey, SegmentRow, forget, shorter_than, store};
use crate::{
AsyncVectorRead, AsyncVectorWrite, Connected, Embedding, Field, Hits, Identifiable, Ranked,
Scored, SlotHandle, Transformer, VectorColumn, VectorEngine, VectorRecord,
retrieve::{
EqSeen, HashSeen, RetrieveError, Seen, SqlGather, VectorQuery, filled, ranked_keys,
vector_keyed,
},
store::column::testkit::{Col, Fallback, FallbackRead, Refusal},
};
const DIM: usize = 4;
const EMBEDDING: VectorColumn = VectorColumn::new("embedding");
pub(super) struct Voiceprint {
key: SegmentKey,
embedding: Embedding<f32, DIM>,
}
impl Voiceprint {
pub(super) fn of(media: &str, id: i64, coordinates: [f32; DIM]) -> Self {
Self {
key: SegmentKey {
media: media.to_owned(),
id,
},
embedding: Embedding::new(Arc::from(coordinates.to_vec())).expect("the fixture declares DIM"),
}
}
}
impl Identifiable for Voiceprint {
type Id = <SegmentKey as Identifiable>::Id;
fn id_ref(&self) -> &i64 {
Identifiable::id_ref(&self.key)
}
}
impl Transformer for Voiceprint {
type Key = SegmentKey;
type Lineage = <Segment as Transformer>::Lineage;
type Record = ();
type Graphql = ();
fn key(&self) -> SegmentKey {
self.key.clone()
}
}
impl VectorRecord for Voiceprint {
type Element = f32;
type Filter = ();
fn vectors(&self) -> impl Iterator<Item = (VectorColumn, &[f32])> {
core::iter::once((EMBEDDING, self.embedding.coordinates()))
}
}
impl Voiceprint {
fn coordinates(&self) -> &[f32] {
self
.vectors()
.find(|(column, _)| *column == EMBEDDING)
.expect("the declaration elects it")
.1
}
}
pub(super) struct Voiceprints(Vec<Voiceprint>);
fn addressed(key: &SegmentKey) -> Vec<Col> {
let mut buf = Voiceprints::buffer();
key.write(&mut buf);
Voiceprints::finalize(buf)
}
impl VectorEngine for Voiceprints {
type Options = ();
type SearchOptions = ();
type Error = Refusal;
type Score = f32;
type Element = f32;
type Address = Vec<Col>;
type Connection = ();
type WriteBuffer = Fallback;
type ReadBuffer = FallbackRead;
fn buffer() -> Self::WriteBuffer {
Fallback::default()
}
fn finalize(buf: Self::WriteBuffer) -> Self::Address {
buf.0
}
fn open(address: Self::Address) -> Self::ReadBuffer {
Fallback(address).read()
}
}
impl AsyncVectorWrite for Voiceprints {
type Doc = Voiceprint;
type WriteContext<'a> = ();
fn upsert(
&self,
_ctx: Connected<'_, (), Self::WriteContext<'_>>,
_doc: Self::Doc,
) -> impl Future<Output = Result<(), Self::Error>> + Send {
core::future::ready(Ok(()))
}
}
impl AsyncVectorRead for Voiceprints {
type ReadContext<'a> = ();
fn knn(
&self,
_ctx: Connected<'_, (), Self::ReadContext<'_>>,
column: VectorColumn,
query: &[Self::Element],
_k: ::core::num::NonZeroUsize,
_options: ::core::option::Option<&Self::SearchOptions>,
) -> impl Future<Output = Result<Hits<Self::Address, Self::Score>, Self::Error>> + Send {
let mut hits: Vec<Scored<Vec<Col>, f32>> = self
.0
.iter()
.filter(|_| column == EMBEDDING)
.map(|held| Scored::new(addressed(&held.key), cosine(held.coordinates(), query)))
.collect();
hits.sort_by(|a, b| b.score().total_cmp(&a.score()));
core::future::ready(Ok(hits))
}
fn get(
&self,
_ctx: Connected<'_, (), Self::ReadContext<'_>>,
column: VectorColumn,
address: &Self::Address,
) -> impl Future<Output = Result<Vec<Vec<Self::Element>>, Self::Error>> + Send {
core::future::ready(Ok(
self
.0
.iter()
.filter(|held| column == EMBEDDING && *address == addressed(&held.key))
.map(|held| held.coordinates().to_vec())
.collect(),
))
}
}
fn cosine(a: &[f32], b: &[f32]) -> f32 {
let dot: f32 = a.iter().zip(b).map(|(x, y)| x * y).sum();
let norm = |v: &[f32]| v.iter().map(|x| x * x).sum::<f32>().sqrt();
let scale = norm(a) * norm(b);
if scale == 0.0 { 0.0 } else { dot / scale }
}
pub(super) fn index(
extra: impl IntoIterator<Item = Voiceprint>,
) -> SlotHandle<Voiceprints, Voiceprint> {
let mut held = vec![
Voiceprint::of("m1", 1, [1.0, 0.00, 0.0, 0.0]),
Voiceprint::of("m1", 2, [1.0, 0.20, 0.0, 0.0]),
Voiceprint::of("m2", 3, [1.0, 0.45, 0.0, 0.0]),
Voiceprint::of("m2", 4, [1.0, 0.80, 0.0, 0.0]),
Voiceprint::of("m3", 5, [1.0, 1.40, 0.0, 0.0]),
Voiceprint::of("m3", 6, [1.0, 2.60, 0.0, 0.0]),
];
held.extend(extra);
SlotHandle::mint(Voiceprints(held))
}
pub(super) fn query(floor: f32) -> VectorQuery<f32, f32, DIM> {
VectorQuery::new(
Embedding::new(Arc::from(vec![1.0, 0.0, 0.0, 0.0])).expect("the fixture declares DIM"),
floor,
)
}
pub(super) async fn retrieve_keys<D>(
slot: &mut SlotHandle<Voiceprints, Voiceprint>,
query: &VectorQuery<f32, f32, DIM>,
limit: usize,
) -> Vec<Ranked<SegmentKey, f32>>
where
D: Seen<SegmentKey>,
{
let Some(k) = NonZeroUsize::new(limit) else {
return Vec::new();
};
let hits = slot
.knn(
Connected::new(&(), ()),
EMBEDDING,
query.coordinates(),
k,
None,
)
.await
.expect("the toy index cannot fail");
let hits = vector_keyed::<Voiceprints, SegmentKey>(hits).expect("the addresses read back");
ranked_keys::<_, _, D>(hits, query.floor(), limit)
}
async fn retrieve(
slot: &mut SlotHandle<Voiceprints, Voiceprint>,
db: &mut sqlx::sqlite::SqliteConnection,
query: &VectorQuery<f32, f32, DIM>,
filter: Option<&SegmentFilter<Sqlite>>,
limit: usize,
) -> Result<
Vec<Ranked<Segment, f32>>,
RetrieveError<core::convert::Infallible, crate::query::sql::SqlQueryError>,
> {
let ranked = retrieve_keys::<HashSeen<_>>(slot, query, usize::MAX).await;
let mut gather = SqlGather::<SegmentRow, Sqlite>::new(db);
filled(ranked, &mut gather, filter, limit)
.await
.map_err(RetrieveError::Gather)
}
fn ids(page: &[Ranked<Segment, f32>]) -> Vec<i64> {
page.iter().map(|hit| hit.node_ref().id).collect()
}
#[tokio::test]
async fn the_floor_turns_k_into_a_ceiling() {
let mut engine = index([]);
let unfloored = retrieve_keys::<HashSeen<_>>(&mut engine, &query(0.0), usize::MAX).await;
assert_eq!(unfloored.len(), 6, "knn answers with everything it holds");
let floored = retrieve_keys::<HashSeen<_>>(&mut engine, &query(0.9), usize::MAX).await;
assert_eq!(
floored
.iter()
.map(|hit| hit.node_ref().id)
.collect::<Vec<_>>(),
vec![1, 2, 3]
);
assert_eq!(
floored.iter().map(Ranked::rank).collect::<Vec<_>>(),
vec![1, 2, 3]
);
}
#[tokio::test]
async fn a_duplicate_key_collapses_and_leaves_its_rank_behind() {
let mut engine = index([Voiceprint::of("m2", 3, [1.0, 1.90, 0.0, 0.0])]);
let ranked = retrieve_keys::<HashSeen<_>>(&mut engine, &query(0.0), usize::MAX).await;
assert_eq!(ranked.len(), 6, "seven hits, one duplicate key");
assert_eq!(
ranked
.iter()
.map(|hit| hit.node_ref().id)
.collect::<Vec<_>>(),
vec![1, 2, 3, 4, 5, 6]
);
assert_eq!(
ranked.iter().map(Ranked::rank).collect::<Vec<_>>(),
vec![1, 2, 3, 4, 5, 7]
);
let kept = ranked
.iter()
.find(|hit| hit.node_ref().id == 3)
.expect("the survivor");
assert!(kept.score() > 0.9, "{kept:?}");
}
#[tokio::test]
async fn tier_b_answers_whole_entities_in_engine_order() {
let mut engine = index([]);
let mut db = store().await;
let page = retrieve(&mut engine, &mut db, &query(0.0), None, 10)
.await
.expect("the retrieve runs");
assert_eq!(ids(&page), vec![1, 2, 3, 4, 5, 6]);
assert_eq!(
page.iter().map(Ranked::rank).collect::<Vec<_>>(),
vec![1, 2, 3, 4, 5, 6]
);
assert_eq!(
page[0].node_ref().transcript,
"a long sunset over the harbour"
);
}
#[tokio::test]
async fn a_filter_forces_a_second_key_batch() {
let mut engine = index([]);
let mut db = store().await;
let short = shorter_than(15);
let page = retrieve(&mut engine, &mut db, &query(0.0), Some(&short), 2)
.await
.expect("the retrieve runs");
assert_eq!(ids(&page), vec![2, 4]);
assert_eq!(
page.iter().map(Ranked::rank).collect::<Vec<_>>(),
vec![2, 4]
);
}
#[tokio::test]
async fn a_store_missing_row_is_skipped_and_the_page_fills() {
let mut engine = index([]);
let mut db = store().await;
forget(&mut db, 2).await;
let page = retrieve(&mut engine, &mut db, &query(0.0), None, 10)
.await
.expect("the retrieve runs");
assert_eq!(ids(&page), vec![1, 3, 4, 5, 6]);
assert_eq!(
page.iter().map(Ranked::rank).collect::<Vec<_>>(),
vec![1, 3, 4, 5, 6]
);
}
#[tokio::test]
async fn a_floored_retrieve_answers_short() {
let mut engine = index([]);
let mut db = store().await;
let page = retrieve(&mut engine, &mut db, &query(0.97), None, 10)
.await
.expect("the retrieve runs");
assert_eq!(ids(&page), vec![1, 2]);
}
#[tokio::test]
async fn the_equality_mechanism_answers_the_same_stream() {
let mut engine = index([]);
let hashed = retrieve_keys::<HashSeen<_>>(&mut engine, &query(0.0), usize::MAX).await;
let listed = retrieve_keys::<EqSeen<_>>(&mut engine, &query(0.0), usize::MAX).await;
assert_eq!(hashed, listed);
}