use core::{
num::NonZeroUsize,
pin::pin,
task::{Context, Poll, Waker},
};
use std::sync::{Arc, Mutex};
use smol_bytes::Utf8Bytes;
use super::{AsyncVectorStore, VectorColumn, VectorEngine};
use crate::{
AsyncVectorRead, AsyncVectorWrite, ColumnReader, ColumnWriter, Connected, Embedding, Field, Hits,
Identifiable, Scored, Transformer, VectorRecord,
store::column::testkit::{Col, Fallback, FallbackRead, Refusal},
};
const NEIGHBOURS: NonZeroUsize = NonZeroUsize::new(10).expect("10 is not zero");
fn block_on<F: Future>(future: F) -> F::Output {
let mut future = pin!(future);
let mut cx = Context::from_waker(Waker::noop());
loop {
if let Poll::Ready(value) = future.as_mut().poll(&mut cx) {
return value;
}
}
}
const DIM: usize = 8;
struct Keyframe {
media: u64,
ordinal: u32,
embedding: Embedding<f32, DIM>,
}
#[derive(Debug, Clone, PartialEq)]
struct KeyframeKey {
media: u64,
ordinal: u32,
}
impl Field for KeyframeKey {
fn write(&self, w: &mut impl ColumnWriter) {
self.media.write(w);
self.ordinal.write(w);
}
fn read<R: ColumnReader>(r: &mut R) -> Result<Self, R::Error> {
Ok(Self {
media: u64::read(r)?,
ordinal: u32::read(r)?,
})
}
}
#[derive(Debug, PartialEq)]
struct KeyframeLineage {
media: u64,
}
impl Identifiable for Keyframe {
type Id = u32;
fn id_ref(&self) -> &u32 {
&self.ordinal
}
}
impl Transformer for Keyframe {
type Key = KeyframeKey;
type Lineage = KeyframeLineage;
type Record = Self;
type Graphql = ();
fn key(&self) -> KeyframeKey {
KeyframeKey {
media: self.media,
ordinal: self.ordinal,
}
}
}
const EMBEDDING: VectorColumn = VectorColumn::new("embedding");
impl VectorRecord for Keyframe {
type Element = f32;
type Filter = KeyframeLineage;
fn vectors(&self) -> impl Iterator<Item = (VectorColumn, &[f32])> {
core::iter::once((EMBEDDING, self.embedding.coordinates()))
}
}
fn keyframe(media: u64, ordinal: u32) -> Keyframe {
Keyframe {
media,
ordinal,
embedding: Embedding::new(Arc::from(vec![0.5; DIM])).expect("the fixture declares DIM"),
}
}
struct Q8(u8);
struct Quantized(Vec<Q8>);
impl VectorRecord for Quantized {
type Element = Q8;
type Filter = ();
fn vectors(&self) -> impl Iterator<Item = (VectorColumn, &[Q8])> {
core::iter::once((VectorColumn::new("quantized"), self.0.as_slice()))
}
}
fn only(record: &impl VectorRecord<Element = f32>) -> &[f32] {
let mut columns = record.vectors();
let (_, coordinates) = columns.next().expect("one election");
assert!(columns.next().is_none(), "and only one");
coordinates
}
#[test]
fn a_vector_record_states_its_coordinates() {
let frame = keyframe(7, 12);
assert_eq!(
frame.vectors().collect::<Vec<_>>(),
[(EMBEDDING, &[0.5; DIM][..])],
"one election, under the name it was elected by",
);
assert_eq!(only(&frame).len(), Embedding::<f32, DIM>::DIM);
fn coordinates_of<T: VectorRecord>(record: &T) -> Vec<(VectorColumn, &[T::Element])> {
record.vectors().collect()
}
assert_eq!(coordinates_of(&frame)[0].1.len(), DIM);
}
#[test]
fn the_embedding_face_carries_no_identity() {
fn width_of<T: VectorRecord>(record: &T) -> usize {
record.vectors().map(|(_, held)| held.len()).sum()
}
let quantized = Quantized(vec![Q8(1), Q8(2), Q8(3)]);
assert_eq!(width_of(&quantized), 3);
assert_eq!(quantized.vectors().next().expect("one election").1[2].0, 3);
}
#[test]
fn the_unused_face_answers_the_whole_seam() {
assert_eq!(
block_on(AsyncVectorWrite::upsert(&(), Connected::new(&(), ()), ())),
Ok(())
);
assert_eq!(
block_on(AsyncVectorRead::knn(
&(),
Connected::new(&(), ()),
VectorColumn::new("nothing"),
&[],
NEIGHBOURS,
None
)),
Ok(Vec::new())
);
fn write_then_read<E: VectorEngine>() -> (Result<i64, E::Error>, Result<(), E::Error>) {
let mut buffer = E::buffer();
7_i64.write(&mut buffer);
let mut phase = E::open(E::finalize(buffer));
let read = i64::read(&mut phase);
(read, phase.finish())
}
assert_eq!(write_then_read::<()>(), (Err(crate::NoColumns), Ok(())));
}
#[test]
fn neither_verb_projects_a_record() {
fn index_anything<D>(db: &D, doc: D::Doc) -> impl Future<Output = Result<(), D::Error>>
where
D: AsyncVectorWrite<Connection = (), WriteContext<'static> = ()>,
{
db.upsert(Connected::new(&(), ()), doc)
}
fn search<D>(
db: &D,
column: VectorColumn,
query: &[D::Element],
) -> impl Future<Output = Result<Hits<D::Address, D::Score>, D::Error>>
where
D: AsyncVectorRead<Connection = (), ReadContext<'static> = ()>,
D::SearchOptions: 'static,
{
db.knn(Connected::new(&(), ()), column, query, NEIGHBOURS, None)
}
assert_eq!(block_on(index_anything::<()>(&(), ())), Ok(()));
assert_eq!(block_on(search::<()>(&(), EMBEDDING, &[])), Ok(Vec::new()));
}
struct MemoryIndex {
widths: Mutex<Vec<usize>>,
hit: Vec<Col>,
}
impl VectorEngine for MemoryIndex {
type Options = ();
type SearchOptions = ();
type Error = Refusal;
type Score = f32;
type Element = f32;
type Address = Vec<Col>;
type Connection = ();
type WriteBuffer = Fallback;
type ReadBuffer = FallbackRead;
fn buffer() -> Self::WriteBuffer {
Fallback::default()
}
fn finalize(buf: Self::WriteBuffer) -> Self::Address {
buf.0
}
fn open(address: Self::Address) -> Self::ReadBuffer {
Fallback(address).read()
}
}
impl AsyncVectorWrite for MemoryIndex {
type WriteContext<'a> = ();
type Doc = usize;
fn upsert(
&self,
_ctx: Connected<'_, Self::Connection, Self::WriteContext<'_>>,
doc: Self::Doc,
) -> impl Future<Output = Result<(), Self::Error>> + Send {
self
.widths
.lock()
.expect("the toy index is not poisoned")
.push(doc);
core::future::ready(Ok(()))
}
}
impl AsyncVectorRead for MemoryIndex {
type ReadContext<'a> = ();
fn knn(
&self,
_ctx: Connected<'_, Self::Connection, Self::ReadContext<'_>>,
_column: VectorColumn,
query: &[Self::Element],
_k: NonZeroUsize,
_options: ::core::option::Option<&Self::SearchOptions>,
) -> impl Future<Output = Result<Hits<Self::Address, Self::Score>, Self::Error>> + Send {
core::future::ready(Ok(vec![Scored::new(self.hit.clone(), query.len() as f32)]))
}
fn get(
&self,
_ctx: Connected<'_, Self::Connection, Self::ReadContext<'_>>,
_column: VectorColumn,
_address: &Self::Address,
) -> impl Future<Output = Result<Vec<Vec<Self::Element>>, Self::Error>> + Send {
core::future::ready(Ok(Vec::new()))
}
}
fn addressed(key: &KeyframeKey) -> Vec<Col> {
let mut buf = MemoryIndex::buffer();
key.write(&mut buf);
MemoryIndex::finalize(buf)
}
#[test]
fn a_written_face_takes_the_entry_and_answers_with_addresses() {
let frame = keyframe(7, 12);
let index = MemoryIndex {
widths: Mutex::new(Vec::new()),
hit: addressed(&frame.key()),
};
assert_eq!(
block_on(index.upsert(Connected::new(&(), ()), only(&frame).len())),
Ok(())
);
assert_eq!(
*index.widths.lock().expect("the toy index is not poisoned"),
vec![DIM]
);
let hits = block_on(index.knn(
Connected::new(&(), ()),
EMBEDDING,
only(&frame),
NEIGHBOURS,
None,
))
.expect("the toy index cannot fail");
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].data_ref(), &addressed(&frame.key()));
assert_eq!(hits[0].score(), DIM as f32);
let mut phase = MemoryIndex::open(hits[0].clone().into_data());
assert_eq!(
KeyframeKey::read(&mut phase).expect("the address is this key's"),
frame.key()
);
assert_eq!(phase.finish(), Ok(()));
}
struct RecordingStore {
minted: Mutex<Vec<(Utf8Bytes, usize)>>,
}
impl AsyncVectorStore for RecordingStore {
type Options = ();
type Error = core::convert::Infallible;
type Handle = ();
async fn open(_options: Self::Options) -> Result<Self, Self::Error> {
Ok(Self {
minted: Mutex::new(Vec::new()),
})
}
fn vector(&self, slot: impl Into<Utf8Bytes>, dim: usize) -> Self::Handle {
self
.minted
.lock()
.expect("the recording store is not poisoned")
.push((slot.into(), dim));
}
}
#[test]
fn a_declared_width_reaches_the_slot_through_the_field_type() {
let store = block_on(RecordingStore::open(())).expect("the recording store opens");
store.vector("keyframe_embedding", Embedding::<f32, DIM>::DIM);
store.vector(String::from("caption_embedding"), 4 * 96);
let minted = store
.minted
.lock()
.expect("the recording store is not poisoned");
let minted: Vec<(&str, usize)> = minted.iter().map(|(slot, dim)| (&**slot, *dim)).collect();
assert_eq!(
minted,
vec![("keyframe_embedding", DIM), ("caption_embedding", 384)]
);
}