use super::{Dedup, Elected, ElectionLabel, IndexGraphql, IndexModel, Projected, Upstream};
use crate::{
engine::{Engine, FieldEngines},
model::{Column, GraphqlKind, NodeKind, Projection, Role},
prefix::EmissionPrefix,
};
fn ident(name: &str) -> syn::Ident {
syn::Ident::new(name, proc_macro2::Span::call_site())
}
fn column(name: &str, ty: &str, project: Projection) -> Column {
Column {
name: ident(name),
source: ident(name),
ty: syn::parse_str(ty).expect("a type"),
project,
}
}
fn projected(field: &str, ty: &str) -> Projected {
Projected {
field: ident(field),
from: ident(field),
ty: syn::parse_str(ty).expect("a type"),
}
}
fn stored(field: &str, engines: &[&str], fanout: bool) -> FieldEngines {
FieldEngines {
field: ident(field),
engines: engines
.iter()
.map(|word| Engine::bare(ident(word)))
.collect(),
fanout,
}
}
fn elects(field: &str, label: ElectionLabel) -> Elected {
Elected::new(
ident(field),
label,
syn::parse_str("Embedding<f32, 4>").expect("a type"),
None,
)
}
fn document() -> IndexModel {
IndexModel {
prefix: EmissionPrefix::default(),
numbers: crate::index::Numbers::default(),
field_numbers: Vec::new(),
index: ident("SearchDocument"),
kind: NodeKind::Fulltext,
vis: syn::Visibility::Public(syn::token::Pub::default()),
krate: syn::parse_quote!(::ingraph),
source: Upstream {
node: syn::parse_quote!(asr::Segment),
projections: vec![projected("title", "String"), projected("ordinal", "u32")],
},
dedup: Dedup::Hash,
embeddings: Vec::new(),
labelled: Vec::new(),
searchable: vec![ident("title")],
engines: vec![stored("title", &["tantivy", "hnsw"], true)],
stores: Vec::new(),
extension: None,
data: false,
keys: false,
subscribe: false,
graphql: Some(IndexGraphql::of(GraphqlKind::AsyncGraphql)),
rename: None,
slot: None,
fields: vec![
column("title", "String", Projection::Borrowed),
column("ordinal", "u32", Projection::Copied),
],
}
}
fn frames() -> IndexModel {
IndexModel {
prefix: EmissionPrefix::default(),
numbers: crate::index::Numbers::default(),
field_numbers: Vec::new(),
index: ident("ClipVectors"),
kind: NodeKind::Vector,
vis: syn::Visibility::Public(syn::token::Pub::default()),
krate: syn::parse_quote!(::ingraph),
source: Upstream {
node: syn::parse_quote!(media::Clip),
projections: vec![
projected("frames", "Vec<Embedding<f32, 4>>"),
projected("poster", "Embedding<f32, 4>"),
],
},
dedup: Dedup::Hash,
embeddings: vec![
elects("frames", ElectionLabel::List),
elects("poster", ElectionLabel::Embedding),
],
labelled: Vec::new(),
searchable: Vec::new(),
engines: vec![
stored("frames", &["lance", "hnsw"], true),
stored("poster", &["lance"], false),
],
stores: Vec::new(),
extension: None,
data: false,
keys: false,
subscribe: false,
graphql: Some(IndexGraphql::of(GraphqlKind::AsyncGraphql)),
rename: None,
slot: None,
fields: vec![
column("frames", "Vec<Embedding<f32, 4>>", Projection::Borrowed),
column("poster", "Embedding<f32, 4>", Projection::Borrowed),
],
}
}
fn artifacts() -> IndexModel {
IndexModel {
prefix: EmissionPrefix::default(),
kind: NodeKind::Blob,
searchable: Vec::new(),
engines: Vec::new(),
..document()
}
}
fn emission(model: &IndexModel) -> syn::File {
let file: syn::File = syn::parse2(model.emit()).expect("the emission is a parsable file");
syn::File {
items: crate::tests::ungate(file.items),
..file
}
}
fn declared_methods(file: &syn::File) -> Vec<String> {
items(file)
.into_iter()
.filter_map(|item| match item {
syn::Item::Impl(declared) if declared.trait_.is_none() => Some(declared),
_ => None,
})
.flat_map(|declared| &declared.items)
.filter_map(|item| match item {
syn::ImplItem::Fn(method) => Some(method.sig.ident.to_string()),
_ => None,
})
.collect()
}
fn scoped_types(file: &syn::File) -> Vec<String> {
file
.items
.iter()
.filter_map(|item| match item {
syn::Item::Struct(item) => Some(item.ident.to_string()),
syn::Item::Enum(item) => Some(item.ident.to_string()),
_ => None,
})
.collect()
}
fn items(file: &syn::File) -> Vec<&syn::Item> {
let mut found = Vec::new();
for item in &file.items {
found.push(item);
if let syn::Item::Const(anonymous) = item
&& let syn::Expr::Block(block) = &*anonymous.expr
{
found.extend(
block
.block
.stmts
.iter()
.filter_map(|statement| match statement {
syn::Stmt::Item(inner) => Some(inner),
_ => None,
}),
);
}
}
found
}
#[test]
fn the_enumeration_names_exactly_the_methods_the_emission_declares() {
for model in [document(), frames(), artifacts()] {
let declared = declared_methods(&emission(&model));
let named: Vec<String> = model
.emitted_methods()
.iter()
.map(|emitted| emitted.name.to_string())
.collect();
assert_eq!(named, declared, "`{}`", model.index);
}
}
#[test]
fn the_enumeration_names_exactly_the_types_the_emission_declares() {
for model in [document(), frames(), artifacts()] {
let declared = scoped_types(&emission(&model));
let named: Vec<String> = model
.emitted_types()
.iter()
.map(|declared| declared.name.to_string())
.collect();
assert_eq!(named, declared, "`{}`", model.index);
}
}
#[test]
fn the_fan_door_is_enumerated_under_the_field_it_fans() {
let names: Vec<String> = document()
.emitted_methods()
.iter()
.map(|emitted| emitted.name.to_string())
.collect();
assert!(names.contains(&"fan_title".to_owned()), "{names:?}");
assert!(
!names.contains(&"fan_ordinal".to_owned()),
"and only for a field that declared a fan: {names:?}",
);
}
#[test]
fn a_portable_field_is_enumerated_with_no_fan_door() {
let mut portable = document();
portable.engines = vec![stored("title", &["tantivy", "hnsw"], false)];
let names: Vec<String> = portable
.emitted_methods()
.iter()
.map(|emitted| emitted.name.to_string())
.collect();
assert!(
!names.iter().any(|name| name.starts_with("fan_")),
"{names:?}",
);
}
#[test]
fn the_read_surface_is_enumerated_election_by_election() {
let names: Vec<String> = frames()
.emitted_methods()
.iter()
.map(|emitted| emitted.name.to_string())
.collect();
for door in [
"retrieve_keys_by_frames",
"retrieve_by_frames",
"get_frames",
"retrieve_similar_by_frames",
"retrieve_keys_by_frames_with",
"retrieve_keys_by_poster",
"retrieve_by_poster",
"get_poster",
"retrieve_similar_by_poster",
"fold_by_frames",
] {
assert!(names.contains(&door.to_owned()), "`{door}`: {names:?}");
}
for absent in [
"retrieve_keys_by_poster_with",
"fold_by_poster",
"ask_keys_by_frames",
"ask_by_frames",
"ask_keys_by_poster",
"ask_by_poster",
] {
assert!(!names.contains(&absent.to_owned()), "`{absent}`: {names:?}");
}
}
#[test]
fn the_full_text_subject_is_enumerated_unsuffixed() {
let names: Vec<String> = document()
.emitted_methods()
.iter()
.map(|emitted| emitted.name.to_string())
.collect();
assert!(
names.contains(&"retrieve_keys".to_owned()) && names.contains(&"retrieve".to_owned()),
"{names:?}",
);
assert!(
!names.iter().any(|name| name.starts_with("get_")),
"a full-text index elects no column, so there is nothing to read back by key: {names:?}",
);
}
#[test]
fn a_copy_field_named_after_a_fan_door_is_two_of_one_name() {
let mut declared = document();
declared
.fields
.push(column("fan_title", "u32", Projection::Copied));
declared
.source
.projections
.push(projected("fan_title", "u32"));
assert_eq!(taken(&declared, "fan_title"), 2, "the fan and the accessor");
let mut apart = document();
apart
.fields
.push(column("banner", "u32", Projection::Copied));
apart.source.projections.push(projected("banner", "u32"));
assert_eq!(taken(&apart, "fan_title"), 1, "the fan alone");
}
#[test]
fn a_copy_field_named_after_a_by_key_read_is_two_of_one_name() {
let mut declared = frames();
declared
.fields
.push(column("get_poster", "u32", Projection::Copied));
declared
.source
.projections
.push(projected("get_poster", "u32"));
assert_eq!(taken(&declared, "get_poster"), 2);
assert_eq!(taken(&frames(), "get_poster"), 1, "the door alone");
}
fn taken(model: &IndexModel, name: &str) -> usize {
model
.emitted_methods()
.iter()
.filter(|emitted| emitted.name == name)
.count()
}
fn minted(model: &IndexModel, name: &str) -> usize {
model
.emitted_types()
.iter()
.filter(|declared| declared.name == name)
.count()
}
#[test]
fn two_slots_can_mint_one_marker_and_two_distinct_names() {
let mut declared = document();
declared.engines = vec![stored("a", &["b_c"], false), stored("a_b", &["c"], false)];
assert_eq!(minted(&declared, "SearchDocumentABC"), 2);
let names: Vec<String> = declared
.slots()
.iter()
.map(|slot| slot.name(&declared.index))
.collect();
assert_eq!(
names,
["search_document__a__b_c", "search_document__a_b__c"],
"the `__` join keeps the boundary the camel-case one cannot",
);
}
#[test]
fn a_slot_marker_can_land_on_an_election_marker() {
let mut declared = frames();
declared.embeddings = vec![elects("image", ElectionLabel::Embedding)];
declared.engines = vec![stored("by", &["image"], false)];
assert_eq!(minted(&declared, "ClipVectorsByImage"), 2);
}
#[test]
fn two_elections_can_mint_one_marker() {
let mut declared = frames();
declared.embeddings = vec![
elects("image_1", ElectionLabel::Embedding),
elects("image1", ElectionLabel::Embedding),
];
declared.engines = Vec::new();
assert_eq!(minted(&declared, "ClipVectorsByImage1"), 2);
}
#[test]
fn an_ordinary_declaration_mints_every_marker_once() {
for model in [document(), frames(), artifacts()] {
let mut names: Vec<String> = model
.emitted_types()
.iter()
.map(|declared| declared.name.to_string())
.collect();
let minted = names.len();
names.sort_unstable();
names.dedup();
assert_eq!(names.len(), minted, "`{}`: {names:?}", model.index);
}
}
#[test]
fn a_marker_arrives_with_the_pairing_that_minted_it() {
let mut declared = document();
declared.engines = vec![stored("a", &["b_c"], false), stored("a_b", &["c"], false)];
let markers = |model: &IndexModel| -> Vec<String> {
model
.emitted_types()
.iter()
.filter(|declared| !matches!(declared.role, Role::Fuse | Role::Queries))
.map(|declared| declared.role.to_string())
.collect()
};
assert_eq!(
markers(&declared),
["the `a` × `b_c` slot marker", "the `a_b` × `c` slot marker"],
);
assert_eq!(
markers(&frames()),
[
"the `frames` election's marker",
"the `poster` election's marker",
"the `frames` × `lance` slot marker",
"the `frames` × `hnsw` slot marker",
"the `poster` × `lance` slot marker",
],
"elections first, then slots — the emitter's own walk",
);
}
#[test]
fn a_marker_names_the_field_a_refusal_points_at() {
let at: Vec<String> = frames()
.emitted_types()
.iter()
.filter_map(|declared| declared.role.at())
.map(ToString::to_string)
.collect();
assert_eq!(
at,
["frames", "poster", "frames", "frames", "poster"],
"one per marker, and never the engine word",
);
}