use std::collections::HashMap;
use crate::db::collection::{Field, Shape};
use crate::db::schema::{
CalibrateMode, CodecSpec, FieldSpec, FieldSpecKind, Schema, TextBinding, VectorBinding,
VectorInline,
};
use crate::db::schema_doc;
use crate::error::{Error, Result};
use crate::file::{EmbeddingSpaceSpec, ValiseFile};
use crate::format::catalog::{
AnalyzerDesc, FieldDesc, FieldSchemaDesc, FieldSource, IdfVariant, PunctuationPolicy,
RetrievalProfileDesc, RetrievalProfileParams, RetrievalProfileType, Stemming, StopwordsPolicy,
TextSpaceDesc, Tokenizer, UnicodeNormalization, VectorMetric,
};
use crate::format::dtype::Dtype;
use crate::format::{
AnalyzerId, CodecId, CollectionId, EmbeddingSpaceId, FieldSchemaId, RetrievalProfileId,
TextSpaceId,
};
pub(crate) const PROVIDER: &str = "valise-db";
pub(crate) const RESERVED_PREFIX: char = '~';
const AUTO_SPACE_PREFIX: &str = "~auto/";
pub(crate) fn auto_space_name(collection: &str, field: &str) -> String {
format!("{AUTO_SPACE_PREFIX}{collection}/{field}")
}
pub(crate) fn reject_reserved_name(what: &str, name: &str) -> Result<()> {
if name.starts_with(RESERVED_PREFIX) {
return Err(Error::Format(format!(
"{what} name '{name}' is reserved: names starting with '~' are internal"
)));
}
Ok(())
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum Metric {
Cosine,
Dot,
L2,
}
impl Metric {
pub(crate) fn to_engine(self) -> VectorMetric {
match self {
Metric::Cosine => VectorMetric::Cosine,
Metric::Dot => VectorMetric::InnerProduct,
Metric::L2 => VectorMetric::L2,
}
}
pub(crate) fn from_engine(m: VectorMetric) -> Self {
match m {
VectorMetric::Cosine => Metric::Cosine,
VectorMetric::InnerProduct => Metric::Dot,
VectorMetric::L2 => Metric::L2,
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum Lang {
English,
Raw,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum SpaceKind {
Vector { dim: u32 },
Text,
}
#[derive(Clone, Debug)]
pub struct Space {
name: String,
kind: SpaceKind,
}
pub type SpaceRef = Space;
impl Space {
pub(crate) fn vector(name: &str, dim: u32) -> Self {
Self {
name: name.to_owned(),
kind: SpaceKind::Vector { dim },
}
}
pub(crate) fn text(name: &str) -> Self {
Self {
name: name.to_owned(),
kind: SpaceKind::Text,
}
}
#[must_use]
pub fn name(&self) -> &str {
&self.name
}
#[must_use]
pub fn is_text(&self) -> bool {
matches!(self.kind, SpaceKind::Text)
}
#[must_use]
pub fn is_vector(&self) -> bool {
matches!(self.kind, SpaceKind::Vector { .. })
}
pub(crate) fn dim(&self) -> Option<u32> {
match self.kind {
SpaceKind::Vector { dim } => Some(dim),
SpaceKind::Text => None,
}
}
}
#[derive(Clone, Debug)]
pub struct SpaceInfo {
pub name: String,
pub kind: SpaceKind,
pub auto: bool,
}
pub(crate) enum VecState {
Ready { space_id: EmbeddingSpaceId },
Deferred {
sample_target: usize,
codec: CodecSpec,
},
}
pub(crate) struct VectorSpaceEntry {
pub dim: u32,
pub metric: Metric,
pub state: VecState,
}
pub(crate) struct TextSpaceEntry {
pub text_space_id: TextSpaceId,
}
pub(crate) struct CollectionRecord {
pub collection_id: CollectionId,
pub shape: Option<Shape>,
pub doc: Option<schema_doc::SchemaDoc>,
}
#[derive(Default)]
pub(crate) struct SchemaRegistry {
pub vector_spaces: HashMap<String, VectorSpaceEntry>,
pub text_spaces: HashMap<String, TextSpaceEntry>,
pub collections: HashMap<String, CollectionRecord>,
}
impl SchemaRegistry {
pub fn rebuild(&mut self, valise: &ValiseFile) -> Result<()> {
for es in valise.embedding_spaces() {
if es.provider == PROVIDER {
self.vector_spaces.insert(
es.model.clone(),
VectorSpaceEntry {
dim: es.dimension,
metric: Metric::from_engine(es.metric),
state: VecState::Ready {
space_id: es.embedding_space_id,
},
},
);
}
}
for ts in valise.text_spaces() {
self.text_spaces.insert(
ts.name.clone(),
TextSpaceEntry {
text_space_id: ts.text_space_id,
},
);
}
for c in valise.collections() {
if c.name.starts_with(RESERVED_PREFIX) {
continue;
}
self.collections.insert(
c.name.clone(),
CollectionRecord {
collection_id: c.collection_id,
shape: None,
doc: None,
},
);
}
for (coll_name, doc) in schema_doc::load_all(valise)? {
if !self.collections.contains_key(&coll_name) {
continue;
}
if let Some(shape) = self.shape_from_doc(&doc)
&& let Some(rec) = self.collections.get_mut(&coll_name)
{
rec.shape = Some(shape);
rec.doc = Some(doc);
}
}
Ok(())
}
fn shape_from_doc(&mut self, doc: &schema_doc::SchemaDoc) -> Option<Shape> {
let mut fields = Vec::with_capacity(doc.fields.len());
for f in &doc.fields {
let field = match f.kind.as_str() {
schema_doc::KIND_TEXT => {
if !self.text_spaces.contains_key(&f.space) {
return None;
}
Field::Text(Space::text(&f.space))
}
schema_doc::KIND_VECTOR => {
if let Some(entry) = self.vector_spaces.get(&f.space) {
Field::Vector(Space::vector(&f.space, entry.dim))
} else {
let spec = f.vector_spec()?;
let metric = schema_doc::metric_from_str(&spec.metric).ok()?;
let codec = spec.codec.to_codec_spec().ok()?;
self.vector_spaces.insert(
f.space.clone(),
VectorSpaceEntry {
dim: spec.dim,
metric,
state: VecState::Deferred {
sample_target: spec.calibrate.sample_target(),
codec,
},
},
);
Field::Vector(Space::vector(&f.space, spec.dim))
}
}
_ => return None,
};
fields.push((f.name.clone(), field));
}
Some(Shape { fields })
}
pub fn define_space(
&mut self,
valise: &mut ValiseFile,
name: &str,
spec: &FieldSpec,
) -> Result<Space> {
match &spec.kind {
FieldSpecKind::Text(t) => match &t.binding {
TextBinding::Inline { lang } => self.define_text_space(valise, name, *lang),
TextBinding::Shared(s) => Err(Error::Format(format!(
"define_space: '{name}' cannot be defined from a binding to space '{}'",
s.name()
))),
},
FieldSpecKind::Vector(v) => {
if let Some(method) = v.shared_misuse {
return Err(shared_misuse_error("define_space", name, method));
}
match &v.binding {
VectorBinding::Inline(inline) => self.define_vector_space(valise, name, inline),
VectorBinding::Shared(s) => Err(Error::Format(format!(
"define_space: '{name}' cannot be defined from a binding to space '{}'",
s.name()
))),
}
}
}
}
pub fn define_vector_space(
&mut self,
valise: &mut ValiseFile,
name: &str,
spec: &VectorInline,
) -> Result<Space> {
if let Some(existing) = self.vector_spaces.get(name) {
if existing.dim != spec.dim {
return Err(Error::Format(format!(
"define_space: '{name}' already defined with dim {} (got {})",
existing.dim, spec.dim
)));
}
if existing.metric != spec.metric {
return Err(Error::Format(format!(
"define_space: '{name}' already defined with metric {:?} (got {:?})",
existing.metric, spec.metric
)));
}
return Ok(Space::vector(name, existing.dim));
}
ensure_sketchable_dim(spec.dim)?;
let state = if let CodecSpec::Prebuilt(params) = &spec.codec.spec {
if params.dimension() != spec.dim {
return Err(Error::Format(format!(
"define_space: '{name}' dim {} disagrees with prebuilt params dimension {}",
spec.dim,
params.dimension()
)));
}
let codec = valise.register_codec(params.clone())?;
let space_id = register_embedding_space(valise, name, spec.dim, spec.metric, codec)?;
VecState::Ready { space_id }
} else {
match &spec.calibrate.mode {
CalibrateMode::Auto { sample } => VecState::Deferred {
sample_target: (*sample).max(1),
codec: spec.codec.spec.clone(),
},
CalibrateMode::Now(sample) => {
let codec = register_codec_for(valise, spec.dim, &spec.codec.spec, sample)?;
let space_id =
register_embedding_space(valise, name, spec.dim, spec.metric, codec)?;
VecState::Ready { space_id }
}
}
};
self.vector_spaces.insert(
name.to_owned(),
VectorSpaceEntry {
dim: spec.dim,
metric: spec.metric,
state,
},
);
Ok(Space::vector(name, spec.dim))
}
pub fn define_text_space(
&mut self,
valise: &mut ValiseFile,
name: &str,
lang: Lang,
) -> Result<Space> {
if self.text_spaces.contains_key(name) {
return Ok(Space::text(name));
}
let analyzer_id = valise.register_analyzer(analyzer_desc(lang))?;
let field_schema_id = valise.register_field_schema(field_schema_desc())?;
let profile_id = valise.register_retrieval_profile(bm25_profile_desc())?;
let text_space_id = valise.register_text_space(TextSpaceDesc {
text_space_id: TextSpaceId(0),
name: name.to_owned(),
analyzer_id,
field_schema_id,
default_profile_id: profile_id,
flags: 0,
enabled_retrievers: 0,
})?;
self.text_spaces
.insert(name.to_owned(), TextSpaceEntry { text_space_id });
Ok(Space::text(name))
}
pub fn bind_schema(
&mut self,
valise: &mut ValiseFile,
coll: &str,
schema: &Schema,
) -> Result<Shape> {
validate_schema(coll, schema)?;
let mut fields = Vec::with_capacity(schema.fields.len());
for (fname, spec) in &schema.fields {
let field = match &spec.kind {
FieldSpecKind::Text(t) => match &t.binding {
TextBinding::Inline { lang } => {
let space =
self.define_text_space(valise, &auto_space_name(coll, fname), *lang)?;
Field::Text(space)
}
TextBinding::Shared(s) => Field::Text(s.clone()),
},
FieldSpecKind::Vector(v) => match &v.binding {
VectorBinding::Inline(inline) => {
let space = self.define_vector_space(
valise,
&auto_space_name(coll, fname),
inline,
)?;
Field::Vector(space)
}
VectorBinding::Shared(s) => Field::Vector(s.clone()),
},
};
fields.push((fname.clone(), field));
}
let shape = Shape { fields };
shape.validate()?;
Ok(shape)
}
pub fn vector_space_id(&self, name: &str) -> Option<EmbeddingSpaceId> {
match self.vector_spaces.get(name)?.state {
VecState::Ready { space_id } => Some(space_id),
VecState::Deferred { .. } => None,
}
}
pub fn text_space_id(&self, name: &str) -> Option<TextSpaceId> {
self.text_spaces.get(name).map(|t| t.text_space_id)
}
pub fn space_infos(&self) -> Vec<SpaceInfo> {
let mut out: Vec<SpaceInfo> = self
.vector_spaces
.iter()
.map(|(name, e)| SpaceInfo {
name: name.clone(),
kind: SpaceKind::Vector { dim: e.dim },
auto: name.starts_with(AUTO_SPACE_PREFIX),
})
.chain(self.text_spaces.keys().map(|name| SpaceInfo {
name: name.clone(),
kind: SpaceKind::Text,
auto: name.starts_with(AUTO_SPACE_PREFIX),
}))
.collect();
out.sort_by(|a, b| a.name.cmp(&b.name));
out
}
}
fn validate_schema(coll: &str, schema: &Schema) -> Result<()> {
let mut text_fields = 0usize;
let mut seen_names: Vec<&str> = Vec::with_capacity(schema.fields.len());
let mut seen_vec_spaces: Vec<String> = Vec::new();
for (name, spec) in &schema.fields {
if seen_names.contains(&name.as_str()) {
return Err(Error::Format(format!(
"schema: duplicate field name '{name}'"
)));
}
seen_names.push(name);
match &spec.kind {
FieldSpecKind::Text(t) => {
if let TextBinding::Shared(s) = &t.binding
&& !s.is_text()
{
return Err(Error::Format(format!(
"schema: field '{name}' is declared text but bound to vector space '{}'",
s.name()
)));
}
text_fields += 1;
}
FieldSpecKind::Vector(v) => {
if let Some(method) = v.shared_misuse {
return Err(shared_misuse_error("schema", name, method));
}
let space_name = match &v.binding {
VectorBinding::Inline(_) => auto_space_name(coll, name),
VectorBinding::Shared(s) => {
if !s.is_vector() {
return Err(Error::Format(format!(
"schema: field '{name}' is declared vector but bound to text space '{}'",
s.name()
)));
}
s.name().to_owned()
}
};
if seen_vec_spaces.contains(&space_name) {
return Err(Error::Format(format!(
"schema: vector fields must use distinct spaces; '{space_name}' is reused"
)));
}
seen_vec_spaces.push(space_name);
}
}
}
if text_fields > 1 {
return Err(Error::Unsupported(
"schema: at most one text field per collection in v1 (use the doc→chunk pattern for multiple text fields)".into(),
));
}
Ok(())
}
fn shared_misuse_error(context: &str, name: &str, method: &'static str) -> Error {
Error::Format(format!(
"{context}: field '{name}' binds a shared space — .{method}() is configured on the \
space at define_space, not on the binding"
))
}
#[derive(Debug, PartialEq, Eq)]
struct FieldSig {
name: String,
kind: &'static str,
space: String,
}
impl std::fmt::Display for FieldSig {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{} '{}' (space '{}')", self.kind, self.name, self.space)
}
}
fn schema_signature(coll: &str, schema: &Schema) -> Vec<FieldSig> {
schema
.fields
.iter()
.map(|(name, spec)| match &spec.kind {
FieldSpecKind::Text(t) => FieldSig {
name: name.clone(),
kind: "text",
space: match &t.binding {
TextBinding::Inline { .. } => auto_space_name(coll, name),
TextBinding::Shared(s) => s.name().to_owned(),
},
},
FieldSpecKind::Vector(v) => FieldSig {
name: name.clone(),
kind: "vector",
space: match &v.binding {
VectorBinding::Inline(_) => auto_space_name(coll, name),
VectorBinding::Shared(s) => s.name().to_owned(),
},
},
})
.collect()
}
fn shape_signature(shape: &Shape) -> Vec<FieldSig> {
shape
.fields
.iter()
.map(|(name, field)| match field {
Field::Text(s) => FieldSig {
name: name.clone(),
kind: "text",
space: s.name().to_owned(),
},
Field::Vector(s) => FieldSig {
name: name.clone(),
kind: "vector",
space: s.name().to_owned(),
},
})
.collect()
}
pub(crate) fn check_redeclare(coll: &str, existing: &Shape, schema: &Schema) -> Result<()> {
let old = shape_signature(existing);
let new = schema_signature(coll, schema);
if new.len() < old.len() {
return Err(Error::SchemaMismatch {
collection: coll.to_owned(),
detail: format!(
"re-declared with {} field(s), but {} were previously declared",
new.len(),
old.len()
),
});
}
for (i, (o, n)) in old.iter().zip(new.iter()).enumerate() {
if o != n {
return Err(Error::SchemaMismatch {
collection: coll.to_owned(),
detail: format!("field {i} was declared as {o}, re-declared as {n}"),
});
}
}
Ok(())
}
fn ensure_sketchable_dim(dim: u32) -> Result<()> {
if dim == 0 || !dim.is_multiple_of(64) {
return Err(Error::Unsupported(format!(
"vector space dimension must be a positive multiple of 64 (got {dim})"
)));
}
Ok(())
}
pub(crate) fn register_codec_for(
valise: &mut ValiseFile,
dim: u32,
spec: &CodecSpec,
sample: &[Vec<f32>],
) -> Result<CodecId> {
match spec {
CodecSpec::Qam {
amp_bits,
phase_bits,
} => valise.register_codec_qam_from_sample_with_bits(
dim as usize,
*amp_bits,
*phase_bits,
sample,
),
CodecSpec::Upq { cells, design } => valise.register_codec_upq_from_sample_with_options(
dim as usize,
*cells,
design.to_engine(),
sample,
),
CodecSpec::Prebuilt(params) => valise.register_codec(params.clone()),
}
}
pub(crate) fn register_embedding_space(
valise: &mut ValiseFile,
name: &str,
dim: u32,
metric: Metric,
codec: CodecId,
) -> Result<EmbeddingSpaceId> {
valise.register_embedding_space(EmbeddingSpaceSpec {
provider: PROVIDER.to_owned(),
model: name.to_owned(),
dimension: dim,
metric: metric.to_engine(),
normalized: matches!(metric, Metric::Cosine),
dtype: Dtype::F32,
primary_codec_id: Some(codec),
secondary_codec_id: None,
})
}
pub(crate) fn analyzer_desc(lang: Lang) -> AnalyzerDesc {
match lang {
Lang::English => AnalyzerDesc {
analyzer_id: AnalyzerId(0),
unicode_normalization: UnicodeNormalization::Nfkc,
case_fold: true,
accent_fold: false,
tokenizer: Tokenizer::UnicodeWords,
stemming: Stemming::Porter2English,
stopword_set_ref: None,
stopword_query_only: false,
stopwords: StopwordsPolicy::EnglishLucene,
english_possessive_strip: true,
min_token_len: 1,
max_token_len: 64,
shingle_size: 0,
ngram_min: None,
ngram_max: None,
punctuation_policy: PunctuationPolicy::Drop,
},
Lang::Raw => AnalyzerDesc {
analyzer_id: AnalyzerId(0),
unicode_normalization: UnicodeNormalization::None,
case_fold: false,
accent_fold: false,
tokenizer: Tokenizer::Whitespace,
stemming: Stemming::None,
stopword_set_ref: None,
stopword_query_only: true,
stopwords: StopwordsPolicy::None,
english_possessive_strip: false,
min_token_len: 0,
max_token_len: 0,
shingle_size: 0,
ngram_min: None,
ngram_max: None,
punctuation_policy: PunctuationPolicy::Keep,
},
}
}
pub(crate) fn field_schema_desc() -> FieldSchemaDesc {
FieldSchemaDesc {
field_schema_id: FieldSchemaId(0),
fields: vec![FieldDesc {
field_id: 1,
name: "body".to_owned(),
source: FieldSource::SearchText,
store_positions: true,
store_term_freq: true,
store_set_membership: true,
weight: 1.0,
}],
}
}
pub(crate) fn bm25_profile_desc() -> RetrievalProfileDesc {
RetrievalProfileDesc {
profile_id: RetrievalProfileId(0),
profile_type: RetrievalProfileType::Bm25,
params: RetrievalProfileParams::Bm25 {
k1: 1.2,
b: 0.75,
idf_variant: IdfVariant::RobertsonSparckJones,
},
}
}