use qdrant_edge::bm25_embed::EdgeBm25Config;
use qql_core::error::QqlError;
use qql_embed::{Bm25TextConfig, SparseVector};
use super::FastEmbedder;
use super::err;
pub(crate) fn to_qql_sparse(sv: qdrant_edge::SparseVector) -> SparseVector {
SparseVector {
indices: sv.indices,
values: sv.values,
}
}
pub(crate) fn edge_bm25_config(text: &Bm25TextConfig) -> Result<EdgeBm25Config, QqlError> {
use ordered_float::NotNan;
use qdrant_edge::TokenizerType;
let params = &text.params;
let k = NotNan::new(params.k1()).map_err(|e| err(format!("invalid bm25 k1: {e}")))?;
let b = NotNan::new(params.b()).map_err(|e| err(format!("invalid bm25 b: {e}")))?;
let avg_len =
NotNan::new(params.avg_len()).map_err(|e| err(format!("invalid bm25 avg_len: {e}")))?;
let tokenizer = match text.tokenizer {
qql_embed::Tokenizer::Word => TokenizerType::Word,
qql_embed::Tokenizer::Whitespace => TokenizerType::Whitespace,
qql_embed::Tokenizer::Prefix => TokenizerType::Prefix,
qql_embed::Tokenizer::Multilingual => TokenizerType::Multilingual,
};
let mut cfg = EdgeBm25Config {
k,
b,
avg_len,
tokenizer,
language: Some(text.language.name().to_string()),
lowercase: Some(text.lowercase),
ascii_folding: Some(text.ascii_folding),
min_token_len: text.min_token_len,
max_token_len: text.max_token_len,
..Default::default()
};
if text.stopwords.is_some() || text.stemmer.is_some() {
cfg = merge_text_overrides(cfg, text)?;
}
Ok(cfg)
}
fn merge_text_overrides(
cfg: EdgeBm25Config,
text: &Bm25TextConfig,
) -> Result<EdgeBm25Config, QqlError> {
let mut json = serde_json::to_value(&cfg)
.map_err(|e| err(format!("edge BM25 config serialize failed: {e}")))?;
if let Some(selection) = &text.stopwords {
let languages: Vec<&str> = selection
.languages
.iter()
.map(|language| language.name())
.collect();
json["stopwords"] = serde_json::json!({
"languages": languages,
"custom": selection.custom,
});
}
if let Some(stemmer) = &text.stemmer {
json["stemmer"] = match stemmer {
qql_embed::Stemmer::Snowball(language) => {
if language.stem_algorithm().is_none() {
return Err(err(format!(
"edge BM25 stemmer unavailable for language {:?}",
language.name()
)));
}
serde_json::json!({"type": "snowball", "language": language.name()})
}
qql_embed::Stemmer::Armenian => {
serde_json::json!({"type": "snowball", "language": "armenian"})
}
qql_embed::Stemmer::Tamil => {
serde_json::json!({"type": "snowball", "language": "tamil"})
}
qql_embed::Stemmer::Disabled => serde_json::json!({"type": "none"}),
};
}
serde_json::from_value(json).map_err(|e| err(format!("edge BM25 override shape mismatch: {e}")))
}
pub(crate) fn ensure_sparse_model_allowed(
embedder: &FastEmbedder,
model: &str,
) -> Result<(), QqlError> {
let Some(ref sparse) = embedder.sparse else {
if !model.is_empty() && !model.eq_ignore_ascii_case("default") {
return Err(qql_embed::sparse_model_unsupported_error(model));
}
return Ok(());
};
if !(embedder.accepts_sparse_model(model)
|| model.is_empty()
|| model.eq_ignore_ascii_case("default"))
{
return Err(err(format!(
"local sparse embedder is locked to '{}' ({}); cannot satisfy MODEL '{model}'",
sparse.model_name, sparse.model_code
)));
}
Ok(())
}
pub(crate) async fn embed_sparse_fastembed_batch(
embedder: &FastEmbedder,
texts: Vec<String>,
) -> Result<Vec<SparseVector>, QqlError> {
let Some(ref sparse) = embedder.sparse else {
return Err(err("no fastembed sparse model configured"));
};
let model_arc = sparse.model.clone();
let embeddings = tokio::task::spawn_blocking(move || {
let mut model = model_arc
.lock()
.map_err(|e| err(format!("fastembed sparse mutex poisoned: {e}")))?;
model
.embed(texts, None)
.map_err(|e| err(format!("fastembed SparseTextEmbedding failed: {e}")))
})
.await
.map_err(|e| err(format!("spawn_blocking failed: {e}")))??;
Ok(embeddings
.into_iter()
.map(|e| SparseVector {
indices: e.indices.iter().map(|&i| i as u32).collect(),
values: e.values.clone(),
})
.collect())
}