1use std::collections::{HashMap, HashSet};
4
5use khive_db::pool::NoteSearchFallbackCause;
6use lattice_embed::{EmbeddingModel, EmbeddingRole, EmbeddingService, TokenCount, MAX_TEXT_BYTES};
7use uuid::Uuid;
8
9use crate::config::{parse_embedding_model_alias, sanitize_key};
10use crate::curation::note_fts_document;
11use crate::embedder_registry::with_embedding_admission;
12use crate::error::{RuntimeError, RuntimeResult};
13use crate::runtime::{KhiveRuntime, NamespaceToken};
14use khive_retrieval::hybrid::{combine_leg_first_appearance, fuse_labelled, HitLabel};
15use khive_retrieval::HYBRID_CANDIDATE_MULTIPLIER as CANDIDATE_MULTIPLIER;
16use khive_score::DeterministicScore;
17use khive_storage::types::{
18 PageRequest, TextFilter, TextQueryMode, TextSearchHit, TextSearchRequest, VectorRecord,
19 VectorSearchHit,
20};
21use khive_storage::ContentRef;
22use khive_storage::EntityFilter;
23use khive_types::SubstrateKind;
24
25const CUSTOM_QUERY_ROLE_MODEL: EmbeddingModel = EmbeddingModel::AllMiniLmL6V2;
31
32pub use khive_retrieval::{
33 HybridSearchOutcome, RankScoreKind, SearchHit, SearchSignals, SearchSource,
34};
35
36mod single_engine;
37
38#[cfg(test)]
39mod request_stop_tests;
40
41#[cfg(test)]
42mod token_report_tests;
43
44pub(crate) const EMBEDDING_BATCH_PAGE_SIZE: usize = 256;
46
47#[cfg(any(test, feature = "fault-injection"))]
49std::thread_local! {
50 static BACKFILL_READER_FAIL: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
51}
52
53#[cfg(any(test, feature = "fault-injection"))]
59pub fn arm_backfill_reader_fail() {
60 BACKFILL_READER_FAIL.with(|c| c.set(true));
61}
62
63const RRF_K: usize = 10;
70
71pub const EMBEDDING_INPUT_TRUNCATED_WARNING: &str =
73 "embedding input was truncated to the embedder maximum; full content was stored unchanged";
74
75#[derive(Clone, Debug)]
78pub struct DocumentEmbeddingOutcome {
79 pub model_name: String,
80 pub vector: Vec<f32>,
81 pub prepared_text_fingerprint: Option<ContentRef>,
83 pub source_bytes: usize,
84 pub embedded_bytes: usize,
85 pub provider_token_count: Option<TokenCount>,
88 pub truncated: bool,
89}
90
91fn document_vector_or_refusal(outcome: DocumentEmbeddingOutcome) -> RuntimeResult<Vec<f32>> {
94 if !outcome.truncated {
95 return Ok(outcome.vector);
96 }
97 let token_count = outcome
98 .provider_token_count
99 .filter(|count| count.truncated());
100 let byte_cut = outcome.source_bytes > outcome.embedded_bytes;
101 let detail = match (byte_cut, token_count) {
102 (true, Some(count)) => format!(
103 "from {} to {} bytes and from {} to {} provider tokens for the byte-bounded input",
104 outcome.source_bytes, outcome.embedded_bytes, count.before_truncation, count.embedded
105 ),
106 (true, None) => format!(
107 "from {} to {} bytes",
108 outcome.source_bytes, outcome.embedded_bytes
109 ),
110 (false, Some(count)) => format!(
111 "from {} to {} provider tokens",
112 count.before_truncation, count.embedded
113 ),
114 (false, None) => "by an unspecified input limit".to_string(),
115 };
116 Err(RuntimeError::InvalidInput(format!(
117 "embedding input truncated {detail}; use embed_document_outcome to inspect the bounded vector"
118 )))
119}
120
121#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
123pub struct EmbeddingTruncationReport {
124 pub truncated: u64,
125 pub discarded_bytes: u64,
126 #[serde(default, skip_serializing_if = "counter_is_zero")]
127 pub discarded_tokens: u64,
128 #[serde(default, skip_serializing_if = "counter_is_zero")]
129 pub token_reporting_unavailable: u64,
130}
131
132fn counter_is_zero(value: &u64) -> bool {
133 *value == 0
134}
135
136impl EmbeddingTruncationReport {
137 pub fn observe(&mut self, outcome: &DocumentEmbeddingOutcome) {
138 if outcome.truncated {
139 self.truncated = self.truncated.saturating_add(1);
140 self.discarded_bytes = self
141 .discarded_bytes
142 .saturating_add(outcome.source_bytes.saturating_sub(outcome.embedded_bytes) as u64);
143 }
144 if let Some(count) = outcome.provider_token_count {
145 self.discarded_tokens = self
146 .discarded_tokens
147 .saturating_add(count.before_truncation.saturating_sub(count.embedded) as u64);
148 } else {
149 self.token_reporting_unavailable = self.token_reporting_unavailable.saturating_add(1);
150 }
151 }
152
153 #[must_use]
154 pub const fn any_truncated(&self) -> bool {
155 self.truncated > 0
156 }
157
158 pub fn merge(&mut self, other: Self) {
160 self.truncated = self.truncated.saturating_add(other.truncated);
161 self.discarded_bytes = self.discarded_bytes.saturating_add(other.discarded_bytes);
162 self.discarded_tokens = self.discarded_tokens.saturating_add(other.discarded_tokens);
163 self.token_reporting_unavailable = self
164 .token_reporting_unavailable
165 .saturating_add(other.token_reporting_unavailable);
166 }
167}
168
169struct DocumentEmbeddings {
170 vectors: Vec<Vec<f32>>,
171 token_counts: Option<Vec<TokenCount>>,
172}
173
174async fn document_embeddings(
175 service: &dyn EmbeddingService,
176 texts: &[String],
177 model: EmbeddingModel,
178) -> RuntimeResult<DocumentEmbeddings> {
179 let result = with_embedding_admission(async {
180 match service
181 .embed_with_report(texts, model, EmbeddingRole::Passage)
182 .await
183 {
184 Ok(report) => Ok(DocumentEmbeddings {
185 vectors: report.embeddings,
186 token_counts: Some(report.token_counts),
187 }),
188 Err(lattice_embed::EmbedError::Unsupported(_)) => service
189 .embed_passage(texts, model)
190 .await
191 .map(|vectors| DocumentEmbeddings {
192 vectors,
193 token_counts: None,
194 }),
195 Err(error) => Err(error),
196 }
197 })
198 .await?;
199 if let Some(counts) = &result.token_counts {
200 if result.vectors.len() != texts.len() || counts.len() != texts.len() {
201 return Err(RuntimeError::Internal(format!(
202 "embed_with_report returned {} vectors and {} token counts for {} inputs",
203 result.vectors.len(),
204 counts.len(),
205 texts.len(),
206 )));
207 }
208 if counts
209 .iter()
210 .any(|count| count.embedded > count.before_truncation)
211 {
212 return Err(RuntimeError::Internal(
213 "embed_with_report returned an embedded token count above the input count".into(),
214 ));
215 }
216 }
217 Ok(result)
218}
219
220pub fn document_embedding_budget(model_name: &str) -> usize {
222 parse_embedding_model_alias(model_name)
223 .and_then(|model| model.document_instruction())
224 .map_or(MAX_TEXT_BYTES, |prefix| {
225 MAX_TEXT_BYTES.saturating_sub(prefix.len())
226 })
227}
228
229pub fn bounded_embedding_input(text: &str, max_bytes: usize) -> (&str, bool) {
231 if text.len() <= max_bytes {
232 return (text, false);
233 }
234
235 let end = text
236 .char_indices()
237 .map(|(index, _)| index)
238 .take_while(|index| *index <= max_bytes)
239 .last()
240 .unwrap_or(0);
241 (&text[..end], true)
242}
243
244fn prepared_document_fingerprint(text: &str, model: EmbeddingModel) -> ContentRef {
245 match model.document_instruction() {
246 Some(prefix) => VectorRecord::fingerprint_text(&format!("{prefix}{text}")),
247 None => VectorRecord::fingerprint_text(text),
248 }
249}
250
251impl KhiveRuntime {
252 fn require_default_embedder(&self) -> RuntimeResult<&str> {
253 let model_name = self.default_embedder_name();
254 if model_name.is_empty() {
255 return Err(RuntimeError::Unconfigured("embedding_model".into()));
256 }
257 Ok(model_name)
258 }
259
260 pub async fn embed(&self, text: &str) -> RuntimeResult<Vec<f32>> {
265 let model_name = self.require_default_embedder()?;
266 self.embed_with_model(model_name, text).await
267 }
268
269 pub async fn embed_with_model(&self, model_name: &str, text: &str) -> RuntimeResult<Vec<f32>> {
285 let model = parse_embedding_model_alias(model_name);
286 let service = self.embedder(model_name).await?;
287 let emb_model = model.unwrap_or_default();
288 crate::usage::count(crate::usage::UsageUnit::EmbedCalls, 1);
292 let out = with_embedding_admission(service.embed_one(text, emb_model)).await;
293 out
294 }
295
296 pub async fn embed_document_with_model_outcome(
315 &self,
316 model_name: &str,
317 text: &str,
318 ) -> RuntimeResult<DocumentEmbeddingOutcome> {
319 self.embed_document_with_model_outcome_inner(None, model_name, text)
320 .await
321 }
322
323 pub(crate) async fn embed_document_with_model_outcome_for_token(
324 &self,
325 token: &NamespaceToken,
326 model_name: &str,
327 text: &str,
328 ) -> RuntimeResult<DocumentEmbeddingOutcome> {
329 self.embed_document_with_model_outcome_inner(Some(token), model_name, text)
330 .await
331 }
332
333 async fn embed_document_with_model_outcome_inner(
334 &self,
335 token: Option<&NamespaceToken>,
336 model_name: &str,
337 text: &str,
338 ) -> RuntimeResult<DocumentEmbeddingOutcome> {
339 let model = parse_embedding_model_alias(model_name);
340 let (service, audited_document_preparation) = self
341 .embedder_with_input_attestation(model_name, token)
342 .await?;
343 let emb_model = model.unwrap_or_default();
344 let source_bytes = text.len();
345 let (text, truncated) =
346 bounded_embedding_input(text, document_embedding_budget(model_name));
347 let embedded_bytes = text.len();
348 crate::usage::count(crate::usage::UsageUnit::EmbedCalls, 1);
350 let embeddings =
351 document_embeddings(service.as_ref(), &[text.to_string()], emb_model).await?;
352 let provider_token_count = embeddings.token_counts.map(|counts| counts[0]);
353 let mut vectors = embeddings.vectors;
354 if vectors.len() != 1 {
355 return Err(RuntimeError::Internal(format!(
356 "embed_passage returned {} vectors for 1 input",
357 vectors.len()
358 )));
359 }
360 let out = vectors.pop().expect("checked len == 1 above");
361 Ok(DocumentEmbeddingOutcome {
362 model_name: model_name.to_owned(),
363 vector: out,
364 prepared_text_fingerprint: audited_document_preparation
365 .then(|| prepared_document_fingerprint(text, emb_model)),
366 source_bytes,
367 embedded_bytes,
368 provider_token_count,
369 truncated: truncated || provider_token_count.is_some_and(|count| count.truncated()),
370 })
371 }
372
373 pub async fn embed_query_with_model(
387 &self,
388 model_name: &str,
389 text: &str,
390 ) -> RuntimeResult<Vec<f32>> {
391 self.embed_query_with_model_inner(None, model_name, text)
392 .await
393 }
394
395 pub(crate) async fn embed_query_with_model_for_token(
396 &self,
397 token: &NamespaceToken,
398 model_name: &str,
399 text: &str,
400 ) -> RuntimeResult<Vec<f32>> {
401 self.embed_query_with_model_inner(Some(token), model_name, text)
402 .await
403 }
404
405 async fn embed_query_with_model_inner(
406 &self,
407 token: Option<&NamespaceToken>,
408 model_name: &str,
409 text: &str,
410 ) -> RuntimeResult<Vec<f32>> {
411 let _phase = khive_storage::request_context::enter_search_phase(
412 khive_storage::request_context::SearchPhase::VectorEmbedding,
413 );
414 let model = parse_embedding_model_alias(model_name);
415 let service = match token {
416 Some(token) => self.embedder_with_token(token, model_name).await?,
417 None => self.embedder(model_name).await?,
418 };
419 let texts = [text.to_string()];
420 let emb_model = model.unwrap_or(CUSTOM_QUERY_ROLE_MODEL);
421 crate::usage::count(crate::usage::UsageUnit::EmbedCalls, 1);
423 let embeddings = match model {
424 Some(
425 EmbeddingModel::BgeSmallEnV15
426 | EmbeddingModel::BgeBaseEnV15
427 | EmbeddingModel::BgeLargeEnV15,
428 ) => with_embedding_admission(service.embed(&texts, emb_model)).await,
429 _ => with_embedding_admission(service.embed_query(&texts, emb_model)).await,
430 };
431 let out = embeddings?
432 .into_iter()
433 .next()
434 .ok_or_else(|| RuntimeError::Internal("embed_query returned empty vec".into()))?;
435 Ok(out)
436 }
437
438 pub async fn embed_document(&self, text: &str) -> RuntimeResult<Vec<f32>> {
447 let outcome = self.embed_document_outcome(text).await?;
448 document_vector_or_refusal(outcome)
449 }
450
451 pub async fn embed_document_outcome(
453 &self,
454 text: &str,
455 ) -> RuntimeResult<DocumentEmbeddingOutcome> {
456 let model_name = self.require_default_embedder()?;
457 self.embed_document_with_model_outcome(model_name, text)
458 .await
459 }
460
461 pub async fn embed_query(&self, text: &str) -> RuntimeResult<Vec<f32>> {
468 let model_name = self.require_default_embedder()?;
469 self.embed_query_with_model(model_name, text).await
470 }
471
472 async fn embed_query_for_token(
473 &self,
474 token: &NamespaceToken,
475 text: &str,
476 ) -> RuntimeResult<Vec<f32>> {
477 let model_name = self.require_default_embedder()?;
478 self.embed_query_with_model_for_token(token, model_name, text)
479 .await
480 }
481
482 pub async fn embed_batch(&self, texts: &[String]) -> RuntimeResult<Vec<Vec<f32>>> {
490 if texts.is_empty() {
491 return Ok(vec![]);
492 }
493 let model_name = self.require_default_embedder()?;
494 self.embed_batch_with_model(model_name, texts).await
495 }
496
497 pub async fn embed_batch_with_model(
502 &self,
503 model_name: &str,
504 texts: &[String],
505 ) -> RuntimeResult<Vec<Vec<f32>>> {
506 if texts.is_empty() {
507 return Ok(vec![]);
508 }
509 let model = parse_embedding_model_alias(model_name);
510 let service = self.embedder(model_name).await?;
511 let emb_model = model.unwrap_or_default();
512 let out = with_embedding_admission(service.embed(texts, emb_model)).await;
513 crate::usage::count(crate::usage::UsageUnit::EmbedCalls, texts.len() as u64);
514 out
515 }
516
517 pub async fn embed_document_batch_with_model(
531 &self,
532 model_name: &str,
533 texts: &[String],
534 ) -> RuntimeResult<Vec<Vec<f32>>> {
535 if texts.is_empty() {
536 return Ok(vec![]);
537 }
538 let outcomes = self
539 .embed_document_batch_with_model_outcomes(model_name, texts)
540 .await?;
541 let mut report = EmbeddingTruncationReport::default();
542 for outcome in &outcomes {
543 report.observe(outcome);
544 }
545 if report.any_truncated() {
546 return Err(RuntimeError::InvalidInput(format!(
547 "embedding input truncated for {} documents ({} discarded bytes, {} discarded tokens); use embed_document_batch_with_model_outcomes to inspect the bounded vectors",
548 report.truncated, report.discarded_bytes, report.discarded_tokens
549 )));
550 }
551 Ok(outcomes.into_iter().map(|outcome| outcome.vector).collect())
552 }
553
554 pub async fn embed_document_batch_with_model_outcomes(
555 &self,
556 model_name: &str,
557 texts: &[String],
558 ) -> RuntimeResult<Vec<DocumentEmbeddingOutcome>> {
559 self.embed_document_batch_with_model_outcomes_inner(None, model_name, texts)
560 .await
561 }
562
563 pub(crate) async fn embed_document_batch_with_model_outcomes_for_token(
564 &self,
565 token: &NamespaceToken,
566 model_name: &str,
567 texts: &[String],
568 ) -> RuntimeResult<Vec<DocumentEmbeddingOutcome>> {
569 self.embed_document_batch_with_model_outcomes_inner(Some(token), model_name, texts)
570 .await
571 }
572
573 async fn embed_document_batch_with_model_outcomes_inner(
574 &self,
575 token: Option<&NamespaceToken>,
576 model_name: &str,
577 texts: &[String],
578 ) -> RuntimeResult<Vec<DocumentEmbeddingOutcome>> {
579 if texts.is_empty() {
580 return Ok(vec![]);
581 }
582 let model = parse_embedding_model_alias(model_name);
583 let (service, audited_document_preparation) = self
584 .embedder_with_input_attestation(model_name, token)
585 .await?;
586 let emb_model = model.unwrap_or_default();
587 let budget = document_embedding_budget(model_name);
588 if token.is_some() {
589 crate::usage::count(crate::usage::UsageUnit::EmbedCalls, texts.len() as u64);
591 }
592 let out = if texts.iter().all(|text| text.len() <= budget) {
593 document_embeddings(service.as_ref(), texts, emb_model).await
594 } else {
595 let bounded_texts: Vec<String> = texts
596 .iter()
597 .map(|text| bounded_embedding_input(text, budget).0.to_owned())
598 .collect();
599 document_embeddings(service.as_ref(), &bounded_texts, emb_model).await
600 };
601 if token.is_none() {
602 crate::usage::count(crate::usage::UsageUnit::EmbedCalls, texts.len() as u64);
603 }
604 let embeddings = out?;
605 let vectors = embeddings.vectors;
606 if vectors.len() != texts.len() {
607 return Err(RuntimeError::Internal(format!(
608 "embed_passage returned {} vectors for {} inputs",
609 vectors.len(),
610 texts.len()
611 )));
612 }
613 Ok(texts
614 .iter()
615 .zip(vectors)
616 .enumerate()
617 .map(|(index, (text, vector))| {
618 let (bounded, truncated) = bounded_embedding_input(text, budget);
619 let provider_token_count =
620 embeddings.token_counts.as_ref().map(|counts| counts[index]);
621 DocumentEmbeddingOutcome {
622 model_name: model_name.to_owned(),
623 vector,
624 prepared_text_fingerprint: audited_document_preparation
625 .then(|| prepared_document_fingerprint(bounded, emb_model)),
626 source_bytes: text.len(),
627 embedded_bytes: bounded.len(),
628 provider_token_count,
629 truncated: truncated
630 || provider_token_count.is_some_and(|count| count.truncated()),
631 }
632 })
633 .collect())
634 }
635
636 pub async fn embed_document_batch(&self, texts: &[String]) -> RuntimeResult<Vec<Vec<f32>>> {
645 if texts.is_empty() {
646 return Ok(vec![]);
647 }
648 let model_name = self.require_default_embedder()?;
649 self.embed_document_batch_with_model(model_name, texts)
650 .await
651 }
652
653 pub async fn embed_document_batch_outcomes(
655 &self,
656 texts: &[String],
657 ) -> RuntimeResult<Vec<DocumentEmbeddingOutcome>> {
658 if texts.is_empty() {
659 return Ok(vec![]);
660 }
661 let model_name = self.require_default_embedder()?;
662 self.embed_document_batch_with_model_outcomes(model_name, texts)
663 .await
664 }
665
666 pub async fn embed_query_batch_with_model(
673 &self,
674 model_name: &str,
675 texts: &[String],
676 ) -> RuntimeResult<Vec<Vec<f32>>> {
677 if texts.is_empty() {
678 return Ok(vec![]);
679 }
680 let model = parse_embedding_model_alias(model_name);
681 let service = self.embedder(model_name).await?;
682 let emb_model = model.unwrap_or(CUSTOM_QUERY_ROLE_MODEL);
683 let out = match model {
684 Some(
685 EmbeddingModel::BgeSmallEnV15
686 | EmbeddingModel::BgeBaseEnV15
687 | EmbeddingModel::BgeLargeEnV15,
688 ) => with_embedding_admission(service.embed(texts, emb_model)).await,
689 _ => with_embedding_admission(service.embed_query(texts, emb_model)).await,
690 };
691 crate::usage::count(crate::usage::UsageUnit::EmbedCalls, texts.len() as u64);
692 out
693 }
694
695 pub(crate) async fn note_search_vector_search(
699 &self,
700 token: &NamespaceToken,
701 query_embedding: Option<Vec<f32>>,
702 query_text: &str,
703 top_k: u32,
704 ) -> RuntimeResult<Vec<VectorSearchHit>> {
705 let embedding = match query_embedding {
706 Some(embedding) => embedding,
707 None => self.embed_query_for_token(token, query_text).await?,
708 };
709 let model = self.default_embedder_name();
710 let fallback_cause = if model.is_empty() {
711 NoteSearchFallbackCause::NoModel
712 } else if let Some(provider) = self.note_search_ann_provider()? {
713 if let Some(hits) = provider.search(token, model, &embedding, top_k).await? {
714 crate::note_search_ann::record_ann_route(self.backend().pool());
715 crate::usage::count(crate::usage::UsageUnit::VectorPasses, 1);
716 return Ok(hits);
717 }
718 NoteSearchFallbackCause::NoEligibleGraph
719 } else {
720 NoteSearchFallbackCause::NoProvider
721 };
722 crate::note_search_ann::record_fallback_route(self.backend().pool(), fallback_cause);
723 self.vector_search(
724 token,
725 Some(embedding),
726 None,
727 top_k,
728 Some(SubstrateKind::Note),
729 )
730 .await
731 }
732
733 #[allow(clippy::too_many_arguments)]
779 pub async fn hybrid_search(
780 &self,
781 token: &NamespaceToken,
782 query_text: &str,
783 query_vector: Option<Vec<f32>>,
784 limit: u32,
785 entity_kind: Option<&str>,
786 entity_type: Option<&str>,
787 tags_any: &[String],
788 properties_filter: Option<&serde_json::Value>,
789 ) -> RuntimeResult<Vec<SearchHit>> {
790 self.hybrid_search_with_text_mode(
791 token,
792 query_text,
793 query_vector,
794 limit,
795 entity_kind,
796 entity_type,
797 tags_any,
798 properties_filter,
799 TextQueryMode::Plain,
800 )
801 .await
802 }
803
804 #[allow(clippy::too_many_arguments)]
806 pub async fn hybrid_search_with_text_mode(
807 &self,
808 token: &NamespaceToken,
809 query_text: &str,
810 query_vector: Option<Vec<f32>>,
811 limit: u32,
812 entity_kind: Option<&str>,
813 entity_type: Option<&str>,
814 tags_any: &[String],
815 properties_filter: Option<&serde_json::Value>,
816 text_mode: TextQueryMode,
817 ) -> RuntimeResult<Vec<SearchHit>> {
818 let (hits, _vector_error) = self
819 .hybrid_search_inner(
820 token,
821 query_text,
822 query_vector,
823 limit,
824 entity_kind,
825 entity_type,
826 tags_any,
827 properties_filter,
828 text_mode,
829 None,
830 false,
831 None,
832 )
833 .await?;
834 Ok(hits)
835 }
836
837 pub async fn hybrid_search_each_kind(
848 &self,
849 token: &NamespaceToken,
850 query_text: &str,
851 query_vector: Option<Vec<f32>>,
852 limit: u32,
853 entity_kinds: &[&str],
854 ) -> RuntimeResult<Vec<Vec<SearchHit>>> {
855 if entity_kinds.is_empty() {
856 return Ok(Vec::new());
857 }
858 let candidates = limit.saturating_mul(CANDIDATE_MULTIPLIER).max(limit);
859 let (vector_hits, _vector_error) = self
860 .hybrid_vector_stage(token, query_text, query_vector, candidates, None, false)
861 .await?;
862 let mut per_kind = Vec::with_capacity(entity_kinds.len());
863 for &kind in entity_kinds {
864 let (hits, _vector_error) = self
865 .hybrid_search_inner(
866 token,
867 query_text,
868 None,
869 limit,
870 Some(kind),
871 None,
872 &[],
873 None,
874 TextQueryMode::Plain,
875 None,
876 false,
877 Some(vector_hits.clone()),
878 )
879 .await?;
880 per_kind.push(hits);
881 }
882 Ok(per_kind)
883 }
884
885 #[allow(clippy::too_many_arguments)]
890 pub(crate) async fn hybrid_search_with_vector_similarity_floor(
891 &self,
892 token: &NamespaceToken,
893 query_text: &str,
894 query_vector: Option<Vec<f32>>,
895 limit: u32,
896 entity_kind: Option<&str>,
897 entity_type: Option<&str>,
898 tags_any: &[String],
899 properties_filter: Option<&serde_json::Value>,
900 vector_similarity_floor: f64,
901 ) -> RuntimeResult<Vec<SearchHit>> {
902 let (hits, _vector_error) = self
903 .hybrid_search_inner(
904 token,
905 query_text,
906 query_vector,
907 limit,
908 entity_kind,
909 entity_type,
910 tags_any,
911 properties_filter,
912 TextQueryMode::Plain,
913 Some(vector_similarity_floor),
914 false,
915 None,
916 )
917 .await?;
918 Ok(hits)
919 }
920
921 #[allow(clippy::too_many_arguments)]
932 pub async fn hybrid_search_outcome(
933 &self,
934 token: &NamespaceToken,
935 query_text: &str,
936 limit: u32,
937 entity_kind: Option<&str>,
938 entity_type: Option<&str>,
939 tags_any: &[String],
940 properties_filter: Option<&serde_json::Value>,
941 ) -> RuntimeResult<HybridSearchOutcome> {
942 self.hybrid_search_outcome_with_text_mode(
943 token,
944 query_text,
945 limit,
946 entity_kind,
947 entity_type,
948 tags_any,
949 properties_filter,
950 TextQueryMode::Plain,
951 )
952 .await
953 }
954
955 #[allow(clippy::too_many_arguments)]
957 pub async fn hybrid_search_outcome_with_text_mode(
958 &self,
959 token: &NamespaceToken,
960 query_text: &str,
961 limit: u32,
962 entity_kind: Option<&str>,
963 entity_type: Option<&str>,
964 tags_any: &[String],
965 properties_filter: Option<&serde_json::Value>,
966 text_mode: TextQueryMode,
967 ) -> RuntimeResult<HybridSearchOutcome> {
968 let (hits, vector_error) = self
969 .hybrid_search_inner(
970 token,
971 query_text,
972 None,
973 limit,
974 entity_kind,
975 entity_type,
976 tags_any,
977 properties_filter,
978 text_mode,
979 None,
980 true,
981 None,
982 )
983 .await?;
984 Ok(HybridSearchOutcome { hits, vector_error })
985 }
986
987 #[allow(clippy::too_many_arguments)]
990 async fn hybrid_search_inner(
991 &self,
992 token: &NamespaceToken,
993 query_text: &str,
994 query_vector: Option<Vec<f32>>,
995 limit: u32,
996 entity_kind: Option<&str>,
997 entity_type: Option<&str>,
998 tags_any: &[String],
999 properties_filter: Option<&serde_json::Value>,
1000 text_mode: TextQueryMode,
1001 vector_similarity_floor: Option<f64>,
1002 tolerate_vector_error: bool,
1003 vector_pool: Option<Vec<VectorSearchHit>>,
1004 ) -> RuntimeResult<(Vec<SearchHit>, Option<String>)> {
1005 khive_storage::await_request_read_phase("hybrid_search", async {
1006 let candidates = limit.saturating_mul(CANDIDATE_MULTIPLIER).max(limit);
1007
1008 let visible_ns: Vec<String> = token
1009 .visible_namespaces()
1010 .iter()
1011 .map(|ns| ns.as_str().to_owned())
1012 .collect();
1013 let text_store = self.text(token)?;
1017 let text_fut = async {
1018 let _phase = khive_storage::request_context::enter_search_phase(
1019 khive_storage::request_context::SearchPhase::TextSearch,
1020 );
1021 text_store
1022 .search(TextSearchRequest {
1023 query: query_text.to_string(),
1024 mode: text_mode,
1025 filter: Some(TextFilter {
1026 namespaces: visible_ns.clone(),
1027 record_kinds: entity_kind
1034 .map(|kind| vec![kind.to_string()])
1035 .unwrap_or_default(),
1036 ..TextFilter::default()
1037 }),
1038 top_k: candidates,
1039 snippet_chars: 200,
1040 })
1041 .await
1042 };
1043 let text_fut = crate::stage_seam::text_stage(text_fut);
1044 let vector_fut = async {
1046 let _phase = khive_storage::request_context::enter_search_phase(
1047 khive_storage::request_context::SearchPhase::VectorSearch,
1048 );
1049 match vector_pool {
1050 Some(pool) => Ok((pool, None)),
1051 None => {
1052 self.hybrid_vector_stage(
1053 token,
1054 query_text,
1055 query_vector,
1056 candidates,
1057 vector_similarity_floor,
1058 tolerate_vector_error,
1059 )
1060 .await
1061 }
1062 }
1063 };
1064 let (text_search_result, vector_result) = tokio::join!(text_fut, vector_fut);
1065 let text_hits = crate::error::fts_text_leg_or_err(
1070 text_search_result.map_err(RuntimeError::from),
1071 "hybrid_search",
1072 query_text,
1073 )?;
1074 let (vector_hits, vector_error) = vector_result?;
1075
1076 let fusion_limit = text_hits.len().saturating_add(vector_hits.len());
1081 let mut fused = {
1082 let _phase = khive_storage::request_context::enter_search_phase(
1083 khive_storage::request_context::SearchPhase::Fusion,
1084 );
1085 rrf_fuse(text_hits, vector_hits, fusion_limit, query_text)
1086 };
1087
1088 if !fused.is_empty() {
1091 let candidate_ids: Vec<Uuid> = fused.iter().map(|h| h.entity_id).collect();
1092 let _phase = khive_storage::request_context::enter_search_phase(
1093 khive_storage::request_context::SearchPhase::EntityHydration,
1094 );
1095 let alive_page = self
1096 .entities(token)?
1097 .query_entities(
1098 token.namespace().as_str(),
1099 EntityFilter {
1100 ids: candidate_ids,
1101 kinds: entity_kind.map(|k| vec![k.to_string()]).unwrap_or_default(),
1102 entity_types: entity_type
1103 .map(|t| vec![t.to_string()])
1104 .unwrap_or_default(),
1105 namespaces: visible_ns,
1106 tags_any: tags_any.to_vec(),
1107 ..EntityFilter::default()
1108 },
1109 PageRequest {
1110 offset: 0,
1111 limit: u32::try_from(fused.len()).unwrap_or(u32::MAX),
1112 },
1113 )
1114 .await?;
1115 drop(_phase);
1116 let mut entity_meta: HashMap<Uuid, (String, Option<String>)> = HashMap::new();
1117 let mut alive: HashSet<Uuid> = HashSet::new();
1118 for e in alive_page.items {
1119 if let Some(pf) = properties_filter {
1122 if !properties_match(e.properties.as_ref(), pf) {
1123 continue;
1124 }
1125 }
1126 alive.insert(e.id);
1127 entity_meta.insert(e.id, (e.name, e.description));
1128 }
1129
1130 fused.retain(|h| alive.contains(&h.entity_id));
1131
1132 for hit in &mut fused {
1134 if let Some((name, description)) = entity_meta.get(&hit.entity_id) {
1135 if hit.title.is_none() {
1136 hit.title = Some(name.clone());
1137 }
1138 if hit.snippet.is_none() {
1139 hit.snippet = description.clone();
1140 }
1141 }
1142 }
1143 }
1144
1145 fused.truncate(limit as usize);
1146 Ok((fused, vector_error))
1147 })
1148 .await?
1149 }
1150
1151 async fn embed_backfill_page(
1152 &self,
1153 token: &NamespaceToken,
1154 model_name: &str,
1155 inputs: &[(Uuid, String)],
1156 ) -> Vec<Option<DocumentEmbeddingOutcome>> {
1157 let texts: Vec<String> = inputs.iter().map(|(_, text)| text.clone()).collect();
1158 match self
1159 .embed_document_batch_with_model_outcomes_for_token(token, model_name, &texts)
1160 .await
1161 {
1162 Ok(outcomes) => outcomes.into_iter().map(Some).collect(),
1163 Err(error) => {
1164 tracing::warn!(
1165 model = %model_name,
1166 error = %error,
1167 "backfill_missing_embeddings: batch embed failed; retrying records individually"
1168 );
1169 let mut outcomes = Vec::with_capacity(inputs.len());
1170 for (id, text) in inputs {
1171 match self
1172 .embed_document_with_model_outcome_for_token(token, model_name, text)
1173 .await
1174 {
1175 Ok(outcome) => outcomes.push(Some(outcome)),
1176 Err(error) => {
1177 tracing::warn!(
1178 id = %id,
1179 model = %model_name,
1180 error = %error,
1181 "backfill_missing_embeddings: record embed failed"
1182 );
1183 outcomes.push(None);
1184 }
1185 }
1186 }
1187 outcomes
1188 }
1189 }
1190 }
1191
1192 pub async fn backfill_missing_embeddings(&self, token: &NamespaceToken) -> RuntimeResult<u64> {
1205 use khive_storage::types::{SqlRow, SqlStatement, SqlValue};
1206
1207 let model_names = self.registered_embedding_model_names();
1208 if model_names.is_empty() {
1209 tracing::debug!(
1210 "backfill_missing_embeddings: no embedding models registered, skipping"
1211 );
1212 return Ok(0);
1213 }
1214
1215 let ns = token.namespace().as_str().to_string();
1216 let mut total_backfilled = 0u64;
1217
1218 for model_name in &model_names {
1219 let mut model_truncation = EmbeddingTruncationReport::default();
1220 match self.vectors_for_model(token, model_name) {
1221 Ok(_) => {}
1222 Err(error) => {
1223 tracing::warn!(model = %model_name, error = %error,
1224 "backfill_missing_embeddings: vector store unavailable");
1225 continue;
1226 }
1227 };
1228 let vec_table = format!("vec_{}", sanitize_key(model_name));
1230
1231 const PAGE_SIZE: usize = EMBEDDING_BATCH_PAGE_SIZE;
1235 let mut entity_total = 0usize;
1236 let mut entity_cursor = String::new();
1237 loop {
1238 let entity_sql = SqlStatement {
1239 sql: format!(
1240 "SELECT id FROM entities \
1241 WHERE namespace = ?1 AND deleted_at IS NULL AND id > ?3 \
1242 AND id NOT IN (\
1243 SELECT subject_id FROM {vec_table} \
1244 WHERE namespace = ?1 AND embedding_model = ?2 \
1245 ) ORDER BY id LIMIT {PAGE_SIZE}"
1246 ),
1247 params: vec![
1248 SqlValue::Text(ns.clone()),
1249 SqlValue::Text(model_name.clone()),
1250 SqlValue::Text(entity_cursor.clone()),
1251 ],
1252 label: Some("backfill_entities".into()),
1253 };
1254
1255 let entity_rows: Vec<SqlRow> = {
1256 let sql = self.sql();
1257 let reader_result = sql.reader().await;
1258 #[cfg(any(test, feature = "fault-injection"))]
1259 let reader_result = if BACKFILL_READER_FAIL.with(|c| c.get()) {
1260 BACKFILL_READER_FAIL.with(|c| c.set(false));
1261 Err(khive_storage::StorageError::Pool {
1262 operation: "reader".into(),
1263 message: "injected failure".into(),
1264 })
1265 } else {
1266 reader_result
1267 };
1268 let mut reader = reader_result.map_err(RuntimeError::Storage)?;
1269 reader
1270 .query_all(entity_sql)
1271 .await
1272 .map_err(RuntimeError::Storage)?
1273 };
1274
1275 let batch_len = entity_rows.len();
1276 entity_total += batch_len;
1277 if batch_len == 0 {
1278 break;
1279 }
1280 entity_cursor = match entity_rows.last().and_then(|row| row.columns.first()) {
1281 Some(column) => match &column.value {
1282 SqlValue::Text(id) => id.clone(),
1283 _ => {
1284 return Err(RuntimeError::Internal(
1285 "backfill entity ID is not text".into(),
1286 ))
1287 }
1288 },
1289 None => {
1290 return Err(RuntimeError::Internal(
1291 "backfill entity page is empty".into(),
1292 ))
1293 }
1294 };
1295
1296 let entity_store = self.entities(token)?;
1297 let mut entities = Vec::with_capacity(batch_len);
1298 let mut inputs = Vec::with_capacity(batch_len);
1299 for row in &entity_rows {
1300 let Some(SqlValue::Text(id)) = row.columns.first().map(|column| &column.value)
1301 else {
1302 continue;
1303 };
1304 let Ok(id) = id.parse::<Uuid>() else { continue };
1305 let Some(entity) = entity_store.get_entity(id).await? else {
1306 continue;
1307 };
1308 if entity.namespace != ns || entity.deleted_at.is_some() {
1309 continue;
1310 }
1311 let text = crate::curation::entity_embedding_text(&entity);
1312 if text.trim().is_empty() {
1313 continue;
1314 }
1315 inputs.push((id, text));
1316 entities.push(entity);
1317 }
1318 let outcomes = self.embed_backfill_page(token, model_name, &inputs).await;
1319 for (entity, outcome) in entities.into_iter().zip(outcomes) {
1320 let Some(outcome) = outcome else { continue };
1321 model_truncation.observe(&outcome);
1322 match self
1323 .publish_entity_vector_revision(token, &entity, model_name, &outcome.vector)
1324 .await
1325 {
1326 Ok(true) => total_backfilled += 1,
1327 Ok(false) => {}
1328 Err(error) => tracing::warn!(
1329 id = %entity.id, model = %model_name, error = %error,
1330 "backfill_missing_embeddings: entity vector insert failed"
1331 ),
1332 }
1333 }
1334
1335 if batch_len < PAGE_SIZE {
1336 break;
1337 }
1338 }
1339
1340 let text_store = self.text_for_notes(token).ok();
1342 let note_store = self.notes(token).ok();
1343 let mut note_total = 0usize;
1344 let mut note_cursor = String::new();
1347 loop {
1348 let note_sql = SqlStatement {
1351 sql: format!(
1352 "SELECT id FROM notes \
1353 WHERE namespace = ?1 AND deleted_at IS NULL AND id > ?3 \
1354 AND id NOT IN (\
1355 SELECT subject_id FROM {vec_table} \
1356 WHERE namespace = ?1 AND embedding_model = ?2 \
1357 ) ORDER BY id LIMIT {PAGE_SIZE}"
1358 ),
1359 params: vec![
1360 SqlValue::Text(ns.clone()),
1361 SqlValue::Text(model_name.clone()),
1362 SqlValue::Text(note_cursor.clone()),
1363 ],
1364 label: Some("backfill_notes".into()),
1365 };
1366
1367 let note_rows: Vec<SqlRow> = {
1368 let sql = self.sql();
1369 let reader_result = sql.reader().await;
1370 #[cfg(any(test, feature = "fault-injection"))]
1371 let reader_result = if BACKFILL_READER_FAIL.with(|c| c.get()) {
1372 BACKFILL_READER_FAIL.with(|c| c.set(false));
1373 Err(khive_storage::StorageError::Pool {
1374 operation: "reader".into(),
1375 message: "injected failure".into(),
1376 })
1377 } else {
1378 reader_result
1379 };
1380 let mut reader = reader_result.map_err(RuntimeError::Storage)?;
1381 reader
1382 .query_all(note_sql)
1383 .await
1384 .map_err(RuntimeError::Storage)?
1385 };
1386
1387 let batch_len = note_rows.len();
1388 note_total += batch_len;
1389 if batch_len == 0 {
1390 break;
1391 }
1392 note_cursor = match note_rows.last().and_then(|row| row.columns.first()) {
1393 Some(column) => match &column.value {
1394 SqlValue::Text(id) => id.clone(),
1395 _ => {
1396 return Err(RuntimeError::Internal(
1397 "backfill note ID is not text".into(),
1398 ))
1399 }
1400 },
1401 None => {
1402 return Err(RuntimeError::Internal("backfill note page is empty".into()))
1403 }
1404 };
1405
1406 let mut notes_for_batch = Vec::with_capacity(batch_len);
1407 let mut inputs = Vec::with_capacity(batch_len);
1408 for row in ¬e_rows {
1409 let id_str = row.columns.first().and_then(|c| {
1410 if let SqlValue::Text(s) = &c.value {
1411 Some(s.clone())
1412 } else {
1413 None
1414 }
1415 });
1416
1417 let Some(id_str) = id_str else {
1418 continue;
1419 };
1420 let Ok(id) = id_str.parse::<Uuid>() else {
1421 continue;
1422 };
1423
1424 let note = match ¬e_store {
1425 Some(store) => match store.get_note(id).await {
1426 Ok(Some(n)) => n,
1427 _ => continue,
1428 },
1429 None => continue,
1430 };
1431
1432 if note.content.trim().is_empty() {
1433 continue;
1434 }
1435
1436 if model_names.first().map(|n| n.as_str()) == Some(model_name.as_str()) {
1439 if let Some(ref ts) = text_store {
1440 if let Err(e) = ts.upsert_document(note_fts_document(¬e)).await {
1441 tracing::warn!(id = %id, error = %e,
1442 "backfill_missing_embeddings: note FTS upsert failed");
1443 }
1444 }
1445 }
1446
1447 if !self
1448 .embedding_models_for_note_kind(¬e.kind)
1449 .contains(model_name)
1450 {
1451 continue;
1452 }
1453
1454 inputs.push((id, note.content.clone()));
1455 notes_for_batch.push(note);
1456 }
1457
1458 let outcomes = self.embed_backfill_page(token, model_name, &inputs).await;
1459 for (note, outcome) in notes_for_batch.into_iter().zip(outcomes) {
1460 let Some(outcome) = outcome else { continue };
1461 model_truncation.observe(&outcome);
1462 match self
1463 .publish_note_vector_revision(token, ¬e, model_name, &outcome.vector)
1464 .await
1465 {
1466 Ok(true) => total_backfilled += 1,
1467 Ok(false) => {}
1468 Err(error) => tracing::warn!(
1469 id = %note.id, model = %model_name, error = %error,
1470 "backfill_missing_embeddings: note vector insert failed"
1471 ),
1472 }
1473 }
1474
1475 if batch_len < PAGE_SIZE {
1476 break;
1477 }
1478 }
1479
1480 tracing::info!(
1481 model = %model_name,
1482 namespace = %ns,
1483 entities = entity_total,
1484 notes = note_total,
1485 truncated = model_truncation.truncated,
1486 discarded_bytes = model_truncation.discarded_bytes,
1487 "backfill_missing_embeddings: model pass complete"
1488 );
1489 }
1490
1491 tracing::info!(
1492 namespace = %ns,
1493 total_backfilled = total_backfilled,
1494 "backfill_missing_embeddings: finished"
1495 );
1496
1497 Ok(total_backfilled)
1498 }
1499
1500 pub async fn sweep_orphan_vectors(
1516 &self,
1517 token: &NamespaceToken,
1518 max_delete_per_model: u32,
1519 dry_run: bool,
1520 ) -> RuntimeResult<u64> {
1521 use khive_storage::types::OrphanSweepConfig;
1522 use khive_storage::StorageError;
1523
1524 let model_names = self.registered_embedding_model_names();
1525 if model_names.is_empty() {
1526 tracing::debug!("sweep_orphan_vectors: no embedding models registered, skipping");
1527 return Ok(0);
1528 }
1529
1530 let ns = token.namespace().as_str().to_string();
1531 let mut total_deleted = 0u64;
1532
1533 for model_name in &model_names {
1534 let store = match self.vectors_for_model(token, model_name) {
1535 Ok(s) => s,
1536 Err(e) => {
1537 tracing::warn!(
1538 model = %model_name,
1539 error = %e,
1540 "sweep_orphan_vectors: failed to get vector store, skipping model"
1541 );
1542 continue;
1543 }
1544 };
1545
1546 let caps = store.capabilities();
1547 if !caps.supports_orphan_sweep {
1548 tracing::debug!(
1549 model = %model_name,
1550 "sweep_orphan_vectors: backend does not support orphan sweep, skipping"
1551 );
1552 continue;
1553 }
1554
1555 let config = OrphanSweepConfig {
1556 subject_id_allowlist: None,
1557 namespaces: vec![ns.clone()],
1558 substrate_kinds: vec![],
1559 max_delete: max_delete_per_model,
1560 dry_run,
1561 };
1562
1563 match store.orphan_sweep(&config).await {
1564 Ok(result) => {
1565 tracing::info!(
1566 model = %model_name,
1567 namespace = %ns,
1568 scanned = result.scanned,
1569 deleted = result.deleted,
1570 would_delete = result.would_delete,
1571 dry_run = dry_run,
1572 "sweep_orphan_vectors: sweep complete"
1573 );
1574 total_deleted += result.deleted;
1575 }
1576 Err(StorageError::Unsupported { .. }) => {
1577 tracing::debug!(
1578 model = %model_name,
1579 "sweep_orphan_vectors: backend returned Unsupported, skipping"
1580 );
1581 }
1582 Err(e) => {
1583 tracing::warn!(
1584 model = %model_name,
1585 error = %e,
1586 "sweep_orphan_vectors: sweep failed, continuing with other models"
1587 );
1588 }
1589 }
1590 }
1591
1592 tracing::info!(
1593 namespace = %ns,
1594 total_deleted = total_deleted,
1595 dry_run = dry_run,
1596 "sweep_orphan_vectors: finished"
1597 );
1598
1599 Ok(total_deleted)
1600 }
1601}
1602
1603pub(crate) fn properties_match(
1608 properties: Option<&serde_json::Value>,
1609 filter: &serde_json::Value,
1610) -> bool {
1611 let required = match filter.as_object() {
1612 Some(obj) if !obj.is_empty() => obj,
1613 _ => return true,
1614 };
1615 let actual = match properties.and_then(serde_json::Value::as_object) {
1616 Some(obj) => obj,
1617 None => return false,
1618 };
1619 required
1620 .iter()
1621 .all(|(k, v)| actual.get(k).is_some_and(|av| av == v))
1622}
1623
1624const EXACT_MATCH_BOOST: f64 = 0.5;
1628
1629fn rrf_fuse(
1638 text_hits: Vec<TextSearchHit>,
1639 vector_hits: Vec<VectorSearchHit>,
1640 limit: usize,
1641 query_text: &str,
1642) -> Vec<SearchHit> {
1643 let mut text_arm = Vec::new();
1644 for (rank, hit) in text_hits.into_iter().enumerate() {
1645 let label = HitLabel {
1646 rank,
1647 signals: SearchSignals {
1648 vector_similarity: None,
1649 keyword_score: Some(hit.score),
1650 },
1651 source: SearchSource::Text,
1652 title: hit.title,
1653 snippet: hit.snippet,
1654 };
1655 text_arm.push((hit.subject_id, label));
1656 }
1657 let mut vector_arm = Vec::new();
1658 for (rank, hit) in vector_hits.into_iter().enumerate() {
1659 let label = HitLabel {
1660 rank,
1661 signals: SearchSignals {
1662 vector_similarity: Some(hit.score),
1663 keyword_score: None,
1664 },
1665 source: SearchSource::Vector,
1666 title: None,
1667 snippet: None,
1668 };
1669 vector_arm.push((hit.subject_id, label));
1670 }
1671
1672 let fused = fuse_labelled(
1673 vec![text_arm, vector_arm],
1674 RRF_K,
1675 combine_leg_first_appearance,
1676 );
1677
1678 let query_lower = query_text.to_lowercase();
1680 let boost = DeterministicScore::from_f64(EXACT_MATCH_BOOST);
1681 let mut hits = Vec::with_capacity(fused.len());
1682 for (entity_id, score, label) in fused {
1683 let title_lower = label.title.as_deref().map(str::to_lowercase);
1684 let exact = title_lower.as_deref() == Some(query_lower.as_str());
1685 let score = if exact { score + boost } else { score };
1686 hits.push(SearchHit {
1687 entity_id,
1688 score,
1689 rank_score_kind: RankScoreKind::Rrf,
1690 signals: label.signals,
1691 source: label.source,
1692 title: label.title,
1693 snippet: label.snippet,
1694 });
1695 }
1696
1697 hits.sort_by(|a, b| {
1698 khive_score::cmp_desc_then_id(a.score, &a.entity_id, b.score, &b.entity_id)
1699 });
1700 hits.truncate(limit);
1701 hits
1702}
1703
1704#[cfg(test)]
1705mod rrf_fuse_label_tests;
1706
1707#[cfg(test)]
1708#[path = "retrieval_tests.rs"]
1709mod tests;