1use std::collections::{HashMap, HashSet};
4
5use khive_db::pool::NoteSearchFallbackCause;
6use lattice_embed::{EmbeddingModel, EmbeddingRole, EmbeddingService, TokenCount, MAX_TEXT_BYTES};
7use uuid::Uuid;
8
9use crate::config::{parse_embedding_model_alias, sanitize_key};
10use crate::curation::note_fts_document;
11use crate::embedder_registry::with_embedding_admission;
12use crate::error::{RuntimeError, RuntimeResult};
13use crate::runtime::{KhiveRuntime, NamespaceToken};
14use khive_retrieval::hybrid::{combine_leg_first_appearance, fuse_labelled, HitLabel};
15use khive_score::DeterministicScore;
16use khive_storage::types::{
17 PageRequest, TextFilter, TextQueryMode, TextSearchHit, TextSearchRequest, VectorRecord,
18 VectorSearchHit,
19};
20use khive_storage::ContentRef;
21use khive_storage::EntityFilter;
22use khive_types::SubstrateKind;
23
24const CUSTOM_QUERY_ROLE_MODEL: EmbeddingModel = EmbeddingModel::AllMiniLmL6V2;
30
31pub use khive_retrieval::{
32 HybridSearchOutcome, RankScoreKind, SearchHit, SearchSignals, SearchSource,
33};
34
35mod single_engine;
36
37#[cfg(test)]
38mod request_stop_tests;
39
40#[cfg(test)]
41mod token_report_tests;
42
43pub(crate) const EMBEDDING_BATCH_PAGE_SIZE: usize = 256;
45
46#[cfg(any(test, feature = "fault-injection"))]
48std::thread_local! {
49 static BACKFILL_READER_FAIL: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
50}
51
52#[cfg(any(test, feature = "fault-injection"))]
58pub fn arm_backfill_reader_fail() {
59 BACKFILL_READER_FAIL.with(|c| c.set(true));
60}
61
62const RRF_K: usize = 10;
69
70const CANDIDATE_MULTIPLIER: u32 = 4;
72
73pub const EMBEDDING_INPUT_TRUNCATED_WARNING: &str =
75 "embedding input was truncated to the embedder maximum; full content was stored unchanged";
76
77#[derive(Clone, Debug)]
80pub struct DocumentEmbeddingOutcome {
81 pub model_name: String,
82 pub vector: Vec<f32>,
83 pub prepared_text_fingerprint: Option<ContentRef>,
85 pub source_bytes: usize,
86 pub embedded_bytes: usize,
87 pub provider_token_count: Option<TokenCount>,
90 pub truncated: bool,
91}
92
93fn document_vector_or_refusal(outcome: DocumentEmbeddingOutcome) -> RuntimeResult<Vec<f32>> {
96 if !outcome.truncated {
97 return Ok(outcome.vector);
98 }
99 let token_count = outcome
100 .provider_token_count
101 .filter(|count| count.truncated());
102 let byte_cut = outcome.source_bytes > outcome.embedded_bytes;
103 let detail = match (byte_cut, token_count) {
104 (true, Some(count)) => format!(
105 "from {} to {} bytes and from {} to {} provider tokens for the byte-bounded input",
106 outcome.source_bytes, outcome.embedded_bytes, count.before_truncation, count.embedded
107 ),
108 (true, None) => format!(
109 "from {} to {} bytes",
110 outcome.source_bytes, outcome.embedded_bytes
111 ),
112 (false, Some(count)) => format!(
113 "from {} to {} provider tokens",
114 count.before_truncation, count.embedded
115 ),
116 (false, None) => "by an unspecified input limit".to_string(),
117 };
118 Err(RuntimeError::InvalidInput(format!(
119 "embedding input truncated {detail}; use embed_document_outcome to inspect the bounded vector"
120 )))
121}
122
123#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
125pub struct EmbeddingTruncationReport {
126 pub truncated: u64,
127 pub discarded_bytes: u64,
128 #[serde(default, skip_serializing_if = "counter_is_zero")]
129 pub discarded_tokens: u64,
130 #[serde(default, skip_serializing_if = "counter_is_zero")]
131 pub token_reporting_unavailable: u64,
132}
133
134fn counter_is_zero(value: &u64) -> bool {
135 *value == 0
136}
137
138impl EmbeddingTruncationReport {
139 pub fn observe(&mut self, outcome: &DocumentEmbeddingOutcome) {
140 if outcome.truncated {
141 self.truncated = self.truncated.saturating_add(1);
142 self.discarded_bytes = self
143 .discarded_bytes
144 .saturating_add(outcome.source_bytes.saturating_sub(outcome.embedded_bytes) as u64);
145 }
146 if let Some(count) = outcome.provider_token_count {
147 self.discarded_tokens = self
148 .discarded_tokens
149 .saturating_add(count.before_truncation.saturating_sub(count.embedded) as u64);
150 } else {
151 self.token_reporting_unavailable = self.token_reporting_unavailable.saturating_add(1);
152 }
153 }
154
155 #[must_use]
156 pub const fn any_truncated(&self) -> bool {
157 self.truncated > 0
158 }
159
160 pub fn merge(&mut self, other: Self) {
162 self.truncated = self.truncated.saturating_add(other.truncated);
163 self.discarded_bytes = self.discarded_bytes.saturating_add(other.discarded_bytes);
164 self.discarded_tokens = self.discarded_tokens.saturating_add(other.discarded_tokens);
165 self.token_reporting_unavailable = self
166 .token_reporting_unavailable
167 .saturating_add(other.token_reporting_unavailable);
168 }
169}
170
171struct DocumentEmbeddings {
172 vectors: Vec<Vec<f32>>,
173 token_counts: Option<Vec<TokenCount>>,
174}
175
176async fn document_embeddings(
177 service: &dyn EmbeddingService,
178 texts: &[String],
179 model: EmbeddingModel,
180) -> RuntimeResult<DocumentEmbeddings> {
181 let result = with_embedding_admission(async {
182 match service
183 .embed_with_report(texts, model, EmbeddingRole::Passage)
184 .await
185 {
186 Ok(report) => Ok(DocumentEmbeddings {
187 vectors: report.embeddings,
188 token_counts: Some(report.token_counts),
189 }),
190 Err(lattice_embed::EmbedError::Unsupported(_)) => service
191 .embed_passage(texts, model)
192 .await
193 .map(|vectors| DocumentEmbeddings {
194 vectors,
195 token_counts: None,
196 }),
197 Err(error) => Err(error),
198 }
199 })
200 .await?;
201 if let Some(counts) = &result.token_counts {
202 if result.vectors.len() != texts.len() || counts.len() != texts.len() {
203 return Err(RuntimeError::Internal(format!(
204 "embed_with_report returned {} vectors and {} token counts for {} inputs",
205 result.vectors.len(),
206 counts.len(),
207 texts.len(),
208 )));
209 }
210 if counts
211 .iter()
212 .any(|count| count.embedded > count.before_truncation)
213 {
214 return Err(RuntimeError::Internal(
215 "embed_with_report returned an embedded token count above the input count".into(),
216 ));
217 }
218 }
219 Ok(result)
220}
221
222pub fn document_embedding_budget(model_name: &str) -> usize {
224 parse_embedding_model_alias(model_name)
225 .and_then(|model| model.document_instruction())
226 .map_or(MAX_TEXT_BYTES, |prefix| {
227 MAX_TEXT_BYTES.saturating_sub(prefix.len())
228 })
229}
230
231pub fn bounded_embedding_input(text: &str, max_bytes: usize) -> (&str, bool) {
233 if text.len() <= max_bytes {
234 return (text, false);
235 }
236
237 let end = text
238 .char_indices()
239 .map(|(index, _)| index)
240 .take_while(|index| *index <= max_bytes)
241 .last()
242 .unwrap_or(0);
243 (&text[..end], true)
244}
245
246fn prepared_document_fingerprint(text: &str, model: EmbeddingModel) -> ContentRef {
247 match model.document_instruction() {
248 Some(prefix) => VectorRecord::fingerprint_text(&format!("{prefix}{text}")),
249 None => VectorRecord::fingerprint_text(text),
250 }
251}
252
253impl KhiveRuntime {
254 fn require_default_embedder(&self) -> RuntimeResult<&str> {
255 let model_name = self.default_embedder_name();
256 if model_name.is_empty() {
257 return Err(RuntimeError::Unconfigured("embedding_model".into()));
258 }
259 Ok(model_name)
260 }
261
262 pub async fn embed(&self, text: &str) -> RuntimeResult<Vec<f32>> {
267 let model_name = self.require_default_embedder()?;
268 self.embed_with_model(model_name, text).await
269 }
270
271 pub async fn embed_with_model(&self, model_name: &str, text: &str) -> RuntimeResult<Vec<f32>> {
287 let model = parse_embedding_model_alias(model_name);
288 let service = self.embedder(model_name).await?;
289 let emb_model = model.unwrap_or_default();
290 crate::usage::count(crate::usage::UsageUnit::EmbedCalls, 1);
294 let out = with_embedding_admission(service.embed_one(text, emb_model)).await;
295 out
296 }
297
298 pub async fn embed_document_with_model_outcome(
317 &self,
318 model_name: &str,
319 text: &str,
320 ) -> RuntimeResult<DocumentEmbeddingOutcome> {
321 self.embed_document_with_model_outcome_inner(None, model_name, text)
322 .await
323 }
324
325 pub(crate) async fn embed_document_with_model_outcome_for_token(
326 &self,
327 token: &NamespaceToken,
328 model_name: &str,
329 text: &str,
330 ) -> RuntimeResult<DocumentEmbeddingOutcome> {
331 self.embed_document_with_model_outcome_inner(Some(token), model_name, text)
332 .await
333 }
334
335 async fn embed_document_with_model_outcome_inner(
336 &self,
337 token: Option<&NamespaceToken>,
338 model_name: &str,
339 text: &str,
340 ) -> RuntimeResult<DocumentEmbeddingOutcome> {
341 let model = parse_embedding_model_alias(model_name);
342 let (service, audited_document_preparation) = self
343 .embedder_with_input_attestation(model_name, token)
344 .await?;
345 let emb_model = model.unwrap_or_default();
346 let source_bytes = text.len();
347 let (text, truncated) =
348 bounded_embedding_input(text, document_embedding_budget(model_name));
349 let embedded_bytes = text.len();
350 crate::usage::count(crate::usage::UsageUnit::EmbedCalls, 1);
352 let embeddings =
353 document_embeddings(service.as_ref(), &[text.to_string()], emb_model).await?;
354 let provider_token_count = embeddings.token_counts.map(|counts| counts[0]);
355 let mut vectors = embeddings.vectors;
356 if vectors.len() != 1 {
357 return Err(RuntimeError::Internal(format!(
358 "embed_passage returned {} vectors for 1 input",
359 vectors.len()
360 )));
361 }
362 let out = vectors.pop().expect("checked len == 1 above");
363 Ok(DocumentEmbeddingOutcome {
364 model_name: model_name.to_owned(),
365 vector: out,
366 prepared_text_fingerprint: audited_document_preparation
367 .then(|| prepared_document_fingerprint(text, emb_model)),
368 source_bytes,
369 embedded_bytes,
370 provider_token_count,
371 truncated: truncated || provider_token_count.is_some_and(|count| count.truncated()),
372 })
373 }
374
375 pub async fn embed_query_with_model(
389 &self,
390 model_name: &str,
391 text: &str,
392 ) -> RuntimeResult<Vec<f32>> {
393 self.embed_query_with_model_inner(None, model_name, text)
394 .await
395 }
396
397 pub(crate) async fn embed_query_with_model_for_token(
398 &self,
399 token: &NamespaceToken,
400 model_name: &str,
401 text: &str,
402 ) -> RuntimeResult<Vec<f32>> {
403 self.embed_query_with_model_inner(Some(token), model_name, text)
404 .await
405 }
406
407 async fn embed_query_with_model_inner(
408 &self,
409 token: Option<&NamespaceToken>,
410 model_name: &str,
411 text: &str,
412 ) -> RuntimeResult<Vec<f32>> {
413 let _phase = khive_storage::request_context::enter_search_phase(
414 khive_storage::request_context::SearchPhase::VectorEmbedding,
415 );
416 let model = parse_embedding_model_alias(model_name);
417 let service = match token {
418 Some(token) => self.embedder_with_token(token, model_name).await?,
419 None => self.embedder(model_name).await?,
420 };
421 let texts = [text.to_string()];
422 let emb_model = model.unwrap_or(CUSTOM_QUERY_ROLE_MODEL);
423 crate::usage::count(crate::usage::UsageUnit::EmbedCalls, 1);
425 let embeddings = match model {
426 Some(
427 EmbeddingModel::BgeSmallEnV15
428 | EmbeddingModel::BgeBaseEnV15
429 | EmbeddingModel::BgeLargeEnV15,
430 ) => with_embedding_admission(service.embed(&texts, emb_model)).await,
431 _ => with_embedding_admission(service.embed_query(&texts, emb_model)).await,
432 };
433 let out = embeddings?
434 .into_iter()
435 .next()
436 .ok_or_else(|| RuntimeError::Internal("embed_query returned empty vec".into()))?;
437 Ok(out)
438 }
439
440 pub async fn embed_document(&self, text: &str) -> RuntimeResult<Vec<f32>> {
449 let outcome = self.embed_document_outcome(text).await?;
450 document_vector_or_refusal(outcome)
451 }
452
453 pub async fn embed_document_outcome(
455 &self,
456 text: &str,
457 ) -> RuntimeResult<DocumentEmbeddingOutcome> {
458 let model_name = self.require_default_embedder()?;
459 self.embed_document_with_model_outcome(model_name, text)
460 .await
461 }
462
463 pub async fn embed_query(&self, text: &str) -> RuntimeResult<Vec<f32>> {
470 let model_name = self.require_default_embedder()?;
471 self.embed_query_with_model(model_name, text).await
472 }
473
474 async fn embed_query_for_token(
475 &self,
476 token: &NamespaceToken,
477 text: &str,
478 ) -> RuntimeResult<Vec<f32>> {
479 let model_name = self.require_default_embedder()?;
480 self.embed_query_with_model_for_token(token, model_name, text)
481 .await
482 }
483
484 pub async fn embed_batch(&self, texts: &[String]) -> RuntimeResult<Vec<Vec<f32>>> {
492 if texts.is_empty() {
493 return Ok(vec![]);
494 }
495 let model_name = self.require_default_embedder()?;
496 self.embed_batch_with_model(model_name, texts).await
497 }
498
499 pub async fn embed_batch_with_model(
504 &self,
505 model_name: &str,
506 texts: &[String],
507 ) -> RuntimeResult<Vec<Vec<f32>>> {
508 if texts.is_empty() {
509 return Ok(vec![]);
510 }
511 let model = parse_embedding_model_alias(model_name);
512 let service = self.embedder(model_name).await?;
513 let emb_model = model.unwrap_or_default();
514 let out = with_embedding_admission(service.embed(texts, emb_model)).await;
515 crate::usage::count(crate::usage::UsageUnit::EmbedCalls, texts.len() as u64);
516 out
517 }
518
519 pub async fn embed_document_batch_with_model(
533 &self,
534 model_name: &str,
535 texts: &[String],
536 ) -> RuntimeResult<Vec<Vec<f32>>> {
537 if texts.is_empty() {
538 return Ok(vec![]);
539 }
540 let outcomes = self
541 .embed_document_batch_with_model_outcomes(model_name, texts)
542 .await?;
543 let mut report = EmbeddingTruncationReport::default();
544 for outcome in &outcomes {
545 report.observe(outcome);
546 }
547 if report.any_truncated() {
548 return Err(RuntimeError::InvalidInput(format!(
549 "embedding input truncated for {} documents ({} discarded bytes, {} discarded tokens); use embed_document_batch_with_model_outcomes to inspect the bounded vectors",
550 report.truncated, report.discarded_bytes, report.discarded_tokens
551 )));
552 }
553 Ok(outcomes.into_iter().map(|outcome| outcome.vector).collect())
554 }
555
556 pub async fn embed_document_batch_with_model_outcomes(
557 &self,
558 model_name: &str,
559 texts: &[String],
560 ) -> RuntimeResult<Vec<DocumentEmbeddingOutcome>> {
561 self.embed_document_batch_with_model_outcomes_inner(None, model_name, texts)
562 .await
563 }
564
565 pub(crate) async fn embed_document_batch_with_model_outcomes_for_token(
566 &self,
567 token: &NamespaceToken,
568 model_name: &str,
569 texts: &[String],
570 ) -> RuntimeResult<Vec<DocumentEmbeddingOutcome>> {
571 self.embed_document_batch_with_model_outcomes_inner(Some(token), model_name, texts)
572 .await
573 }
574
575 async fn embed_document_batch_with_model_outcomes_inner(
576 &self,
577 token: Option<&NamespaceToken>,
578 model_name: &str,
579 texts: &[String],
580 ) -> RuntimeResult<Vec<DocumentEmbeddingOutcome>> {
581 if texts.is_empty() {
582 return Ok(vec![]);
583 }
584 let model = parse_embedding_model_alias(model_name);
585 let (service, audited_document_preparation) = self
586 .embedder_with_input_attestation(model_name, token)
587 .await?;
588 let emb_model = model.unwrap_or_default();
589 let budget = document_embedding_budget(model_name);
590 if token.is_some() {
591 crate::usage::count(crate::usage::UsageUnit::EmbedCalls, texts.len() as u64);
593 }
594 let out = if texts.iter().all(|text| text.len() <= budget) {
595 document_embeddings(service.as_ref(), texts, emb_model).await
596 } else {
597 let bounded_texts: Vec<String> = texts
598 .iter()
599 .map(|text| bounded_embedding_input(text, budget).0.to_owned())
600 .collect();
601 document_embeddings(service.as_ref(), &bounded_texts, emb_model).await
602 };
603 if token.is_none() {
604 crate::usage::count(crate::usage::UsageUnit::EmbedCalls, texts.len() as u64);
605 }
606 let embeddings = out?;
607 let vectors = embeddings.vectors;
608 if vectors.len() != texts.len() {
609 return Err(RuntimeError::Internal(format!(
610 "embed_passage returned {} vectors for {} inputs",
611 vectors.len(),
612 texts.len()
613 )));
614 }
615 Ok(texts
616 .iter()
617 .zip(vectors)
618 .enumerate()
619 .map(|(index, (text, vector))| {
620 let (bounded, truncated) = bounded_embedding_input(text, budget);
621 let provider_token_count =
622 embeddings.token_counts.as_ref().map(|counts| counts[index]);
623 DocumentEmbeddingOutcome {
624 model_name: model_name.to_owned(),
625 vector,
626 prepared_text_fingerprint: audited_document_preparation
627 .then(|| prepared_document_fingerprint(bounded, emb_model)),
628 source_bytes: text.len(),
629 embedded_bytes: bounded.len(),
630 provider_token_count,
631 truncated: truncated
632 || provider_token_count.is_some_and(|count| count.truncated()),
633 }
634 })
635 .collect())
636 }
637
638 pub async fn embed_document_batch(&self, texts: &[String]) -> RuntimeResult<Vec<Vec<f32>>> {
647 if texts.is_empty() {
648 return Ok(vec![]);
649 }
650 let model_name = self.require_default_embedder()?;
651 self.embed_document_batch_with_model(model_name, texts)
652 .await
653 }
654
655 pub async fn embed_document_batch_outcomes(
657 &self,
658 texts: &[String],
659 ) -> RuntimeResult<Vec<DocumentEmbeddingOutcome>> {
660 if texts.is_empty() {
661 return Ok(vec![]);
662 }
663 let model_name = self.require_default_embedder()?;
664 self.embed_document_batch_with_model_outcomes(model_name, texts)
665 .await
666 }
667
668 pub async fn embed_query_batch_with_model(
675 &self,
676 model_name: &str,
677 texts: &[String],
678 ) -> RuntimeResult<Vec<Vec<f32>>> {
679 if texts.is_empty() {
680 return Ok(vec![]);
681 }
682 let model = parse_embedding_model_alias(model_name);
683 let service = self.embedder(model_name).await?;
684 let emb_model = model.unwrap_or(CUSTOM_QUERY_ROLE_MODEL);
685 let out = match model {
686 Some(
687 EmbeddingModel::BgeSmallEnV15
688 | EmbeddingModel::BgeBaseEnV15
689 | EmbeddingModel::BgeLargeEnV15,
690 ) => with_embedding_admission(service.embed(texts, emb_model)).await,
691 _ => with_embedding_admission(service.embed_query(texts, emb_model)).await,
692 };
693 crate::usage::count(crate::usage::UsageUnit::EmbedCalls, texts.len() as u64);
694 out
695 }
696
697 pub(crate) async fn note_search_vector_search(
701 &self,
702 token: &NamespaceToken,
703 query_embedding: Option<Vec<f32>>,
704 query_text: &str,
705 top_k: u32,
706 ) -> RuntimeResult<Vec<VectorSearchHit>> {
707 let embedding = match query_embedding {
708 Some(embedding) => embedding,
709 None => self.embed_query_for_token(token, query_text).await?,
710 };
711 let model = self.default_embedder_name();
712 let fallback_cause = if model.is_empty() {
713 NoteSearchFallbackCause::NoModel
714 } else if let Some(provider) = self.note_search_ann_provider()? {
715 if let Some(hits) = provider.search(token, model, &embedding, top_k).await? {
716 crate::note_search_ann::record_ann_route(self.backend().pool());
717 crate::usage::count(crate::usage::UsageUnit::VectorPasses, 1);
718 return Ok(hits);
719 }
720 NoteSearchFallbackCause::NoEligibleGraph
721 } else {
722 NoteSearchFallbackCause::NoProvider
723 };
724 crate::note_search_ann::record_fallback_route(self.backend().pool(), fallback_cause);
725 self.vector_search(
726 token,
727 Some(embedding),
728 None,
729 top_k,
730 Some(SubstrateKind::Note),
731 )
732 .await
733 }
734
735 #[allow(clippy::too_many_arguments)]
781 pub async fn hybrid_search(
782 &self,
783 token: &NamespaceToken,
784 query_text: &str,
785 query_vector: Option<Vec<f32>>,
786 limit: u32,
787 entity_kind: Option<&str>,
788 entity_type: Option<&str>,
789 tags_any: &[String],
790 properties_filter: Option<&serde_json::Value>,
791 ) -> RuntimeResult<Vec<SearchHit>> {
792 self.hybrid_search_with_text_mode(
793 token,
794 query_text,
795 query_vector,
796 limit,
797 entity_kind,
798 entity_type,
799 tags_any,
800 properties_filter,
801 TextQueryMode::Plain,
802 )
803 .await
804 }
805
806 #[allow(clippy::too_many_arguments)]
808 pub async fn hybrid_search_with_text_mode(
809 &self,
810 token: &NamespaceToken,
811 query_text: &str,
812 query_vector: Option<Vec<f32>>,
813 limit: u32,
814 entity_kind: Option<&str>,
815 entity_type: Option<&str>,
816 tags_any: &[String],
817 properties_filter: Option<&serde_json::Value>,
818 text_mode: TextQueryMode,
819 ) -> RuntimeResult<Vec<SearchHit>> {
820 let (hits, _vector_error) = self
821 .hybrid_search_inner(
822 token,
823 query_text,
824 query_vector,
825 limit,
826 entity_kind,
827 entity_type,
828 tags_any,
829 properties_filter,
830 text_mode,
831 None,
832 false,
833 None,
834 )
835 .await?;
836 Ok(hits)
837 }
838
839 pub async fn hybrid_search_each_kind(
850 &self,
851 token: &NamespaceToken,
852 query_text: &str,
853 query_vector: Option<Vec<f32>>,
854 limit: u32,
855 entity_kinds: &[&str],
856 ) -> RuntimeResult<Vec<Vec<SearchHit>>> {
857 if entity_kinds.is_empty() {
858 return Ok(Vec::new());
859 }
860 let candidates = limit.saturating_mul(CANDIDATE_MULTIPLIER).max(limit);
861 let (vector_hits, _vector_error) = self
862 .hybrid_vector_stage(token, query_text, query_vector, candidates, None, false)
863 .await?;
864 let mut per_kind = Vec::with_capacity(entity_kinds.len());
865 for &kind in entity_kinds {
866 let (hits, _vector_error) = self
867 .hybrid_search_inner(
868 token,
869 query_text,
870 None,
871 limit,
872 Some(kind),
873 None,
874 &[],
875 None,
876 TextQueryMode::Plain,
877 None,
878 false,
879 Some(vector_hits.clone()),
880 )
881 .await?;
882 per_kind.push(hits);
883 }
884 Ok(per_kind)
885 }
886
887 #[allow(clippy::too_many_arguments)]
892 pub(crate) async fn hybrid_search_with_vector_similarity_floor(
893 &self,
894 token: &NamespaceToken,
895 query_text: &str,
896 query_vector: Option<Vec<f32>>,
897 limit: u32,
898 entity_kind: Option<&str>,
899 entity_type: Option<&str>,
900 tags_any: &[String],
901 properties_filter: Option<&serde_json::Value>,
902 vector_similarity_floor: f64,
903 ) -> RuntimeResult<Vec<SearchHit>> {
904 let (hits, _vector_error) = self
905 .hybrid_search_inner(
906 token,
907 query_text,
908 query_vector,
909 limit,
910 entity_kind,
911 entity_type,
912 tags_any,
913 properties_filter,
914 TextQueryMode::Plain,
915 Some(vector_similarity_floor),
916 false,
917 None,
918 )
919 .await?;
920 Ok(hits)
921 }
922
923 #[allow(clippy::too_many_arguments)]
934 pub async fn hybrid_search_outcome(
935 &self,
936 token: &NamespaceToken,
937 query_text: &str,
938 limit: u32,
939 entity_kind: Option<&str>,
940 entity_type: Option<&str>,
941 tags_any: &[String],
942 properties_filter: Option<&serde_json::Value>,
943 ) -> RuntimeResult<HybridSearchOutcome> {
944 self.hybrid_search_outcome_with_text_mode(
945 token,
946 query_text,
947 limit,
948 entity_kind,
949 entity_type,
950 tags_any,
951 properties_filter,
952 TextQueryMode::Plain,
953 )
954 .await
955 }
956
957 #[allow(clippy::too_many_arguments)]
959 pub async fn hybrid_search_outcome_with_text_mode(
960 &self,
961 token: &NamespaceToken,
962 query_text: &str,
963 limit: u32,
964 entity_kind: Option<&str>,
965 entity_type: Option<&str>,
966 tags_any: &[String],
967 properties_filter: Option<&serde_json::Value>,
968 text_mode: TextQueryMode,
969 ) -> RuntimeResult<HybridSearchOutcome> {
970 let (hits, vector_error) = self
971 .hybrid_search_inner(
972 token,
973 query_text,
974 None,
975 limit,
976 entity_kind,
977 entity_type,
978 tags_any,
979 properties_filter,
980 text_mode,
981 None,
982 true,
983 None,
984 )
985 .await?;
986 Ok(HybridSearchOutcome { hits, vector_error })
987 }
988
989 #[allow(clippy::too_many_arguments)]
992 async fn hybrid_search_inner(
993 &self,
994 token: &NamespaceToken,
995 query_text: &str,
996 query_vector: Option<Vec<f32>>,
997 limit: u32,
998 entity_kind: Option<&str>,
999 entity_type: Option<&str>,
1000 tags_any: &[String],
1001 properties_filter: Option<&serde_json::Value>,
1002 text_mode: TextQueryMode,
1003 vector_similarity_floor: Option<f64>,
1004 tolerate_vector_error: bool,
1005 vector_pool: Option<Vec<VectorSearchHit>>,
1006 ) -> RuntimeResult<(Vec<SearchHit>, Option<String>)> {
1007 khive_storage::await_request_read_phase("hybrid_search", async {
1008 let candidates = limit.saturating_mul(CANDIDATE_MULTIPLIER).max(limit);
1009
1010 let visible_ns: Vec<String> = token
1011 .visible_namespaces()
1012 .iter()
1013 .map(|ns| ns.as_str().to_owned())
1014 .collect();
1015 let text_store = self.text(token)?;
1019 let text_fut = async {
1020 let _phase = khive_storage::request_context::enter_search_phase(
1021 khive_storage::request_context::SearchPhase::TextSearch,
1022 );
1023 text_store
1024 .search(TextSearchRequest {
1025 query: query_text.to_string(),
1026 mode: text_mode,
1027 filter: Some(TextFilter {
1028 namespaces: visible_ns.clone(),
1029 record_kinds: entity_kind
1036 .map(|kind| vec![kind.to_string()])
1037 .unwrap_or_default(),
1038 ..TextFilter::default()
1039 }),
1040 top_k: candidates,
1041 snippet_chars: 200,
1042 })
1043 .await
1044 };
1045 let text_fut = crate::stage_seam::text_stage(text_fut);
1046 let vector_fut = async {
1048 let _phase = khive_storage::request_context::enter_search_phase(
1049 khive_storage::request_context::SearchPhase::VectorSearch,
1050 );
1051 match vector_pool {
1052 Some(pool) => Ok((pool, None)),
1053 None => {
1054 self.hybrid_vector_stage(
1055 token,
1056 query_text,
1057 query_vector,
1058 candidates,
1059 vector_similarity_floor,
1060 tolerate_vector_error,
1061 )
1062 .await
1063 }
1064 }
1065 };
1066 let (text_search_result, vector_result) = tokio::join!(text_fut, vector_fut);
1067 let text_hits = crate::error::fts_text_leg_or_err(
1072 text_search_result.map_err(RuntimeError::from),
1073 "hybrid_search",
1074 query_text,
1075 )?;
1076 let (vector_hits, vector_error) = vector_result?;
1077
1078 let fusion_limit = text_hits.len().saturating_add(vector_hits.len());
1083 let mut fused = {
1084 let _phase = khive_storage::request_context::enter_search_phase(
1085 khive_storage::request_context::SearchPhase::Fusion,
1086 );
1087 rrf_fuse(text_hits, vector_hits, fusion_limit, query_text)
1088 };
1089
1090 if !fused.is_empty() {
1093 let candidate_ids: Vec<Uuid> = fused.iter().map(|h| h.entity_id).collect();
1094 let _phase = khive_storage::request_context::enter_search_phase(
1095 khive_storage::request_context::SearchPhase::EntityHydration,
1096 );
1097 let alive_page = self
1098 .entities(token)?
1099 .query_entities(
1100 token.namespace().as_str(),
1101 EntityFilter {
1102 ids: candidate_ids,
1103 kinds: entity_kind.map(|k| vec![k.to_string()]).unwrap_or_default(),
1104 entity_types: entity_type
1105 .map(|t| vec![t.to_string()])
1106 .unwrap_or_default(),
1107 namespaces: visible_ns,
1108 tags_any: tags_any.to_vec(),
1109 ..EntityFilter::default()
1110 },
1111 PageRequest {
1112 offset: 0,
1113 limit: u32::try_from(fused.len()).unwrap_or(u32::MAX),
1114 },
1115 )
1116 .await?;
1117 drop(_phase);
1118 let mut entity_meta: HashMap<Uuid, (String, Option<String>)> = HashMap::new();
1119 let mut alive: HashSet<Uuid> = HashSet::new();
1120 for e in alive_page.items {
1121 if let Some(pf) = properties_filter {
1124 if !properties_match(e.properties.as_ref(), pf) {
1125 continue;
1126 }
1127 }
1128 alive.insert(e.id);
1129 entity_meta.insert(e.id, (e.name, e.description));
1130 }
1131
1132 fused.retain(|h| alive.contains(&h.entity_id));
1133
1134 for hit in &mut fused {
1136 if let Some((name, description)) = entity_meta.get(&hit.entity_id) {
1137 if hit.title.is_none() {
1138 hit.title = Some(name.clone());
1139 }
1140 if hit.snippet.is_none() {
1141 hit.snippet = description.clone();
1142 }
1143 }
1144 }
1145 }
1146
1147 fused.truncate(limit as usize);
1148 Ok((fused, vector_error))
1149 })
1150 .await?
1151 }
1152
1153 async fn embed_backfill_page(
1154 &self,
1155 token: &NamespaceToken,
1156 model_name: &str,
1157 inputs: &[(Uuid, String)],
1158 ) -> Vec<Option<DocumentEmbeddingOutcome>> {
1159 let texts: Vec<String> = inputs.iter().map(|(_, text)| text.clone()).collect();
1160 match self
1161 .embed_document_batch_with_model_outcomes_for_token(token, model_name, &texts)
1162 .await
1163 {
1164 Ok(outcomes) => outcomes.into_iter().map(Some).collect(),
1165 Err(error) => {
1166 tracing::warn!(
1167 model = %model_name,
1168 error = %error,
1169 "backfill_missing_embeddings: batch embed failed; retrying records individually"
1170 );
1171 let mut outcomes = Vec::with_capacity(inputs.len());
1172 for (id, text) in inputs {
1173 match self
1174 .embed_document_with_model_outcome_for_token(token, model_name, text)
1175 .await
1176 {
1177 Ok(outcome) => outcomes.push(Some(outcome)),
1178 Err(error) => {
1179 tracing::warn!(
1180 id = %id,
1181 model = %model_name,
1182 error = %error,
1183 "backfill_missing_embeddings: record embed failed"
1184 );
1185 outcomes.push(None);
1186 }
1187 }
1188 }
1189 outcomes
1190 }
1191 }
1192 }
1193
1194 pub async fn backfill_missing_embeddings(&self, token: &NamespaceToken) -> RuntimeResult<u64> {
1207 use khive_storage::types::{SqlRow, SqlStatement, SqlValue};
1208
1209 let model_names = self.registered_embedding_model_names();
1210 if model_names.is_empty() {
1211 tracing::debug!(
1212 "backfill_missing_embeddings: no embedding models registered, skipping"
1213 );
1214 return Ok(0);
1215 }
1216
1217 let ns = token.namespace().as_str().to_string();
1218 let mut total_backfilled = 0u64;
1219
1220 for model_name in &model_names {
1221 let mut model_truncation = EmbeddingTruncationReport::default();
1222 match self.vectors_for_model(token, model_name) {
1223 Ok(_) => {}
1224 Err(error) => {
1225 tracing::warn!(model = %model_name, error = %error,
1226 "backfill_missing_embeddings: vector store unavailable");
1227 continue;
1228 }
1229 };
1230 let vec_table = format!("vec_{}", sanitize_key(model_name));
1232
1233 const PAGE_SIZE: usize = EMBEDDING_BATCH_PAGE_SIZE;
1237 let mut entity_total = 0usize;
1238 let mut entity_cursor = String::new();
1239 loop {
1240 let entity_sql = SqlStatement {
1241 sql: format!(
1242 "SELECT id FROM entities \
1243 WHERE namespace = ?1 AND deleted_at IS NULL AND id > ?3 \
1244 AND id NOT IN (\
1245 SELECT subject_id FROM {vec_table} \
1246 WHERE namespace = ?1 AND embedding_model = ?2 \
1247 ) ORDER BY id LIMIT {PAGE_SIZE}"
1248 ),
1249 params: vec![
1250 SqlValue::Text(ns.clone()),
1251 SqlValue::Text(model_name.clone()),
1252 SqlValue::Text(entity_cursor.clone()),
1253 ],
1254 label: Some("backfill_entities".into()),
1255 };
1256
1257 let entity_rows: Vec<SqlRow> = {
1258 let sql = self.sql();
1259 let reader_result = sql.reader().await;
1260 #[cfg(any(test, feature = "fault-injection"))]
1261 let reader_result = if BACKFILL_READER_FAIL.with(|c| c.get()) {
1262 BACKFILL_READER_FAIL.with(|c| c.set(false));
1263 Err(khive_storage::StorageError::Pool {
1264 operation: "reader".into(),
1265 message: "injected failure".into(),
1266 })
1267 } else {
1268 reader_result
1269 };
1270 let mut reader = reader_result.map_err(RuntimeError::Storage)?;
1271 reader
1272 .query_all(entity_sql)
1273 .await
1274 .map_err(RuntimeError::Storage)?
1275 };
1276
1277 let batch_len = entity_rows.len();
1278 entity_total += batch_len;
1279 if batch_len == 0 {
1280 break;
1281 }
1282 entity_cursor = match entity_rows.last().and_then(|row| row.columns.first()) {
1283 Some(column) => match &column.value {
1284 SqlValue::Text(id) => id.clone(),
1285 _ => {
1286 return Err(RuntimeError::Internal(
1287 "backfill entity ID is not text".into(),
1288 ))
1289 }
1290 },
1291 None => {
1292 return Err(RuntimeError::Internal(
1293 "backfill entity page is empty".into(),
1294 ))
1295 }
1296 };
1297
1298 let entity_store = self.entities(token)?;
1299 let mut entities = Vec::with_capacity(batch_len);
1300 let mut inputs = Vec::with_capacity(batch_len);
1301 for row in &entity_rows {
1302 let Some(SqlValue::Text(id)) = row.columns.first().map(|column| &column.value)
1303 else {
1304 continue;
1305 };
1306 let Ok(id) = id.parse::<Uuid>() else { continue };
1307 let Some(entity) = entity_store.get_entity(id).await? else {
1308 continue;
1309 };
1310 if entity.namespace != ns || entity.deleted_at.is_some() {
1311 continue;
1312 }
1313 let text = crate::curation::entity_embedding_text(&entity);
1314 if text.trim().is_empty() {
1315 continue;
1316 }
1317 inputs.push((id, text));
1318 entities.push(entity);
1319 }
1320 let outcomes = self.embed_backfill_page(token, model_name, &inputs).await;
1321 for (entity, outcome) in entities.into_iter().zip(outcomes) {
1322 let Some(outcome) = outcome else { continue };
1323 model_truncation.observe(&outcome);
1324 match self
1325 .publish_entity_vector_revision(token, &entity, model_name, &outcome.vector)
1326 .await
1327 {
1328 Ok(true) => total_backfilled += 1,
1329 Ok(false) => {}
1330 Err(error) => tracing::warn!(
1331 id = %entity.id, model = %model_name, error = %error,
1332 "backfill_missing_embeddings: entity vector insert failed"
1333 ),
1334 }
1335 }
1336
1337 if batch_len < PAGE_SIZE {
1338 break;
1339 }
1340 }
1341
1342 let text_store = self.text_for_notes(token).ok();
1344 let note_store = self.notes(token).ok();
1345 let mut note_total = 0usize;
1346 let mut note_cursor = String::new();
1349 loop {
1350 let note_sql = SqlStatement {
1353 sql: format!(
1354 "SELECT id FROM notes \
1355 WHERE namespace = ?1 AND deleted_at IS NULL AND id > ?3 \
1356 AND id NOT IN (\
1357 SELECT subject_id FROM {vec_table} \
1358 WHERE namespace = ?1 AND embedding_model = ?2 \
1359 ) ORDER BY id LIMIT {PAGE_SIZE}"
1360 ),
1361 params: vec![
1362 SqlValue::Text(ns.clone()),
1363 SqlValue::Text(model_name.clone()),
1364 SqlValue::Text(note_cursor.clone()),
1365 ],
1366 label: Some("backfill_notes".into()),
1367 };
1368
1369 let note_rows: Vec<SqlRow> = {
1370 let sql = self.sql();
1371 let reader_result = sql.reader().await;
1372 #[cfg(any(test, feature = "fault-injection"))]
1373 let reader_result = if BACKFILL_READER_FAIL.with(|c| c.get()) {
1374 BACKFILL_READER_FAIL.with(|c| c.set(false));
1375 Err(khive_storage::StorageError::Pool {
1376 operation: "reader".into(),
1377 message: "injected failure".into(),
1378 })
1379 } else {
1380 reader_result
1381 };
1382 let mut reader = reader_result.map_err(RuntimeError::Storage)?;
1383 reader
1384 .query_all(note_sql)
1385 .await
1386 .map_err(RuntimeError::Storage)?
1387 };
1388
1389 let batch_len = note_rows.len();
1390 note_total += batch_len;
1391 if batch_len == 0 {
1392 break;
1393 }
1394 note_cursor = match note_rows.last().and_then(|row| row.columns.first()) {
1395 Some(column) => match &column.value {
1396 SqlValue::Text(id) => id.clone(),
1397 _ => {
1398 return Err(RuntimeError::Internal(
1399 "backfill note ID is not text".into(),
1400 ))
1401 }
1402 },
1403 None => {
1404 return Err(RuntimeError::Internal("backfill note page is empty".into()))
1405 }
1406 };
1407
1408 let mut notes_for_batch = Vec::with_capacity(batch_len);
1409 let mut inputs = Vec::with_capacity(batch_len);
1410 for row in ¬e_rows {
1411 let id_str = row.columns.first().and_then(|c| {
1412 if let SqlValue::Text(s) = &c.value {
1413 Some(s.clone())
1414 } else {
1415 None
1416 }
1417 });
1418
1419 let Some(id_str) = id_str else {
1420 continue;
1421 };
1422 let Ok(id) = id_str.parse::<Uuid>() else {
1423 continue;
1424 };
1425
1426 let note = match ¬e_store {
1427 Some(store) => match store.get_note(id).await {
1428 Ok(Some(n)) => n,
1429 _ => continue,
1430 },
1431 None => continue,
1432 };
1433
1434 if note.content.trim().is_empty() {
1435 continue;
1436 }
1437
1438 if model_names.first().map(|n| n.as_str()) == Some(model_name.as_str()) {
1441 if let Some(ref ts) = text_store {
1442 if let Err(e) = ts.upsert_document(note_fts_document(¬e)).await {
1443 tracing::warn!(id = %id, error = %e,
1444 "backfill_missing_embeddings: note FTS upsert failed");
1445 }
1446 }
1447 }
1448
1449 if !self
1450 .embedding_models_for_note_kind(¬e.kind)
1451 .contains(model_name)
1452 {
1453 continue;
1454 }
1455
1456 inputs.push((id, note.content.clone()));
1457 notes_for_batch.push(note);
1458 }
1459
1460 let outcomes = self.embed_backfill_page(token, model_name, &inputs).await;
1461 for (note, outcome) in notes_for_batch.into_iter().zip(outcomes) {
1462 let Some(outcome) = outcome else { continue };
1463 model_truncation.observe(&outcome);
1464 match self
1465 .publish_note_vector_revision(token, ¬e, model_name, &outcome.vector)
1466 .await
1467 {
1468 Ok(true) => total_backfilled += 1,
1469 Ok(false) => {}
1470 Err(error) => tracing::warn!(
1471 id = %note.id, model = %model_name, error = %error,
1472 "backfill_missing_embeddings: note vector insert failed"
1473 ),
1474 }
1475 }
1476
1477 if batch_len < PAGE_SIZE {
1478 break;
1479 }
1480 }
1481
1482 tracing::info!(
1483 model = %model_name,
1484 namespace = %ns,
1485 entities = entity_total,
1486 notes = note_total,
1487 truncated = model_truncation.truncated,
1488 discarded_bytes = model_truncation.discarded_bytes,
1489 "backfill_missing_embeddings: model pass complete"
1490 );
1491 }
1492
1493 tracing::info!(
1494 namespace = %ns,
1495 total_backfilled = total_backfilled,
1496 "backfill_missing_embeddings: finished"
1497 );
1498
1499 Ok(total_backfilled)
1500 }
1501
1502 pub async fn sweep_orphan_vectors(
1518 &self,
1519 token: &NamespaceToken,
1520 max_delete_per_model: u32,
1521 dry_run: bool,
1522 ) -> RuntimeResult<u64> {
1523 use khive_storage::types::OrphanSweepConfig;
1524 use khive_storage::StorageError;
1525
1526 let model_names = self.registered_embedding_model_names();
1527 if model_names.is_empty() {
1528 tracing::debug!("sweep_orphan_vectors: no embedding models registered, skipping");
1529 return Ok(0);
1530 }
1531
1532 let ns = token.namespace().as_str().to_string();
1533 let mut total_deleted = 0u64;
1534
1535 for model_name in &model_names {
1536 let store = match self.vectors_for_model(token, model_name) {
1537 Ok(s) => s,
1538 Err(e) => {
1539 tracing::warn!(
1540 model = %model_name,
1541 error = %e,
1542 "sweep_orphan_vectors: failed to get vector store, skipping model"
1543 );
1544 continue;
1545 }
1546 };
1547
1548 let caps = store.capabilities();
1549 if !caps.supports_orphan_sweep {
1550 tracing::debug!(
1551 model = %model_name,
1552 "sweep_orphan_vectors: backend does not support orphan sweep, skipping"
1553 );
1554 continue;
1555 }
1556
1557 let config = OrphanSweepConfig {
1558 subject_id_allowlist: None,
1559 namespaces: vec![ns.clone()],
1560 substrate_kinds: vec![],
1561 max_delete: max_delete_per_model,
1562 dry_run,
1563 };
1564
1565 match store.orphan_sweep(&config).await {
1566 Ok(result) => {
1567 tracing::info!(
1568 model = %model_name,
1569 namespace = %ns,
1570 scanned = result.scanned,
1571 deleted = result.deleted,
1572 would_delete = result.would_delete,
1573 dry_run = dry_run,
1574 "sweep_orphan_vectors: sweep complete"
1575 );
1576 total_deleted += result.deleted;
1577 }
1578 Err(StorageError::Unsupported { .. }) => {
1579 tracing::debug!(
1580 model = %model_name,
1581 "sweep_orphan_vectors: backend returned Unsupported, skipping"
1582 );
1583 }
1584 Err(e) => {
1585 tracing::warn!(
1586 model = %model_name,
1587 error = %e,
1588 "sweep_orphan_vectors: sweep failed, continuing with other models"
1589 );
1590 }
1591 }
1592 }
1593
1594 tracing::info!(
1595 namespace = %ns,
1596 total_deleted = total_deleted,
1597 dry_run = dry_run,
1598 "sweep_orphan_vectors: finished"
1599 );
1600
1601 Ok(total_deleted)
1602 }
1603}
1604
1605pub(crate) fn properties_match(
1610 properties: Option<&serde_json::Value>,
1611 filter: &serde_json::Value,
1612) -> bool {
1613 let required = match filter.as_object() {
1614 Some(obj) if !obj.is_empty() => obj,
1615 _ => return true,
1616 };
1617 let actual = match properties.and_then(serde_json::Value::as_object) {
1618 Some(obj) => obj,
1619 None => return false,
1620 };
1621 required
1622 .iter()
1623 .all(|(k, v)| actual.get(k).is_some_and(|av| av == v))
1624}
1625
1626const EXACT_MATCH_BOOST: f64 = 0.5;
1630
1631fn rrf_fuse(
1640 text_hits: Vec<TextSearchHit>,
1641 vector_hits: Vec<VectorSearchHit>,
1642 limit: usize,
1643 query_text: &str,
1644) -> Vec<SearchHit> {
1645 let mut text_arm = Vec::new();
1646 for (rank, hit) in text_hits.into_iter().enumerate() {
1647 let label = HitLabel {
1648 rank,
1649 signals: SearchSignals {
1650 vector_similarity: None,
1651 keyword_score: Some(hit.score),
1652 },
1653 source: SearchSource::Text,
1654 title: hit.title,
1655 snippet: hit.snippet,
1656 };
1657 text_arm.push((hit.subject_id, label));
1658 }
1659 let mut vector_arm = Vec::new();
1660 for (rank, hit) in vector_hits.into_iter().enumerate() {
1661 let label = HitLabel {
1662 rank,
1663 signals: SearchSignals {
1664 vector_similarity: Some(hit.score),
1665 keyword_score: None,
1666 },
1667 source: SearchSource::Vector,
1668 title: None,
1669 snippet: None,
1670 };
1671 vector_arm.push((hit.subject_id, label));
1672 }
1673
1674 let fused = fuse_labelled(
1675 vec![text_arm, vector_arm],
1676 RRF_K,
1677 combine_leg_first_appearance,
1678 );
1679
1680 let query_lower = query_text.to_lowercase();
1682 let boost = DeterministicScore::from_f64(EXACT_MATCH_BOOST);
1683 let mut hits = Vec::with_capacity(fused.len());
1684 for (entity_id, score, label) in fused {
1685 let title_lower = label.title.as_deref().map(str::to_lowercase);
1686 let exact = title_lower.as_deref() == Some(query_lower.as_str());
1687 let score = if exact { score + boost } else { score };
1688 hits.push(SearchHit {
1689 entity_id,
1690 score,
1691 rank_score_kind: RankScoreKind::Rrf,
1692 signals: label.signals,
1693 source: label.source,
1694 title: label.title,
1695 snippet: label.snippet,
1696 });
1697 }
1698
1699 hits.sort_by(|a, b| {
1700 khive_score::cmp_desc_then_id(a.score, &a.entity_id, b.score, &b.entity_id)
1701 });
1702 hits.truncate(limit);
1703 hits
1704}
1705
1706#[cfg(test)]
1707mod rrf_fuse_label_tests;
1708
1709#[cfg(test)]
1710#[path = "retrieval_tests.rs"]
1711mod tests;