khive_pack_knowledge/
lib.rs1mod feedback;
4pub(crate) mod handlers;
5pub(crate) mod knowledge;
6mod lint;
7mod pack;
8mod vocab;
9
10pub use knowledge::snapshot_invalidation::invalidate_legacy_vamana_snapshots;
11pub use pack::KnowledgePack;
12
13use khive_runtime::{KhiveRuntime, NamespaceToken, RuntimeError};
14use serde_json::{json, Value};
15
16#[derive(Debug, Clone, Copy)]
18pub struct KnowledgeReindexOptions {
19 pub atoms: bool,
21 pub sections: bool,
23 pub drop_existing: bool,
25 pub rebuild_ann: bool,
27 pub batch_size: Option<u32>,
29}
30
31pub async fn reindex_knowledge(
50 runtime: &KhiveRuntime,
51 token: &NamespaceToken,
52 opts: KnowledgeReindexOptions,
53 on_atom_progress: Option<&(dyn Fn(u64, u64) + Send + Sync)>,
54 on_section_progress: Option<&(dyn Fn(u64, u64) + Send + Sync)>,
55) -> Result<Value, RuntimeError> {
56 let mut atoms_indexed = 0u64;
57 let mut failed = 0u64;
58 let mut ann_failed = false;
59 let mut truncation_by_model = serde_json::Map::new();
60 if opts.atoms {
61 let ann = knowledge::vamana::new_shared();
62 let mut params = serde_json::Map::new();
63 params.insert("rebuild_ann".into(), Value::Bool(opts.rebuild_ann));
64 params.insert("insert_only".into(), Value::Bool(!opts.drop_existing));
65 if let Some(bs) = opts.batch_size {
66 params.insert("batch_size".into(), Value::from(bs));
67 }
68 let result = knowledge::KnowledgeHandlers::index(
69 runtime,
70 token,
71 Value::Object(params),
72 &ann,
73 on_atom_progress,
74 )
75 .await?;
76 atoms_indexed = result.get("indexed").and_then(|n| n.as_u64()).unwrap_or(0);
77 failed = result.get("failed").and_then(|n| n.as_u64()).unwrap_or(0);
78 ann_failed = result
79 .get("ann_failed")
80 .and_then(|b| b.as_bool())
81 .unwrap_or(false);
82 truncation_by_model = result
83 .get("truncation_by_model")
84 .and_then(Value::as_object)
85 .cloned()
86 .unwrap_or_default();
87 }
88
89 let mut sections_indexed = 0u64;
90 let mut sections_failed = 0u64;
91 let mut sections_superseded = 0u64;
92 if opts.sections {
93 let batch = opts
94 .batch_size
95 .map(|b| b as usize)
96 .unwrap_or(knowledge::util::DEFAULT_EMBED_BATCH);
97 let (indexed, _skipped, superseded, sec_failed, section_truncation) =
98 knowledge::sections_index::embed_sections(
99 runtime,
100 token,
101 opts.drop_existing,
102 batch,
103 on_section_progress,
104 None,
105 )
106 .await?;
107 sections_indexed = indexed as u64;
108 sections_failed = sec_failed as u64;
109 sections_superseded = superseded as u64;
110 if section_truncation.any_truncated() {
111 let model = runtime.default_embedder_name().to_string();
112 let existing = truncation_by_model
113 .remove(&model)
114 .and_then(|value| {
115 serde_json::from_value::<khive_runtime::retrieval::EmbeddingTruncationReport>(
116 value,
117 )
118 .ok()
119 })
120 .unwrap_or_default();
121 let mut combined: khive_runtime::retrieval::EmbeddingTruncationReport = existing;
122 combined.merge(section_truncation);
123 truncation_by_model.insert(
124 model,
125 serde_json::to_value(combined)
126 .expect("EmbeddingTruncationReport serialization cannot fail"),
127 );
128 }
129 }
130
131 Ok(json!({
132 "atoms_indexed": atoms_indexed,
133 "sections_indexed": sections_indexed,
134 "failed": failed,
135 "ann_failed": ann_failed,
136 "sections_failed": sections_failed,
137 "sections_superseded": sections_superseded,
138 "truncation_by_model": truncation_by_model,
139 }))
140}
141
142pub async fn rebuild_knowledge_fts_indexes(runtime: &KhiveRuntime) -> Result<Value, RuntimeError> {
154 knowledge::index_handler::rebuild_fts_indexes(runtime).await
155}