khive_pack_knowledge/
lib.rs1pub(crate) mod handlers;
4pub(crate) mod knowledge;
5mod pack;
6mod vocab;
7
8pub use pack::KnowledgePack;
9
10use khive_runtime::{KhiveRuntime, NamespaceToken, RuntimeError};
11use serde_json::{json, Value};
12
13#[derive(Debug, Clone, Copy)]
15pub struct KnowledgeReindexOptions {
16 pub atoms: bool,
18 pub sections: bool,
20 pub drop_existing: bool,
22 pub rebuild_ann: bool,
24 pub batch_size: Option<u32>,
26}
27
28pub async fn reindex_knowledge(
40 runtime: &KhiveRuntime,
41 token: &NamespaceToken,
42 opts: KnowledgeReindexOptions,
43 on_atom_progress: Option<&(dyn Fn(u64, u64) + Send + Sync)>,
44 on_section_progress: Option<&(dyn Fn(u64, u64) + Send + Sync)>,
45) -> Result<Value, RuntimeError> {
46 let mut atoms_indexed = 0u64;
47 let mut failed = 0u64;
48 let mut ann_failed = false;
49 let mut truncation_by_model = serde_json::Map::new();
50 if opts.atoms {
51 let ann = knowledge::vamana::new_shared();
52 let mut params = serde_json::Map::new();
53 params.insert("rebuild_ann".into(), Value::Bool(opts.rebuild_ann));
54 params.insert("insert_only".into(), Value::Bool(!opts.drop_existing));
55 if let Some(bs) = opts.batch_size {
56 params.insert("batch_size".into(), Value::from(bs));
57 }
58 let result = knowledge::KnowledgeHandlers::index(
59 runtime,
60 token,
61 Value::Object(params),
62 &ann,
63 on_atom_progress,
64 )
65 .await?;
66 atoms_indexed = result.get("indexed").and_then(|n| n.as_u64()).unwrap_or(0);
67 failed = result.get("failed").and_then(|n| n.as_u64()).unwrap_or(0);
68 ann_failed = result
69 .get("ann_failed")
70 .and_then(|b| b.as_bool())
71 .unwrap_or(false);
72 truncation_by_model = result
73 .get("truncation_by_model")
74 .and_then(Value::as_object)
75 .cloned()
76 .unwrap_or_default();
77 }
78
79 let mut sections_indexed = 0u64;
80 let mut sections_failed = 0u64;
81 if opts.sections {
82 let batch = opts
83 .batch_size
84 .map(|b| b as usize)
85 .unwrap_or(knowledge::util::DEFAULT_EMBED_BATCH);
86 let (indexed, _skipped, sec_failed, section_truncation) =
87 knowledge::sections_index::embed_sections(
88 runtime,
89 token,
90 opts.drop_existing,
91 batch,
92 on_section_progress,
93 None,
94 )
95 .await?;
96 sections_indexed = indexed as u64;
97 sections_failed = sec_failed as u64;
98 if section_truncation.any_truncated() {
99 let model = runtime.default_embedder_name().to_string();
100 let existing = truncation_by_model
101 .remove(&model)
102 .and_then(|value| {
103 serde_json::from_value::<khive_runtime::retrieval::EmbeddingTruncationReport>(
104 value,
105 )
106 .ok()
107 })
108 .unwrap_or_default();
109 let mut combined: khive_runtime::retrieval::EmbeddingTruncationReport = existing;
110 combined.merge(section_truncation);
111 truncation_by_model.insert(
112 model,
113 serde_json::to_value(combined)
114 .expect("EmbeddingTruncationReport serialization cannot fail"),
115 );
116 }
117 }
118
119 Ok(json!({
120 "atoms_indexed": atoms_indexed,
121 "sections_indexed": sections_indexed,
122 "failed": failed,
123 "ann_failed": ann_failed,
124 "sections_failed": sections_failed,
125 "truncation_by_model": truncation_by_model,
126 }))
127}