Skip to main content

khive_pack_knowledge/
lib.rs

1//! pack-knowledge — knowledge corpus verbs for khive.
2
3mod feedback;
4pub(crate) mod handlers;
5pub(crate) mod knowledge;
6mod pack;
7mod vocab;
8
9pub use pack::KnowledgePack;
10
11use khive_runtime::{KhiveRuntime, NamespaceToken, RuntimeError};
12use serde_json::{json, Value};
13
14/// Options for [`reindex_knowledge`].
15#[derive(Debug, Clone, Copy)]
16pub struct KnowledgeReindexOptions {
17    /// Embed atoms (and rebuild the atom Vamana ANN).
18    pub atoms: bool,
19    /// Embed sections into `knowledge_sections.embedding` (ADR-051).
20    pub sections: bool,
21    /// Re-embed everything; when false, only fill missing vectors.
22    pub drop_existing: bool,
23    /// Rebuild the atom Vamana ANN snapshot (only meaningful with `atoms`).
24    pub rebuild_ann: bool,
25    /// Records per embedding batch.
26    pub batch_size: Option<u32>,
27}
28
29/// Reindex the knowledge corpus for `token`'s namespace: embed atoms and/or
30/// sections with the default embedder and (optionally) rebuild the atom Vamana
31/// ANN snapshot.
32///
33/// Library entry for `kkernel reindex` — callable without an MCP server. Every
34/// effect is scoped to `token`'s namespace; the whole-database FTS rebuild is
35/// deliberately not an option here but a separate operator entry point,
36/// [`rebuild_knowledge_fts_indexes`], so that holding a namespace token never
37/// grants a cost that spans every namespace.
38///
39/// Knowledge is single-model: atom indexing, section indexing, and search all
40/// use the default embedder. Returns `{atoms_indexed, sections_indexed, failed,
41/// ann_failed, sections_failed, truncation_by_model}`.
42///
43/// Optional progress callbacks receive `(processed, total)` after each batch.
44pub async fn reindex_knowledge(
45    runtime: &KhiveRuntime,
46    token: &NamespaceToken,
47    opts: KnowledgeReindexOptions,
48    on_atom_progress: Option<&(dyn Fn(u64, u64) + Send + Sync)>,
49    on_section_progress: Option<&(dyn Fn(u64, u64) + Send + Sync)>,
50) -> Result<Value, RuntimeError> {
51    let mut atoms_indexed = 0u64;
52    let mut failed = 0u64;
53    let mut ann_failed = false;
54    let mut truncation_by_model = serde_json::Map::new();
55    if opts.atoms {
56        let ann = knowledge::vamana::new_shared();
57        let mut params = serde_json::Map::new();
58        params.insert("rebuild_ann".into(), Value::Bool(opts.rebuild_ann));
59        params.insert("insert_only".into(), Value::Bool(!opts.drop_existing));
60        if let Some(bs) = opts.batch_size {
61            params.insert("batch_size".into(), Value::from(bs));
62        }
63        let result = knowledge::KnowledgeHandlers::index(
64            runtime,
65            token,
66            Value::Object(params),
67            &ann,
68            on_atom_progress,
69        )
70        .await?;
71        atoms_indexed = result.get("indexed").and_then(|n| n.as_u64()).unwrap_or(0);
72        failed = result.get("failed").and_then(|n| n.as_u64()).unwrap_or(0);
73        ann_failed = result
74            .get("ann_failed")
75            .and_then(|b| b.as_bool())
76            .unwrap_or(false);
77        truncation_by_model = result
78            .get("truncation_by_model")
79            .and_then(Value::as_object)
80            .cloned()
81            .unwrap_or_default();
82    }
83
84    let mut sections_indexed = 0u64;
85    let mut sections_failed = 0u64;
86    if opts.sections {
87        let batch = opts
88            .batch_size
89            .map(|b| b as usize)
90            .unwrap_or(knowledge::util::DEFAULT_EMBED_BATCH);
91        let (indexed, _skipped, sec_failed, section_truncation) =
92            knowledge::sections_index::embed_sections(
93                runtime,
94                token,
95                opts.drop_existing,
96                batch,
97                on_section_progress,
98                None,
99            )
100            .await?;
101        sections_indexed = indexed as u64;
102        sections_failed = sec_failed as u64;
103        if section_truncation.any_truncated() {
104            let model = runtime.default_embedder_name().to_string();
105            let existing = truncation_by_model
106                .remove(&model)
107                .and_then(|value| {
108                    serde_json::from_value::<khive_runtime::retrieval::EmbeddingTruncationReport>(
109                        value,
110                    )
111                    .ok()
112                })
113                .unwrap_or_default();
114            let mut combined: khive_runtime::retrieval::EmbeddingTruncationReport = existing;
115            combined.merge(section_truncation);
116            truncation_by_model.insert(
117                model,
118                serde_json::to_value(combined)
119                    .expect("EmbeddingTruncationReport serialization cannot fail"),
120            );
121        }
122    }
123
124    Ok(json!({
125        "atoms_indexed": atoms_indexed,
126        "sections_indexed": sections_indexed,
127        "failed": failed,
128        "ann_failed": ann_failed,
129        "sections_failed": sections_failed,
130        "truncation_by_model": truncation_by_model,
131    }))
132}
133
134/// Rebuild and rank-1 integrity-check both external-content knowledge FTS
135/// indexes (`fts_knowledge`, `fts_sections`).
136///
137/// Operator entry point for `kkernel reindex --rebuild-fts`. Both indexes are
138/// global — they cover every namespace in the database — so this takes no
139/// namespace token and is never reachable from a namespace-scoped verb: the
140/// `knowledge.index` MCP verb does not accept it, and [`reindex_knowledge`]
141/// does not offer it as an option, because neither has per-caller admission
142/// control to bound a whole-database cost. The writer batch is atomic, and the
143/// returned `{indexes, elapsed_ms, integrity_ok}` reports what the rebuild
144/// actually did.
145pub async fn rebuild_knowledge_fts_indexes(runtime: &KhiveRuntime) -> Result<Value, RuntimeError> {
146    knowledge::index_handler::rebuild_fts_indexes(runtime).await
147}