Skip to main content

khive_pack_knowledge/
lib.rs

1//! pack-knowledge — knowledge corpus verbs for khive.
2
3mod feedback;
4pub(crate) mod handlers;
5pub(crate) mod knowledge;
6mod lint;
7mod pack;
8mod vocab;
9
10pub use knowledge::snapshot_invalidation::invalidate_legacy_vamana_snapshots;
11pub use pack::KnowledgePack;
12
13use khive_runtime::{KhiveRuntime, NamespaceToken, RuntimeError};
14use serde_json::{json, Value};
15
16/// Options for [`reindex_knowledge`].
17#[derive(Debug, Clone, Copy)]
18pub struct KnowledgeReindexOptions {
19    /// Embed atoms (and rebuild the atom Vamana ANN).
20    pub atoms: bool,
21    /// Embed sections into `knowledge_sections.embedding` (ADR-051).
22    pub sections: bool,
23    /// Re-embed everything; when false, only fill missing vectors.
24    pub drop_existing: bool,
25    /// Rebuild the atom Vamana ANN snapshot (only meaningful with `atoms`).
26    pub rebuild_ann: bool,
27    /// Records per embedding batch.
28    pub batch_size: Option<u32>,
29}
30
31/// Reindex the knowledge corpus for `token`'s namespace: embed atoms and/or
32/// sections with the default embedder and (optionally) rebuild the atom Vamana
33/// ANN snapshot.
34///
35/// Library entry for `kkernel reindex` — callable without an MCP server. Every
36/// effect is scoped to `token`'s namespace; the whole-database FTS rebuild is
37/// deliberately not an option here but a separate operator entry point,
38/// [`rebuild_knowledge_fts_indexes`], so that holding a namespace token never
39/// grants a cost that spans every namespace.
40///
41/// Knowledge is single-model: atom indexing, section indexing, and search all
42/// use the default embedder. Returns `{atoms_indexed, sections_indexed, failed,
43/// ann_failed, sections_failed, sections_superseded, truncation_by_model}`.
44/// `sections_superseded` counts conditional vector writes rejected because the
45/// section or its atom changed while embedding was in flight; these are not
46/// storage failures.
47///
48/// Optional progress callbacks receive `(processed, total)` after each batch.
49pub async fn reindex_knowledge(
50    runtime: &KhiveRuntime,
51    token: &NamespaceToken,
52    opts: KnowledgeReindexOptions,
53    on_atom_progress: Option<&(dyn Fn(u64, u64) + Send + Sync)>,
54    on_section_progress: Option<&(dyn Fn(u64, u64) + Send + Sync)>,
55) -> Result<Value, RuntimeError> {
56    let mut atoms_indexed = 0u64;
57    let mut failed = 0u64;
58    let mut ann_failed = false;
59    let mut truncation_by_model = serde_json::Map::new();
60    if opts.atoms {
61        let ann = knowledge::vamana::new_shared();
62        let mut params = serde_json::Map::new();
63        params.insert("rebuild_ann".into(), Value::Bool(opts.rebuild_ann));
64        params.insert("insert_only".into(), Value::Bool(!opts.drop_existing));
65        if let Some(bs) = opts.batch_size {
66            params.insert("batch_size".into(), Value::from(bs));
67        }
68        let result = knowledge::KnowledgeHandlers::index(
69            runtime,
70            token,
71            Value::Object(params),
72            &ann,
73            on_atom_progress,
74        )
75        .await?;
76        atoms_indexed = result.get("indexed").and_then(|n| n.as_u64()).unwrap_or(0);
77        failed = result.get("failed").and_then(|n| n.as_u64()).unwrap_or(0);
78        ann_failed = result
79            .get("ann_failed")
80            .and_then(|b| b.as_bool())
81            .unwrap_or(false);
82        truncation_by_model = result
83            .get("truncation_by_model")
84            .and_then(Value::as_object)
85            .cloned()
86            .unwrap_or_default();
87    }
88
89    let mut sections_indexed = 0u64;
90    let mut sections_failed = 0u64;
91    let mut sections_superseded = 0u64;
92    if opts.sections {
93        let batch = opts
94            .batch_size
95            .map(|b| b as usize)
96            .unwrap_or(knowledge::util::DEFAULT_EMBED_BATCH);
97        let (indexed, _skipped, superseded, sec_failed, section_truncation) =
98            knowledge::sections_index::embed_sections(
99                runtime,
100                token,
101                opts.drop_existing,
102                batch,
103                on_section_progress,
104                None,
105            )
106            .await?;
107        sections_indexed = indexed as u64;
108        sections_failed = sec_failed as u64;
109        sections_superseded = superseded as u64;
110        if section_truncation.any_truncated() {
111            let model = runtime.default_embedder_name().to_string();
112            let existing = truncation_by_model
113                .remove(&model)
114                .and_then(|value| {
115                    serde_json::from_value::<khive_runtime::retrieval::EmbeddingTruncationReport>(
116                        value,
117                    )
118                    .ok()
119                })
120                .unwrap_or_default();
121            let mut combined: khive_runtime::retrieval::EmbeddingTruncationReport = existing;
122            combined.merge(section_truncation);
123            truncation_by_model.insert(
124                model,
125                serde_json::to_value(combined)
126                    .expect("EmbeddingTruncationReport serialization cannot fail"),
127            );
128        }
129    }
130
131    Ok(json!({
132        "atoms_indexed": atoms_indexed,
133        "sections_indexed": sections_indexed,
134        "failed": failed,
135        "ann_failed": ann_failed,
136        "sections_failed": sections_failed,
137        "sections_superseded": sections_superseded,
138        "truncation_by_model": truncation_by_model,
139    }))
140}
141
142/// Rebuild and rank-1 integrity-check both external-content knowledge FTS
143/// indexes (`fts_knowledge`, `fts_sections`).
144///
145/// Operator entry point for `kkernel reindex --rebuild-fts`. Both indexes are
146/// global — they cover every namespace in the database — so this takes no
147/// namespace token and is never reachable from a namespace-scoped verb: the
148/// `knowledge.index` MCP verb does not accept it, and [`reindex_knowledge`]
149/// does not offer it as an option, because neither has per-caller admission
150/// control to bound a whole-database cost. The writer batch is atomic, and the
151/// returned `{indexes, elapsed_ms, integrity_ok}` reports what the rebuild
152/// actually did.
153pub async fn rebuild_knowledge_fts_indexes(runtime: &KhiveRuntime) -> Result<Value, RuntimeError> {
154    knowledge::index_handler::rebuild_fts_indexes(runtime).await
155}