1use std::collections::HashMap;
4use std::sync::Mutex;
5
6use async_trait::async_trait;
7use serde_json::Value;
8use uuid::Uuid;
9
10use khive_brain_core::SectionPosteriorState;
11use khive_runtime::pack::{PackByIdResolver, PackRuntime};
12use khive_runtime::resource_admission::{
13 AdmittedJob, BodyOutcome, FinalizationOutcome, JobOutcome, TrackedJob, WorkClass,
14};
15use khive_runtime::{
16 KhiveRuntime, Namespace, NamespaceToken, PackSchemaPlan, Resolved, RuntimeError, VerbRegistry,
17};
18use khive_storage::types::{SqlStatement, SqlValue};
19use khive_storage::{PhaseCancelledPayload, PhaseCompletedPayload, PhaseStartedPayload};
20use khive_types::{EventKind, HandlerDef, Pack};
21
22use crate::knowledge::util::{atom_from_row, atom_to_json, domain_from_row, domain_to_json};
23use crate::knowledge::vamana;
24use crate::knowledge::KnowledgeHandlers;
25use crate::vocab::{KNOWLEDGE_HANDLERS, KNOWLEDGE_SCHEMA_PLAN_STMTS};
26
27pub struct KnowledgePack {
29 pub(crate) runtime: KhiveRuntime,
30 pub(crate) ann: vamana::SharedAnn,
31 pub(crate) section_posteriors: Mutex<HashMap<String, SectionPosteriorState>>,
37 pub(crate) brain_profile: Option<String>,
43}
44
45impl Pack for KnowledgePack {
46 const NAME: &'static str = "knowledge";
47 const NOTE_KINDS: &'static [&'static str] = &[];
48 const ENTITY_KINDS: &'static [&'static str] = &[];
49 const BRAIN_CONSUMER_KINDS: &'static [&'static str] =
50 &[khive_brain_core::ConsumerKind::KnowledgeCompose.as_str()];
51 const HANDLERS: &'static [HandlerDef] = &KNOWLEDGE_HANDLERS;
52 const REQUIRES: &'static [&'static str] = &["kg"];
53 const SCHEMA_PLAN: Option<PackSchemaPlan> = Some(PackSchemaPlan {
54 pack: "knowledge",
55 statements: &KNOWLEDGE_SCHEMA_PLAN_STMTS,
56 });
57}
58
59impl KnowledgePack {
60 pub fn new(runtime: KhiveRuntime) -> Self {
62 Self::new_with_index_role(runtime, true)
63 }
64
65 pub fn new_with_index_role(runtime: KhiveRuntime, builds_corpus_indexes: bool) -> Self {
70 let brain_profile = runtime.config().brain_profile.clone();
71 Self {
72 runtime,
73 ann: vamana::new_shared_for_role(builds_corpus_indexes),
74 section_posteriors: Mutex::new(HashMap::new()),
75 brain_profile,
76 }
77 }
78}
79
80struct KnowledgePackFactory;
81
82impl khive_runtime::PackFactory for KnowledgePackFactory {
83 khive_runtime::pack_factory_metadata!(KnowledgePack);
84
85 fn create(&self, runtime: KhiveRuntime) -> Box<dyn khive_runtime::PackRuntime> {
86 Box::new(KnowledgePack::new_with_index_role(
87 runtime,
88 khive_runtime::daemon::is_warm_index_host(),
89 ))
90 }
91
92 fn create_resolver(
93 &self,
94 runtime: KhiveRuntime,
95 ) -> Option<Box<dyn khive_runtime::PackByIdResolver>> {
96 Some(Box::new(KnowledgePack::new(runtime)))
97 }
98}
99
100inventory::submit! { khive_runtime::PackRegistration(&KnowledgePackFactory) }
101
102async fn run_knowledge_embedder_warm(
104 runtime: KhiveRuntime,
105 token: Option<NamespaceToken>,
106 job: AdmittedJob,
107) -> JobOutcome<Result<Vec<f32>, RuntimeError>> {
108 let phase_entry = std::time::Instant::now();
109 let entry_cpu = khive_runtime::process_resource_usage();
110 let mut start_attempted = false;
111 let start_finished = if let Some(token) = &token {
112 match job
113 .run_body(async {
114 start_attempted = true;
116 khive_runtime::emit_phase_event(
117 &runtime,
118 token,
119 "knowledge.embedder_warm",
120 EventKind::PhaseStarted,
121 PhaseStartedPayload {
122 work_class: "warm".into(),
123 phase: "knowledge_embedder_warm".into(),
124 corpus_size: None,
125 },
126 )
127 .await;
128 })
129 .await
130 {
131 BodyOutcome::Finished(()) => true,
132 BodyOutcome::Cancelled => false,
133 }
134 } else {
135 true
136 };
137 if !start_finished && !start_attempted {
138 return JobOutcome::Cancelled;
140 }
141 let (phase_start, cpu_start, result) = if start_finished {
142 let phase_start = std::time::Instant::now();
144 let cpu_start = khive_runtime::process_resource_usage();
145 let result = job
146 .run_body(runtime.embed("__khive_knowledge_warm__"))
147 .await;
148 (phase_start, cpu_start, result)
149 } else {
150 tracing::warn!(
151 phase = "knowledge_embedder_warm",
152 start_attempted,
153 start_finished,
154 start_append_persistence = "unknown",
155 "warm start append was interrupted before returning",
156 );
157 (phase_entry, entry_cpu, BodyOutcome::Cancelled)
158 };
159 let cancelled = match &result {
160 BodyOutcome::Cancelled => true,
161 BodyOutcome::Finished(Err(error)) => khive_runtime::is_benign_shutdown_cancellation(error),
162 BodyOutcome::Finished(Ok(_)) => false,
163 };
164 if let BodyOutcome::Finished(Err(error)) = &result {
165 tracing::warn!(error = %error, "knowledge_embedder_warm: warm operation returned an error");
166 }
167 let finalization = if let Some(token) = &token {
168 let wall_us = phase_start.elapsed().as_micros() as i64;
169 let cpu_us =
170 khive_runtime::cpu_delta_us(cpu_start, khive_runtime::process_resource_usage());
171 job.finalize(async {
173 if cancelled {
174 khive_runtime::emit_phase_event(
175 &runtime,
176 token,
177 "knowledge.embedder_warm",
178 EventKind::PhaseCancelled,
179 PhaseCancelledPayload {
180 work_class: "warm".into(),
181 phase: "knowledge_embedder_warm".into(),
182 wall_us,
183 cpu_us,
184 },
185 )
186 .await;
187 } else {
188 khive_runtime::emit_phase_event(
189 &runtime,
190 token,
191 "knowledge.embedder_warm",
192 EventKind::PhaseCompleted,
193 PhaseCompletedPayload {
194 work_class: "warm".into(),
195 phase: "knowledge_embedder_warm".into(),
196 wall_us,
197 cpu_us,
198 },
199 )
200 .await;
201 }
202 })
203 .await
204 } else {
205 job.no_telemetry()
206 };
207 match finalization {
208 FinalizationOutcome::AttemptTimedOut
209 | FinalizationOutcome::NoAttemptBeforeDeadline
210 | FinalizationOutcome::DeadlineOverflow => {
211 tracing::warn!(
212 ?finalization,
213 "knowledge_embedder_warm: terminal append attempt did not finish"
214 );
215 return JobOutcome::FinalizationTimedOut;
216 }
217 FinalizationOutcome::AlreadyFinalized => {
218 tracing::warn!("knowledge_embedder_warm: terminal finalization was already claimed");
219 }
220 FinalizationOutcome::AttemptReturned | FinalizationOutcome::NoTelemetryToken => {}
221 }
222 match result {
223 BodyOutcome::Finished(result) => JobOutcome::Completed(result),
224 BodyOutcome::Cancelled => JobOutcome::Cancelled,
225 }
226}
227
228#[async_trait]
229impl PackRuntime for KnowledgePack {
230 khive_runtime::pack_runtime_metadata!();
231
232 async fn warm(&self) {
233 if !self.runtime.is_read_only() {
237 crate::knowledge::vamana::warm_known_snapshots(&self.runtime, &self.ann).await;
238 crate::knowledge::vamana::start_rotation_watcher(&self.runtime, &self.ann);
239 }
240 if !self.runtime.default_embedder_name().is_empty() {
241 let runtime = self.runtime.background_view();
242 let domain = runtime.background_domain();
243 let tracking = TrackedJob::register("knowledge_embedder_warm");
245 let token = match runtime.authorize(khive_runtime::Namespace::local()) {
246 Ok(token) => Some(token),
247 Err(error) => {
248 tracing::warn!(error = %error, "knowledge_embedder_warm: warmup proceeds without telemetry");
249 None
250 }
251 };
252 tokio::spawn(async move {
255 let admitted = match domain.admit(WorkClass::Warm, &tracking).await {
256 Ok(job) => job,
257 Err(error) => {
258 tracing::warn!(error = %error, "knowledge_embedder_warm: admission refused");
259 return;
260 }
261 };
262 match domain.spawn_supervised(admitted, move |job| {
263 run_knowledge_embedder_warm(runtime, token, job)
264 }) {
265 Ok(task) => task.detach(),
266 Err(error) => {
267 tracing::warn!(error = %error, "knowledge_embedder_warm: supervisor submission refused");
268 }
269 }
270 });
271 }
272 }
273
274 async fn dispatch(
275 &self,
276 verb: &str,
277 params: Value,
278 registry: &VerbRegistry,
279 token: &NamespaceToken,
280 ) -> Result<Value, RuntimeError> {
281 match verb {
282 "knowledge.upsert_atoms" => {
283 KnowledgeHandlers::upsert_atoms(&self.runtime, token, params).await
284 }
285 "knowledge.upsert_domains" => {
286 KnowledgeHandlers::upsert_domains(&self.runtime, token, params).await
287 }
288 "knowledge.get" => KnowledgeHandlers::get(&self.runtime, token, params).await,
289 "knowledge.list" => KnowledgeHandlers::list(&self.runtime, token, params).await,
290 "knowledge.export" => KnowledgeHandlers::export(&self.runtime, token, params).await,
291 "knowledge.lint" => self.handle_lint(token, params, registry).await,
292 "knowledge.delete_atoms" => {
293 KnowledgeHandlers::delete_atoms(&self.runtime, token, params).await
294 }
295 "knowledge.stats" => KnowledgeHandlers::stats(&self.runtime, token, params).await,
296 "knowledge.eval_retrieval" => {
297 KnowledgeHandlers::eval_retrieval(&self.runtime, token, params, &self.ann).await
298 }
299 "knowledge.index" => {
300 KnowledgeHandlers::index(&self.runtime, token, params, &self.ann, None).await
301 }
302 "knowledge.fold" => KnowledgeHandlers::fold(&self.runtime, token, params).await,
303 "knowledge.search" => {
304 KnowledgeHandlers::search(&self.runtime, token, params, &self.ann).await
305 }
306 "knowledge.suggest" => {
307 KnowledgeHandlers::suggest(&self.runtime, token, params, &self.ann).await
308 }
309 "knowledge.compose" => {
310 let effective_token = match params.get("namespace") {
316 None => token.clone(),
317 Some(Value::String(ns_str)) => {
318 let ns = Namespace::parse(ns_str).map_err(|e| {
319 RuntimeError::InvalidInput(format!("invalid namespace {ns_str:?}: {e}"))
320 })?;
321 if &ns != token.namespace() {
322 return Err(RuntimeError::InvalidInput(
323 "knowledge.compose namespace does not match authorized token namespace"
324 .to_string(),
325 ));
326 }
327 token.with_namespace(ns)
331 }
332 Some(_) => {
333 return Err(RuntimeError::InvalidInput(
334 "invalid namespace: expected a string".to_string(),
335 ));
336 }
337 };
338 let type_weights = self
339 .resolve_compose_type_weights(registry, &effective_token)
340 .await;
341 KnowledgeHandlers::compose(
342 &self.runtime,
343 &effective_token,
344 params,
345 &self.ann,
346 type_weights,
347 )
348 .await
349 }
350 "knowledge.edit" => {
351 KnowledgeHandlers::edit(&self.runtime, token, params, &self.ann).await
352 }
353 "knowledge.import" => {
354 KnowledgeHandlers::import(&self.runtime, token, params, &self.ann).await
355 }
356 "knowledge.challenge" => {
357 KnowledgeHandlers::challenge(&self.runtime, token, params).await
358 }
359 "knowledge.adjudicate" => {
360 KnowledgeHandlers::adjudicate(&self.runtime, token, params).await
361 }
362 "knowledge.learn" => self.handle_learn(token, params).await,
363 "knowledge.cite" => self.handle_cite(token, params).await,
364 "knowledge.topic" => self.handle_topic(token, params).await,
365 "knowledge.feedback" => self.handle_feedback(token, params, registry).await,
366 _ => Err(RuntimeError::InvalidInput(format!(
367 "knowledge pack does not handle verb {verb:?}"
368 ))),
369 }
370 }
371}
372
373#[async_trait]
374impl PackByIdResolver for KnowledgePack {
375 async fn resolve_by_id(&self, id: Uuid) -> Result<Option<Resolved>, RuntimeError> {
381 let sql = self.runtime.sql();
382 let id_str = id.to_string();
383
384 let mut reader = sql
385 .reader()
386 .await
387 .map_err(|e| RuntimeError::Internal(format!("knowledge resolve_by_id reader: {e}")))?;
388
389 let domain_row = reader
391 .query_row(SqlStatement {
392 sql: khive_runtime::sql!("knowledge_resolver_live_domain").into(),
393 params: vec![SqlValue::Text(id_str.clone())],
394 label: Some("knowledge.resolve_by_id.domain".into()),
395 })
396 .await
397 .map_err(|e| {
398 RuntimeError::Internal(format!("knowledge resolve_by_id domain query: {e}"))
399 })?;
400
401 if let Some(row) = domain_row {
402 if let Some(domain) = domain_from_row(&row) {
403 return Ok(Some(Resolved::PackRecord {
404 pack: "knowledge".into(),
405 kind: "domain".into(),
406 data: domain_to_json(&domain),
407 }));
408 }
409 }
410
411 let atom_row = reader
413 .query_row(SqlStatement {
414 sql: khive_runtime::sql!("knowledge_resolver_live_atom").into(),
415 params: vec![SqlValue::Text(id_str)],
416 label: Some("knowledge.resolve_by_id.atom".into()),
417 })
418 .await
419 .map_err(|e| {
420 RuntimeError::Internal(format!("knowledge resolve_by_id atom query: {e}"))
421 })?;
422
423 if let Some(row) = atom_row {
424 if let Some(atom) = atom_from_row(&row) {
425 return Ok(Some(Resolved::PackRecord {
426 pack: "knowledge".into(),
427 kind: "atom".into(),
428 data: atom_to_json(&atom),
429 }));
430 }
431 }
432
433 Ok(None)
434 }
435
436 async fn resolve_by_id_including_deleted(
440 &self,
441 id: Uuid,
442 ) -> Result<Option<Resolved>, RuntimeError> {
443 let sql = self.runtime.sql();
444 let id_str = id.to_string();
445
446 let mut reader = sql.reader().await.map_err(|e| {
447 RuntimeError::Internal(format!(
448 "knowledge resolve_by_id_including_deleted reader: {e}"
449 ))
450 })?;
451
452 let domain_row = reader
453 .query_row(SqlStatement {
454 sql: khive_runtime::sql!("knowledge_resolver_domain_including_deleted").into(),
455 params: vec![SqlValue::Text(id_str.clone())],
456 label: Some("knowledge.resolve_by_id_incl_deleted.domain".into()),
457 })
458 .await
459 .map_err(|e| {
460 RuntimeError::Internal(format!(
461 "knowledge resolve_by_id_including_deleted domain query: {e}"
462 ))
463 })?;
464
465 if let Some(row) = domain_row {
466 if let Some(domain) = domain_from_row(&row) {
467 return Ok(Some(Resolved::PackRecord {
468 pack: "knowledge".into(),
469 kind: "domain".into(),
470 data: domain_to_json(&domain),
471 }));
472 }
473 }
474
475 let atom_row = reader
476 .query_row(SqlStatement {
477 sql: khive_runtime::sql!("knowledge_resolver_atom_including_deleted").into(),
478 params: vec![SqlValue::Text(id_str)],
479 label: Some("knowledge.resolve_by_id_incl_deleted.atom".into()),
480 })
481 .await
482 .map_err(|e| {
483 RuntimeError::Internal(format!(
484 "knowledge resolve_by_id_including_deleted atom query: {e}"
485 ))
486 })?;
487
488 if let Some(row) = atom_row {
489 if let Some(atom) = atom_from_row(&row) {
490 return Ok(Some(Resolved::PackRecord {
491 pack: "knowledge".into(),
492 kind: "atom".into(),
493 data: atom_to_json(&atom),
494 }));
495 }
496 }
497
498 Ok(None)
499 }
500
501 fn private_record_verbs(&self) -> &'static [&'static str] {
502 &[
503 "knowledge.upsert_atoms",
504 "knowledge.upsert_domains",
505 "knowledge.edit",
506 ]
507 }
508
509 async fn delete_by_id(&self, id: Uuid, hard: bool) -> Result<serde_json::Value, RuntimeError> {
515 let sql = self.runtime.sql();
516 let id_str = id.to_string();
517
518 let kind = match self.resolve_by_id_including_deleted(id).await? {
520 Some(Resolved::PackRecord { kind, .. }) => kind,
521 _ => {
522 return Err(RuntimeError::NotFound(format!(
523 "knowledge record not found: {id}"
524 )));
525 }
526 };
527
528 let now_us = std::time::SystemTime::now()
529 .duration_since(std::time::UNIX_EPOCH)
530 .unwrap_or_default()
531 .as_micros() as i64;
532
533 let mut writer = sql
534 .writer()
535 .await
536 .map_err(|e| RuntimeError::Internal(format!("knowledge delete_by_id writer: {e}")))?;
537
538 match kind.as_str() {
539 "domain" => {
540 if hard {
541 writer
545 .execute_batch(vec![
546 SqlStatement {
547 sql: khive_runtime::sql!("knowledge_atom_sections_delete").into(),
548 params: vec![SqlValue::Text(id_str.clone())],
549 label: Some(
550 "knowledge.delete_by_id.domain_mirror_sections.hard".into(),
551 ),
552 },
553 SqlStatement {
554 sql: khive_runtime::sql!("knowledge_atom_hard_delete").into(),
555 params: vec![SqlValue::Text(id_str.clone())],
556 label: Some("knowledge.delete_by_id.domain_mirror.hard".into()),
557 },
558 SqlStatement {
559 sql: khive_runtime::sql!("knowledge_domain_hard_delete").into(),
560 params: vec![SqlValue::Text(id_str.clone())],
561 label: Some("knowledge.delete_by_id.domain.hard".into()),
562 },
563 ])
564 .await
565 .map_err(|e| {
566 RuntimeError::Internal(format!(
567 "knowledge delete_by_id domain hard: {e}"
568 ))
569 })?;
570 } else {
571 writer
572 .execute(SqlStatement {
573 sql: khive_runtime::sql!("knowledge_domain_soft_delete").into(),
574 params: vec![SqlValue::Integer(now_us), SqlValue::Text(id_str.clone())],
575 label: Some("knowledge.delete_by_id.domain.soft".into()),
576 })
577 .await
578 .map_err(|e| {
579 RuntimeError::Internal(format!(
580 "knowledge delete_by_id domain soft: {e}"
581 ))
582 })?;
583 writer
585 .execute(SqlStatement {
586 sql: khive_runtime::sql!("knowledge_atom_soft_delete").into(),
587 params: vec![SqlValue::Integer(now_us), SqlValue::Text(id_str.clone())],
588 label: Some("knowledge.delete_by_id.domain_mirror.soft".into()),
589 })
590 .await
591 .map_err(|e| {
592 RuntimeError::Internal(format!(
593 "knowledge delete_by_id domain mirror soft: {e}"
594 ))
595 })?;
596 }
597 }
598 "atom" => {
599 if hard {
600 writer
602 .execute_batch(vec![
603 SqlStatement {
604 sql: khive_runtime::sql!("knowledge_atom_sections_delete").into(),
605 params: vec![SqlValue::Text(id_str.clone())],
606 label: Some("knowledge.delete_by_id.atom_sections.hard".into()),
607 },
608 SqlStatement {
609 sql: khive_runtime::sql!("knowledge_atom_hard_delete").into(),
610 params: vec![SqlValue::Text(id_str.clone())],
611 label: Some("knowledge.delete_by_id.atom.hard".into()),
612 },
613 ])
614 .await
615 .map_err(|e| {
616 RuntimeError::Internal(format!("knowledge delete_by_id atom hard: {e}"))
617 })?;
618 } else {
619 writer
620 .execute(SqlStatement {
621 sql: khive_runtime::sql!("knowledge_atom_soft_delete").into(),
622 params: vec![SqlValue::Integer(now_us), SqlValue::Text(id_str.clone())],
623 label: Some("knowledge.delete_by_id.atom.soft".into()),
624 })
625 .await
626 .map_err(|e| {
627 RuntimeError::Internal(format!("knowledge delete_by_id atom soft: {e}"))
628 })?;
629 }
630 }
631 other => {
632 return Err(RuntimeError::Internal(format!(
633 "knowledge delete_by_id: unexpected kind {other:?}"
634 )));
635 }
636 }
637
638 Ok(serde_json::json!({
639 "deleted": true,
640 "id": id_str,
641 "kind": kind,
642 "hard": hard,
643 }))
644 }
645}
646
647#[cfg(test)]
648#[path = "pack_tests.rs"]
649mod tests;
650
651#[cfg(test)]
652#[path = "warm_admission_tests.rs"]
653mod warm_admission_tests;