use std::collections::HashMap;
use std::sync::Mutex;
use async_trait::async_trait;
use serde_json::Value;
use uuid::Uuid;
use khive_brain_core::SectionPosteriorState;
use khive_runtime::pack::{PackByIdResolver, PackRuntime};
use khive_runtime::resource_admission::{
AdmittedJob, BodyOutcome, FinalizationOutcome, JobOutcome, TrackedJob, WorkClass,
};
use khive_runtime::{
KhiveRuntime, Namespace, NamespaceToken, PackSchemaPlan, Resolved, RuntimeError, VerbRegistry,
};
use khive_storage::types::{SqlStatement, SqlValue};
use khive_storage::{PhaseCancelledPayload, PhaseCompletedPayload, PhaseStartedPayload};
use khive_types::{EventKind, HandlerDef, Pack};
use crate::knowledge::util::{atom_from_row, atom_to_json, domain_from_row, domain_to_json};
use crate::knowledge::vamana;
use crate::knowledge::KnowledgeHandlers;
use crate::vocab::{KNOWLEDGE_HANDLERS, KNOWLEDGE_SCHEMA_PLAN_STMTS};
pub struct KnowledgePack {
pub(crate) runtime: KhiveRuntime,
pub(crate) ann: vamana::SharedAnn,
pub(crate) section_posteriors: Mutex<HashMap<String, SectionPosteriorState>>,
pub(crate) brain_profile: Option<String>,
}
impl Pack for KnowledgePack {
const NAME: &'static str = "knowledge";
const NOTE_KINDS: &'static [&'static str] = &[];
const ENTITY_KINDS: &'static [&'static str] = &[];
const BRAIN_CONSUMER_KINDS: &'static [&'static str] =
&[khive_brain_core::ConsumerKind::KnowledgeCompose.as_str()];
const HANDLERS: &'static [HandlerDef] = &KNOWLEDGE_HANDLERS;
const REQUIRES: &'static [&'static str] = &["kg"];
const SCHEMA_PLAN: Option<PackSchemaPlan> = Some(PackSchemaPlan {
pack: "knowledge",
statements: &KNOWLEDGE_SCHEMA_PLAN_STMTS,
});
}
impl KnowledgePack {
pub fn new(runtime: KhiveRuntime) -> Self {
Self::new_with_index_role(runtime, true)
}
pub fn new_with_index_role(runtime: KhiveRuntime, builds_corpus_indexes: bool) -> Self {
let brain_profile = runtime.config().brain_profile.clone();
Self {
runtime,
ann: vamana::new_shared_for_role(builds_corpus_indexes),
section_posteriors: Mutex::new(HashMap::new()),
brain_profile,
}
}
}
struct KnowledgePackFactory;
impl khive_runtime::PackFactory for KnowledgePackFactory {
khive_runtime::pack_factory_metadata!(KnowledgePack);
fn create(&self, runtime: KhiveRuntime) -> Box<dyn khive_runtime::PackRuntime> {
Box::new(KnowledgePack::new_with_index_role(
runtime,
khive_runtime::daemon::is_warm_index_host(),
))
}
fn create_resolver(
&self,
runtime: KhiveRuntime,
) -> Option<Box<dyn khive_runtime::PackByIdResolver>> {
Some(Box::new(KnowledgePack::new(runtime)))
}
}
inventory::submit! { khive_runtime::PackRegistration(&KnowledgePackFactory) }
async fn run_knowledge_embedder_warm(
runtime: KhiveRuntime,
token: Option<NamespaceToken>,
job: AdmittedJob,
) -> JobOutcome<Result<Vec<f32>, RuntimeError>> {
let phase_entry = std::time::Instant::now();
let entry_cpu = khive_runtime::process_resource_usage();
let mut start_attempted = false;
let start_finished = if let Some(token) = &token {
match job
.run_body(async {
start_attempted = true;
khive_runtime::emit_phase_event(
&runtime,
token,
"knowledge.embedder_warm",
EventKind::PhaseStarted,
PhaseStartedPayload {
work_class: "warm".into(),
phase: "knowledge_embedder_warm".into(),
corpus_size: None,
},
)
.await;
})
.await
{
BodyOutcome::Finished(()) => true,
BodyOutcome::Cancelled => false,
}
} else {
true
};
if !start_finished && !start_attempted {
return JobOutcome::Cancelled;
}
let (phase_start, cpu_start, result) = if start_finished {
let phase_start = std::time::Instant::now();
let cpu_start = khive_runtime::process_resource_usage();
let result = job
.run_body(runtime.embed("__khive_knowledge_warm__"))
.await;
(phase_start, cpu_start, result)
} else {
tracing::warn!(
phase = "knowledge_embedder_warm",
start_attempted,
start_finished,
start_append_persistence = "unknown",
"warm start append was interrupted before returning",
);
(phase_entry, entry_cpu, BodyOutcome::Cancelled)
};
let cancelled = match &result {
BodyOutcome::Cancelled => true,
BodyOutcome::Finished(Err(error)) => khive_runtime::is_benign_shutdown_cancellation(error),
BodyOutcome::Finished(Ok(_)) => false,
};
if let BodyOutcome::Finished(Err(error)) = &result {
tracing::warn!(error = %error, "knowledge_embedder_warm: warm operation returned an error");
}
let finalization = if let Some(token) = &token {
let wall_us = phase_start.elapsed().as_micros() as i64;
let cpu_us =
khive_runtime::cpu_delta_us(cpu_start, khive_runtime::process_resource_usage());
job.finalize(async {
if cancelled {
khive_runtime::emit_phase_event(
&runtime,
token,
"knowledge.embedder_warm",
EventKind::PhaseCancelled,
PhaseCancelledPayload {
work_class: "warm".into(),
phase: "knowledge_embedder_warm".into(),
wall_us,
cpu_us,
},
)
.await;
} else {
khive_runtime::emit_phase_event(
&runtime,
token,
"knowledge.embedder_warm",
EventKind::PhaseCompleted,
PhaseCompletedPayload {
work_class: "warm".into(),
phase: "knowledge_embedder_warm".into(),
wall_us,
cpu_us,
},
)
.await;
}
})
.await
} else {
job.no_telemetry()
};
match finalization {
FinalizationOutcome::AttemptTimedOut
| FinalizationOutcome::NoAttemptBeforeDeadline
| FinalizationOutcome::DeadlineOverflow => {
tracing::warn!(
?finalization,
"knowledge_embedder_warm: terminal append attempt did not finish"
);
return JobOutcome::FinalizationTimedOut;
}
FinalizationOutcome::AlreadyFinalized => {
tracing::warn!("knowledge_embedder_warm: terminal finalization was already claimed");
}
FinalizationOutcome::AttemptReturned | FinalizationOutcome::NoTelemetryToken => {}
}
match result {
BodyOutcome::Finished(result) => JobOutcome::Completed(result),
BodyOutcome::Cancelled => JobOutcome::Cancelled,
}
}
#[async_trait]
impl PackRuntime for KnowledgePack {
khive_runtime::pack_runtime_metadata!();
async fn warm(&self) {
if !self.runtime.is_read_only() {
crate::knowledge::vamana::warm_known_snapshots(&self.runtime, &self.ann).await;
crate::knowledge::vamana::start_rotation_watcher(&self.runtime, &self.ann);
}
if !self.runtime.default_embedder_name().is_empty() {
let runtime = self.runtime.background_view();
let domain = runtime.background_domain();
let tracking = TrackedJob::register("knowledge_embedder_warm");
let token = match runtime.authorize(khive_runtime::Namespace::local()) {
Ok(token) => Some(token),
Err(error) => {
tracing::warn!(error = %error, "knowledge_embedder_warm: warmup proceeds without telemetry");
None
}
};
tokio::spawn(async move {
let admitted = match domain.admit(WorkClass::Warm, &tracking).await {
Ok(job) => job,
Err(error) => {
tracing::warn!(error = %error, "knowledge_embedder_warm: admission refused");
return;
}
};
match domain.spawn_supervised(admitted, move |job| {
run_knowledge_embedder_warm(runtime, token, job)
}) {
Ok(task) => task.detach(),
Err(error) => {
tracing::warn!(error = %error, "knowledge_embedder_warm: supervisor submission refused");
}
}
});
}
}
async fn dispatch(
&self,
verb: &str,
params: Value,
registry: &VerbRegistry,
token: &NamespaceToken,
) -> Result<Value, RuntimeError> {
match verb {
"knowledge.upsert_atoms" => {
KnowledgeHandlers::upsert_atoms(&self.runtime, token, params).await
}
"knowledge.upsert_domains" => {
KnowledgeHandlers::upsert_domains(&self.runtime, token, params).await
}
"knowledge.get" => KnowledgeHandlers::get(&self.runtime, token, params).await,
"knowledge.list" => KnowledgeHandlers::list(&self.runtime, token, params).await,
"knowledge.export" => KnowledgeHandlers::export(&self.runtime, token, params).await,
"knowledge.lint" => self.handle_lint(token, params, registry).await,
"knowledge.delete_atoms" => {
KnowledgeHandlers::delete_atoms(&self.runtime, token, params).await
}
"knowledge.stats" => KnowledgeHandlers::stats(&self.runtime, token, params).await,
"knowledge.eval_retrieval" => {
KnowledgeHandlers::eval_retrieval(&self.runtime, token, params, &self.ann).await
}
"knowledge.index" => {
KnowledgeHandlers::index(&self.runtime, token, params, &self.ann, None).await
}
"knowledge.fold" => KnowledgeHandlers::fold(&self.runtime, token, params).await,
"knowledge.search" => {
KnowledgeHandlers::search(&self.runtime, token, params, &self.ann).await
}
"knowledge.suggest" => {
KnowledgeHandlers::suggest(&self.runtime, token, params, &self.ann).await
}
"knowledge.compose" => {
let effective_token = match params.get("namespace") {
None => token.clone(),
Some(Value::String(ns_str)) => {
let ns = Namespace::parse(ns_str).map_err(|e| {
RuntimeError::InvalidInput(format!("invalid namespace {ns_str:?}: {e}"))
})?;
if &ns != token.namespace() {
return Err(RuntimeError::InvalidInput(
"knowledge.compose namespace does not match authorized token namespace"
.to_string(),
));
}
token.with_namespace(ns)
}
Some(_) => {
return Err(RuntimeError::InvalidInput(
"invalid namespace: expected a string".to_string(),
));
}
};
let type_weights = self
.resolve_compose_type_weights(registry, &effective_token)
.await;
KnowledgeHandlers::compose(
&self.runtime,
&effective_token,
params,
&self.ann,
type_weights,
)
.await
}
"knowledge.edit" => {
KnowledgeHandlers::edit(&self.runtime, token, params, &self.ann).await
}
"knowledge.import" => {
KnowledgeHandlers::import(&self.runtime, token, params, &self.ann).await
}
"knowledge.challenge" => {
KnowledgeHandlers::challenge(&self.runtime, token, params).await
}
"knowledge.adjudicate" => {
KnowledgeHandlers::adjudicate(&self.runtime, token, params).await
}
"knowledge.learn" => self.handle_learn(token, params).await,
"knowledge.cite" => self.handle_cite(token, params).await,
"knowledge.topic" => self.handle_topic(token, params).await,
"knowledge.feedback" => self.handle_feedback(token, params, registry).await,
_ => Err(RuntimeError::InvalidInput(format!(
"knowledge pack does not handle verb {verb:?}"
))),
}
}
}
#[async_trait]
impl PackByIdResolver for KnowledgePack {
async fn resolve_by_id(&self, id: Uuid) -> Result<Option<Resolved>, RuntimeError> {
let sql = self.runtime.sql();
let id_str = id.to_string();
let mut reader = sql
.reader()
.await
.map_err(|e| RuntimeError::Internal(format!("knowledge resolve_by_id reader: {e}")))?;
let domain_row = reader
.query_row(SqlStatement {
sql: khive_runtime::sql!("knowledge_resolver_live_domain").into(),
params: vec![SqlValue::Text(id_str.clone())],
label: Some("knowledge.resolve_by_id.domain".into()),
})
.await
.map_err(|e| {
RuntimeError::Internal(format!("knowledge resolve_by_id domain query: {e}"))
})?;
if let Some(row) = domain_row {
if let Some(domain) = domain_from_row(&row) {
return Ok(Some(Resolved::PackRecord {
pack: "knowledge".into(),
kind: "domain".into(),
data: domain_to_json(&domain),
}));
}
}
let atom_row = reader
.query_row(SqlStatement {
sql: khive_runtime::sql!("knowledge_resolver_live_atom").into(),
params: vec![SqlValue::Text(id_str)],
label: Some("knowledge.resolve_by_id.atom".into()),
})
.await
.map_err(|e| {
RuntimeError::Internal(format!("knowledge resolve_by_id atom query: {e}"))
})?;
if let Some(row) = atom_row {
if let Some(atom) = atom_from_row(&row) {
return Ok(Some(Resolved::PackRecord {
pack: "knowledge".into(),
kind: "atom".into(),
data: atom_to_json(&atom),
}));
}
}
Ok(None)
}
async fn resolve_by_id_including_deleted(
&self,
id: Uuid,
) -> Result<Option<Resolved>, RuntimeError> {
let sql = self.runtime.sql();
let id_str = id.to_string();
let mut reader = sql.reader().await.map_err(|e| {
RuntimeError::Internal(format!(
"knowledge resolve_by_id_including_deleted reader: {e}"
))
})?;
let domain_row = reader
.query_row(SqlStatement {
sql: khive_runtime::sql!("knowledge_resolver_domain_including_deleted").into(),
params: vec![SqlValue::Text(id_str.clone())],
label: Some("knowledge.resolve_by_id_incl_deleted.domain".into()),
})
.await
.map_err(|e| {
RuntimeError::Internal(format!(
"knowledge resolve_by_id_including_deleted domain query: {e}"
))
})?;
if let Some(row) = domain_row {
if let Some(domain) = domain_from_row(&row) {
return Ok(Some(Resolved::PackRecord {
pack: "knowledge".into(),
kind: "domain".into(),
data: domain_to_json(&domain),
}));
}
}
let atom_row = reader
.query_row(SqlStatement {
sql: khive_runtime::sql!("knowledge_resolver_atom_including_deleted").into(),
params: vec![SqlValue::Text(id_str)],
label: Some("knowledge.resolve_by_id_incl_deleted.atom".into()),
})
.await
.map_err(|e| {
RuntimeError::Internal(format!(
"knowledge resolve_by_id_including_deleted atom query: {e}"
))
})?;
if let Some(row) = atom_row {
if let Some(atom) = atom_from_row(&row) {
return Ok(Some(Resolved::PackRecord {
pack: "knowledge".into(),
kind: "atom".into(),
data: atom_to_json(&atom),
}));
}
}
Ok(None)
}
fn private_record_verbs(&self) -> &'static [&'static str] {
&[
"knowledge.upsert_atoms",
"knowledge.upsert_domains",
"knowledge.edit",
]
}
async fn delete_by_id(&self, id: Uuid, hard: bool) -> Result<serde_json::Value, RuntimeError> {
let sql = self.runtime.sql();
let id_str = id.to_string();
let kind = match self.resolve_by_id_including_deleted(id).await? {
Some(Resolved::PackRecord { kind, .. }) => kind,
_ => {
return Err(RuntimeError::NotFound(format!(
"knowledge record not found: {id}"
)));
}
};
let now_us = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_micros() as i64;
let mut writer = sql
.writer()
.await
.map_err(|e| RuntimeError::Internal(format!("knowledge delete_by_id writer: {e}")))?;
match kind.as_str() {
"domain" => {
if hard {
writer
.execute_batch(vec![
SqlStatement {
sql: khive_runtime::sql!("knowledge_atom_sections_delete").into(),
params: vec![SqlValue::Text(id_str.clone())],
label: Some(
"knowledge.delete_by_id.domain_mirror_sections.hard".into(),
),
},
SqlStatement {
sql: khive_runtime::sql!("knowledge_atom_hard_delete").into(),
params: vec![SqlValue::Text(id_str.clone())],
label: Some("knowledge.delete_by_id.domain_mirror.hard".into()),
},
SqlStatement {
sql: khive_runtime::sql!("knowledge_domain_hard_delete").into(),
params: vec![SqlValue::Text(id_str.clone())],
label: Some("knowledge.delete_by_id.domain.hard".into()),
},
])
.await
.map_err(|e| {
RuntimeError::Internal(format!(
"knowledge delete_by_id domain hard: {e}"
))
})?;
} else {
writer
.execute(SqlStatement {
sql: khive_runtime::sql!("knowledge_domain_soft_delete").into(),
params: vec![SqlValue::Integer(now_us), SqlValue::Text(id_str.clone())],
label: Some("knowledge.delete_by_id.domain.soft".into()),
})
.await
.map_err(|e| {
RuntimeError::Internal(format!(
"knowledge delete_by_id domain soft: {e}"
))
})?;
writer
.execute(SqlStatement {
sql: khive_runtime::sql!("knowledge_atom_soft_delete").into(),
params: vec![SqlValue::Integer(now_us), SqlValue::Text(id_str.clone())],
label: Some("knowledge.delete_by_id.domain_mirror.soft".into()),
})
.await
.map_err(|e| {
RuntimeError::Internal(format!(
"knowledge delete_by_id domain mirror soft: {e}"
))
})?;
}
}
"atom" => {
if hard {
writer
.execute_batch(vec![
SqlStatement {
sql: khive_runtime::sql!("knowledge_atom_sections_delete").into(),
params: vec![SqlValue::Text(id_str.clone())],
label: Some("knowledge.delete_by_id.atom_sections.hard".into()),
},
SqlStatement {
sql: khive_runtime::sql!("knowledge_atom_hard_delete").into(),
params: vec![SqlValue::Text(id_str.clone())],
label: Some("knowledge.delete_by_id.atom.hard".into()),
},
])
.await
.map_err(|e| {
RuntimeError::Internal(format!("knowledge delete_by_id atom hard: {e}"))
})?;
} else {
writer
.execute(SqlStatement {
sql: khive_runtime::sql!("knowledge_atom_soft_delete").into(),
params: vec![SqlValue::Integer(now_us), SqlValue::Text(id_str.clone())],
label: Some("knowledge.delete_by_id.atom.soft".into()),
})
.await
.map_err(|e| {
RuntimeError::Internal(format!("knowledge delete_by_id atom soft: {e}"))
})?;
}
}
other => {
return Err(RuntimeError::Internal(format!(
"knowledge delete_by_id: unexpected kind {other:?}"
)));
}
}
Ok(serde_json::json!({
"deleted": true,
"id": id_str,
"kind": kind,
"hard": hard,
}))
}
}
#[cfg(test)]
#[path = "pack_tests.rs"]
mod tests;
#[cfg(test)]
#[path = "warm_admission_tests.rs"]
mod warm_admission_tests;