#[cfg(test)]
use super::{atom_from_row, Atom};
use super::{
domain_from_row, embed_cosine_scores, estimate_compose_item_tokens, hex_prefix_to_uuid_pattern,
is_timeout, row_i64, row_str, sql_err, Domain, EntityFilter, HashMap, HashSet, KhiveRuntime,
NamespaceToken, PageRequest, QueryEmbeddingCache, RuntimeError, SqlStatement, SqlValue, Uuid,
SERVABLE_SECTION,
};
pub(super) struct ScoredTextItem {
pub(super) id: String,
pub(super) slug: String,
pub(super) name: String,
pub(super) text: String,
pub(super) score: f32,
}
fn compare_scored_text_items(a: &ScoredTextItem, b: &ScoredTextItem) -> std::cmp::Ordering {
khive_score::cmp_desc_then_id_f32(a.score, &a.slug, b.score, &b.slug)
}
pub(super) async fn load_domain_by_id_or_slug(
runtime: &KhiveRuntime,
ns: &str,
id_or_slug: &str,
) -> Result<Domain, RuntimeError> {
let sql = runtime.sql();
let mut reader = sql
.reader()
.await
.map_err(|e| sql_err("compose domain reader", e))?;
let id = id_or_slug.trim().to_string();
let row = if id.parse::<Uuid>().is_ok() {
reader
.query_row(SqlStatement {
sql: khive_runtime::sql!("knowledge_compose_domain_id").into(),
params: vec![SqlValue::Text(id.clone()), SqlValue::Text(ns.to_owned())],
label: None,
})
.await
.map_err(|e| sql_err("compose domain by id", e))?
} else {
let by_slug = reader
.query_row(SqlStatement {
sql: khive_runtime::sql!("knowledge_compose_domain_slug").into(),
params: vec![SqlValue::Text(id.clone()), SqlValue::Text(ns.to_owned())],
label: None,
})
.await
.map_err(|e| sql_err("compose domain by slug", e))?;
if by_slug.is_some() {
by_slug
} else {
let is_hex = id.len() >= 8
&& id.len() <= 36
&& id.chars().all(|c| c.is_ascii_hexdigit() || c == '-');
if is_hex {
let pattern = format!("{}%", hex_prefix_to_uuid_pattern(&id));
let rows = reader
.query_all(SqlStatement {
sql: khive_runtime::sql!("knowledge_compose_domain_prefix").into(),
params: vec![SqlValue::Text(pattern), SqlValue::Text(ns.to_owned())],
label: None,
})
.await
.map_err(|e| sql_err("compose domain by prefix", e))?;
if rows.len() > 1 {
return Err(RuntimeError::InvalidInput(format!(
"ambiguous domain prefix {id:?} matches multiple domains"
)));
}
rows.into_iter().next()
} else {
None
}
}
};
row.and_then(|r| domain_from_row(&r))
.ok_or_else(|| RuntimeError::NotFound(format!("domain not found: {id:?}")))
}
#[cfg(test)]
pub(super) async fn load_atom_by_id_or_slug(
runtime: &KhiveRuntime,
ns: &str,
id_or_slug: &str,
) -> Result<Atom, RuntimeError> {
let sql = runtime.sql();
let mut reader = sql
.reader()
.await
.map_err(|e| sql_err("compose atom reader", e))?;
let id = id_or_slug.trim().to_string();
let row = if id.parse::<Uuid>().is_ok() {
reader
.query_row(SqlStatement {
sql: khive_runtime::sql!("knowledge_compose_atom_id").into(),
params: vec![SqlValue::Text(id.clone()), SqlValue::Text(ns.to_owned())],
label: None,
})
.await
.map_err(|e| sql_err("compose atom by id", e))?
} else {
let by_slug = reader
.query_row(SqlStatement {
sql: khive_runtime::sql!("knowledge_compose_atom_slug").into(),
params: vec![SqlValue::Text(id.clone()), SqlValue::Text(ns.to_owned())],
label: None,
})
.await
.map_err(|e| sql_err("compose atom by slug", e))?;
if by_slug.is_some() {
by_slug
} else {
let is_hex = id.len() >= 8
&& id.len() <= 36
&& id.chars().all(|c| c.is_ascii_hexdigit() || c == '-');
if is_hex {
let pattern = format!("{}%", hex_prefix_to_uuid_pattern(&id));
let rows = reader
.query_all(SqlStatement {
sql: khive_runtime::sql!("knowledge_compose_atom_prefix").into(),
params: vec![SqlValue::Text(pattern), SqlValue::Text(ns.to_owned())],
label: None,
})
.await
.map_err(|e| sql_err("compose atom by prefix", e))?;
if rows.len() > 1 {
return Err(RuntimeError::InvalidInput(format!(
"ambiguous atom prefix {id:?} matches multiple atoms"
)));
}
rows.into_iter().next()
} else {
None
}
}
};
row.and_then(|r| atom_from_row(&r))
.ok_or_else(|| RuntimeError::NotFound(format!("atom not found: {id:?}")))
}
pub(super) fn parse_domain_members(domain: &Domain) -> Result<Vec<String>, RuntimeError> {
if domain.members.is_empty() || domain.members == "[]" {
return Ok(Vec::new());
}
serde_json::from_str::<Vec<String>>(&domain.members).map_err(|e| {
RuntimeError::Internal(format!(
"domain {:?} has invalid members JSON: {e}",
domain.slug
))
})
}
#[derive(Debug, Default)]
pub(super) struct DomainMemberSizing {
pub(super) tokens: usize,
pub(super) live_members: usize,
}
pub(super) async fn load_domain_member_token_sizes(
runtime: &KhiveRuntime,
ns: &str,
domain_ids: &[String],
) -> Result<(HashMap<String, DomainMemberSizing>, bool), RuntimeError> {
let mut sizes: HashMap<String, DomainMemberSizing> = HashMap::new();
if domain_ids.is_empty() {
return Ok((sizes, false));
}
let placeholders = domain_ids
.iter()
.enumerate()
.map(|(i, _)| format!("?{}", i + 2))
.collect::<Vec<_>>()
.join(",");
let mut params = vec![SqlValue::Text(ns.to_owned())];
params.extend(domain_ids.iter().cloned().map(SqlValue::Text));
let sql = runtime.sql();
let mut reader = match sql.reader().await {
Ok(reader) => reader,
Err(e) if is_timeout(&e) => return Ok((sizes, true)),
Err(e) => return Err(sql_err("suggest member size reader", e)),
};
let rows = match reader
.query_all(SqlStatement {
sql: format!(
"SELECT DISTINCT d.id AS domain_id, a.id AS atom_id, a.name, a.content \
FROM knowledge_domains AS d \
LEFT JOIN json_each(d.members) AS member ON 1 = 1 \
LEFT JOIN knowledge_atoms AS a \
ON a.namespace = d.namespace \
AND a.slug = member.value \
AND a.deleted_at IS NULL \
WHERE d.namespace = ?1 \
AND d.id IN ({placeholders}) \
AND d.deleted_at IS NULL"
),
params,
label: None,
})
.await
{
Ok(rows) => rows,
Err(e) if is_timeout(&e) => return Ok((sizes, true)),
Err(e) => return Err(sql_err("suggest member size query", e)),
};
for row in rows {
let Some(domain_id) = row_str(&row, "domain_id") else {
continue;
};
let sizing = sizes.entry(domain_id).or_default();
let Some(content) = row_str(&row, "content") else {
continue;
};
let name = row_str(&row, "name").unwrap_or_default();
sizing.tokens = sizing
.tokens
.saturating_add(estimate_compose_item_tokens(&name, &content));
sizing.live_members = sizing.live_members.saturating_add(1);
}
Ok((sizes, false))
}
pub(super) async fn load_atom_body_line_counts(
runtime: &KhiveRuntime,
ns: &str,
atom_ids: &[String],
) -> Result<Option<HashMap<String, usize>>, RuntimeError> {
let mut counts: HashMap<String, usize> = atom_ids.iter().map(|id| (id.clone(), 0)).collect();
if atom_ids.is_empty() {
return Ok(Some(counts));
}
let placeholders = atom_ids
.iter()
.enumerate()
.map(|(i, _)| format!("?{}", i + 2))
.collect::<Vec<_>>()
.join(",");
let mut params = vec![SqlValue::Text(ns.to_owned())];
params.extend(atom_ids.iter().cloned().map(SqlValue::Text));
let sql = runtime.sql();
let mut reader = match sql.reader().await {
Ok(reader) => reader,
Err(e) if is_timeout(&e) => return Ok(None),
Err(e) => return Err(sql_err("search body line count reader", e)),
};
let rows = match reader
.query_all(SqlStatement {
sql: format!(
"SELECT atom_id, \
SUM(CASE WHEN content = '' THEN 0 \
ELSE length(content) \
- length(replace(content, char(10), '')) \
+ (CASE WHEN substr(content, -1) = char(10) \
THEN 0 ELSE 1 END) \
END) AS body_lines \
FROM knowledge_sections \
WHERE namespace = ?1 AND atom_id IN ({placeholders}) AND {SERVABLE_SECTION} \
GROUP BY atom_id"
),
params,
label: None,
})
.await
{
Ok(rows) => rows,
Err(e) if is_timeout(&e) => return Ok(None),
Err(e) => return Err(sql_err("search body line count query", e)),
};
for row in rows {
let Some(atom_id) = row_str(&row, "atom_id") else {
continue;
};
let Some(body_lines) = row_i64(&row, "body_lines") else {
continue;
};
if let Ok(body_lines) = usize::try_from(body_lines) {
counts.insert(atom_id, body_lines);
}
}
Ok(Some(counts))
}
pub(super) async fn rerank_text_items(
runtime: &KhiveRuntime,
query: &str,
query_embedding: &mut QueryEmbeddingCache,
items: &mut [ScoredTextItem],
) -> Result<(), RuntimeError> {
if items.is_empty() {
return Ok(());
}
let texts: Vec<String> = items.iter().map(|item| item.text.clone()).collect();
if let Some(cosines) = embed_cosine_scores(runtime, query, query_embedding, &texts).await? {
for (item, cos) in items.iter_mut().zip(cosines.iter()) {
item.score = cos.max(0.0);
}
items.sort_by(compare_scored_text_items);
}
Ok(())
}
const KG_BLEND_ENTITY_KINDS: [&str; 2] = ["concept", "document"];
pub(super) const KG_BLEND_CAP: usize = 5;
pub(super) struct KgEntityHit {
pub(super) id: String,
pub(super) kind: String,
pub(super) name: String,
pub(super) description: String,
pub(super) score: f32,
}
fn compare_kg_entity_hits(a: &KgEntityHit, b: &KgEntityHit) -> std::cmp::Ordering {
khive_score::cmp_desc_then_id_f32(a.score, &a.id, b.score, &b.id)
}
pub(super) async fn search_kg_entities(
runtime: &KhiveRuntime,
token: &NamespaceToken,
ns: &str,
query: &str,
query_embedding: &mut QueryEmbeddingCache,
cap: usize,
min_score: f32,
) -> Result<Vec<KgEntityHit>, RuntimeError> {
let Some(query_vector) = query_embedding.role_specific.as_deref() else {
return Ok(Vec::new());
};
let candidate_k = ((cap * 4) as u32).max(20);
let mut candidate_ids: Vec<Uuid> = Vec::new();
let mut seen: HashSet<Uuid> = HashSet::new();
let hits_by_kind = runtime
.hybrid_search_each_kind(
token,
query,
Some(query_vector.to_vec()),
candidate_k,
&KG_BLEND_ENTITY_KINDS,
)
.await?;
for hit in hits_by_kind.into_iter().flatten() {
if seen.insert(hit.entity_id) {
candidate_ids.push(hit.entity_id);
}
}
if candidate_ids.is_empty() {
return Ok(Vec::new());
}
let visible_ns: Vec<String> = token
.visible_namespace_strs()
.iter()
.map(|s| s.to_string())
.collect();
let entities_page = runtime
.entities(token)?
.query_entities(
ns,
EntityFilter {
ids: candidate_ids.clone(),
kinds: KG_BLEND_ENTITY_KINDS
.iter()
.map(|k| k.to_string())
.collect(),
namespaces: visible_ns,
..EntityFilter::default()
},
PageRequest {
offset: 0,
limit: candidate_ids.len() as u32,
},
)
.await?;
if entities_page.items.is_empty() {
return Ok(Vec::new());
}
let texts: Vec<String> = entities_page
.items
.iter()
.map(|e| format!("{} {}", e.name, e.description.as_deref().unwrap_or("")))
.collect();
let cosines = match embed_cosine_scores(runtime, query, query_embedding, &texts).await? {
Some(c) => c,
None => return Ok(Vec::new()),
};
let mut hits: Vec<KgEntityHit> = entities_page
.items
.iter()
.zip(cosines.iter())
.map(|(e, &score)| KgEntityHit {
id: e.id.to_string(),
kind: e.kind.clone(),
name: e.name.clone(),
description: e.description.clone().unwrap_or_default(),
score: score.max(0.0),
})
.collect();
hits.sort_by(compare_kg_entity_hits);
hits.retain(|h| h.score >= min_score);
hits.truncate(cap);
Ok(hits)
}
#[cfg(test)]
#[path = "compose_ordering_tests.rs"]
mod ordering_tests;