use super::{
atom_embed_text_fields, filter_hits_by_status, filter_hits_by_type, has_domain_mirror_tag,
is_read_timeout, is_timeout, json, row_bool, row_str, search_ann_with_warm_wait,
status_sql_clause, vamana, AnnAvailability, AnnSearchState, HashMap, HashSet, KhiveRuntime,
LexicalBound, LexicalTimeout, NamespaceToken, QueryEmbeddingCache, RoleSpecificEmbedding,
RuntimeError, ScoreProvenance, ScoredHit, SearchCoreOutcome, SearchCtx, SqlStatement, SqlValue,
Value, HYDRATION_ID_CHUNK,
};
pub(super) fn hydrate_atoms_statement(ns: &str, ids: &[String]) -> SqlStatement {
let placeholders = ids
.iter()
.enumerate()
.map(|(i, _)| format!("?{}", i + 2))
.collect::<Vec<_>>()
.join(",");
let mut params = vec![SqlValue::Text(ns.to_owned())];
params.extend(ids.iter().cloned().map(SqlValue::Text));
SqlStatement {
sql: format!(
"SELECT id, slug, name, content, tags, finalized, status FROM knowledge_atoms \
WHERE id IN ({placeholders}) AND +namespace = ?1 AND deleted_at IS NULL"
),
params,
label: None,
}
}
pub(super) fn hydrate_domains_statement(ns: &str, ids: &[String]) -> SqlStatement {
let placeholders = ids
.iter()
.enumerate()
.map(|(i, _)| format!("?{}", i + 2))
.collect::<Vec<_>>()
.join(",");
let mut params = vec![SqlValue::Text(ns.to_owned())];
params.extend(ids.iter().cloned().map(SqlValue::Text));
SqlStatement {
sql: format!(
"SELECT id, slug, name, description, tags, status FROM knowledge_domains \
WHERE id IN ({placeholders}) AND +namespace = ?1 AND deleted_at IS NULL"
),
params,
label: None,
}
}
pub(super) fn phase_b_hydration_statement(
ns: &str,
rowids: &[i64],
statuses: &[String],
exclude_statuses: &[&str],
type_clause: &str,
) -> SqlStatement {
let rowid_placeholders = rowids
.iter()
.enumerate()
.map(|(i, _)| format!("?{}", i + 2))
.collect::<Vec<_>>()
.join(",");
let (status_clause, status_params) =
status_sql_clause(statuses, exclude_statuses, 2 + rowids.len());
let mut params = vec![SqlValue::Text(ns.to_owned())];
params.extend(rowids.iter().map(|rowid| SqlValue::Integer(*rowid)));
params.extend(status_params);
SqlStatement {
sql: format!(
"SELECT a.*, a.rowid AS rowid FROM knowledge_atoms AS a \
WHERE a.rowid IN ({rowid_placeholders}) \
AND +a.namespace = ?1 \
AND a.deleted_at IS NULL{status_clause}{type_clause}"
),
params,
label: None,
}
}
pub(super) async fn hydrate_empty_hits(
runtime: &KhiveRuntime,
ns: &str,
hits: &mut Vec<ScoredHit>,
) -> usize {
let ids: Vec<String> = hits
.iter()
.filter(|hit| hit.slug.is_empty())
.map(|hit| hit.id.clone())
.collect();
if ids.is_empty() {
return 0;
}
let sql = runtime.sql();
let mut reader = match sql.reader().await {
Ok(r) => r,
Err(error) => {
tracing::warn!(
target: "khive_pack_knowledge::knowledge::search",
namespace = ns,
requested = ids.len(),
error = %error,
"knowledge candidate hydration could not acquire a reader"
);
hits.retain(|hit| !hit.slug.is_empty());
return ids.len();
}
};
let mut atom_rows = Vec::new();
for chunk in ids.chunks(HYDRATION_ID_CHUNK) {
match reader.query_all(hydrate_atoms_statement(ns, chunk)).await {
Ok(rows) => atom_rows.extend(rows),
Err(error) => {
tracing::warn!(
target: "khive_pack_knowledge::knowledge::search",
namespace = ns,
requested = chunk.len(),
error = %error,
"knowledge atom candidate hydration chunk degraded"
);
}
}
}
let mut atom_rows_by_id: HashMap<String, khive_storage::types::SqlRow> = HashMap::new();
for row in atom_rows {
if let Some(id) = row_str(&row, "id") {
atom_rows_by_id.insert(id, row);
}
}
for hit in hits.iter_mut().filter(|hit| hit.slug.is_empty()) {
if let Some(row) = atom_rows_by_id.get(&hit.id) {
hit.slug = row_str(row, "slug").unwrap_or_default();
hit.name = row_str(row, "name").unwrap_or_default();
hit.content = row_str(row, "content");
hit.tags = row_str(row, "tags");
hit.atom_embed_text = Some(atom_embed_text_fields(
&hit.name,
hit.content.as_deref().unwrap_or(""),
hit.tags.as_deref().unwrap_or("[]"),
));
hit.finalized = row_bool(row, "finalized");
hit.status = row_str(row, "status");
hit.is_domain = has_domain_mirror_tag(hit.tags.as_deref().unwrap_or(""));
}
}
let missing_ids: Vec<String> = hits
.iter()
.filter(|hit| hit.slug.is_empty())
.map(|hit| hit.id.clone())
.collect();
if missing_ids.is_empty() {
return 0;
}
let mut domain_rows = Vec::new();
for chunk in missing_ids.chunks(HYDRATION_ID_CHUNK) {
match reader.query_all(hydrate_domains_statement(ns, chunk)).await {
Ok(rows) => domain_rows.extend(rows),
Err(error) => {
tracing::warn!(
target: "khive_pack_knowledge::knowledge::search",
namespace = ns,
requested = chunk.len(),
error = %error,
"knowledge domain candidate hydration chunk degraded"
);
}
}
}
let mut domain_rows_by_id: HashMap<String, khive_storage::types::SqlRow> = HashMap::new();
for row in domain_rows {
if let Some(id) = row_str(&row, "id") {
domain_rows_by_id.insert(id, row);
}
}
for hit in hits.iter_mut().filter(|hit| hit.slug.is_empty()) {
if let Some(row) = domain_rows_by_id.get(&hit.id) {
hit.slug = row_str(row, "slug").unwrap_or_default();
hit.name = row_str(row, "name").unwrap_or_default();
hit.content = row_str(row, "description");
hit.tags = row_str(row, "tags");
hit.atom_embed_text = None;
hit.finalized = false;
hit.is_domain = true;
hit.status = row_str(row, "status");
}
}
let failed = hits.iter().filter(|hit| hit.slug.is_empty()).count();
hits.retain(|hit| !hit.slug.is_empty());
if failed > 0 {
tracing::warn!(
target: "khive_pack_knowledge::knowledge::search",
namespace = ns,
requested = ids.len(),
failed,
"knowledge candidate hydration returned a degraded pool"
);
}
failed
}
pub(super) fn attach_hydration_degradation(out: &mut Value, hydration_failures: usize) {
if hydration_failures == 0 {
return;
}
if !out
.get("degraded")
.is_some_and(serde_json::Value::is_object)
{
out["degraded"] = json!({});
}
out["degraded"]["hydration_failures"] = json!(hydration_failures);
}
pub(super) fn attach_lexical_timeout_degradation(out: &mut Value, timeouts: &[LexicalTimeout]) {
if timeouts.is_empty() {
return;
}
if !out
.get("degraded")
.is_some_and(serde_json::Value::is_object)
{
out["degraded"] = json!({});
}
out["degraded"]["lexical_timeout"] = json!(true);
out["degraded"]["lexical_timeout_instrumented"] = json!(true);
if timeouts
.iter()
.any(|detail| detail.bound == LexicalBound::OrderingProbe)
{
out["degraded"]["lexical_ordering_probe_timeout"] = json!(true);
}
let details: Vec<_> = timeouts
.iter()
.filter(|detail| detail.phase.public())
.take(3)
.collect();
if !details.is_empty() {
out["degraded"]["lexical_timeout_details"] = json!(details);
}
}
pub(super) fn attach_body_lines_timeout_degradation(out: &mut Value) {
if !out
.get("degraded")
.is_some_and(serde_json::Value::is_object)
{
out["degraded"] = json!({});
}
out["degraded"]["body_lines_timeout"] = json!(true);
}
pub(super) fn attach_member_sizing_timeout_degradation(out: &mut Value, excluded: &[Value]) {
if excluded.is_empty() {
return;
}
if !out
.get("degraded")
.is_some_and(serde_json::Value::is_object)
{
out["degraded"] = json!({});
}
out["degraded"]["member_sizing_timeout"] = json!({
"excluded": excluded,
"note": "no measurement was produced (timeout or missing canonical domain), so these domains \
were withheld from `results` — their cost is unknown and a \
budgeted knowledge.fold selection cannot safely admit an unpriced \
item. Each entry keeps its id/name/rank/score for reference; \
measure size separately before folding one of these in.",
});
}
struct EligibleAnnSearchState {
hits: Vec<ScoredHit>,
availability: AnnAvailability,
hydration_failures: usize,
}
async fn search_eligible_ann_with_refill(
ctx: &SearchCtx<'_>,
token: &NamespaceToken,
ann: &vamana::SharedAnn,
key: &vamana::AnnKey,
query_embedding: &[f32],
target_eligible: usize,
initial_k: usize,
) -> Result<EligibleAnnSearchState, RuntimeError> {
let runtime = ctx.runtime;
let target_eligible = target_eligible.max(1);
let mut request_k = initial_k.max(target_eligible).max(1);
loop {
khive_storage::ensure_request_read_active("knowledge.search")?;
let AnnSearchState {
hits: raw_hits,
availability,
source_exhausted,
} = search_ann_with_warm_wait(runtime, token, ann, key, query_embedding, request_k).await;
let mut seen = HashSet::with_capacity(raw_hits.len());
let mut hits: Vec<ScoredHit> = raw_hits
.into_iter()
.filter(|(id, _)| seen.insert(*id))
.map(|(id, score)| ScoredHit {
id: id.to_string(),
slug: String::new(),
name: String::new(),
content: None,
tags: None,
atom_embed_text: None,
finalized: false,
is_domain: false,
status: None,
score,
provenance: ScoreProvenance::ann(),
})
.collect();
let hydration_failures = hydrate_empty_hits(runtime, ctx.ns, &mut hits).await;
khive_storage::ensure_request_read_active("knowledge.search")?;
filter_hits_by_status(&mut hits, ctx.statuses, ctx.exclude_statuses);
filter_hits_by_type(&mut hits, ctx.type_filter);
if hits.len() >= target_eligible || source_exhausted {
hits.truncate(target_eligible);
return Ok(EligibleAnnSearchState {
hits,
availability,
hydration_failures,
});
}
let next_k = request_k.saturating_mul(2);
if next_k == request_k {
hits.truncate(target_eligible);
return Ok(EligibleAnnSearchState {
hits,
availability,
hydration_failures,
});
}
request_k = next_k;
}
}
pub(super) struct CandidateStageOutcome {
pub(super) ann_hits: Vec<ScoredHit>,
pub(super) ann_availability: Option<AnnAvailability>,
pub(super) hydration_failures: usize,
pub(super) query_embedding: QueryEmbeddingCache,
}
pub(super) struct CandidateStageOptions {
pub(super) query_embedding: QueryEmbeddingCache,
pub(super) ann_target: usize,
pub(super) ann_initial_k: usize,
pub(super) operation: &'static str,
}
pub(super) async fn run_candidate_stages<F>(
ctx: &SearchCtx<'_>,
token: &NamespaceToken,
ann: &vamana::SharedAnn,
raw_query: &str,
options: CandidateStageOptions,
lexical: F,
) -> Result<(SearchCoreOutcome, CandidateStageOutcome), RuntimeError>
where
F: std::future::Future<Output = Result<SearchCoreOutcome, RuntimeError>>,
{
vamana::ensure_ann_background(ctx.runtime, token, ann);
let ann_stage = async {
let mut query_embedding = options.query_embedding;
let mut ann_hits = Vec::new();
let mut ann_availability = None;
let mut hydration_failures = 0;
if query_embedding.role_specific.is_not_attempted() {
query_embedding.role_specific = match khive_storage::await_request_read_phase(
options.operation,
ctx.runtime.embed_query(raw_query),
)
.await
{
Ok(Ok(vector)) => RoleSpecificEmbedding::Vector(vector),
Ok(Err(_)) => RoleSpecificEmbedding::Failed,
Err(error) if is_timeout(&error) => RoleSpecificEmbedding::Failed,
Err(error) => return Err(error.into()),
};
}
if let Some(vector) = query_embedding.role_specific.as_deref() {
let key = vamana::AnnKey::new(ctx.ns, ctx.runtime.default_embedder_name());
match khive_storage::await_request_read_phase(
options.operation,
search_eligible_ann_with_refill(
ctx,
token,
ann,
&key,
vector,
options.ann_target,
options.ann_initial_k,
),
)
.await
{
Ok(Ok(state)) => {
ann_hits = state.hits;
ann_availability = Some(state.availability);
hydration_failures = state.hydration_failures;
}
Ok(Err(error)) if is_read_timeout(&error) => {}
Ok(Err(error)) => return Err(error),
Err(error) if is_timeout(&error) => {}
Err(error) => return Err(error.into()),
}
}
Ok(CandidateStageOutcome {
ann_hits,
ann_availability,
hydration_failures,
query_embedding,
})
};
let (ann_result, lexical_result) = tokio::join!(ann_stage, lexical);
let ann_result = ann_result?;
Ok((lexical_result?, ann_result))
}