use std::collections::BTreeMap;
use std::path::{Component, Path, PathBuf};
use khive_fs::tree_discovery::{walk_tree_reported, ReportedWalkLimits, WalkFailureKind, WalkFile};
use serde_json::{json, Value};
use uuid::Uuid;
use khive_runtime::{KhiveRuntime, NamespaceToken, RuntimeError};
use khive_storage::types::{SqlStatement, SqlValue};
use khive_storage::{StorageCapability, StorageError};
use super::schema::{
AdjudicateParams, ChallengeParams, EditParams, ImportParams, Section, SectionType,
};
use super::sections_index::embed_sections;
use super::util::resolve_atom_id;
use super::util::{
content_hash, deser, has_domain_mirror_tag, new_id, now_us, row_str, sql_err,
validate_atom_content, validate_section_content,
};
use super::vamana;
use super::KnowledgeHandlers;
fn count_tokens(text: &str) -> i64 {
text.split_whitespace().count() as i64
}
fn parse_section_type(s: &str) -> Result<SectionType, RuntimeError> {
SectionType::from_str_loose(s).ok_or_else(|| {
RuntimeError::InvalidInput(format!(
"unknown section_type {s:?}; valid values: {}",
SectionType::NAMES.join(", ")
))
})
}
pub(super) fn section_from_row(row: &khive_storage::types::SqlRow) -> Option<Section> {
let id: Uuid = row_str(row, "id")?.parse().ok()?;
let st_str = row_str(row, "section_type")?;
let retired = SectionType::is_retired_name(&st_str);
let section_type = if retired {
st_str
} else {
SectionType::from_str_loose(&st_str)?.as_str().to_owned()
};
Some(Section {
id,
atom_id: row_str(row, "atom_id")?,
namespace: row_str(row, "namespace")?,
section_type,
retired,
heading: row_str(row, "heading").unwrap_or_default(),
content: row_str(row, "content").unwrap_or_default(),
content_hash: row_str(row, "content_hash").unwrap_or_default(),
status: row_str(row, "status").unwrap_or_else(|| "draft".into()),
tokens: super::util::row_i64(row, "tokens").unwrap_or(0),
sort_order: super::util::row_i64(row, "sort_order").unwrap_or(0),
created_at: super::util::row_i64(row, "created_at").unwrap_or(0),
updated_at: super::util::row_i64(row, "updated_at").unwrap_or(0),
})
}
pub(super) fn section_to_json(s: &Section) -> Value {
let mut out = json!({
"id": s.id.to_string(),
"atom_id": s.atom_id,
"namespace": s.namespace,
"section_type": s.section_type,
"heading": s.heading,
"content": s.content,
"content_hash": s.content_hash,
"status": s.status,
"tokens": s.tokens,
"sort_order": s.sort_order,
"created_at": s.created_at,
"updated_at": s.updated_at,
});
if s.retired {
out["retired"] = json!(true);
}
out
}
const MAX_IMPORT_DEPTH: usize = 32;
const MAX_IMPORT_ENTRIES: usize = 100_000;
const MAX_IMPORT_FILES: usize = 10_000;
const MAX_IMPORT_FILE_BYTES: u64 = 10 * 1024 * 1024;
const MAX_IMPORT_TOTAL_BYTES: u64 = 256 * 1024 * 1024;
#[derive(Clone, Copy, Debug)]
struct ImportTraversalLimits {
max_depth: usize,
max_entries: usize,
max_files: usize,
}
const IMPORT_TRAVERSAL_LIMITS: ImportTraversalLimits = ImportTraversalLimits {
max_depth: MAX_IMPORT_DEPTH,
max_entries: MAX_IMPORT_ENTRIES,
max_files: MAX_IMPORT_FILES,
};
#[derive(Debug)]
struct ImportDiscovery {
files: Vec<PathBuf>,
walk_files: Vec<WalkFile>,
entries_visited: usize,
files_skipped: usize,
}
fn traversal_error(
path: &Path,
entries_visited: usize,
error: impl std::fmt::Display,
) -> RuntimeError {
RuntimeError::InvalidInput(format!(
"knowledge.import traversal failed at {:?} after {entries_visited} entries: {error}",
path
))
}
fn traversal_limit_error(
limit_kind: &str,
path: &Path,
depth: usize,
entries_visited: usize,
files_discovered: usize,
limits: ImportTraversalLimits,
) -> RuntimeError {
RuntimeError::InvalidInput(format!(
"knowledge.import traversal {limit_kind} limit exceeded at {:?}: \
depth={depth} max_depth={} entries_visited={entries_visited} max_entries={} \
files_discovered={files_discovered} max_files={}",
path, limits.max_depth, limits.max_entries, limits.max_files
))
}
fn normalize_import_root(path: &Path) -> PathBuf {
path.components().collect()
}
fn collect_md_files_with_limits(
root: &Path,
limits: ImportTraversalLimits,
) -> Result<ImportDiscovery, RuntimeError> {
let report = walk_tree_reported(
root,
ReportedWalkLimits {
max_depth: limits.max_depth,
max_entries: limits.max_entries,
max_files: limits.max_files,
},
|path| path.extension().and_then(|extension| extension.to_str()) == Some("md"),
)
.map_err(|failure| {
let path = if failure.path.as_os_str().is_empty() {
root.to_path_buf()
} else {
root.join(&failure.path)
};
let progress = failure.progress;
let limit = |kind| {
traversal_limit_error(
kind,
&path,
failure.depth,
progress.entries_visited,
progress.files_discovered,
limits,
)
};
match failure.kind {
WalkFailureKind::Io(error) => traversal_error(&path, progress.entries_visited, error),
WalkFailureKind::EntryLimit => limit("entry"),
WalkFailureKind::DepthLimit => limit("depth"),
WalkFailureKind::FileLimit => limit("markdown file"),
WalkFailureKind::EntryChanged => traversal_error(
&path,
progress.entries_visited,
"directory entry changed during admission",
),
}
})?;
Ok(ImportDiscovery {
files: report
.files
.iter()
.map(|file| root.join(file.relative_path()))
.collect(),
walk_files: report.files,
entries_visited: report.progress.entries_visited,
files_skipped: report.progress.files_skipped,
})
}
fn collect_md_files(root: &Path) -> Result<ImportDiscovery, RuntimeError> {
collect_md_files_with_limits(root, IMPORT_TRAVERSAL_LIMITS)
}
pub(super) fn to_slug(stem: &str) -> String {
stem.to_ascii_lowercase()
.chars()
.map(|c| {
if c.is_ascii_alphanumeric() || c == '-' {
c
} else {
'-'
}
})
.collect::<String>()
.split('-')
.filter(|s| !s.is_empty())
.collect::<Vec<_>>()
.join("-")
}
#[derive(Debug)]
struct ImportSourceIdentity {
slug: String,
source_path: String,
}
fn import_source_identity(
root: &Path,
file: &Path,
root_is_file: bool,
) -> Result<ImportSourceIdentity, RuntimeError> {
let relative = if root_is_file {
file.file_name().map(PathBuf::from).ok_or_else(|| {
RuntimeError::InvalidInput(format!("import file has no filename: {file:?}"))
})?
} else {
file.strip_prefix(root).map(PathBuf::from).map_err(|_| {
RuntimeError::InvalidInput(format!(
"import file {file:?} is outside traversal root {root:?}"
))
})?
};
let mut source_components = Vec::new();
for component in relative.components() {
let Component::Normal(component) = component else {
return Err(RuntimeError::InvalidInput(format!(
"import source path is not root-relative: {relative:?}"
)));
};
let component = component.to_str().ok_or_else(|| {
RuntimeError::InvalidInput(format!(
"import source path is not valid UTF-8: {relative:?}"
))
})?;
source_components.push(component.to_string());
}
let source_path = source_components.join("/");
let filename = source_components.last().ok_or_else(|| {
RuntimeError::InvalidInput(format!("import source path is empty: {relative:?}"))
})?;
let stem = Path::new(filename)
.file_stem()
.and_then(|stem| stem.to_str())
.ok_or_else(|| {
RuntimeError::InvalidInput(format!(
"import source filename has no valid UTF-8 stem: {relative:?}"
))
})?;
let mut slug_components = source_components[..source_components.len() - 1]
.iter()
.map(|component| to_slug(component))
.collect::<Vec<_>>();
slug_components.push(to_slug(stem));
if slug_components.iter().any(String::is_empty) {
return Err(RuntimeError::InvalidInput(format!(
"import source path normalizes to an empty slug component: {source_path:?}"
)));
}
Ok(ImportSourceIdentity {
slug: slug_components.join("--"),
source_path,
})
}
#[derive(Debug)]
struct PreparedSection {
section_type: SectionType,
heading: String,
content: String,
}
#[derive(Debug)]
struct PreparedImportFile {
slug: String,
source_path: String,
canonical_id: Option<String>,
name: String,
tags: Vec<String>,
atom_content: String,
properties: serde_json::Map<String, Value>,
source_uri: String,
source_type: &'static str,
sections: Vec<PreparedSection>,
sections_discovered: usize,
sections_skipped: usize,
sections_unknown_type: usize,
}
#[derive(Debug, Default)]
struct ImportFrontmatter {
canonical_id: Option<String>,
name: Option<String>,
tags: Vec<String>,
properties: serde_json::Map<String, Value>,
}
fn frontmatter_error(source_path: &str, message: impl std::fmt::Display) -> RuntimeError {
RuntimeError::InvalidInput(format!(
"invalid YAML frontmatter in import source {source_path:?}: {message}"
))
}
fn frontmatter_string(
value: Option<Value>,
field: &str,
source_path: &str,
) -> Result<Option<String>, RuntimeError> {
match value {
None | Some(Value::Null) => Ok(None),
Some(Value::String(value)) => {
let value = value.trim();
if value.is_empty() {
Err(frontmatter_error(
source_path,
format!("{field:?} must not be empty"),
))
} else {
Ok(Some(value.to_string()))
}
}
Some(_) => Err(frontmatter_error(
source_path,
format!("{field:?} must be a string or null"),
)),
}
}
fn frontmatter_tags(value: Option<Value>, source_path: &str) -> Result<Vec<String>, RuntimeError> {
let raw_tags = match value {
None | Some(Value::Null) => return Ok(Vec::new()),
Some(Value::String(value)) => value
.split(',')
.map(str::trim)
.filter(|tag| !tag.is_empty())
.map(str::to_string)
.collect(),
Some(Value::Array(values)) => values
.into_iter()
.map(|value| match value {
Value::String(value) if !value.trim().is_empty() => Ok(value.trim().to_string()),
Value::String(_) => Err(frontmatter_error(
source_path,
"\"tags\" entries must not be empty",
)),
_ => Err(frontmatter_error(
source_path,
"\"tags\" entries must be strings",
)),
})
.collect::<Result<Vec<_>, _>>()?,
Some(_) => {
return Err(frontmatter_error(
source_path,
"\"tags\" must be a string, an array of strings, or null",
))
}
};
Ok(raw_tags)
}
fn split_import_frontmatter<'a>(
content: &'a str,
source_path: &str,
) -> Result<(ImportFrontmatter, &'a str), RuntimeError> {
let mut lines = content.split_inclusive('\n');
let Some(first_line) = lines.next() else {
return Ok((ImportFrontmatter::default(), content));
};
let delimiter = first_line.trim_end_matches(['\r', '\n']);
if delimiter != "---" {
return Ok((ImportFrontmatter::default(), content));
}
let header_start = first_line.len();
let mut cursor = header_start;
let mut header_end = None;
let mut body_start = None;
for line in lines {
let line_without_ending = line.trim_end_matches(['\r', '\n']);
if line_without_ending == "---" || line_without_ending == "..." {
header_end = Some(cursor);
body_start = Some(cursor + line.len());
break;
}
cursor += line.len();
}
let (header_end, body_start) = header_end.zip(body_start).ok_or_else(|| {
frontmatter_error(source_path, "missing closing `---` or `...` delimiter")
})?;
let header = &content[header_start..header_end];
let yaml = if header.trim().is_empty() {
Value::Object(serde_json::Map::new())
} else {
let yaml: serde_yaml::Value =
serde_yaml::from_str(header).map_err(|error| frontmatter_error(source_path, error))?;
serde_json::to_value(yaml).map_err(|error| frontmatter_error(source_path, error))?
};
let Value::Object(mut fields) = yaml else {
return Err(frontmatter_error(
source_path,
"the top-level value must be a mapping",
));
};
let mut canonical_id = None;
for alias in ["id", "atlas_id", "atlas-id"] {
if let Some(value) = frontmatter_string(fields.remove(alias), alias, source_path)? {
if let Some(existing) = &canonical_id {
if existing != &value {
return Err(frontmatter_error(
source_path,
format!("identity aliases disagree: {existing:?} does not match {value:?}"),
));
}
} else {
canonical_id = Some(value);
}
}
}
let explicit_name = frontmatter_string(fields.remove("name"), "name", source_path)?;
let title = frontmatter_string(fields.remove("title"), "title", source_path)?;
let name = explicit_name.or(title);
let tags = frontmatter_tags(fields.remove("tags"), source_path)?;
let mut properties = match fields.remove("properties") {
None | Some(Value::Null) => serde_json::Map::new(),
Some(Value::Object(properties)) => properties,
Some(_) => {
return Err(frontmatter_error(
source_path,
"\"properties\" must be a mapping or null",
))
}
};
properties.extend(fields);
Ok((
ImportFrontmatter {
canonical_id,
name,
tags,
properties,
},
&content[body_start..],
))
}
fn import_open_error(source_path: &str, error: impl std::fmt::Display) -> RuntimeError {
RuntimeError::InvalidInput(format!(
"failed to open import source {source_path:?}: {error}"
))
}
fn import_nonregular_error(source_path: &str) -> RuntimeError {
RuntimeError::InvalidInput(format!(
"import source {source_path:?} is not a regular file on the opened handle \
(symlink or special-file swap after validation)"
))
}
#[cfg(unix)]
fn open_import_file_handle(file: &Path, source_path: &str) -> Result<std::fs::File, RuntimeError> {
use khive_fs::opened_file::{open_regular_file_nofollow, ContainedOpenError};
open_regular_file_nofollow(file).map_err(|error| match error {
ContainedOpenError::Open(error) => import_open_error(source_path, error),
ContainedOpenError::Metadata(error) => RuntimeError::InvalidInput(format!(
"failed to inspect opened import source {source_path:?}: {error}"
)),
ContainedOpenError::NotRegular => import_nonregular_error(source_path),
ContainedOpenError::Resolve(error) => RuntimeError::InvalidInput(format!(
"failed to resolve opened import source {source_path:?}: {error}"
)),
ContainedOpenError::Escapes { opened } => RuntimeError::InvalidInput(format!(
"opened import source {source_path:?} escaped its root: {opened:?}"
)),
})
}
#[cfg(not(unix))]
fn open_import_file_handle(file: &Path, source_path: &str) -> Result<std::fs::File, RuntimeError> {
std::fs::File::open(file).map_err(|error| import_open_error(source_path, error))
}
fn read_import_file(
file: &Path,
source_path: &str,
total_bytes: &mut u64,
) -> Result<String, RuntimeError> {
read_opened_import_file(
open_import_file_handle(file, source_path)?,
source_path,
total_bytes,
)
}
fn read_opened_import_file(
mut handle: std::fs::File,
source_path: &str,
total_bytes: &mut u64,
) -> Result<String, RuntimeError> {
use std::io::Read;
let metadata = handle.metadata().map_err(|error| {
RuntimeError::InvalidInput(format!(
"failed to inspect opened import source {source_path:?}: {error}"
))
})?;
if !metadata.is_file() {
return Err(import_nonregular_error(source_path));
}
if metadata.len() > MAX_IMPORT_FILE_BYTES {
return Err(RuntimeError::InvalidInput(format!(
"import source {source_path:?} is {} bytes, exceeding the per-file cap of \
{MAX_IMPORT_FILE_BYTES} bytes",
metadata.len()
)));
}
let mut bytes = Vec::with_capacity(metadata.len() as usize);
handle
.by_ref()
.take(MAX_IMPORT_FILE_BYTES + 1)
.read_to_end(&mut bytes)
.map_err(|error| {
RuntimeError::InvalidInput(format!(
"failed to read import source {source_path:?}: {error}"
))
})?;
if bytes.len() as u64 > MAX_IMPORT_FILE_BYTES {
return Err(RuntimeError::InvalidInput(format!(
"import source {source_path:?} grew beyond the per-file cap of \
{MAX_IMPORT_FILE_BYTES} bytes while being read"
)));
}
*total_bytes = total_bytes.saturating_add(bytes.len() as u64);
if *total_bytes > MAX_IMPORT_TOTAL_BYTES {
return Err(RuntimeError::InvalidInput(format!(
"knowledge.import aggregate size {total_bytes} bytes exceeds the total-import \
cap of {MAX_IMPORT_TOTAL_BYTES} bytes at {source_path:?}"
)));
}
String::from_utf8(bytes).map_err(|_| {
RuntimeError::InvalidInput(format!("import source {source_path:?} is not valid UTF-8"))
})
}
enum ImportReadSource<'a> {
RootFile(&'a Path),
Discovered(&'a WalkFile),
}
fn prepare_import_file(
source: ImportReadSource<'_>,
identity: ImportSourceIdentity,
chunk_strategy: &str,
total_bytes: &mut u64,
) -> Result<PreparedImportFile, RuntimeError> {
let content = match source {
ImportReadSource::RootFile(file) => {
read_import_file(file, &identity.source_path, total_bytes)?
}
ImportReadSource::Discovered(file) => {
let handle = file
.open_read()
.map_err(|error| import_open_error(&identity.source_path, error))?;
read_opened_import_file(handle, &identity.source_path, total_bytes)?
}
};
let (frontmatter, markdown_body) = split_import_frontmatter(&content, &identity.source_path)?;
let (atom_name, atom_body, parsed_sections) = parse_atlas_md(markdown_body);
let canonical_id = frontmatter.canonical_id;
let slug = if let Some(id) = &canonical_id {
let slug = to_slug(id);
if slug.is_empty() {
return Err(frontmatter_error(
&identity.source_path,
format!("canonical identity {id:?} normalizes to an empty slug"),
));
}
slug
} else {
identity.slug
};
let atlas_id = canonical_id.clone().or_else(|| extract_atlas_id(&content));
let name = frontmatter.name.unwrap_or_else(|| {
if atom_name.is_empty() {
slug.rsplit("--").next().unwrap_or(&slug).replace('-', " ")
} else {
atom_name
}
});
let atom_content = if chunk_strategy == "atom" {
markdown_body.to_string()
} else if atom_body.split_whitespace().count() >= super::util::MIN_ATOM_CONTENT_WORDS {
atom_body
} else {
parsed_sections
.iter()
.map(|(_, _, body)| body.as_str())
.collect::<Vec<_>>()
.join("\n\n")
};
validate_atom_content(&atom_content)?;
let source_uri = atlas_id
.as_ref()
.map(|id| format!("atlas:{id}"))
.unwrap_or_else(|| format!("file:{}", identity.source_path));
let source_type = "imported";
let mut properties = frontmatter.properties;
properties.insert(
"source_path".to_string(),
Value::String(identity.source_path.clone()),
);
if let Some(id) = atlas_id {
properties.insert("atlas_id".to_string(), Value::String(id));
}
let sections_discovered = parsed_sections.len();
let (sections, sections_skipped, sections_unknown_type) = if chunk_strategy == "section" {
let mut prepared = Vec::new();
let mut skipped = 0usize;
let mut unknown_type = 0usize;
for (index, (section_type, heading, content)) in parsed_sections.into_iter().enumerate() {
let record = format!("section[{index}]");
let Some(section_type) = section_type else {
tracing::warn!(
source_path = %identity.source_path,
heading = %heading,
"knowledge.import refused a section whose heading names no section type"
);
unknown_type = unknown_type.saturating_add(1);
continue;
};
if content.chars().count() < super::util::MIN_SECTION_CONTENT_LEN {
skipped = skipped.saturating_add(1);
continue;
}
validate_section_content(&content)?;
khive_runtime::secret_gate::check_at(&heading, &record, "heading")?;
khive_runtime::secret_gate::check_at(&content, &record, "content")?;
prepared.push(PreparedSection {
section_type,
heading,
content,
});
}
(prepared, skipped, unknown_type)
} else {
(Vec::new(), 0, 0)
};
khive_runtime::secret_gate::check_at(&slug, "file", "slug")?;
khive_runtime::secret_gate::check_at(&name, "file", "name")?;
khive_runtime::secret_gate::check_tags_at(&frontmatter.tags, "file", "tags")?;
khive_runtime::secret_gate::check_at(&atom_content, "file", "content")?;
let properties_value = Value::Object(properties.clone());
khive_runtime::secret_gate::check_json_at(&properties_value, "file", "properties")?;
khive_runtime::secret_gate::reject_reserved_secret_gate_property(Some(&properties_value))?;
khive_runtime::secret_gate::check_at(&source_uri, "file", "source_uri")?;
khive_runtime::secret_gate::check_at(source_type, "file", "source_type")?;
Ok(PreparedImportFile {
slug,
source_path: identity.source_path,
canonical_id,
name,
tags: frontmatter.tags,
atom_content,
properties,
source_uri,
source_type,
sections,
sections_discovered,
sections_skipped,
sections_unknown_type,
})
}
fn existing_atom_identity_keys(row: &khive_storage::types::SqlRow) -> Vec<String> {
let mut identifiers = Vec::new();
if let Some(source_uri) = row_str(row, "source_uri") {
if let Some(id) = source_uri.strip_prefix("atlas:") {
identifiers.push(id.to_string());
}
}
let properties = match row.get("properties") {
Some(SqlValue::Text(properties)) => serde_json::from_str::<Value>(properties).ok(),
Some(SqlValue::Json(properties)) => Some(properties.clone()),
_ => None,
};
if let Some(Value::Object(properties)) = properties {
for key in ["atlas_id", "atlas-id", "id"] {
if let Some(Value::String(id)) = properties.get(key) {
identifiers.push(id.clone());
}
}
}
identifiers
.into_iter()
.map(|id| to_slug(id.trim()))
.filter(|id| !id.is_empty())
.collect()
}
async fn validate_existing_import_identities(
runtime: &KhiveRuntime,
token: &NamespaceToken,
prepared_files: &[PreparedImportFile],
) -> Result<(), RuntimeError> {
if prepared_files.is_empty() {
return Ok(());
}
let namespace = token.namespace().as_str().to_string();
let mut reader = runtime
.sql()
.reader()
.await
.map_err(|error| sql_err("knowledge.import identity preflight reader", error))?;
let mut slugs_by_identity: BTreeMap<String, Vec<String>> = BTreeMap::new();
if prepared_files
.iter()
.any(|prepared| prepared.canonical_id.is_some())
{
let rows = reader
.query_all(SqlStatement {
sql: khive_runtime::sql!("knowledge_import_existing_identities").into(),
params: vec![SqlValue::Text(namespace.clone())],
label: Some("knowledge.import.identity_preflight".into()),
})
.await
.map_err(|error| sql_err("knowledge.import identity preflight", error))?;
for row in &rows {
let Some(existing_slug) = row_str(row, "slug") else {
continue;
};
for identity in existing_atom_identity_keys(row) {
let slugs = slugs_by_identity.entry(identity).or_default();
if !slugs.contains(&existing_slug) {
slugs.push(existing_slug.clone());
}
}
}
}
for prepared in prepared_files {
if let Some(canonical_id) = &prepared.canonical_id {
if let Some(existing_slugs) = slugs_by_identity.get(&prepared.slug) {
if let Some(existing_slug) = existing_slugs
.iter()
.find(|existing_slug| *existing_slug != &prepared.slug)
{
return Err(RuntimeError::InvalidInput(format!(
"canonical import identity {canonical_id:?} in {:?} maps to slug {:?}, \
but existing atom {:?} claims the same identity; refusing to create a duplicate",
prepared.source_path, prepared.slug, existing_slug
)));
}
}
}
let existing = reader
.query_row(SqlStatement {
sql: khive_runtime::sql!("knowledge_import_atom_tags").into(),
params: vec![
SqlValue::Text(prepared.slug.clone()),
SqlValue::Text(namespace.clone()),
],
label: Some("knowledge.import.target_preflight".into()),
})
.await
.map_err(|error| sql_err("knowledge.import target preflight", error))?;
if let Some(row) = existing {
let slug = &prepared.slug;
if has_domain_mirror_tag(&row_str(&row, "tags").unwrap_or_default()) {
return Err(RuntimeError::InvalidInput(format!(
"atom slug {slug:?} collides with a domain mirror; use upsert_domains instead"
)));
}
if super::util::row_i64(&row, "deleted_at").is_some() {
return Err(RuntimeError::InvalidInput(format!(
"atom slug {slug:?} was previously deleted; choose a new slug"
)));
}
}
}
Ok(())
}
fn extract_atlas_id(content: &str) -> Option<String> {
content.lines().take(32).find_map(|line| {
let trimmed = line.trim();
trimmed
.strip_prefix("atlas_id:")
.or_else(|| trimmed.strip_prefix("atlas-id:"))
.map(str::trim)
.filter(|s| !s.is_empty())
.map(str::to_string)
})
}
type ParsedSection = (Option<SectionType>, String, String);
fn split_type_marker(heading: &str) -> Option<(&str, &str)> {
let without_close = heading.strip_suffix('}')?;
let open = without_close.rfind('{')?;
let marker = &without_close[open + 1..];
let is_token = marker
.bytes()
.next()
.is_some_and(|b| b.is_ascii_lowercase())
&& marker.bytes().all(|b| b.is_ascii_lowercase() || b == b'_');
if !is_token {
return None;
}
Some((without_close[..open].trim_end(), marker))
}
fn section_heading_type(heading: &str) -> (Option<SectionType>, String) {
match split_type_marker(heading) {
Some((display, marker)) => (marker.parse().ok(), display.to_string()),
None => (SectionType::from_str_loose(heading), heading.to_string()),
}
}
fn parse_atlas_md(content: &str) -> (String, String, Vec<ParsedSection>) {
let mut name = String::new();
let mut pre_body = String::new();
let mut sections: Vec<ParsedSection> = Vec::new();
let mut current_heading: Option<(Option<SectionType>, String)> = None;
let mut current_body = String::new();
let mut in_pre = true;
for line in content.lines() {
if let Some(rest) = line.strip_prefix("# ") {
if name.is_empty() && current_heading.is_none() && current_body.trim().is_empty() {
name = rest.trim().to_string();
continue;
}
}
if let Some(rest) = line.strip_prefix("## ") {
if let Some((stype, heading)) = current_heading.take() {
sections.push((stype, heading, current_body.trim_end().to_string()));
current_body.clear();
} else if in_pre {
pre_body = current_body.trim_end().to_string();
current_body.clear();
in_pre = false;
}
current_heading = Some(section_heading_type(rest.trim()));
continue;
}
current_body.push_str(line);
current_body.push('\n');
}
if let Some((stype, heading)) = current_heading {
sections.push((stype, heading, current_body.trim_end().to_string()));
} else {
pre_body = current_body.trim_end().to_string();
}
(name, pre_body, sections)
}
impl KnowledgeHandlers {
pub(crate) async fn edit(
runtime: &KhiveRuntime,
token: &NamespaceToken,
params: Value,
ann: &vamana::SharedAnn,
) -> Result<Value, RuntimeError> {
let p: EditParams = deser(params)?;
if p.sections.is_empty() {
return Err(RuntimeError::InvalidInput(
"sections must not be empty".into(),
));
}
let ns = token.namespace().as_str().to_owned();
let sql = runtime.sql();
let atom_id = {
let mut reader = sql
.reader()
.await
.map_err(|e| sql_err("edit atom reader", e))?;
let id = p.id.trim().to_string();
let row = if let Ok(uuid) = id.parse::<Uuid>() {
reader
.query_row(SqlStatement {
sql: khive_runtime::sql!("knowledge_atom_resolve_id").into(),
params: vec![
SqlValue::Text(uuid.as_hyphenated().to_string()),
SqlValue::Text(ns.clone()),
],
label: None,
})
.await
.map_err(|e| sql_err("edit atom lookup by id", e))?
} else {
reader
.query_row(SqlStatement {
sql: khive_runtime::sql!("knowledge_atom_resolve_slug").into(),
params: vec![SqlValue::Text(id.clone()), SqlValue::Text(ns.clone())],
label: None,
})
.await
.map_err(|e| sql_err("edit atom lookup by slug", e))?
};
row.and_then(|r| row_str(&r, "id"))
.ok_or_else(|| RuntimeError::NotFound(format!("atom not found: {:?}", p.id)))?
};
let now = now_us();
let mut upserted = 0usize;
let mut section_results: Vec<Value> = Vec::with_capacity(p.sections.len());
let mut section_types = Vec::with_capacity(p.sections.len());
for (index, su) in p.sections.iter().enumerate() {
let record = format!("section[{index}]");
let stype = parse_section_type(&su.section_type)?;
validate_section_content(&su.content)?;
khive_runtime::secret_gate::check_at(&su.content, &record, "content")?;
if let Some(ref h) = su.heading {
khive_runtime::secret_gate::check_at(h, &record, "heading")?;
}
section_types.push(stype);
}
for (su, stype) in p.sections.iter().zip(section_types) {
let heading = su.heading.as_deref().unwrap_or(stype.as_str()).to_string();
let tokens = count_tokens(&su.content);
let sort_order = su.sort_order.unwrap_or_else(|| {
SectionType::ALL
.iter()
.position(|&t| t == stype)
.unwrap_or(9) as i64
});
let hash = content_hash(&su.content);
let mut reader = sql
.reader()
.await
.map_err(|e| sql_err("edit section reader", e))?;
let existing_section = reader
.query_row(SqlStatement {
sql: khive_runtime::sql!("knowledge_section_hash_probe").into(),
params: vec![
SqlValue::Text(atom_id.clone()),
SqlValue::Text(hash.clone()),
],
label: None,
})
.await
.map_err(|e| sql_err("edit section lookup", e))?;
let section_id = existing_section
.as_ref()
.and_then(|r| row_str(r, "id"))
.unwrap_or_else(new_id);
let mut writer = sql
.writer()
.await
.map_err(|e| sql_err("edit section writer", e))?;
if existing_section.is_some() {
writer
.execute(SqlStatement {
sql: khive_runtime::sql!("knowledge_section_metadata_update").into(),
params: vec![
SqlValue::Text(stype.as_str().to_string()),
SqlValue::Text(heading.clone()),
SqlValue::Integer(tokens),
SqlValue::Integer(sort_order),
SqlValue::Integer(now),
SqlValue::Text(section_id.clone()),
],
label: None,
})
.await
.map_err(|e| sql_err("edit section update", e))?;
} else {
writer
.execute(SqlStatement {
sql: khive_runtime::sql!("knowledge_section_insert").into(),
params: vec![
SqlValue::Text(section_id.clone()),
SqlValue::Text(atom_id.clone()),
SqlValue::Text(ns.clone()),
SqlValue::Text(stype.as_str().to_string()),
SqlValue::Text(heading.clone()),
SqlValue::Text(su.content.clone()),
SqlValue::Text(hash.clone()),
SqlValue::Integer(tokens),
SqlValue::Integer(sort_order),
SqlValue::Integer(now),
SqlValue::Integer(now),
],
label: None,
})
.await
.map_err(|e| sql_err("edit section insert", e))?;
}
upserted += 1;
section_results.push(json!({
"id": section_id,
"atom_id": atom_id,
"section_type": stype.as_str(),
"heading": heading,
"tokens": tokens,
"content_hash": hash,
}));
}
let (_, _, _, _, section_truncation) =
embed_sections(runtime, token, false, 32, None, Some(&atom_id)).await?;
let (atom_vector_refreshed, mut truncation_by_model) = {
let atom_params = serde_json::json!({
"ids": [atom_id],
"rebuild_ann": false,
"insert_only": false,
});
let result = KnowledgeHandlers::index(runtime, token, atom_params, ann, None).await?;
let failed = result.get("failed").and_then(|v| v.as_u64()).unwrap_or(0);
let skipped_no_embedder = result.get("reason").is_some();
if failed > 0 {
tracing::warn!(
atom_id = %atom_id,
failed = failed,
"knowledge.edit: atom vector refresh failed; \
hybrid recall for this atom may be stale until next reindex"
);
}
let truncation_by_model = match result.get("truncation_by_model") {
Some(value) => serde_json::from_value::<
BTreeMap<String, khive_runtime::retrieval::EmbeddingTruncationReport>,
>(value.clone())
.map_err(|e| {
RuntimeError::Internal(format!(
"knowledge.edit: invalid knowledge.index truncation report: {e}"
))
})?,
None => BTreeMap::new(),
};
(failed == 0 && !skipped_no_embedder, truncation_by_model)
};
let default_model_name = runtime.default_embedder_name();
if !default_model_name.is_empty() {
truncation_by_model
.entry(default_model_name.to_string())
.or_default()
.merge(section_truncation);
}
let any_truncated = truncation_by_model
.values()
.any(khive_runtime::retrieval::EmbeddingTruncationReport::any_truncated);
let mut response = json!({
"atom_id": atom_id,
"upserted": upserted,
"sections": section_results,
"atom_vector_refreshed": atom_vector_refreshed,
"truncation_by_model": truncation_by_model,
});
if any_truncated {
response
.as_object_mut()
.expect("edit response is an object")
.insert(
"warnings".to_string(),
json!([khive_runtime::retrieval::EMBEDDING_INPUT_TRUNCATED_WARNING]),
);
}
Ok(response)
}
pub(crate) async fn import(
runtime: &KhiveRuntime,
token: &NamespaceToken,
params: Value,
ann: &vamana::SharedAnn,
) -> Result<Value, RuntimeError> {
let p: ImportParams = deser(params)?;
let path_str = p.path.trim().to_string();
if path_str.is_empty() {
return Err(RuntimeError::InvalidInput("path must not be empty".into()));
}
let chunk_strategy = p
.chunk_strategy
.as_deref()
.unwrap_or("section")
.to_ascii_lowercase();
if !["section", "atom"].contains(&chunk_strategy.as_str()) {
return Err(RuntimeError::InvalidInput(format!(
"unknown chunk_strategy {:?}; valid: section | atom",
chunk_strategy
)));
}
let format = p.format.as_deref().unwrap_or("atlas_md");
if format != "atlas_md" {
return Err(RuntimeError::InvalidInput(format!(
"unknown format {format:?}; only \"atlas_md\" is supported"
)));
}
let normalized_md_path = normalize_import_root(Path::new(&path_str));
let md_path = normalized_md_path.as_path();
let metadata = std::fs::symlink_metadata(md_path).map_err(|error| {
if error.kind() == std::io::ErrorKind::NotFound {
RuntimeError::NotFound(format!("path does not exist: {path_str:?}"))
} else {
RuntimeError::InvalidInput(format!(
"failed to inspect import path {path_str:?}: {error}"
))
}
})?;
if metadata.file_type().is_symlink() {
return Err(RuntimeError::InvalidInput(format!(
"import path must not be a symbolic link: {path_str:?}"
)));
}
let root_is_file = metadata.is_file();
let discovery = if root_is_file {
if md_path.extension().and_then(|extension| extension.to_str()) != Some("md") {
return Err(RuntimeError::InvalidInput(format!(
"import file must have a .md extension: {path_str:?}"
)));
}
ImportDiscovery {
files: vec![md_path.to_path_buf()],
walk_files: Vec::new(),
entries_visited: 1,
files_skipped: 0,
}
} else if metadata.is_dir() {
collect_md_files(md_path)?
} else {
return Err(RuntimeError::InvalidInput(format!(
"path is not a regular file or directory: {path_str:?}"
)));
};
let mut identities = Vec::with_capacity(discovery.files.len());
for file in &discovery.files {
let identity = import_source_identity(md_path, file, root_is_file)?;
identities.push((file.clone(), identity));
}
let mut prepared_files = Vec::with_capacity(identities.len());
let mut total_bytes = 0u64;
for (index, (file, identity)) in identities.into_iter().enumerate() {
let source = if root_is_file {
ImportReadSource::RootFile(&file)
} else {
ImportReadSource::Discovered(&discovery.walk_files[index])
};
prepared_files.push(prepare_import_file(
source,
identity,
&chunk_strategy,
&mut total_bytes,
)?);
}
drop(discovery.walk_files);
let mut sources_by_slug = BTreeMap::new();
for prepared in &prepared_files {
if let Some(previous_source) =
sources_by_slug.insert(prepared.slug.clone(), prepared.source_path.clone())
{
return Err(RuntimeError::InvalidInput(format!(
"normalized slug collision for {:?}: {:?} and {:?}",
prepared.slug, previous_source, prepared.source_path
)));
}
}
validate_existing_import_identities(runtime, token, &prepared_files).await?;
let sections_discovered = prepared_files
.iter()
.map(|prepared| prepared.sections_discovered)
.sum::<usize>();
let sections_skipped = prepared_files
.iter()
.map(|prepared| prepared.sections_skipped)
.sum::<usize>();
let sections_unknown_type = prepared_files
.iter()
.map(|prepared| prepared.sections_unknown_type)
.sum::<usize>();
let mut imported_atoms = 0usize;
let mut imported_sections = 0usize;
for prepared in &prepared_files {
let upsert_params = serde_json::json!({
"atoms": [{
"slug": prepared.slug,
"name": prepared.name,
"tags": prepared.tags,
"content": prepared.atom_content,
"properties": Value::Object(prepared.properties.clone()),
"source_uri": prepared.source_uri,
"source_type": prepared.source_type,
"finalized": true,
}]
});
KnowledgeHandlers::upsert_import_atoms(runtime, token, upsert_params).await?;
imported_atoms += 1;
if !prepared.sections.is_empty() {
let section_updates = prepared
.sections
.iter()
.map(|section| {
json!({
"section_type": section.section_type.as_str(),
"heading": section.heading,
"content": section.content,
})
})
.collect::<Vec<_>>();
let edit_params = json!({
"id": prepared.slug,
"sections": section_updates,
});
let result = KnowledgeHandlers::edit(runtime, token, edit_params, ann).await?;
if let Some(count) = result.get("upserted").and_then(Value::as_u64) {
imported_sections += count as usize;
}
}
}
Ok(json!({
"imported_atoms": imported_atoms,
"imported_sections": imported_sections,
"files_processed": imported_atoms,
"entries_visited": discovery.entries_visited,
"files_discovered": discovery.files.len(),
"files_skipped": discovery.files_skipped,
"traversal_errors": 0,
"sections_discovered": sections_discovered,
"sections_skipped": sections_skipped,
"sections_unknown_type": sections_unknown_type,
}))
}
pub(crate) async fn challenge(
runtime: &KhiveRuntime,
token: &NamespaceToken,
params: Value,
) -> Result<Value, RuntimeError> {
let p: ChallengeParams = deser(params)?;
let ns = token.namespace().as_str().to_owned();
let sql = runtime.sql();
let atom_id = resolve_atom_id(runtime, &ns, &p.atom_id).await?;
let stype = parse_section_type(&p.section_type)?;
let hash = p
.content_hash
.as_ref()
.map(|h| h.trim().to_string())
.filter(|h| !h.is_empty());
let target_hash = Self::resolve_section_hash(
runtime,
&atom_id,
stype,
hash.as_deref(),
"status NOT IN ('disputed','deprecated')",
"section not found, already disputed, or deprecated",
)
.await?;
let atom_id_for_update = atom_id.clone();
let target_hash_for_update = target_hash.clone();
let affected = sql
.atomic_unit(Box::new(move |writer| {
Box::pin(async move {
let affected = writer
.execute(SqlStatement {
sql: khive_runtime::sql!("knowledge_section_dispute").into(),
params: vec![
SqlValue::Text(atom_id_for_update.clone()),
SqlValue::Text(stype.as_str().to_string()),
SqlValue::Text(target_hash_for_update),
],
label: None,
})
.await
.map_err(|e| StorageError::driver(StorageCapability::Sql, "challenge section status", e))?;
if affected == 0 {
return Ok(Box::new(0_u64) as Box<dyn std::any::Any + Send>);
}
let updated_atoms = writer
.execute(SqlStatement {
sql: format!(
"UPDATE knowledge_atoms SET properties=json_set(coalesce(properties,'{{}}'),'$.dispute_count',coalesce(json_extract(properties,'$.dispute_count'),0)+{affected}) WHERE id=?1 AND namespace=?2"
),
params: vec![
SqlValue::Text(atom_id_for_update),
SqlValue::Text(ns.clone()),
],
label: None,
})
.await
.map_err(|e| StorageError::driver(StorageCapability::Sql, "challenge dispute_count increment", e))?;
if updated_atoms != 1 {
return Err(StorageError::Conflict {
capability: StorageCapability::Sql,
operation: "challenge dispute_count".into(),
message: format!("expected one parent atom, updated {updated_atoms}"),
});
}
Ok(Box::new(affected) as Box<dyn std::any::Any + Send>)
})
}))
.await
.map_err(|e| sql_err("challenge transaction", e))?
.downcast::<u64>()
.map_err(|_| RuntimeError::Internal("invalid challenge transaction outcome".into()))?;
let affected = *affected;
if affected == 0 {
return Err(RuntimeError::InvalidInput(
"section not found, already disputed, or deprecated".into(),
));
}
Ok(json!({
"atom_id": atom_id,
"section_type": stype.as_str(),
"content_hash": target_hash,
"disputed": affected,
"reason": p.reason,
}))
}
async fn resolve_section_hash(
runtime: &KhiveRuntime,
atom_id: &str,
stype: SectionType,
hash: Option<&str>,
status_filter: &str,
not_found_msg: &str,
) -> Result<String, RuntimeError> {
let sql = runtime.sql();
let mut reader = sql
.reader()
.await
.map_err(|e| sql_err("section resolve reader", e))?;
let mut query = format!(
"SELECT content_hash FROM knowledge_sections \
WHERE atom_id=?1 AND section_type=?2 AND {status_filter}"
);
let mut params = vec![
SqlValue::Text(atom_id.to_owned()),
SqlValue::Text(stype.as_str().to_string()),
];
if let Some(h) = hash {
query.push_str(" AND content_hash=?3");
params.push(SqlValue::Text(h.to_owned()));
}
let rows = reader
.query_all(SqlStatement {
sql: query,
params,
label: None,
})
.await
.map_err(|e| sql_err("section resolve", e))?;
if rows.is_empty() {
return Err(RuntimeError::InvalidInput(not_found_msg.to_owned()));
}
if hash.is_none() && rows.len() > 1 {
return Err(RuntimeError::InvalidInput(format!(
"atom has {} '{}' sections matching; specify content_hash to target one",
rows.len(),
stype.as_str(),
)));
}
rows.first()
.and_then(|r| row_str(r, "content_hash"))
.ok_or_else(|| RuntimeError::Internal("section row missing content_hash".into()))
}
pub(crate) async fn adjudicate(
runtime: &KhiveRuntime,
token: &NamespaceToken,
params: Value,
) -> Result<Value, RuntimeError> {
let p: AdjudicateParams = deser(params)?;
let ns = token.namespace().as_str().to_owned();
let sql = runtime.sql();
let resolution = p.resolution.trim().to_ascii_lowercase();
if resolution != "accept" && resolution != "reject" {
return Err(RuntimeError::InvalidInput(
"resolution must be \"accept\" or \"reject\"".into(),
));
}
let atom_id = resolve_atom_id(runtime, &ns, &p.atom_id).await?;
let stype = parse_section_type(&p.section_type)?;
let hash = p
.content_hash
.as_ref()
.map(|h| h.trim().to_string())
.filter(|h| !h.is_empty());
let new_status = if resolution == "accept" {
"verified"
} else {
"reviewed"
};
let target_hash = Self::resolve_section_hash(
runtime,
&atom_id,
stype,
hash.as_deref(),
"status='disputed'",
"section not found or not in disputed state",
)
.await?;
let atom_id_for_update = atom_id.clone();
let target_hash_for_update = target_hash.clone();
let affected = sql
.atomic_unit(Box::new(move |writer| {
Box::pin(async move {
let affected = writer
.execute(SqlStatement {
sql: format!(
"UPDATE knowledge_sections SET status='{new_status}' \
WHERE atom_id=?1 AND section_type=?2 AND content_hash=?3 AND status='disputed'"
),
params: vec![
SqlValue::Text(atom_id_for_update.clone()),
SqlValue::Text(stype.as_str().to_string()),
SqlValue::Text(target_hash_for_update),
],
label: None,
})
.await
.map_err(|e| StorageError::driver(StorageCapability::Sql, "adjudicate section status", e))?;
if affected == 0 {
return Ok(Box::new(0_u64) as Box<dyn std::any::Any + Send>);
}
let updated_atoms = writer
.execute(SqlStatement {
sql: format!(
"UPDATE knowledge_atoms SET properties=json_set(coalesce(properties,'{{}}'),'$.dispute_count',CASE WHEN coalesce(json_extract(properties,'$.dispute_count'),0) >= {affected} THEN coalesce(json_extract(properties,'$.dispute_count'),0)-{affected} ELSE 0 END) WHERE id=?1 AND namespace=?2"
),
params: vec![
SqlValue::Text(atom_id_for_update),
SqlValue::Text(ns.clone()),
],
label: None,
})
.await
.map_err(|e| StorageError::driver(StorageCapability::Sql, "adjudicate dispute_count decrement", e))?;
if updated_atoms != 1 {
return Err(StorageError::Conflict {
capability: StorageCapability::Sql,
operation: "adjudicate dispute_count".into(),
message: format!("expected one parent atom, updated {updated_atoms}"),
});
}
Ok(Box::new(affected) as Box<dyn std::any::Any + Send>)
})
}))
.await
.map_err(|e| sql_err("adjudicate transaction", e))?
.downcast::<u64>()
.map_err(|_| RuntimeError::Internal("invalid adjudicate transaction outcome".into()))?;
let affected = *affected;
if affected == 0 {
return Err(RuntimeError::InvalidInput(
"section not found or not in disputed state".into(),
));
}
Ok(json!({
"atom_id": atom_id,
"section_type": stype.as_str(),
"content_hash": target_hash,
"resolution": resolution,
"new_status": new_status,
"resolved": affected,
}))
}
}
#[cfg(test)]
#[path = "sections_marker_tests.rs"]
mod marker_tests;
#[cfg(test)]
#[path = "sections_import_traversal_tests.rs"]
mod import_traversal_tests;
#[cfg(all(test, unix))]
#[path = "import_fs_tests.rs"]
mod import_fs_tests;
#[cfg(test)]
#[path = "sections_import_read_tests.rs"]
mod import_read_tests;