mod cache;
mod fk;
mod junction;
mod manual;
mod nodes;
mod prepass;
mod specs;
mod table_ops;
pub use specs::FlatSpec;
use cache::{parse_in_parallel, CsvCache, IdTypeCache};
use fk::load_fk_edges;
use junction::load_junction_edges;
use manual::load_manual_nodes;
use nodes::{load_node_specs, should_stream_spec};
use specs::collect_specs;
#[cfg(feature = "xlsx")]
use super::input::xlsx::{XlsxConfig, XlsxFile};
use super::input::{
csv::CsvFile,
delimited::{DelimitedConfig, DelimitedFile},
frame::FrameSource,
resolve_input_path, InputRegistry, Source,
};
use super::schema::{Blueprint, FileSpec};
use crate::datatypes::values::DataFrame;
use crate::graph::mutation::maintain;
use crate::graph::schema::{DirGraph, PROVISIONAL_KEY};
use indexmap::IndexMap;
use std::collections::{BTreeMap, HashMap};
use std::path::{Path, PathBuf};
#[derive(Default)]
pub struct BuildInputs {
pub frames: HashMap<String, DataFrame>,
}
pub struct BuildReport {
pub nodes_by_type: BTreeMap<String, usize>,
pub edges_by_type: BTreeMap<String, usize>,
pub edges_actual: BTreeMap<String, usize>,
pub warnings: Vec<String>,
pub errors: Vec<String>,
pub provisional_purged: usize,
}
impl BuildReport {
pub fn render_text(&self, verbose: bool) -> String {
use std::fmt::Write;
if !verbose {
return String::new();
}
let mut out = String::new();
let n_total: usize = self.nodes_by_type.values().sum();
let e_actual: usize = self.edges_actual.values().sum();
let e_input: usize = self.edges_by_type.values().sum();
let _ = writeln!(out, "Loading blueprint...");
for (t, n) in &self.nodes_by_type {
let _ = writeln!(out, " {}: {} nodes", t, n);
}
for (t, n_input) in &self.edges_by_type {
let n_actual = self.edges_actual.get(t).copied().unwrap_or(0);
if n_actual == *n_input {
let _ = writeln!(out, " [{}]: {} edges", t, n_actual);
} else {
let _ = writeln!(
out,
" [{}]: {} edges ({} input rows, {} deduped)",
t,
n_actual,
n_input,
n_input.saturating_sub(n_actual),
);
}
}
if e_actual == e_input {
let _ = writeln!(
out,
"Loaded {} nodes ({} types), {} edges ({} types)",
n_total,
self.nodes_by_type.len(),
e_actual,
self.edges_by_type.len(),
);
} else {
let _ = writeln!(
out,
"Loaded {} nodes ({} types), {} edges ({} types) — {} input rows, {} deduped",
n_total,
self.nodes_by_type.len(),
e_actual,
self.edges_by_type.len(),
e_input,
e_input.saturating_sub(e_actual),
);
}
if self.provisional_purged > 0 {
let _ = writeln!(
out,
" auto_purge: dropped {} unpromoted provisional stub node(s)",
self.provisional_purged
);
}
out
}
}
pub fn build(
graph: &mut DirGraph,
mut blueprint: Blueprint,
blueprint_dir: &Path,
mut inputs: BuildInputs,
) -> Result<BuildReport, String> {
super::validation::validate_compute(&blueprint)?;
super::validation::validate_inputs(&blueprint)?;
let unknown_keys = super::validation::unknown_key_warnings(&blueprint);
let root = blueprint
.settings
.input_root
.as_deref()
.map(|r| {
if Path::new(r).is_absolute() {
PathBuf::from(r)
} else {
blueprint_dir.join(r)
}
})
.unwrap_or_else(|| blueprint_dir.to_path_buf());
super::compute::apply_compute(&mut blueprint, &root)?;
let mut report = BuildReport {
nodes_by_type: BTreeMap::new(),
edges_by_type: BTreeMap::new(),
edges_actual: BTreeMap::new(),
warnings: Vec::new(),
errors: Vec::new(),
provisional_purged: 0,
};
report.warnings.extend(unknown_keys);
report
.warnings
.extend(super::validation::unknown_property_type_warnings(
&blueprint,
));
let profile = std::env::var("KGLITE_BLUEPRINT_PROFILE").is_ok();
let t0 = std::time::Instant::now();
let (core_specs, sub_specs) = collect_specs(&blueprint.nodes);
for spec in core_specs.iter().chain(sub_specs.iter()) {
if spec.spec.properties.contains_key(PROVISIONAL_KEY) {
return Err(format!(
"node type '{}': property '{}' is reserved (auto-vivification marker)",
spec.node_type, PROVISIONAL_KEY
));
}
}
if profile {
eprintln!(
" collect_specs: {} ms ({} core + {} sub)",
t0.elapsed().as_millis(),
core_specs.len(),
sub_specs.len()
);
}
let (registry, csv_cache, frame_warnings) = prepare_inputs(
&blueprint.files,
&core_specs,
&sub_specs,
&root,
&mut inputs,
profile,
)?;
report.warnings.extend(frame_warnings);
let id_types = IdTypeCache::default();
let t = std::time::Instant::now();
load_manual_nodes(
graph,
&core_specs,
&sub_specs,
®istry,
&csv_cache,
&mut report,
)?;
if profile {
eprintln!(" load_manual_nodes: {} ms", t.elapsed().as_millis());
}
let t = std::time::Instant::now();
load_node_specs(
graph,
&core_specs,
®istry,
&csv_cache,
&id_types,
&mut report,
"core nodes",
)?;
if profile {
eprintln!(" load_core_nodes: {} ms", t.elapsed().as_millis());
}
let t = std::time::Instant::now();
load_node_specs(
graph,
&sub_specs,
®istry,
&csv_cache,
&id_types,
&mut report,
"sub-nodes",
)?;
if profile {
eprintln!(" load_sub_nodes: {} ms", t.elapsed().as_millis());
}
for sub in &sub_specs {
if let Some(parent) = &sub.parent {
if graph.type_indices.contains_key(&sub.node_type)
&& graph.type_indices.contains_key(parent)
{
graph
.parent_types_mut()
.insert(sub.node_type.clone(), parent.clone());
}
}
}
let all_specs: Vec<&FlatSpec> = core_specs.iter().chain(sub_specs.iter()).collect();
let t = std::time::Instant::now();
load_fk_edges(
graph,
&all_specs,
®istry,
&csv_cache,
&id_types,
&mut report,
)?;
if profile {
eprintln!(" load_fk_edges: {} ms", t.elapsed().as_millis());
}
let t = std::time::Instant::now();
load_junction_edges(graph, &all_specs, ®istry, &csv_cache, &mut report)?;
if profile {
eprintln!(" load_junction_edges: {} ms", t.elapsed().as_millis());
}
report.warnings.extend(registry.read_warnings());
finish_build(
graph,
&blueprint,
blueprint_dir,
&all_specs,
&mut report,
profile,
)?;
if profile {
eprintln!(" TOTAL build: {} ms", t0.elapsed().as_millis());
}
Ok(report)
}
fn finish_build(
graph: &mut DirGraph,
blueprint: &Blueprint,
blueprint_dir: &Path,
all_specs: &[&FlatSpec],
report: &mut BuildReport,
profile: bool,
) -> Result<(), String> {
if blueprint.settings.auto_purge {
let t = std::time::Instant::now();
let (purged, _edges) = maintain::purge_provisional_nodes(graph);
report.provisional_purged = purged;
if profile {
eprintln!(
" purge_provisional: {} ms ({} purged)",
t.elapsed().as_millis(),
purged
);
}
}
let t = std::time::Instant::now();
stamp_declared_labels(graph, all_specs, report)?;
if profile {
eprintln!(" stamp_declared_labels: {} ms", t.elapsed().as_millis());
}
if let Some(ontology_path) = &blueprint.ontology {
apply_ontology_gate(graph, ontology_path, blueprint_dir, report)?;
}
report.edges_actual = graph
.get_edge_type_counts()
.iter()
.map(|(t, n)| (t.clone(), *n))
.collect();
Ok(())
}
fn prepare_inputs(
files: &IndexMap<String, FileSpec>,
core_specs: &[FlatSpec],
sub_specs: &[FlatSpec],
root: &Path,
inputs: &mut BuildInputs,
profile: bool,
) -> Result<(InputRegistry, CsvCache, Vec<String>), String> {
let (registry, frame_warnings) =
build_input_registry(files, core_specs, sub_specs, root, inputs)?;
let cache = CsvCache::default();
let mut buffered_inputs: Vec<String> = Vec::new();
for s in core_specs.iter().chain(sub_specs.iter()) {
if should_stream_spec(s, ®istry) {
continue;
}
if let Some(name) = s.input.as_deref() {
buffered_inputs.push(name.to_string());
}
}
buffered_inputs.sort();
buffered_inputs.dedup();
let t_preparse = std::time::Instant::now();
parse_in_parallel(&buffered_inputs, ®istry, &cache);
if profile {
eprintln!(
" parse_in_parallel: {} ms ({} distinct files, streamed specs excluded)",
t_preparse.elapsed().as_millis(),
buffered_inputs.len()
);
}
Ok((registry, cache, frame_warnings))
}
fn build_input_registry(
files: &IndexMap<String, FileSpec>,
core_specs: &[FlatSpec],
sub_specs: &[FlatSpec],
root: &Path,
inputs: &mut BuildInputs,
) -> Result<(InputRegistry, Vec<String>), String> {
let mut registry = InputRegistry::default();
let mut frame_warnings: Vec<String> = Vec::new();
for (name, file) in files {
if file.format == "frame" {
let Some(df) = inputs.frames.remove(name) else {
return Err(format!(
"frame '{name}' is declared in `files` but was not passed in `frames=`"
));
};
let (source, warnings) = FrameSource::new(name, &df);
frame_warnings.extend(warnings);
registry.insert(name.clone(), Box::new(source));
continue;
}
let Some(path) = file.path.as_deref() else {
continue;
};
let resolved = resolve_input_path(root, path);
let source: Box<dyn Source> = match file.format.as_str() {
"delimited" => Box::new(DelimitedFile::new(
resolved,
path.to_string(),
DelimitedConfig::from_spec(name, file)?,
)),
#[cfg(feature = "xlsx")]
"xlsx" => Box::new(XlsxFile::new(
resolved,
path.to_string(),
XlsxConfig::from_spec(name, file)?,
)),
_ => Box::new(CsvFile::new(resolved, path.to_string())),
};
registry.insert(name.clone(), source);
}
if !inputs.frames.is_empty() {
let mut names: Vec<&str> = inputs.frames.keys().map(String::as_str).collect();
names.sort_unstable();
let list = names
.iter()
.map(|n| format!("'{n}'"))
.collect::<Vec<_>>()
.join(", ");
return Err(format!(
"frames= contains {list}, which {} not declared in `files`",
if names.len() == 1 { "is" } else { "are" }
));
}
let declare = |name: &str, registry: &mut InputRegistry| {
registry.insert(
name,
Box::new(CsvFile::new(
resolve_input_path(root, name),
name.to_string(),
)),
);
};
for spec in core_specs.iter().chain(sub_specs.iter()) {
if let Some(name) = spec.spec.csv.as_deref() {
declare(name, &mut registry);
}
for junc in spec.spec.connections.junction_edges.values() {
if let Some(name) = junc.csv.as_deref() {
declare(name, &mut registry);
}
}
}
Ok((registry, frame_warnings))
}
fn stamp_declared_labels(
graph: &mut DirGraph,
specs: &[&FlatSpec],
report: &mut BuildReport,
) -> Result<(), String> {
for spec in specs {
if spec.spec.labels.is_empty() {
continue;
}
maintain::preflight_interner_names(graph, spec.spec.labels.iter().map(String::as_str))
.map_err(|e| format!("node '{}': labels: {}", spec.node_type, e))?;
let Some(nodes) = graph.type_indices.get(&spec.node_type) else {
report.warnings.push(format!(
"node '{}': declares labels {:?} but the build produced no nodes of that type",
spec.node_type, spec.spec.labels
));
continue;
};
let indices: Vec<petgraph::graph::NodeIndex> = nodes.iter().collect();
for label in &spec.spec.labels {
let key = graph.interner.get_or_intern(label);
graph.add_node_labels_bulk(&indices, key);
}
}
Ok(())
}
fn apply_ontology_gate(
graph: &mut DirGraph,
ontology_path: &str,
blueprint_dir: &Path,
report: &mut BuildReport,
) -> Result<(), String> {
let resolved = if Path::new(ontology_path).is_absolute() {
PathBuf::from(ontology_path)
} else {
blueprint_dir.join(ontology_path)
};
let text = std::fs::read_to_string(&resolved)
.map_err(|e| format!("ontology: cannot read {}: {e}", resolved.display()))?;
let store = crate::graph::ontology::ontology_from_json(&text)
.map_err(|e| format!("ontology {}: {e}", resolved.display()))?;
report.warnings.extend(
graph
.define_ontology(store)?
.into_iter()
.map(|w| format!("ontology: {w}")),
);
use crate::graph::languages::cypher::executor::ontology_procedures::audit_counts;
use crate::graph::ontology::Enforcement;
let mut errors: Vec<String> = Vec::new();
for line in audit_counts(graph)? {
if line.violations == 0 && line.exempted == 0 {
continue;
}
let tail = if line.exempted > 0 {
format!(" (+{} exempted)", line.exempted)
} else {
String::new()
};
let summary = format!(
"{} {}: {}/{} ({:.1}%) violations{tail}",
line.entity_kind, line.rule, line.violations, line.total, line.pct
);
match line.severity {
Enforcement::Advisory => {}
Enforcement::Warn => report.warnings.push(format!("ontology: {summary}")),
Enforcement::Error if line.violations > 0 => errors.push(summary),
Enforcement::Error => report.warnings.push(format!("ontology: {summary}")),
}
}
if !errors.is_empty() {
return Err(format!(
"ontology gate failed — {} contract(s) violated:\n {}\nFix the data (or lower \
the declaration's enforcement) and rebuild; drill into each rule with the \
matching CALL procedure (node_property_violation for node properties, \
edge_property_violation for edge properties, or e.g. type_domain_violation).",
errors.len(),
errors.join("\n ")
));
}
Ok(())
}
#[cfg(test)]
mod frame_input_tests {
use super::*;
use crate::datatypes::values::{ColumnData, ColumnType};
fn bp(json: &str) -> Blueprint {
serde_json::from_str(json).expect("fixture parses")
}
fn one_int_frame() -> DataFrame {
let mut df = DataFrame::new(Vec::new());
df.add_column(
"id".to_string(),
ColumnType::Int64,
ColumnData::Int64(vec![Some(1), Some(2)]),
)
.unwrap();
df
}
fn frames(pairs: Vec<(&str, DataFrame)>) -> BuildInputs {
BuildInputs {
frames: pairs.into_iter().map(|(k, v)| (k.to_string(), v)).collect(),
}
}
const ONE_FRAME_SPEC: &str = r#"{"files": {"rows": {"format": "frame"}},
"nodes": {"Person": {"file": "rows", "pk": "id"}}}"#;
#[test]
fn a_declared_frame_that_was_not_passed_names_itself() {
let mut graph = DirGraph::new();
let err = build(
&mut graph,
bp(ONE_FRAME_SPEC),
Path::new("."),
BuildInputs::default(),
)
.err()
.expect("a declared frame with no rows cannot build");
assert_eq!(
err,
"frame 'rows' is declared in `files` but was not passed in `frames=`"
);
}
#[test]
fn a_passed_frame_that_was_not_declared_names_itself() {
let mut graph = DirGraph::new();
let err = build(
&mut graph,
bp(ONE_FRAME_SPEC),
Path::new("."),
frames(vec![("rows", one_int_frame()), ("extra", one_int_frame())]),
)
.err()
.expect("a frame the blueprint never declared cannot be read");
assert_eq!(
err,
"frames= contains 'extra', which is not declared in `files`"
);
}
#[test]
fn several_undeclared_frames_are_all_named() {
let mut graph = DirGraph::new();
let err = build(
&mut graph,
bp(ONE_FRAME_SPEC),
Path::new("."),
frames(vec![
("rows", one_int_frame()),
("z", one_int_frame()),
("a", one_int_frame()),
]),
)
.err()
.expect("undeclared frames cannot be read");
assert_eq!(
err,
"frames= contains 'a', 'z', which are not declared in `files`"
);
}
#[test]
fn a_supplied_frame_builds_its_nodes() {
let mut graph = DirGraph::new();
let report = build(
&mut graph,
bp(ONE_FRAME_SPEC),
Path::new("."),
frames(vec![("rows", one_int_frame())]),
)
.unwrap_or_else(|e| panic!("a supplied frame builds: {e}"));
assert!(report.errors.is_empty(), "{:?}", report.errors);
assert_eq!(report.nodes_by_type.get("Person"), Some(&2));
}
#[test]
fn a_frame_column_outside_the_vocabulary_warns_in_the_report() {
let mut df = one_int_frame();
df.add_column(
"m".to_string(),
ColumnType::Map,
ColumnData::Map(vec![None, None]),
)
.unwrap();
let mut graph = DirGraph::new();
let report = build(
&mut graph,
bp(ONE_FRAME_SPEC),
Path::new("."),
frames(vec![("rows", df)]),
)
.unwrap_or_else(|e| panic!("a supplied frame builds: {e}"));
assert!(
report
.warnings
.iter()
.any(|w| w.contains("frame 'rows' column 'm' has type map")),
"{:?}",
report.warnings
);
}
}
#[cfg(test)]
mod render_text_tests {
use super::*;
fn report() -> BuildReport {
BuildReport {
nodes_by_type: [("Person".to_string(), 3), ("Org".to_string(), 1)]
.into_iter()
.collect(),
edges_by_type: [("KNOWS".to_string(), 5), ("WORKS_AT".to_string(), 2)]
.into_iter()
.collect(),
edges_actual: [("KNOWS".to_string(), 4), ("WORKS_AT".to_string(), 2)]
.into_iter()
.collect(),
warnings: vec!["ignored".to_string()],
errors: vec!["ignored".to_string()],
provisional_purged: 2,
}
}
#[test]
fn verbose_text_is_the_summary_the_wheel_prints() {
let expected = [
"Loading blueprint...",
" Org: 1 nodes",
" Person: 3 nodes",
" [KNOWS]: 4 edges (5 input rows, 1 deduped)",
" [WORKS_AT]: 2 edges",
"Loaded 4 nodes (2 types), 6 edges (2 types) \u{2014} 7 input rows, 1 deduped",
" auto_purge: dropped 2 unpromoted provisional stub node(s)",
"",
]
.join("\n");
assert_eq!(report().render_text(true), expected);
}
#[test]
fn a_non_verbose_report_renders_nothing() {
assert_eq!(report().render_text(false), "");
}
#[test]
fn text_without_dedupe_carries_no_input_row_annotation() {
let mut r = report();
r.edges_actual
.insert("KNOWS".to_string(), 5)
.expect("the fixture has this type");
r.provisional_purged = 0;
let expected = [
"Loading blueprint...",
" Org: 1 nodes",
" Person: 3 nodes",
" [KNOWS]: 5 edges",
" [WORKS_AT]: 2 edges",
"Loaded 4 nodes (2 types), 7 edges (2 types)",
"",
]
.join("\n");
assert_eq!(r.render_text(true), expected);
}
}