use std::collections::HashSet;
use indexmap::IndexMap;
use super::expr;
use super::input::{input_format, input_format_names};
use super::schema::{Blueprint, ComputeOp, FileSpec, JunctionEdge, NodeSpec};
pub fn validate_compute(blueprint: &Blueprint) -> Result<(), String> {
let mut known: HashSet<String> = blueprint.nodes.keys().cloned().collect();
for spec in blueprint.nodes.values() {
for sub in spec.sub_nodes.keys() {
known.insert(sub.clone());
}
}
for (i, op) in blueprint.compute.iter().enumerate() {
validate_op(op, &mut known, i).map_err(|e| format!("blueprint compute[{}]: {}", i, e))?;
check_compute_inputs(blueprint, op)
.map_err(|e| format!("blueprint compute[{}]: {}", i, e))?;
}
Ok(())
}
fn refuse_non_csv_compute_input(
source_type: &str,
input_name: &str,
format: &str,
) -> Result<(), String> {
if format == "csv" {
return Ok(());
}
Err(format!(
"source type '{source_type}' reads input '{input_name}' (format '{format}'), but \
compute reads CSV files only. Materialise that input as CSV, or drop the compute op."
))
}
fn check_compute_inputs(blueprint: &Blueprint, op: &ComputeOp) -> Result<(), String> {
let sources: Vec<&str> = match op {
ComputeOp::Derive { from, .. }
| ComputeOp::Filter { from, .. }
| ComputeOp::Chain { from, .. }
| ComputeOp::Aggregate { from, .. } => vec![from.as_str()],
ComputeOp::Calendar { links, .. } => links.iter().map(|l| l.from.as_str()).collect(),
};
for source_type in sources {
let Some(spec) = super::compute::resolve_source_spec(blueprint, source_type) else {
continue;
};
let Some(name) = spec.file.as_deref() else {
continue;
};
if let Some(file) = blueprint.files.get(name) {
refuse_non_csv_compute_input(source_type, name, &file.format)?;
}
}
Ok(())
}
fn gated_format_hint(format: &str) -> String {
const GATED: &[(&str, &str)] = &[("xlsx", "xlsx")];
if input_format(format).is_some() {
return String::new();
}
match GATED.iter().find(|(name, _)| *name == format) {
Some((_, feature)) => format!(
" '{format}' is one this crate reads, but only when built with the `{feature}` \
Cargo feature — rebuild kglite with it (the Python wheel has it)."
),
None => String::new(),
}
}
pub fn validate_inputs(blueprint: &Blueprint) -> Result<(), String> {
for (name, file) in &blueprint.files {
if input_format(&file.format).is_none() {
return Err(format!(
"files '{name}': unknown format '{}' — this build reads {}.{}",
file.format,
input_format_names(),
gated_format_hint(&file.format)
));
}
let format = input_format(&file.format).expect("checked above");
if format.accepted_keys.contains(&"path") && file.path.is_none() {
return Err(format!(
"files '{name}': no 'path' — a '{}' input must name the file it reads.",
file.format
));
}
(format.validate_entry)(name, file)?;
}
fn walk(blueprint: &Blueprint, node_type: &str, spec: &NodeSpec) -> Result<(), String> {
check_spec_input(
&format!("node '{node_type}'"),
spec.csv.as_deref(),
spec.file.as_deref(),
false,
&blueprint.files,
)?;
for (edge_type, junc) in &spec.connections.junction_edges {
check_junction_input(node_type, edge_type, junc, &blueprint.files)?;
}
for (sub_type, sub) in &spec.sub_nodes {
walk(blueprint, sub_type, sub)?;
}
Ok(())
}
for (node_type, spec) in &blueprint.nodes {
walk(blueprint, node_type, spec)?;
}
Ok(())
}
fn check_junction_input(
node_type: &str,
edge_type: &str,
junc: &JunctionEdge,
files: &IndexMap<String, FileSpec>,
) -> Result<(), String> {
check_spec_input(
&format!("junction '{edge_type}' (node '{node_type}')"),
junc.csv.as_deref(),
junc.file.as_deref(),
true,
files,
)
}
fn check_spec_input(
where_: &str,
csv: Option<&str>,
file: Option<&str>,
required: bool,
files: &IndexMap<String, FileSpec>,
) -> Result<(), String> {
match (csv, file) {
(Some(csv), Some(file)) => {
return Err(format!(
"{where_}: both 'csv' and 'file' are set ('{csv}' and '{file}') — a spec reads \
one input. Keep 'file' and drop 'csv', or the other way round."
));
}
(None, None) if required => {
return Err(format!(
"{where_}: neither 'csv' nor 'file' — a junction table has no rows without one."
));
}
(None, Some(name)) if !files.contains_key(name) => {
let declared = if files.is_empty() {
"no inputs are declared in 'files'".to_string()
} else {
format!(
"declared inputs: {}",
files.keys().cloned().collect::<Vec<_>>().join(", ")
)
};
return Err(format!(
"{where_}: \"file\": \"{name}\" is not declared in 'files'; {declared}."
));
}
(Some(csv), None) => {
if let Some(entry) = files.get(csv) {
let same_file = entry.format == "csv" && entry.path.as_deref() == Some(csv);
if !same_file {
return Err(format!(
"{where_}: \"csv\": \"{csv}\" collides with the 'files' entry named \
'{csv}', which reads '{}' as '{}'. Rename that entry, or reference it \
with \"file\": \"{csv}\".",
entry.path.as_deref().unwrap_or("<no path>"),
entry.format
));
}
}
}
_ => {}
}
Ok(())
}
fn validate_op(op: &ComputeOp, known: &mut HashSet<String>, _idx: usize) -> Result<(), String> {
match op {
ComputeOp::Derive { from, set } => {
if !known.contains(from) {
return Err(format!("derive: unknown source type '{}'", from));
}
if set.is_empty() {
return Err("derive: 'set' must declare at least one property".to_string());
}
for (prop, src) in set {
let ast = expr::parse(src)
.map_err(|e| format!("derive '{}': expression parse: {}", prop, e))?;
check_no_aggregate(&ast).map_err(|e| format!("derive '{}': {}", prop, e))?;
}
}
ComputeOp::Filter {
from,
where_expr,
into,
} => {
if !known.contains(from) {
return Err(format!("filter: unknown source type '{}'", from));
}
let ast =
expr::parse(where_expr).map_err(|e| format!("filter 'where' parse: {}", e))?;
check_no_aggregate(&ast).map_err(|e| format!("filter 'where': {}", e))?;
if let Some(new_type) = into {
if known.contains(new_type) {
return Err(format!(
"filter: 'into' type '{}' collides with existing type",
new_type
));
}
known.insert(new_type.clone());
}
}
ComputeOp::Chain {
from,
group_by,
order_by,
edge,
} => {
if !known.contains(from) {
return Err(format!("chain: unknown source type '{}'", from));
}
if group_by.is_empty() {
return Err("chain: 'group_by' must be non-empty".to_string());
}
if order_by.is_empty() {
return Err("chain: 'order_by' required".to_string());
}
if edge.is_empty() {
return Err("chain: 'edge' name required".to_string());
}
}
ComputeOp::Calendar {
node_type,
start,
end,
links,
in_month_edge,
in_quarter_edge,
in_year_edge,
..
} => {
validate_iso_date("start", start)?;
validate_iso_date("end", end)?;
if start > end {
return Err(format!(
"calendar: start ({}) must be <= end ({})",
start, end
));
}
if node_type.is_empty() {
return Err("calendar: node_type required".to_string());
}
if known.contains(node_type) {
return Err(format!(
"calendar: node_type '{}' collides with existing type",
node_type
));
}
known.insert(node_type.clone());
if in_month_edge.is_some() {
known.insert("Month".to_string());
}
if in_quarter_edge.is_some() {
known.insert("Quarter".to_string());
}
if in_year_edge.is_some() {
known.insert("Year".to_string());
}
for link in links {
if !known.contains(&link.from) {
return Err(format!(
"calendar link: unknown source type '{}'",
link.from
));
}
if link.date_col.is_empty() {
return Err(format!(
"calendar link from '{}': 'date_col' required",
link.from
));
}
if link.edge.is_empty() {
return Err(format!(
"calendar link from '{}': 'edge' name required",
link.from
));
}
}
}
ComputeOp::Aggregate {
from,
into,
agg,
edges,
group_by,
..
} => {
if !known.contains(from) {
return Err(format!("aggregate: unknown source type '{}'", from));
}
if known.contains(into) {
return Err(format!(
"aggregate: 'into' type '{}' collides with existing type",
into
));
}
if group_by.is_empty() {
return Err("aggregate: 'group_by' must be non-empty".to_string());
}
if agg.is_empty() {
return Err(
"aggregate: 'agg' must declare at least one aggregated property".to_string(),
);
}
for (prop, src) in agg {
expr::parse(src)
.map_err(|e| format!("aggregate '{}': expression parse: {}", prop, e))?;
}
known.insert(into.clone());
for edge in edges {
if !known.contains(&edge.to) {
return Err(format!(
"aggregate edge → '{}': unknown target type",
edge.to
));
}
if edge.fk.is_empty() {
return Err(format!(
"aggregate edge → '{}': 'fk' name required",
edge.to
));
}
if edge.edge.is_empty() {
return Err(format!(
"aggregate edge → '{}': 'edge' name required",
edge.to
));
}
}
}
}
Ok(())
}
fn check_no_aggregate(e: &expr::Expr) -> Result<(), String> {
match e {
expr::Expr::Call(name, args) => {
if expr::is_aggregate_fn(name) {
return Err(format!(
"aggregate function '{}' not allowed in row-level expression",
name
));
}
for (_kw, arg) in args {
check_no_aggregate(arg)?;
}
Ok(())
}
expr::Expr::Unary(_, inner) => check_no_aggregate(inner),
expr::Expr::Binary(_, lhs, rhs) => {
check_no_aggregate(lhs)?;
check_no_aggregate(rhs)
}
expr::Expr::List(items) => {
for item in items {
check_no_aggregate(item)?;
}
Ok(())
}
expr::Expr::Literal(_) | expr::Expr::Ident(_) => Ok(()),
}
}
fn validate_iso_date(field: &str, val: &str) -> Result<(), String> {
if val.len() != 10 {
return Err(format!(
"calendar '{}': expected YYYY-MM-DD (10 chars), got '{}'",
field, val
));
}
let bytes = val.as_bytes();
for (i, &b) in bytes.iter().enumerate() {
let ok = match i {
4 | 7 => b == b'-',
_ => b.is_ascii_digit(),
};
if !ok {
return Err(format!(
"calendar '{}': expected YYYY-MM-DD, got '{}'",
field, val
));
}
}
Ok(())
}
pub fn unknown_property_type_warnings(blueprint: &Blueprint) -> Vec<String> {
let mut warnings = Vec::new();
fn is_known(ty: &str) -> bool {
super::typing::map_blueprint_type(ty).is_some()
|| matches!(ty, "geometry" | "location.lat" | "location.lon")
}
fn check(warnings: &mut Vec<String>, where_: &str, kind: &str, map: &IndexMap<String, String>) {
for (col, ty) in map {
if !is_known(ty) {
warnings.push(format!(
"{where_}: unknown {kind} value '{ty}' for column '{col}' — not a type \
keyword (string|int|float|bool|date|datetime|list|array|validFrom|validTo) \
or spatial target (geometry|location.lat|location.lon). The value is \
ignored and the column type is inferred; note this map declares types, it \
does not rename columns."
));
}
}
}
fn walk(warnings: &mut Vec<String>, node_type: &str, spec: &NodeSpec) {
check(
warnings,
&format!("node '{node_type}'"),
"properties",
&spec.properties,
);
for (edge_type, fk) in &spec.connections.fk_edges {
check(
warnings,
&format!("fk_edge '{edge_type}' (node '{node_type}')"),
"property_types",
&fk.property_types,
);
}
for (edge_type, junc) in &spec.connections.junction_edges {
check(
warnings,
&format!("junction '{edge_type}' (node '{node_type}')"),
"property_types",
&junc.property_types,
);
}
for (sub_type, sub) in &spec.sub_nodes {
walk(warnings, sub_type, sub);
}
}
for (node_type, spec) in &blueprint.nodes {
walk(&mut warnings, node_type, spec);
}
warnings
}
pub fn unknown_key_warnings(blueprint: &Blueprint) -> Vec<String> {
use super::schema::{
ACCEPTED_BLUEPRINT_KEYS, ACCEPTED_FK_EDGE_KEYS, ACCEPTED_JUNCTION_EDGE_KEYS,
ACCEPTED_NODE_KEYS, ACCEPTED_SETTINGS_KEYS,
};
fn check(
warnings: &mut Vec<String>,
where_: &str,
extra: &IndexMap<String, serde_json::Value>,
accepted: &[&str],
) {
check_but(warnings, where_, extra, accepted, &[]);
}
fn check_but(
warnings: &mut Vec<String>,
where_: &str,
extra: &IndexMap<String, serde_json::Value>,
accepted: &[&str],
read: &[&str],
) {
for key in extra.keys() {
if read.contains(&key.as_str()) {
continue;
}
let hint = crate::graph::mutation::validation::did_you_mean(key, accepted);
let hint = if hint.is_empty() {
let list = accepted
.iter()
.map(|k| format!("'{k}'"))
.collect::<Vec<_>>()
.join(", ");
format!(" Accepted keys: {list}.")
} else {
hint
};
warnings.push(format!(
"{where_}: unknown key '{key}' — the loader does not read it, so anything it \
declares is ignored.{hint}"
));
}
}
fn walk(warnings: &mut Vec<String>, node_type: &str, spec: &NodeSpec) {
check(
warnings,
&format!("node '{node_type}'"),
&spec.extra,
ACCEPTED_NODE_KEYS,
);
for (edge_type, fk) in &spec.connections.fk_edges {
check(
warnings,
&format!("fk_edge '{edge_type}' (node '{node_type}')"),
&fk.extra,
ACCEPTED_FK_EDGE_KEYS,
);
}
for (edge_type, junc) in &spec.connections.junction_edges {
check(
warnings,
&format!("junction '{edge_type}' (node '{node_type}')"),
&junc.extra,
ACCEPTED_JUNCTION_EDGE_KEYS,
);
}
for (sub_type, sub) in &spec.sub_nodes {
walk(warnings, sub_type, sub);
}
}
let mut warnings = Vec::new();
check(
&mut warnings,
"blueprint",
&blueprint.extra,
ACCEPTED_BLUEPRINT_KEYS,
);
check(
&mut warnings,
"settings",
&blueprint.settings.extra,
ACCEPTED_SETTINGS_KEYS,
);
for (name, file) in &blueprint.files {
let Some(format) = input_format(&file.format) else {
continue;
};
check_but(
&mut warnings,
&format!("file '{name}' (format '{}')", file.format),
&file.extra,
format.accepted_keys,
format.knob_keys,
);
if file.path.is_some() && !format.accepted_keys.contains(&"path") {
warnings.push(format!(
"file '{name}' (format '{}'): unknown key 'path' — the loader does not read it, so anything it declares is ignored. Accepted keys: {}.",
file.format,
format
.accepted_keys
.iter()
.map(|k| format!("'{k}'"))
.collect::<Vec<_>>()
.join(", ")
));
}
}
for (node_type, spec) in &blueprint.nodes {
walk(&mut warnings, node_type, spec);
}
warnings
}
#[cfg(test)]
mod tests {
use super::*;
use crate::graph::blueprint::schema::*;
fn bp_from_json(s: &str) -> Blueprint {
serde_json::from_str(s).expect("blueprint JSON parse")
}
#[test]
fn unknown_property_type_values_warn() {
let bp = bp_from_json(
r#"{"nodes": {"Person": {
"csv": "p.csv", "pk": "id",
"properties": {"age": "int", "geom": "geometry", "born": "birthDate"},
"connections": {
"fk_edges": {"IN_ORG": {
"target": "Org", "fk": "org_id",
"properties": ["since"], "property_types": {"since": "sinceWhen"}
}},
"junction_edges": {"KNOWS": {
"csv": "k.csv", "source_fk": "a", "target": "Person", "target_fk": "b",
"property_types": {"from": "validFrom", "to": "renamedTo"}
}}
},
"sub_nodes": {"Pet": {"csv": "pets.csv", "pk": "id",
"properties": {"kind": "sting"}}}
}}}"#,
);
let warnings = unknown_property_type_warnings(&bp);
assert_eq!(warnings.len(), 4, "{warnings:?}");
assert!(warnings
.iter()
.any(|w| w.contains("'birthDate'") && w.contains("node 'Person'")));
assert!(warnings
.iter()
.any(|w| w.contains("'sinceWhen'") && w.contains("fk_edge 'IN_ORG'")));
assert!(warnings
.iter()
.any(|w| w.contains("'renamedTo'") && w.contains("junction 'KNOWS'")));
assert!(warnings
.iter()
.any(|w| w.contains("'sting'") && w.contains("node 'Pet'")));
assert!(!warnings
.iter()
.any(|w| w.contains("'int'") || w.contains("'geometry'") || w.contains("'validFrom'")));
}
#[test]
fn empty_compute_validates() {
let bp = bp_from_json(r#"{"nodes": {}}"#);
validate_compute(&bp).unwrap();
}
#[test]
fn derive_validates_against_existing_type() {
let bp = bp_from_json(
r#"{
"nodes": {"T": {}},
"compute": [{
"op": "derive",
"from": "T",
"set": {"x": "a + b"}
}]
}"#,
);
validate_compute(&bp).unwrap();
}
#[test]
fn derive_rejects_unknown_source() {
let bp = bp_from_json(
r#"{
"nodes": {},
"compute": [{
"op": "derive",
"from": "Ghost",
"set": {"x": "1"}
}]
}"#,
);
let err = validate_compute(&bp).unwrap_err();
assert!(err.contains("Ghost"), "{err}");
}
#[test]
fn derive_rejects_aggregate_fn() {
let bp = bp_from_json(
r#"{
"nodes": {"T": {}},
"compute": [{
"op": "derive",
"from": "T",
"set": {"x": "sum(a)"}
}]
}"#,
);
let err = validate_compute(&bp).unwrap_err();
assert!(err.contains("aggregate function 'sum'"), "{err}");
}
#[test]
fn derive_rejects_bad_expression() {
let bp = bp_from_json(
r#"{
"nodes": {"T": {}},
"compute": [{
"op": "derive",
"from": "T",
"set": {"x": "1 + + 2"}
}]
}"#,
);
let err = validate_compute(&bp).unwrap_err();
assert!(err.contains("parse"), "{err}");
}
#[test]
fn filter_into_registers_new_type() {
let bp = bp_from_json(
r#"{
"nodes": {"MetricFact": {}},
"compute": [
{"op": "filter", "from": "MetricFact",
"where": "tag == 'Revenues'", "into": "AnnualRevenue"},
{"op": "derive", "from": "AnnualRevenue",
"set": {"value_b": "value / 1e9"}}
]
}"#,
);
validate_compute(&bp).unwrap();
}
#[test]
fn filter_into_rejects_collision() {
let bp = bp_from_json(
r#"{
"nodes": {"T": {}, "U": {}},
"compute": [{
"op": "filter", "from": "T", "where": "true", "into": "U"
}]
}"#,
);
assert!(validate_compute(&bp).is_err());
}
#[test]
fn chain_validates_required_fields() {
let bp = bp_from_json(
r#"{
"nodes": {"T": {}},
"compute": [{"op": "chain", "from": "T", "group_by": [],
"order_by": "date", "edge": "NEXT"}]
}"#,
);
assert!(validate_compute(&bp).is_err());
}
#[test]
fn calendar_validates_dates() {
let bp = bp_from_json(
r#"{
"nodes": {},
"compute": [{"op": "calendar", "type": "Date",
"start": "not-a-date", "end": "2030-12-31"}]
}"#,
);
assert!(validate_compute(&bp).is_err());
let bp = bp_from_json(
r#"{
"nodes": {},
"compute": [{"op": "calendar", "type": "Date",
"start": "2030-01-01", "end": "2020-12-31"}]
}"#,
);
assert!(validate_compute(&bp).is_err());
let bp = bp_from_json(
r#"{
"nodes": {},
"compute": [{"op": "calendar", "type": "Date",
"start": "2020-01-01", "end": "2030-12-31"}]
}"#,
);
validate_compute(&bp).unwrap();
}
#[test]
fn calendar_link_registers_after_calendar() {
let bp = bp_from_json(
r#"{
"nodes": {"Transaction": {}},
"compute": [{
"op": "calendar", "type": "Date",
"start": "2020-01-01", "end": "2030-12-31",
"links": [
{"from": "Transaction", "date_col": "transaction_date",
"edge": "ON_DATE"}
]
}]
}"#,
);
validate_compute(&bp).unwrap();
}
#[test]
fn aggregate_validates_into_and_edges() {
let bp = bp_from_json(
r#"{
"nodes": {"Transaction": {}, "Person": {}, "Company": {}},
"compute": [{
"op": "aggregate",
"from": "Transaction",
"group_by": ["person_nid", "issuer_cik"],
"into": "Position",
"agg": {"current_shares": "last(shares_owned_after, by=transaction_date)"},
"edges": [
{"to": "Person", "fk": "person_nid", "edge": "OF_PERSON"},
{"to": "Company", "fk": "issuer_cik", "edge": "AT_COMPANY"}
]
}]
}"#,
);
validate_compute(&bp).unwrap();
}
#[test]
fn aggregate_allows_aggregate_fns() {
let bp = bp_from_json(
r#"{
"nodes": {"T": {}},
"compute": [{
"op": "aggregate", "from": "T", "into": "U",
"group_by": ["k"],
"agg": {"s": "sum(x)", "c": "count(*)"}
}]
}"#,
);
validate_compute(&bp).unwrap();
}
#[test]
fn op_can_reference_earlier_created_type() {
let bp = bp_from_json(
r#"{
"nodes": {"T": {}},
"compute": [
{"op": "aggregate", "from": "T", "into": "Summary",
"group_by": ["k"], "agg": {"n": "count(*)"}},
{"op": "derive", "from": "Summary",
"set": {"n_scaled": "n * 100"}}
]
}"#,
);
validate_compute(&bp).unwrap();
}
}
#[cfg(test)]
mod input_tests {
use super::*;
fn bp(s: &str) -> Blueprint {
serde_json::from_str(s).expect("blueprint JSON parse")
}
#[test]
fn a_stray_key_in_a_files_entry_warns_against_its_own_format() {
let bp = bp(r#"{"files": {"people": {"path": "p.csv", "delimiter": "\t"}}}"#);
let warnings = unknown_key_warnings(&bp);
assert_eq!(warnings.len(), 1, "{warnings:?}");
let w = &warnings[0];
assert!(w.contains("file 'people' (format 'csv')"), "{w}");
assert!(w.contains("unknown key 'delimiter'"), "{w}");
assert!(w.contains("Accepted keys: 'path', 'format'"), "{w}");
}
#[test]
fn a_well_formed_files_entry_is_silent() {
let bp = bp(r#"{"files": {"people": {"path": "p.csv", "format": "csv"}}}"#);
assert!(unknown_key_warnings(&bp).is_empty());
}
#[test]
fn a_delimited_entrys_knobs_are_not_stray_keys() {
let bp = bp(
r#"{"files": {"taxa": {"path": "nodes.dmp", "format": "delimited",
"delimiter": "\t|\t", "line_suffix": "\t|", "header": false,
"columns": ["id", "parent"], "skip_lines": 0, "encoding": "utf-8",
"prefix_strip": {"id": "x:"}}}}"#,
);
validate_inputs(&bp).expect("a well-formed delimited entry");
assert!(
unknown_key_warnings(&bp).is_empty(),
"{:?}",
unknown_key_warnings(&bp)
);
}
#[test]
fn a_stray_key_on_a_delimited_entry_still_warns() {
let bp = bp(
r#"{"files": {"taxa": {"path": "x.tsv", "format": "delimited",
"delimiter": "\t", "sheet": 2}}}"#,
);
let warnings = unknown_key_warnings(&bp);
assert_eq!(warnings.len(), 1, "{warnings:?}");
assert!(
warnings[0].contains("file 'taxa' (format 'delimited')"),
"{warnings:?}"
);
assert!(warnings[0].contains("unknown key 'sheet'"), "{warnings:?}");
}
#[test]
fn a_delimited_entry_with_an_unreadable_declaration_fails_the_build() {
let bp = bp(r#"{"files": {"taxa": {"path": "x.tsv", "format": "delimited"}}}"#);
let err = validate_inputs(&bp).expect_err("a delimited entry needs a delimiter");
assert!(err.contains("files 'taxa'"), "{err}");
assert!(err.contains("needs a 'delimiter'"), "{err}");
}
#[test]
fn compute_over_a_delimited_input_is_refused() {
let bp = bp(
r#"{"files": {"taxa": {"path": "nodes.dmp", "format": "delimited", "delimiter": "\t"}},
"nodes": {"Taxon": {"file": "taxa", "pk": "id"}},
"compute": [{"op": "derive", "from": "Taxon", "set": {"x": "1"}}]}"#,
);
let err = validate_compute(&bp).expect_err("compute over a delimited input is refused");
assert!(err.contains("input 'taxa' (format 'delimited')"), "{err}");
assert!(err.contains("compute reads CSV files only"), "{err}");
}
#[test]
fn an_unknown_format_suppresses_the_key_warning_and_fails_the_build() {
let bp = bp(r#"{"files": {"t": {"path": "t.parquet", "format": "parquet", "row": 2}}}"#);
assert!(unknown_key_warnings(&bp).is_empty());
let err = validate_inputs(&bp).expect_err("an unreadable format fails the build");
assert!(err.contains("unknown format 'parquet'"), "{err}");
assert!(err.contains("'csv'"), "{err}");
assert!(!err.contains("Cargo feature"), "{err}");
}
#[cfg(not(feature = "xlsx"))]
#[test]
fn a_feature_gated_format_says_which_feature_is_missing() {
let bp = bp(r#"{"files": {"sheet": {"path": "s.xlsx", "format": "xlsx"}}}"#);
let err = validate_inputs(&bp).expect_err("an uncompiled format fails the build");
assert!(err.contains("unknown format 'xlsx'"), "{err}");
assert!(err.contains("`xlsx` Cargo feature"), "{err}");
assert!(err.contains("the Python wheel has it"), "{err}");
}
#[cfg(feature = "xlsx")]
#[test]
fn a_stray_knob_on_an_xlsx_entry_warns_and_the_build_stands() {
let bp = bp(
r#"{"files": {"sheet": {"path": "s.xlsx", "format": "xlsx", "sheet": "drugs",
"header_rows": 3}}}"#,
);
validate_inputs(&bp).expect("a compiled-in format reads");
let warnings = unknown_key_warnings(&bp);
assert_eq!(warnings.len(), 1, "{warnings:?}");
assert!(
warnings[0].contains("unknown key 'header_rows'"),
"{warnings:?}"
);
assert!(warnings[0].contains("'header_row'"), "{warnings:?}");
}
#[test]
fn a_frame_entry_needs_no_path() {
let bp = bp(r#"{"files": {"rows": {"format": "frame"}}}"#);
validate_inputs(&bp).expect("a frame entry names no file");
assert!(unknown_key_warnings(&bp).is_empty());
}
#[test]
fn a_path_on_a_frame_entry_is_a_stray_key() {
let bp = bp(r#"{"files": {"rows": {"format": "frame", "path": "rows.csv"}}}"#);
validate_inputs(&bp).expect("a stray key warns, it does not fail the build");
let warnings = unknown_key_warnings(&bp);
assert_eq!(warnings.len(), 1, "{warnings:?}");
let w = &warnings[0];
assert!(w.contains("file 'rows' (format 'frame')"), "{w}");
assert!(w.contains("unknown key 'path'"), "{w}");
assert!(w.contains("Accepted keys: 'format'"), "{w}");
}
#[test]
fn compute_over_a_frame_input_is_refused() {
let bp = bp(r#"{"files": {"rows": {"format": "frame"}},
"nodes": {"Person": {"file": "rows", "pk": "id"}},
"compute": [{"op": "derive", "from": "Person", "set": {"x": "1"}}]}"#);
let err = validate_compute(&bp).expect_err("compute over a frame is refused");
assert!(err.contains("source type 'Person'"), "{err}");
assert!(err.contains("input 'rows' (format 'frame')"), "{err}");
assert!(err.contains("compute reads CSV files only"), "{err}");
}
#[test]
fn a_csv_entry_must_name_a_path() {
let bp = bp(r#"{"files": {"people": {"format": "csv"}}}"#);
let err = validate_inputs(&bp).expect_err("an entry with no source fails");
assert!(err.contains("files 'people'"), "{err}");
assert!(err.contains("no 'path'"), "{err}");
}
#[test]
fn a_spec_reads_one_input_not_two() {
let bp = bp(r#"{"files": {"p": {"path": "p.csv"}},
"nodes": {"Person": {"csv": "p.csv", "file": "p", "pk": "id"}}}"#);
let err = validate_inputs(&bp).expect_err("csv + file on one spec fails");
assert!(err.contains("node 'Person'"), "{err}");
assert!(err.contains("both 'csv' and 'file'"), "{err}");
}
#[test]
fn an_undeclared_file_name_lists_the_declared_ones() {
let bp = bp(
r#"{"files": {"people": {"path": "p.csv"}, "orgs": {"path": "o.csv"}},
"nodes": {"Person": {"file": "pepole", "pk": "id"}}}"#,
);
let err = validate_inputs(&bp).expect_err("an undeclared name fails");
assert!(err.contains("\"file\": \"pepole\""), "{err}");
assert!(err.contains("declared inputs: people, orgs"), "{err}");
}
#[test]
fn an_entry_shadowing_a_shorthand_for_another_file_is_refused() {
let bp = bp(r#"{"files": {"p.csv": {"path": "other.csv"}},
"nodes": {"Person": {"csv": "p.csv", "pk": "id"}}}"#);
let err = validate_inputs(&bp).expect_err("two files, one registry name");
assert!(err.contains("collides"), "{err}");
assert!(err.contains("other.csv"), "{err}");
}
#[test]
fn an_entry_naming_the_same_file_as_the_shorthand_is_one_input() {
let bp = bp(r#"{"files": {"p.csv": {"path": "p.csv", "format": "csv"}},
"nodes": {"Person": {"csv": "p.csv", "pk": "id"}}}"#);
validate_inputs(&bp).expect("the same file under the same name is one input");
}
#[test]
fn a_junction_must_name_an_input() {
let bp = bp(
r#"{"nodes": {"Person": {"csv": "p.csv", "pk": "id", "connections": {
"junction_edges": {"KNOWS": {
"source_fk": "a", "target": "Person", "target_fk": "b"}}}}}}"#,
);
let err = validate_inputs(&bp).expect_err("a junction with no table fails");
assert!(err.contains("junction 'KNOWS' (node 'Person')"), "{err}");
assert!(err.contains("neither 'csv' nor 'file'"), "{err}");
}
#[test]
fn a_sub_node_spec_is_walked_too() {
let bp = bp(r#"{"nodes": {"Person": {"csv": "p.csv", "pk": "id",
"sub_nodes": {"Pet": {"file": "nowhere", "pk": "id"}}}}}"#);
let err = validate_inputs(&bp).expect_err("a sub-node's input resolves too");
assert!(err.contains("node 'Pet'"), "{err}");
}
#[test]
fn compute_accepts_a_csv_input_and_refuses_any_other_format() {
refuse_non_csv_compute_input("Person", "people", "csv").expect("csv is what compute reads");
let err = refuse_non_csv_compute_input("Person", "sheet", "xlsx")
.expect_err("a non-CSV input is refused");
assert!(err.contains("'Person'"), "{err}");
assert!(err.contains("'sheet'"), "{err}");
assert!(err.contains("'xlsx'"), "{err}");
assert!(err.contains("compute reads CSV files only"), "{err}");
}
#[test]
fn validate_compute_refuses_an_op_over_a_non_csv_input() {
let json = r#"{
"files": {"sheet": {"path": "s.xlsx", "format": "FORMAT"}},
"nodes": {"Person": {"file": "sheet", "pk": "id"}},
"compute": [{"op": "derive", "from": "Person", "set": {"n": "id * 2"}}]
}"#;
validate_compute(&bp(&json.replace("FORMAT", "csv"))).expect("a CSV input is fine");
let err = validate_compute(&bp(&json.replace("FORMAT", "xlsx")))
.expect_err("compute over a non-CSV input is refused");
assert!(err.contains("blueprint compute[0]"), "{err}");
assert!(err.contains("'xlsx'"), "{err}");
}
#[test]
fn validate_compute_checks_a_calendar_link_source() {
let bp = bp(r#"{
"files": {"sheet": {"path": "s.xlsx", "format": "xlsx"}},
"nodes": {"Tx": {"file": "sheet", "pk": "id"}},
"compute": [{"op": "calendar", "start": "2020-01-01", "end": "2020-01-02",
"links": [{"from": "Tx", "date_col": "d", "edge": "ON"}]}]
}"#);
let err = validate_compute(&bp).expect_err("a calendar link source is checked");
assert!(err.contains("'Tx'"), "{err}");
}
}
#[cfg(test)]
mod accepted_key_tests {
use super::*;
use serde_json::{json, Map, Value};
fn fixture_values(level: &str) -> Vec<(&'static str, Value)> {
match level {
"blueprint" => vec![
("settings", json!({})),
("files", json!({})),
("nodes", json!({})),
("compute", json!([])),
("ontology", json!(null)),
],
"settings" => vec![
("input_root", json!(".")),
("root", json!(".")),
("output_path", json!(".")),
("output_file", json!("g.kgl")),
("output", json!("g.kgl")),
("auto_purge", json!(false)),
],
"node" => vec![
("csv", json!("p.csv")),
("file", json!(null)),
("pk", json!("id")),
("title", json!("name")),
("parent", json!("Org")),
("parent_fk", json!("org_id")),
("properties", json!({})),
("labels", json!([])),
("skipped", json!([])),
("filter", json!({})),
("connections", json!({})),
("sub_nodes", json!({})),
("timeseries", json!(null)),
],
"fk_edge" => vec![
("target", json!("Org")),
("fk", json!("org_id")),
("properties", json!([])),
("property_types", json!({})),
("rename", json!({})),
],
"file" => vec![("path", json!("x.csv")), ("format", json!("csv"))],
"file_frame" => vec![("format", json!("frame"))],
"file_delimited" => vec![
("path", json!("nodes.dmp")),
("format", json!("delimited")),
("delimiter", json!("\t|\t")),
("quote", json!(null)),
("header", json!(false)),
("columns", json!(["id", "parent"])),
("skip_lines", json!(0)),
("comment_prefix", json!("#")),
("line_suffix", json!("\t|")),
("encoding", json!("utf-8")),
("prefix_strip", json!({"id": "x:"})),
],
#[cfg(feature = "xlsx")]
"file_xlsx" => vec![
("path", json!("screen.xlsx")),
("format", json!("xlsx")),
("sheet", json!("drugs")),
("header_row", json!(1)),
(
"unpivot",
json!({"id_columns": ["id"], "name_to": "k", "value_to": "v"}),
),
],
"junction_edge" => vec![
("csv", json!("k.csv")),
("file", json!(null)),
("source_fk", json!("a")),
("target", json!("Person")),
("target_type_column", json!(null)),
("target_fk", json!("b")),
("properties", json!([])),
("property_types", json!({})),
("rename", json!({})),
],
other => panic!("no fixture for level {other}"),
}
}
#[test]
fn every_formats_knob_keys_are_accepted_keys_minus_the_struct_fields() {
use super::super::input::INPUT_FORMATS;
let struct_fields = ["path", "format"];
for format in INPUT_FORMATS {
let accepted: HashSet<&str> = format.accepted_keys.iter().copied().collect();
let knobs: HashSet<&str> = format.knob_keys.iter().copied().collect();
assert!(
knobs.is_subset(&accepted),
"format '{}': knob_keys names a key that is not accepted",
format.name
);
for key in &accepted {
assert!(
knobs.contains(key) || struct_fields.contains(key),
"format '{}': accepted key '{key}' is neither a FileSpec field nor a knob \
its reader takes from `extra` — it would warn as a stray key",
format.name
);
}
}
}
const ALIASES: &[(&str, &str)] = &[("root", "input_root"), ("output", "output_file")];
fn object(level: &str) -> Value {
object_with_alias(level, None)
}
fn object_with_alias(level: &str, alias: Option<(&str, &str)>) -> Value {
let mut map = Map::new();
for (key, value) in fixture_values(level) {
if ALIASES.iter().any(|(a, _)| *a == key) {
continue;
}
map.insert(key.to_string(), value);
}
if let Some((alias, canonical)) = alias {
let value = map
.remove(canonical)
.expect("alias substitutes a key the fixture holds");
map.insert(alias.to_string(), value);
}
Value::Object(map)
}
#[test]
fn accepted_key_lists_name_only_keys_the_specs_read() {
use super::super::input::csv::ACCEPTED_FILE_KEYS_CSV;
use super::super::input::delimited::ACCEPTED_FILE_KEYS_DELIMITED;
use super::super::input::frame::ACCEPTED_FILE_KEYS_FRAME;
#[cfg(feature = "xlsx")]
use super::super::input::xlsx::ACCEPTED_FILE_KEYS_XLSX;
use super::super::schema::{
ACCEPTED_BLUEPRINT_KEYS, ACCEPTED_FK_EDGE_KEYS, ACCEPTED_JUNCTION_EDGE_KEYS,
ACCEPTED_NODE_KEYS, ACCEPTED_SETTINGS_KEYS,
};
for (level, accepted) in [
("blueprint", ACCEPTED_BLUEPRINT_KEYS),
("settings", ACCEPTED_SETTINGS_KEYS),
("file", ACCEPTED_FILE_KEYS_CSV),
("file_delimited", ACCEPTED_FILE_KEYS_DELIMITED),
("file_frame", ACCEPTED_FILE_KEYS_FRAME),
#[cfg(feature = "xlsx")]
("file_xlsx", ACCEPTED_FILE_KEYS_XLSX),
("node", ACCEPTED_NODE_KEYS),
("fk_edge", ACCEPTED_FK_EDGE_KEYS),
("junction_edge", ACCEPTED_JUNCTION_EDGE_KEYS),
] {
let fixture: HashSet<&str> =
fixture_values(level).into_iter().map(|(k, _)| k).collect();
let listed: HashSet<&str> = accepted.iter().copied().collect();
assert_eq!(
listed, fixture,
"{level}: ACCEPTED list and this test's fixture disagree"
);
}
let blueprint: Value = object("blueprint");
let mut blueprint = blueprint;
blueprint["settings"] = object("settings");
let files = json!({
"in": object("file"),
"delim": object("file_delimited"),
"rows": object("file_frame"),
});
#[cfg(feature = "xlsx")]
let files = {
let mut files = files;
files["sheet"] = object("file_xlsx");
files
};
blueprint["files"] = files;
let mut node = object("node");
node["connections"] = json!({
"fk_edges": {"IN_ORG": object("fk_edge")},
"junction_edges": {"KNOWS": object("junction_edge")},
});
node["sub_nodes"] = json!({"Alias": object("node")});
blueprint["nodes"] = json!({"Person": node});
let parsed: Blueprint =
serde_json::from_value(blueprint).expect("every accepted key parses");
assert!(
unknown_key_warnings(&parsed).is_empty(),
"a listed key did not reach its struct field: {:?}",
unknown_key_warnings(&parsed)
);
for (alias, canonical) in ALIASES {
let mut blueprint = object("blueprint");
blueprint["settings"] = object_with_alias("settings", Some((alias, canonical)));
let parsed: Blueprint =
serde_json::from_value(blueprint).expect("the alias parses on its own");
assert!(
unknown_key_warnings(&parsed).is_empty(),
"settings alias '{alias}' is not an accepted spelling: {:?}",
unknown_key_warnings(&parsed)
);
}
}
}