use serde_json::{Map, Value};
pub const CURATED_CARD_FIELDS: &[&str] = &[
"title",
"description",
"license",
"source",
"created",
"version",
"creators",
"publisher",
"canonical_url",
"sparql_endpoint",
"source_date",
"derived_from",
"doi",
"cite_as",
"keywords",
"theme",
"extra",
"example_queries",
];
pub const UNKNOWN_FIELD_HINT: &str =
"\n (the card file's top level is reserved for rete-defined fields; \
publisher-defined fields go inside the \"extra\" object — \
see docs/dataset-cards.md)";
pub const CARD_EXTRA_MAX_BYTES: usize = 8192;
pub const CARD_EXTRA_MAX_KEYS: usize = 64;
pub const CARD_EXTRA_MAX_KEY_BYTES: usize = 128;
pub const CARD_DESCRIPTION_MAX_BYTES: usize = CARD_EXTRA_MAX_BYTES;
pub const CARD_EXTRA_MAX_DEPTH: usize = 2;
pub fn normalize_string_list(field: &str, values: Vec<String>) -> Result<Vec<String>, String> {
let mut out: Vec<String> = Vec::with_capacity(values.len());
for v in values {
let trimmed = v.trim();
if trimmed.is_empty() {
return Err(format!("card `{field}` entries must be non-empty"));
}
out.push(trimmed.to_string());
}
out.sort();
out.dedup();
Ok(out)
}
pub fn normalize_description(v: &Value) -> Result<String, String> {
let text = match v {
Value::String(s) => s.trim().to_string(),
Value::Array(lines) => {
let mut out = Vec::with_capacity(lines.len());
for l in lines {
out.push(l.as_str().ok_or_else(|| {
"card `description` must be a string, or an array of strings \
(one per line, joined with newlines)"
.to_string()
})?);
}
out.join("\n").trim().to_string()
}
_ => {
return Err(
"card `description` must be a string, or an array of strings \
(one per line, joined with newlines)"
.to_string(),
)
}
};
check_description_len(&text)?;
Ok(text)
}
pub fn check_description_len(text: &str) -> Result<(), String> {
if text.len() > CARD_DESCRIPTION_MAX_BYTES {
return Err(format!(
"card `description` is {} bytes, over the {CARD_DESCRIPTION_MAX_BYTES}-byte cap; \
every reader fetches the card on every open — a description is the dataset's \
abstract, not its documentation, so link out to the long form \
(`source` / `canonical_url`) or put it in the graph",
text.len()
));
}
Ok(())
}
pub fn normalize_themes(themes: Vec<String>) -> Result<Vec<String>, String> {
let themes = normalize_string_list("theme", themes)?;
for t in &themes {
if !(t.starts_with("http://") || t.starts_with("https://")) {
return Err(format!(
"card `theme` entry {t:?} is not an IRI: a theme is an IRI into a \
controlled vocabulary (e.g. the EU data-theme authority, \
http://publications.europa.eu/resource/authority/data-theme/…); \
free-text subjects belong in `keywords`"
));
}
}
Ok(themes)
}
pub fn normalize_extra(extra: Map<String, Value>) -> Result<Map<String, Value>, String> {
if extra.is_empty() {
return Ok(extra);
}
if extra.len() > CARD_EXTRA_MAX_KEYS {
return Err(format!(
"card `extra` has {} keys, over the {CARD_EXTRA_MAX_KEYS}-key cap",
extra.len()
));
}
let extra: Map<String, Value> = extra
.into_iter()
.map(|(k, v)| (k, canonicalize_json(v)))
.collect();
for (k, v) in &extra {
if k.is_empty() {
return Err("card `extra` keys must be non-empty".to_string());
}
if k == "@context" {
return Err("card `extra` key \"@context\" is reserved for a future \
author-supplied JSON-LD mapping"
.to_string());
}
if k.len() > CARD_EXTRA_MAX_KEY_BYTES {
return Err(format!(
"card `extra` key {k:?} is {} bytes, over the {CARD_EXTRA_MAX_KEY_BYTES}-byte cap",
k.len()
));
}
let depth = json_depth(v);
if depth > CARD_EXTRA_MAX_DEPTH {
return Err(format!(
"card `extra` field {k:?} nests {depth} container levels deep, \
over the {CARD_EXTRA_MAX_DEPTH}-level cap"
));
}
}
let bytes = serde_json::to_vec(&extra)
.map_err(|e| format!("card `extra` does not serialize: {e}"))?
.len();
if bytes > CARD_EXTRA_MAX_BYTES {
return Err(format!(
"card `extra` serializes to {bytes} bytes, over the {CARD_EXTRA_MAX_BYTES}-byte cap; \
every reader fetches the card on every open — trim the bag, \
or put bulk data in the graph itself"
));
}
Ok(extra)
}
pub fn json_depth(v: &Value) -> usize {
match v {
Value::Array(a) => 1 + a.iter().map(json_depth).max().unwrap_or(0),
Value::Object(o) => 1 + o.values().map(json_depth).max().unwrap_or(0),
_ => 0,
}
}
pub fn canonicalize_json(v: Value) -> Value {
match v {
Value::Array(a) => Value::Array(a.into_iter().map(canonicalize_json).collect()),
Value::Object(o) => {
let mut sorted: Vec<(String, Value)> = o.into_iter().collect();
sorted.sort_by(|a, b| a.0.cmp(&b.0));
let mut out = Map::new();
for (k, v) in sorted {
out.insert(k, canonicalize_json(v));
}
Value::Object(out)
}
scalar => scalar,
}
}
pub fn validate_curated_card(doc: &Value) -> Result<Value, String> {
let obj = doc
.as_object()
.ok_or_else(|| "a card document must be a JSON object".to_string())?;
for k in obj.keys() {
if !CURATED_CARD_FIELDS.contains(&k.as_str()) {
let expected = CURATED_CARD_FIELDS
.iter()
.map(|f| format!("`{f}`"))
.collect::<Vec<_>>()
.join(", ");
return Err(format!(
"unknown field `{k}`, expected one of {expected}{UNKNOWN_FIELD_HINT}"
));
}
}
let mut out = Map::new();
for (k, v) in obj {
if v.is_null() {
continue; }
let normalized = match k.as_str() {
"description" => Value::String(normalize_description(v)?),
"title" | "license" | "source" | "created" | "version" | "canonical_url"
| "sparql_endpoint" | "source_date" | "doi" | "cite_as" => {
Value::String(expect_string(k, v)?)
}
"derived_from" | "example_queries" => Value::from(expect_string_array(k, v)?),
"keywords" => Value::from(normalize_string_list(
"keywords",
expect_string_array(k, v)?,
)?),
"theme" => Value::from(normalize_themes(expect_string_array(k, v)?)?),
"creators" => {
let arr = v.as_array().ok_or_else(|| {
format!("card `{k}` must be an array of {{name, orcid}} objects")
})?;
let mut people = Vec::with_capacity(arr.len());
for c in arr {
people.push(named_agent("creators", c, "orcid")?);
}
Value::Array(people)
}
"publisher" => named_agent("publisher", v, "ror")?,
"extra" => {
let bag = v
.as_object()
.ok_or_else(|| "card `extra` must be a JSON object".to_string())?;
Value::Object(normalize_extra(bag.clone())?)
}
_ => unreachable!("field list and match arms are kept in step"),
};
let empty = match &normalized {
Value::String(s) => s.is_empty(),
Value::Array(a) => a.is_empty(),
Value::Object(o) => o.is_empty(),
_ => false,
};
if !empty {
out.insert(k.clone(), normalized);
}
}
Ok(Value::Object(out))
}
pub fn compose_curated_card(
curated: Value,
triple_count: u64,
quad_count: u64,
named_graph_count: u64,
term_count: u64,
format_version: u8,
) -> Value {
let mut card = curated;
let obj = match card.as_object_mut() {
Some(o) => o,
None => return card,
};
obj.insert("triple_count".into(), triple_count.into());
obj.insert("quad_count".into(), quad_count.into());
obj.insert("named_graph_count".into(), named_graph_count.into());
obj.insert("term_count".into(), term_count.into());
obj.insert("format_version".into(), format_version.into());
card
}
fn expect_string(field: &str, v: &Value) -> Result<String, String> {
v.as_str()
.map(|s| s.trim().to_string())
.ok_or_else(|| format!("card `{field}` must be a string"))
}
fn expect_string_array(field: &str, v: &Value) -> Result<Vec<String>, String> {
let arr = v
.as_array()
.ok_or_else(|| format!("card `{field}` must be an array of strings"))?;
arr.iter()
.map(|e| {
e.as_str()
.map(|s| s.to_string())
.ok_or_else(|| format!("card `{field}` must be an array of strings"))
})
.collect()
}
fn named_agent(field: &str, v: &Value, id_key: &str) -> Result<Value, String> {
let o = v
.as_object()
.ok_or_else(|| format!("card `{field}` entries must be objects with a `name`"))?;
for k in o.keys() {
if k != "name" && k != id_key {
return Err(format!(
"unknown field `{k}` in card `{field}`, expected one of `name`, `{id_key}`"
));
}
}
let name = o
.get("name")
.and_then(|n| n.as_str())
.ok_or_else(|| format!("card `{field}` entries need a `name`"))?;
let mut out = Map::new();
out.insert("name".into(), Value::String(name.trim().to_string()));
if let Some(id) = o.get(id_key).and_then(|i| i.as_str()) {
if !id.trim().is_empty() {
out.insert(id_key.into(), Value::String(id.trim().to_string()));
}
}
Ok(Value::Object(out))
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn unknown_top_level_key_is_rejected_with_the_bag_pointer() {
let e = validate_curated_card(&json!({"title": "t", "region": "CH"})).unwrap_err();
assert!(e.contains("unknown field `region`"), "{e}");
assert!(e.contains("\"extra\""), "{e}");
assert!(e.contains("`canonical_url`"), "{e}");
}
#[test]
fn theme_must_be_an_iri_and_points_at_keywords() {
let e = validate_curated_card(&json!({"theme": ["physics"]})).unwrap_err();
assert!(e.contains("not an IRI"), "{e}");
assert!(e.contains("keywords"), "{e}");
let ok = validate_curated_card(&json!({"theme": ["https://www.wikidata.org/entity/Q413"]}))
.unwrap();
assert_eq!(ok["theme"][0], "https://www.wikidata.org/entity/Q413");
}
#[test]
fn description_takes_a_string_or_an_array_of_lines() {
let md = "# What's inside\n\n- 4.4M classes\n- 5.1M edges";
let from_string = validate_curated_card(&json!({ "description": md })).unwrap();
let from_lines = validate_curated_card(&json!({
"description": ["# What's inside", "", "- 4.4M classes", "- 5.1M edges"]
}))
.unwrap();
assert_eq!(from_string["description"], json!(md));
assert_eq!(from_lines["description"], json!(md));
let e = validate_curated_card(&json!({"description": ["ok", 7]})).unwrap_err();
assert!(e.contains("array of strings"), "{e}");
let e = validate_curated_card(&json!({"description": 7})).unwrap_err();
assert!(e.contains("array of strings"), "{e}");
}
#[test]
fn description_markdown_survives_validation_untouched() {
let md = "## Contents\n\n1. one\n2. two\n\n> quoted\n\n`code` and [a link](https://example.org/x)";
let ok = validate_curated_card(&json!({ "description": md })).unwrap();
assert_eq!(ok["description"], json!(md));
}
#[test]
fn description_over_the_cap_is_rejected_rather_than_truncated() {
let at = "x".repeat(CARD_DESCRIPTION_MAX_BYTES);
validate_curated_card(&json!({ "description": at })).expect("exactly at the cap");
let over = "x".repeat(CARD_DESCRIPTION_MAX_BYTES + 1);
let e = validate_curated_card(&json!({ "description": over })).unwrap_err();
assert!(
e.contains(&format!("{} bytes", CARD_DESCRIPTION_MAX_BYTES + 1)),
"{e}"
);
assert!(
e.contains("every reader fetches the card on every open"),
"{e}"
);
let lines: Vec<String> = (0..200).map(|_| "y".repeat(100)).collect();
let e = validate_curated_card(&json!({ "description": lines })).unwrap_err();
assert!(e.contains("over the"), "{e}");
}
#[test]
fn keywords_and_theme_are_sorted_and_deduplicated() {
let ok = validate_curated_card(&json!({"keywords": ["zeta", "alpha", "zeta"]})).unwrap();
assert_eq!(ok["keywords"], json!(["alpha", "zeta"]));
}
#[test]
fn extra_limits_are_enforced_and_nested_keys_sorted() {
let ok = validate_curated_card(&json!({"extra": {"review": {"z": 1, "a": 2}}})).unwrap();
assert_eq!(
serde_json::to_string(&ok["extra"]).unwrap(),
r#"{"review":{"a":2,"z":1}}"#
);
let e =
validate_curated_card(&json!({"extra": {"deep": {"a": {"b": {"c": 1}}}}})).unwrap_err();
assert!(e.contains("over the 2-level cap"), "{e}");
let e = validate_curated_card(&json!({"extra": {"@context": "x"}})).unwrap_err();
assert!(e.contains("reserved"), "{e}");
let big = "x".repeat(CARD_EXTRA_MAX_BYTES);
let e = validate_curated_card(&json!({"extra": {"blob": big}})).unwrap_err();
assert!(e.contains("over the 8192-byte cap"), "{e}");
}
#[test]
fn creators_and_publisher_keep_their_identifiers() {
let ok = validate_curated_card(&json!({
"creators": [{"name": " Ada ", "orcid": "https://orcid.org/0000-0002-1825-0097"}],
"publisher": {"name": "CERN", "ror": "https://ror.org/01ggx4157"},
}))
.unwrap();
assert_eq!(ok["creators"][0]["name"], "Ada");
assert_eq!(
ok["creators"][0]["orcid"],
"https://orcid.org/0000-0002-1825-0097"
);
assert_eq!(ok["publisher"]["ror"], "https://ror.org/01ggx4157");
let e = validate_curated_card(&json!({"creators": [{"orcid": "x"}]})).unwrap_err();
assert!(e.contains("need a `name`"), "{e}");
let e =
validate_curated_card(&json!({"publisher": {"name": "X", "isni": "y"}})).unwrap_err();
assert!(e.contains("unknown field `isni`"), "{e}");
}
#[test]
fn empty_and_null_curated_values_are_absent_not_empty() {
let ok = validate_curated_card(&json!({
"title": "t", "description": "", "keywords": [], "doi": null, "extra": {}
}))
.unwrap();
let o = ok.as_object().unwrap();
assert!(o.contains_key("title"));
for absent in ["description", "keywords", "doi", "extra"] {
assert!(
!o.contains_key(absent),
"{absent} should be absent, not empty"
);
}
}
}