use std::collections::{BTreeMap, BTreeSet};
use serde::{Deserialize, Serialize};
use crate::{Header, RDF_TYPE};
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct DatasetCard {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub title: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub license: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub source: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub created: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub version: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub creators: Vec<Creator>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub publisher: Option<Publisher>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub canonical_url: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub sparql_endpoint: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub source_date: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub derived_from: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub doi: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cite_as: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub keywords: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub theme: Vec<String>,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub extra: BTreeMap<String, serde_json::Value>,
pub triple_count: u64,
pub quad_count: u64,
pub named_graph_count: u64,
pub term_count: u64,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub predicates: Vec<(String, u64)>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub classes: Vec<(String, u64)>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub vocabularies: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub example_queries: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub datatypes: Vec<(String, u64)>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub languages: Vec<(String, u64)>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub class_links: Vec<ClassLink>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub top_hubs: Vec<(String, u64)>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub in_hubs: Vec<(String, u64)>,
#[serde(default, skip_serializing_if = "Signals::is_empty")]
pub signals: Signals,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub queries: Vec<ExampleQuery>,
#[serde(default, skip_serializing_if = "Coherence::is_empty")]
pub coherence: Coherence,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub truncated: bool,
#[serde(default, skip_serializing_if = "is_zero_u32")]
pub top_n: u32,
pub format_version: u8,
}
fn is_zero_u32(v: &u32) -> bool {
*v == 0
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Creator {
pub name: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub orcid: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Publisher {
pub name: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ror: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum Tier {
Card,
Summary,
Index,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ExampleQuery {
pub id: String,
pub title: String,
pub dimension: String,
pub question: String,
pub sparql: String,
pub tier: Tier,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub requires: Vec<String>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ClassLink {
pub s_class: String,
pub predicate: String,
pub o_class: String,
pub count: u64,
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Signals {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub label_predicate: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub default_lang: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub base_iri: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub time_predicates: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub numeric_predicates: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub link_predicates: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub annotation_predicates: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub quoting_predicates: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub annotated_statement: Option<(String, String)>,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub geo_wkt: bool,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub geo_latlong: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub temporal_extent: Option<(String, String)>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub spatial_bbox: Option<[f64; 4]>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub text_index: Option<TextIndexSignal>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub permutations: Option<PermutationsSignal>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub quoted_triples: Option<QuotedTriplesSignal>,
}
impl Signals {
pub fn is_empty(&self) -> bool {
*self == Signals::default()
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct TextIndexSignal {
pub present: bool,
#[serde(default, skip_serializing_if = "is_zero_u64")]
pub bytes: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub token_table_bytes: Option<u64>,
}
fn is_zero_u64(v: &u64) -> bool {
*v == 0
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PermutationsSignal {
pub count: u8,
pub names: Vec<String>,
pub merge_join: bool,
}
impl PermutationsSignal {
pub fn probe(header: &Header) -> Self {
let perms = header.perms;
PermutationsSignal {
count: perms.len() as u8,
names: perms.names().into_iter().map(str::to_string).collect(),
merge_join: perms.has_merge_orders(),
}
}
pub fn describe(&self) -> String {
if self.merge_join {
format!(
"{} index permutations ({}) — sort-merge joins available",
self.count,
self.names.join("/")
)
} else {
format!(
"{} index permutations ({}) — same rows, same routing; \
no sort-merge join (hash/probe answers instead)",
self.count,
self.names.join("/")
)
}
}
}
pub const QUOTED_TRIPLE_SURFACE_RDF12: &str = "rdf12";
pub const QUOTED_TRIPLE_SURFACE_RDF_STAR: &str = "rdf-star";
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct QuotedTriplesSignal {
pub present: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub export_surfaces: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub export_default: Option<String>,
}
impl QuotedTriplesSignal {
pub fn probe(header: &Header) -> Self {
if !header.has_quoted_triples() {
return QuotedTriplesSignal {
present: false,
export_surfaces: Vec::new(),
export_default: None,
};
}
QuotedTriplesSignal {
present: true,
export_surfaces: vec![
QUOTED_TRIPLE_SURFACE_RDF12.to_string(),
QUOTED_TRIPLE_SURFACE_RDF_STAR.to_string(),
],
export_default: Some(QUOTED_TRIPLE_SURFACE_RDF12.to_string()),
}
}
pub fn describe(&self) -> Option<String> {
if !self.present {
return None;
}
Some(format!(
"present — `rete export` writes RDF 1.2 triple terms `<<( s p o )>>` \
by default; `--quoted-triple-syntax {QUOTED_TRIPLE_SURFACE_RDF_STAR}` \
writes `<<s p o>>`"
))
}
}
impl TextIndexSignal {
pub fn probe<R: crate::RangeReader + ?Sized>(reader: &R, header: &Header) -> Self {
if header.text_index_len == 0 {
return TextIndexSignal::default();
}
TextIndexSignal {
present: true,
bytes: header.text_index_len,
token_table_bytes: crate::read_text_index_token_table_len_ranged(reader, header),
}
}
pub fn describe(&self) -> String {
if !self.present {
return "no TEXT_INDEX section — CONTAINS/regex still answer, by full scan".to_string();
}
match self.token_table_bytes {
Some(tt) => format!(
"TEXT_INDEX present — {} bytes ({tt} of them the token table a first search reads)",
self.bytes
),
None => format!("TEXT_INDEX present — {} bytes", self.bytes),
}
}
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Coherence {
pub coherent: bool,
pub inconsistency_count: u32,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub by_kind: Vec<(String, u32)>,
pub scope: String,
pub rules: String,
pub materialized: bool,
}
impl Coherence {
pub fn is_empty(&self) -> bool {
*self == Coherence::default()
}
pub fn from_reasoning(r: &crate::Reasoning, materialized: bool) -> Self {
let mut hist: BTreeMap<&str, u32> = BTreeMap::new();
for inc in &r.inconsistencies {
*hist.entry(inc.kind).or_default() += 1;
}
Coherence {
coherent: r.inconsistencies.is_empty(),
inconsistency_count: r.inconsistencies.len() as u32,
by_kind: hist.into_iter().map(|(k, v)| (k.to_string(), v)).collect(),
scope: "default".to_string(),
rules: crate::REASON_RULESET.to_string(),
materialized,
}
}
}
#[derive(Debug, Clone, Default, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CardInput {
pub title: Option<String>,
#[serde(default, deserialize_with = "de_description")]
pub description: Option<String>,
pub license: Option<String>,
pub source: Option<String>,
pub created: Option<String>,
pub version: Option<String>,
#[serde(default)]
pub creators: Vec<Creator>,
pub publisher: Option<Publisher>,
pub canonical_url: Option<String>,
pub sparql_endpoint: Option<String>,
pub source_date: Option<String>,
#[serde(default)]
pub derived_from: Vec<String>,
pub doi: Option<String>,
pub cite_as: Option<String>,
#[serde(default)]
pub keywords: Vec<String>,
#[serde(default)]
pub theme: Vec<String>,
#[serde(default)]
pub extra: BTreeMap<String, serde_json::Value>,
#[serde(default)]
pub example_queries: Vec<String>,
}
fn de_description<'de, D>(d: D) -> Result<Option<String>, D::Error>
where
D: serde::Deserializer<'de>,
{
use serde::de::Error as _;
let v = Option::<serde_json::Value>::deserialize(d)?;
match v {
None | Some(serde_json::Value::Null) => Ok(None),
Some(v) => crate::card_input::normalize_description(&v)
.map(Some)
.map_err(D::Error::custom),
}
}
impl CardInput {
fn fill(self, card: &mut DatasetCard) {
card.title = self.title;
card.description = self.description;
card.license = self.license;
card.source = self.source;
card.created = self.created;
card.version = self.version;
card.creators = self.creators;
card.publisher = self.publisher;
card.canonical_url = self.canonical_url;
card.sparql_endpoint = self.sparql_endpoint;
card.source_date = self.source_date;
card.derived_from = self.derived_from;
card.doi = self.doi;
card.cite_as = self.cite_as;
card.keywords = self.keywords;
card.theme = self.theme;
card.extra = self.extra;
card.example_queries = self.example_queries;
}
pub fn from_json_str(text: &str) -> Result<Self, String> {
let parsed: CardInput = serde_json::from_str(text).map_err(|e| {
let hint = if e.to_string().contains("unknown field") {
crate::card_input::UNKNOWN_FIELD_HINT
} else {
""
};
format!("{e}{hint}")
})?;
parsed.normalize()
}
pub fn normalize(mut self) -> Result<Self, String> {
if let Some(d) = &self.description {
crate::card_input::check_description_len(d)?;
}
self.keywords = crate::card_input::normalize_string_list(
"keywords",
std::mem::take(&mut self.keywords),
)?;
self.theme = crate::card_input::normalize_themes(std::mem::take(&mut self.theme))?;
let bag: serde_json::Map<String, serde_json::Value> =
std::mem::take(&mut self.extra).into_iter().collect();
self.extra = crate::card_input::normalize_extra(bag)?
.into_iter()
.collect();
Ok(self)
}
}
impl DatasetCard {
pub fn to_json_bytes(&self) -> Vec<u8> {
if self.signals.text_index.is_none()
&& self.signals.permutations.is_none()
&& self.signals.quoted_triples.is_none()
{
return serde_json::to_vec(self).expect("DatasetCard serializes");
}
let mut stored = self.clone();
stored.signals.text_index = None;
stored.signals.permutations = None;
stored.signals.quoted_triples = None;
serde_json::to_vec(&stored).expect("DatasetCard serializes")
}
pub fn observe_text_index(&mut self, measured: TextIndexSignal) -> Option<TextIndexSignal> {
self.signals.text_index.replace(measured)
}
pub fn observe_permutations(
&mut self,
measured: PermutationsSignal,
) -> Option<PermutationsSignal> {
self.signals.permutations.replace(measured)
}
pub fn observe_quoted_triples(
&mut self,
measured: QuotedTriplesSignal,
) -> Option<QuotedTriplesSignal> {
self.signals.quoted_triples.replace(measured)
}
pub fn from_json_bytes(b: &[u8]) -> Result<Self, String> {
serde_json::from_slice(b).map_err(|e| format!("malformed dataset card: {e}"))
}
pub fn with_coherence(mut self, r: &crate::Reasoning, materialized: bool) -> Self {
self.coherence = Coherence::from_reasoning(r, materialized);
self
}
pub fn with_final_counts(mut self, counts: crate::ingest::FinalCounts) -> Self {
self.triple_count = counts.default_triples;
self.quad_count = counts.quads;
self
}
}
pub const CARD_TOP_N: usize = 100;
const ANNOTATION_WITNESS_SAMPLE: usize = 4096;
const RDFS_LABEL: &str = "<http://www.w3.org/2000/01/rdf-schema#label>";
const SKOS_PREFLABEL: &str = "<http://www.w3.org/2004/02/skos/core#prefLabel>";
const SCHEMA_NAME: &str = "<http://schema.org/name>";
const FOAF_NAME: &str = "<http://xmlns.com/foaf/0.1/name>";
const DCT_TITLE: &str = "<http://purl.org/dc/terms/title>";
const OWL_SAMEAS: &str = "<http://www.w3.org/2002/07/owl#sameAs>";
const SKOS_EXACTMATCH: &str = "<http://www.w3.org/2004/02/skos/core#exactMatch>";
const SKOS_CLOSEMATCH: &str = "<http://www.w3.org/2004/02/skos/core#closeMatch>";
const RDFS_SEEALSO: &str = "<http://www.w3.org/2000/01/rdf-schema#seeAlso>";
const WGS_LAT: &str = "<http://www.w3.org/2003/01/geo/wgs84_pos#lat>";
const WGS_LONG: &str = "<http://www.w3.org/2003/01/geo/wgs84_pos#long>";
pub const GEO_ASWKT: &str = "<http://www.opengis.net/ont/geosparql#asWKT>";
pub const GEO_HASGEOMETRY: &str = "<http://www.opengis.net/ont/geosparql#hasGeometry>";
const GEO_WKTLITERAL: &str = "http://www.opengis.net/ont/geosparql#wktLiteral";
const XSD_STRING: &str = "http://www.w3.org/2001/XMLSchema#string";
const RDF_LANGSTRING: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#langString";
pub const O_LITERAL: &str = "(literal)";
const LABEL_PREDICATES: &[&str] = &[
SKOS_PREFLABEL,
RDFS_LABEL,
SCHEMA_NAME,
FOAF_NAME,
DCT_TITLE,
];
const LINK_PREDICATES: &[&str] = &[OWL_SAMEAS, SKOS_EXACTMATCH, SKOS_CLOSEMATCH, RDFS_SEEALSO];
pub trait CardTripleSource {
fn replay(&self, f: &mut dyn FnMut(&str, &str, &str));
}
pub fn curated_counts_card(statements: u64, term_count: u64, curated: CardInput) -> DatasetCard {
let mut card = DatasetCard {
triple_count: statements,
quad_count: statements,
named_graph_count: 0,
term_count,
format_version: crate::format::CURRENT_FORMAT_VERSION,
..DatasetCard::default()
};
curated.fill(&mut card);
card
}
pub fn derive_card(
quads: &[(String, String, String, Option<String>)],
term_count: u64,
named_graph_count: u64,
curated: CardInput,
) -> DatasetCard {
struct Quads<'a>(&'a [(String, String, String, Option<String>)]);
impl CardTripleSource for Quads<'_> {
fn replay(&self, f: &mut dyn FnMut(&str, &str, &str)) {
for (s, p, o, g) in self.0 {
if g.is_none() {
f(s, p, o);
}
}
}
}
derive_card_from(
&Quads(quads),
quads.len() as u64,
term_count,
named_graph_count,
curated,
)
}
pub fn derive_card_encoded(
dict: &crate::Dictionary,
triples: &[(u32, u32, u32)],
quad_count: u64,
term_count: u64,
named_graph_count: u64,
curated: CardInput,
) -> DatasetCard {
struct Encoded<'a> {
dict: &'a crate::Dictionary,
triples: &'a [(u32, u32, u32)],
}
impl CardTripleSource for Encoded<'_> {
fn replay(&self, f: &mut dyn FnMut(&str, &str, &str)) {
for &(s, p, o) in self.triples {
let st = self.dict.subject_term(s).unwrap_or_default();
let pt = self.dict.predicate_term(p).unwrap_or_default();
let ot = self.dict.object_term(o).unwrap_or_default();
f(&st, &pt, &ot);
}
}
}
derive_card_from(
&Encoded { dict, triples },
quad_count,
term_count,
named_graph_count,
curated,
)
}
pub fn derive_card_from(
src: &dyn CardTripleSource,
quad_count: u64,
term_count: u64,
named_graph_count: u64,
curated: CardInput,
) -> DatasetCard {
let mut class_of: std::collections::HashMap<String, String> = std::collections::HashMap::new();
let mut quoted_subject_preds: BTreeSet<String> = BTreeSet::new();
let mut annotated_tokens: BTreeMap<String, String> = BTreeMap::new();
src.replay(&mut |s, p, o| {
if p == RDF_TYPE {
class_of.insert(s.to_string(), o.to_string());
}
if crate::terms::is_quoted_triple(s) {
if let Some((_, quoted_p, _)) = crate::ingest::quoted_triple_parts(s) {
quoted_subject_preds.insert(quoted_p);
}
if annotated_tokens.len() < ANNOTATION_WITNESS_SAMPLE {
annotated_tokens
.entry(s.to_string())
.or_insert_with(|| p.to_string());
}
}
});
let classify = |t: &str| -> String {
if let Some(c) = class_of.get(t) {
c.to_string()
} else if t.starts_with('"') {
O_LITERAL.to_string()
} else {
"(untyped)".to_string()
}
};
let mut pred_counts: BTreeMap<String, u64> = BTreeMap::new();
let mut class_counts: BTreeMap<String, u64> = BTreeMap::new();
let mut datatype_counts: BTreeMap<String, u64> = BTreeMap::new();
let mut lang_counts: BTreeMap<String, u64> = BTreeMap::new();
let mut out_degree: BTreeMap<String, u64> = BTreeMap::new();
let mut in_degree: BTreeMap<String, u64> = BTreeMap::new();
let mut link_rows: BTreeMap<(String, String, String), u64> = BTreeMap::new();
let mut subject_ns: BTreeMap<String, u64> = BTreeMap::new();
let mut time_obj_hits: BTreeMap<String, u64> = BTreeMap::new();
let mut num_obj_hits: BTreeMap<String, u64> = BTreeMap::new();
let mut annotation_hits: BTreeMap<String, u64> = BTreeMap::new();
let mut quoting_hits: BTreeMap<String, u64> = BTreeMap::new();
let mut witnessed_pairs: BTreeMap<(String, String), u64> = BTreeMap::new();
let mut time_values: BTreeMap<String, (Option<String>, Option<String>)> = BTreeMap::new();
let mut time_value_dt: BTreeMap<String, String> = BTreeMap::new();
let mut lat_lo = f64::INFINITY;
let mut lat_hi = f64::NEG_INFINITY;
let mut lon_lo = f64::INFINITY;
let mut lon_hi = f64::NEG_INFINITY;
let mut have_lat = false;
let mut have_lon = false;
let mut geo_wkt = false;
let mut triple_count = 0u64;
src.replay(&mut |s, p, o| {
triple_count += 1;
*pred_counts.entry(p.to_string()).or_default() += 1;
*out_degree.entry(s.to_string()).or_default() += 1;
if let Some(ns) = split_namespace(s) {
*subject_ns.entry(ns).or_default() += 1;
}
if !o.starts_with('"') {
*in_degree.entry(o.to_string()).or_default() += 1;
}
if crate::terms::is_quoted_triple(s) {
*annotation_hits.entry(p.to_string()).or_default() += 1;
}
if crate::terms::is_quoted_triple(o) {
*quoting_hits.entry(p.to_string()).or_default() += 1;
}
if !quoted_subject_preds.is_empty() && quoted_subject_preds.contains(p) {
if let Some(about) = annotated_tokens.get(&format!("<<{s} {p} {o}>>")) {
*witnessed_pairs
.entry((p.to_string(), about.clone()))
.or_default() += 1;
}
}
if p == RDF_TYPE {
*class_counts.entry(o.to_string()).or_default() += 1;
return; }
*link_rows
.entry((classify(s), p.to_string(), classify(o)))
.or_default() += 1;
if p == GEO_ASWKT || p == GEO_HASGEOMETRY {
geo_wkt = true;
}
if let Some(lit) = parse_literal(o) {
*datatype_counts
.entry(format!("<{}>", lit.datatype))
.or_default() += 1;
*lang_counts.entry(lit.lang.clone()).or_default() += 1;
if lit.datatype == GEO_WKTLITERAL {
geo_wkt = true;
}
if is_temporal_datatype(&lit.datatype) || looks_like_year(&lit.value) {
*time_obj_hits.entry(p.to_string()).or_default() += 1;
update_time_extent(p, &lit, &mut time_values, &mut time_value_dt);
} else if is_numeric_datatype(&lit.datatype) {
*num_obj_hits.entry(p.to_string()).or_default() += 1;
}
if p == WGS_LAT {
if let Ok(v) = lit.value.parse::<f64>() {
have_lat = true;
lat_lo = lat_lo.min(v);
lat_hi = lat_hi.max(v);
}
} else if p == WGS_LONG {
if let Ok(v) = lit.value.parse::<f64>() {
have_lon = true;
lon_lo = lon_lo.min(v);
lon_hi = lon_hi.max(v);
}
}
}
});
let pred_count = |iri: &str| pred_counts.get(iri).copied().unwrap_or(0);
let label_predicate = LABEL_PREDICATES
.iter()
.filter(|&&iri| pred_count(iri) > 0)
.max_by_key(|&&iri| pred_count(iri))
.map(|s| s.to_string());
let link_predicates: Vec<String> = LINK_PREDICATES
.iter()
.filter(|&&iri| pred_count(iri) > 0)
.map(|s| s.to_string())
.collect();
let mut truncated = false;
let predicates = cap(sort_desc_owned(pred_counts), &mut truncated);
let classes = cap(sort_desc_owned(class_counts), &mut truncated);
let datatypes = cap(sort_desc_owned(datatype_counts), &mut truncated);
let languages = cap(sort_desc_owned(lang_counts), &mut truncated);
let top_hubs = cap(sort_desc_owned(out_degree), &mut truncated);
let in_hubs = cap(sort_desc_owned(in_degree), &mut truncated);
let class_links = cap(sort_links(link_rows), &mut truncated);
let mut vocab: BTreeSet<String> = BTreeSet::new();
for (iri, _) in predicates.iter().chain(classes.iter()) {
if let Some(ns) = split_namespace(iri) {
vocab.insert(ns);
}
}
let vocabularies = cap(vocab.into_iter().collect(), &mut truncated);
let default_lang = languages
.iter()
.find(|(l, _)| !l.is_empty())
.map(|(l, _)| l.clone());
let base_iri = derive_base_iri(&subject_ns);
let mut time_ranked: Vec<(String, u64)> = time_obj_hits.into_iter().collect();
time_ranked.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
let time_predicates = cap(
time_ranked.iter().map(|(p, _)| p.to_string()).collect(),
&mut truncated,
);
let mut num_ranked: Vec<(String, u64)> = num_obj_hits.into_iter().collect();
num_ranked.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
let numeric_predicates = cap(
num_ranked.iter().map(|(p, _)| p.to_string()).collect(),
&mut truncated,
);
let annotation_predicates = cap(
sort_desc_owned(annotation_hits)
.into_iter()
.map(|(p, _)| p)
.collect(),
&mut truncated,
);
let quoting_predicates = cap(
sort_desc_owned(quoting_hits)
.into_iter()
.map(|(p, _)| p)
.collect(),
&mut truncated,
);
let annotated_statement = witnessed_pairs
.into_iter()
.max_by(|a, b| a.1.cmp(&b.1).then_with(|| b.0.cmp(&a.0)))
.map(|(pair, _)| pair);
let geo_latlong = have_lat && have_lon;
let temporal_extent = time_ranked
.first()
.and_then(|(p, _)| time_values.get(p))
.and_then(|(lo, hi)| match (lo, hi) {
(Some(lo), Some(hi)) => Some((lo.clone(), hi.clone())),
_ => None,
});
let spatial_bbox = if geo_latlong && lon_lo.is_finite() && lat_lo.is_finite() {
Some([lon_lo, lat_lo, lon_hi, lat_hi])
} else {
None
};
let signals = Signals {
label_predicate,
default_lang,
base_iri,
time_predicates,
numeric_predicates,
link_predicates,
annotation_predicates,
quoting_predicates,
annotated_statement,
geo_wkt,
geo_latlong,
temporal_extent,
spatial_bbox,
text_index: None,
permutations: None,
quoted_triples: None,
};
let mut card = DatasetCard {
triple_count,
quad_count,
named_graph_count,
term_count,
predicates,
classes,
vocabularies,
datatypes,
languages,
class_links,
top_hubs,
in_hubs,
signals,
queries: Vec::new(),
coherence: Coherence::default(),
truncated,
top_n: CARD_TOP_N as u32,
format_version: crate::CURRENT_FORMAT_VERSION,
..DatasetCard::default()
};
curated.fill(&mut card);
card.queries = crate::card_queries::generate(&card);
card
}
fn cap<T>(mut v: Vec<T>, truncated: &mut bool) -> Vec<T> {
if v.len() > CARD_TOP_N {
v.truncate(CARD_TOP_N);
*truncated = true;
}
v
}
fn sort_desc_owned(counts: BTreeMap<String, u64>) -> Vec<(String, u64)> {
let mut v: Vec<(String, u64)> = counts.into_iter().collect();
v.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
v
}
fn sort_links(rows: BTreeMap<(String, String, String), u64>) -> Vec<ClassLink> {
let mut v: Vec<ClassLink> = rows
.into_iter()
.map(|((s, p, o), count)| ClassLink {
s_class: s,
predicate: p,
o_class: o,
count,
})
.collect();
v.sort_by(|a, b| {
b.count
.cmp(&a.count)
.then_with(|| a.s_class.cmp(&b.s_class))
.then_with(|| a.predicate.cmp(&b.predicate))
.then_with(|| a.o_class.cmp(&b.o_class))
});
v
}
fn derive_base_iri(ns: &BTreeMap<String, u64>) -> Option<String> {
if ns.is_empty() {
return None;
}
let dominant = ns
.iter()
.max_by(|a, b| a.1.cmp(b.1).then_with(|| b.0.cmp(a.0)))
.map(|(k, _)| k.clone())?;
let mut lcp_len = dominant.len();
for k in ns.keys() {
let common = dominant
.bytes()
.zip(k.bytes())
.take_while(|(a, b)| a == b)
.count();
lcp_len = lcp_len.min(common);
}
let lcp = &dominant[..lcp_len];
let base = match lcp.rfind(['/', '#']) {
Some(i) => &lcp[..=i],
None => "",
};
if base.matches('/').count() >= 3 {
Some(base.to_string())
} else {
Some(dominant)
}
}
struct Literal {
value: String,
datatype: String,
lang: String,
}
fn parse_literal(term: &str) -> Option<Literal> {
let bytes = term.as_bytes();
if bytes.first() != Some(&b'"') {
return None;
}
let mut i = 1usize;
let mut esc = false;
let mut close = None;
while i < bytes.len() {
match bytes[i] {
b'\\' => esc = !esc,
b'"' if !esc => {
close = Some(i);
break;
}
_ => esc = false,
}
i += 1;
}
let close = close?;
let value = term[1..close].to_string();
let suffix = &term[close + 1..];
if let Some(lang) = suffix.strip_prefix('@') {
Some(Literal {
value,
datatype: RDF_LANGSTRING.to_string(),
lang: lang.to_string(),
})
} else if let Some(dt) = suffix.strip_prefix("^^") {
let dt = dt.trim_start_matches('<').trim_end_matches('>').to_string();
Some(Literal {
value,
datatype: dt,
lang: String::new(),
})
} else {
Some(Literal {
value,
datatype: XSD_STRING.to_string(),
lang: String::new(),
})
}
}
fn is_temporal_datatype(dt: &str) -> bool {
matches!(
dt,
"http://www.w3.org/2001/XMLSchema#date"
| "http://www.w3.org/2001/XMLSchema#dateTime"
| "http://www.w3.org/2001/XMLSchema#gYear"
| "http://www.w3.org/2001/XMLSchema#gYearMonth"
| "http://www.w3.org/2001/XMLSchema#gMonthDay"
)
}
fn is_numeric_datatype(dt: &str) -> bool {
matches!(
dt,
"http://www.w3.org/2001/XMLSchema#integer"
| "http://www.w3.org/2001/XMLSchema#decimal"
| "http://www.w3.org/2001/XMLSchema#double"
| "http://www.w3.org/2001/XMLSchema#float"
| "http://www.w3.org/2001/XMLSchema#long"
| "http://www.w3.org/2001/XMLSchema#int"
| "http://www.w3.org/2001/XMLSchema#short"
| "http://www.w3.org/2001/XMLSchema#nonNegativeInteger"
| "http://www.w3.org/2001/XMLSchema#positiveInteger"
)
}
fn looks_like_year(value: &str) -> bool {
let v = value.strip_prefix('-').unwrap_or(value);
let head: String = v.chars().take(4).collect();
head.len() == 4 && head.chars().all(|c| c.is_ascii_digit())
}
fn update_time_extent(
pred: &str,
lit: &Literal,
values: &mut BTreeMap<String, (Option<String>, Option<String>)>,
value_dt: &mut BTreeMap<String, String>,
) {
let dt = value_dt
.entry(pred.to_string())
.or_insert_with(|| lit.datatype.clone());
if *dt != lit.datatype {
return; }
let entry = values.entry(pred.to_string()).or_insert((None, None));
match &mut entry.0 {
Some(lo) if lit.value < *lo => *lo = lit.value.clone(),
None => entry.0 = Some(lit.value.clone()),
_ => {}
}
match &mut entry.1 {
Some(hi) if lit.value > *hi => *hi = lit.value.clone(),
None => entry.1 = Some(lit.value.clone()),
_ => {}
}
}
fn split_namespace(term: &str) -> Option<String> {
let iri = term.strip_prefix('<')?.strip_suffix('>')?;
let cut = iri.rfind(['#', '/'])?;
Some(iri[..=cut].to_string())
}
pub fn load_card(bytes: &[u8]) -> Result<Option<DatasetCard>, String> {
let header = Header::from_bytes(bytes).map_err(|e| e.to_string())?;
if header.metadata_len == 0 {
return Ok(None);
}
let start = header.metadata_offset as usize;
let end = start
.checked_add(header.metadata_len as usize)
.filter(|&e| e <= bytes.len())
.ok_or_else(|| "metadata section out of bounds".to_string())?;
Ok(Some(DatasetCard::from_json_bytes(&bytes[start..end])?))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::card_input::{CARD_EXTRA_MAX_BYTES, CARD_EXTRA_MAX_KEYS, CARD_EXTRA_MAX_KEY_BYTES};
const TYPE: &str = RDF_TYPE;
fn q(s: &str, p: &str, o: &str) -> (String, String, String, Option<String>) {
(s.into(), p.into(), o.into(), None)
}
#[test]
fn card_json_round_trips() {
let card = DatasetCard {
title: Some("Demo".into()),
license: Some("CC0".into()),
triple_count: 3,
quad_count: 3,
term_count: 5,
predicates: vec![("<http://ex/p>".into(), 2)],
vocabularies: vec!["http://ex/".into()],
format_version: 1,
..Default::default()
};
let bytes = card.to_json_bytes();
let back = DatasetCard::from_json_bytes(&bytes).unwrap();
assert_eq!(card, back);
let text = String::from_utf8(bytes).unwrap();
assert!(!text.contains("description"));
assert!(!text.contains("classes"));
}
#[test]
fn the_permutation_signal_is_stripped_from_the_stored_card() {
let measured = PermutationsSignal {
count: 3,
names: vec!["SPO".into(), "POS".into(), "OSP".into()],
merge_join: false,
};
let mut card = DatasetCard {
title: Some("Lean".into()),
triple_count: 3,
..Default::default()
};
assert_eq!(card.observe_permutations(measured.clone()), None);
assert_eq!(card.signals.permutations, Some(measured.clone()));
let bytes = card.to_json_bytes();
let text = String::from_utf8(bytes.clone()).unwrap();
assert!(
!text.contains("permutations"),
"the signal must not reach the metadata section: {text}"
);
let back = DatasetCard::from_json_bytes(&bytes).unwrap();
assert_eq!(back.signals.permutations, None, "unknown, not `six`");
assert_eq!(back.title, card.title);
let authored = r#"{"title":"x","signals":{"permutations":{"count":6}}}"#;
assert!(
serde_json::from_str::<CardInput>(authored).is_err(),
"a hand-written card must not be able to assert a permutation set"
);
}
#[test]
fn the_permutation_signal_is_derived_from_the_header() {
let quads = vec![
q("<http://ex/a>", "<http://ex/p>", "<http://ex/b>"),
q("<http://ex/b>", "<http://ex/p>", "<http://ex/c>"),
];
let (six, _) = crate::ingest::assemble_dataset_with_perms(
quads.clone(),
false,
false,
None,
crate::PyramidAlgo::Louvain,
crate::PermSet::ALL,
|_, _| Vec::new(),
);
let (three, _) = crate::ingest::assemble_dataset_with_perms(
quads,
false,
false,
None,
crate::PyramidAlgo::Louvain,
crate::PermSet::CORE,
|_, _| Vec::new(),
);
assert_eq!(six[50], 0);
assert_eq!(three[50], 0b0000_0111);
let h6 = crate::Header::from_bytes(&six).unwrap();
let h3 = crate::Header::from_bytes(&three).unwrap();
let s6 = PermutationsSignal::probe(&h6);
let s3 = PermutationsSignal::probe(&h3);
assert_eq!(s6.count, 6);
assert!(s6.merge_join);
assert_eq!(s3.count, 3);
assert!(!s3.merge_join);
assert_eq!(s3.names, vec!["SPO", "POS", "OSP"]);
assert!(s3.describe().contains("no sort-merge join"));
assert!(three.len() < six.len(), "{} vs {}", three.len(), six.len());
assert_eq!(h3.dictionary_len, h6.dictionary_len);
assert!(h3.root_dir_len < h6.root_dir_len);
}
#[test]
fn the_text_index_signal_is_stripped_from_the_stored_card() {
let measured = TextIndexSignal {
present: true,
bytes: 1_879_287_762,
token_table_bytes: Some(193_295_361),
};
let mut card = DatasetCard {
title: Some("Indexed".into()),
triple_count: 3,
..Default::default()
};
assert_eq!(card.observe_text_index(measured), None);
assert_eq!(card.signals.text_index, Some(measured));
let bytes = card.to_json_bytes();
let text = String::from_utf8(bytes.clone()).unwrap();
assert!(
!text.contains("text_index"),
"the signal must not reach the metadata section: {text}"
);
let back = DatasetCard::from_json_bytes(&bytes).unwrap();
assert_eq!(back.title, card.title);
assert_eq!(back.signals.text_index, None);
assert_eq!(card.signals.text_index, Some(measured));
}
#[test]
fn the_text_index_signal_round_trips_through_a_card_document() {
let mut card = DatasetCard {
title: Some("Indexed".into()),
..Default::default()
};
card.observe_text_index(TextIndexSignal {
present: true,
bytes: 80,
token_table_bytes: Some(68),
});
let doc = serde_json::to_vec(&card).unwrap();
let back: DatasetCard = serde_json::from_slice(&doc).unwrap();
assert_eq!(back.signals.text_index, card.signals.text_index);
let mut none = DatasetCard::default();
none.observe_text_index(TextIndexSignal::default());
let text = serde_json::to_string(&none).unwrap();
assert!(text.contains(r#""text_index":{"present":false}"#), "{text}");
let absent = serde_json::to_string(&DatasetCard::default()).unwrap();
assert!(!absent.contains("text_index"), "{absent}");
}
#[test]
fn the_quoted_triple_signal_is_derived_from_the_header() {
let plain = vec![
q("<http://ex/a>", "<http://ex/p>", "<http://ex/b>"),
q("<http://ex/b>", "<http://ex/p>", "<http://ex/c>"),
];
let quoted = vec![
q("<http://ex/a>", "<http://ex/p>", "<http://ex/b>"),
q(
"<http://ex/claim>",
"<http://ex/states>",
"<<<http://ex/a> <http://ex/p> <http://ex/b>>>",
),
];
let (no, _) = crate::ingest::assemble_dataset(plain, &[]);
let (yes, _) = crate::ingest::assemble_dataset(quoted, &[]);
let hn = crate::Header::from_bytes(&no).unwrap();
let hy = crate::Header::from_bytes(&yes).unwrap();
assert!(!hn.has_quoted_triples());
assert!(hy.has_quoted_triples());
let sn = QuotedTriplesSignal::probe(&hn);
let sy = QuotedTriplesSignal::probe(&hy);
assert!(!sn.present);
assert!(sy.present);
assert_eq!(sn.describe(), None);
let line = sy.describe().expect("a file with quoted triples says so");
assert!(line.contains("<<( s p o )>>"), "{line}");
assert!(line.contains(QUOTED_TRIPLE_SURFACE_RDF_STAR), "{line}");
assert_eq!(
sy.export_surfaces,
vec![QUOTED_TRIPLE_SURFACE_RDF12, QUOTED_TRIPLE_SURFACE_RDF_STAR]
);
assert_eq!(
sy.export_default.as_deref(),
Some(QUOTED_TRIPLE_SURFACE_RDF12)
);
assert!(sn.export_surfaces.is_empty());
assert_eq!(sn.export_default, None);
}
#[test]
fn the_quoted_triple_signal_is_stripped_from_the_stored_card() {
let measured = QuotedTriplesSignal {
present: true,
export_surfaces: vec![
QUOTED_TRIPLE_SURFACE_RDF12.to_string(),
QUOTED_TRIPLE_SURFACE_RDF_STAR.to_string(),
],
export_default: Some(QUOTED_TRIPLE_SURFACE_RDF12.to_string()),
};
let mut card = DatasetCard {
title: Some("Annotated".into()),
triple_count: 3,
..Default::default()
};
assert_eq!(card.observe_quoted_triples(measured.clone()), None);
assert_eq!(card.signals.quoted_triples, Some(measured.clone()));
let bytes = card.to_json_bytes();
let text = String::from_utf8(bytes.clone()).unwrap();
assert!(
!text.contains("quoted_triples"),
"the signal must not reach the metadata section: {text}"
);
let back = DatasetCard::from_json_bytes(&bytes).unwrap();
assert_eq!(back.title, card.title);
assert_eq!(back.signals.quoted_triples, None);
assert_eq!(card.signals.quoted_triples, Some(measured));
let authored = r#"{"title":"x","signals":{"quoted_triples":{"present":true}}}"#;
assert!(
serde_json::from_str::<CardInput>(authored).is_err(),
"a hand-written card must not be able to assert a quoted-triple set"
);
}
#[test]
fn the_quoted_triple_signal_round_trips_through_a_card_document() {
let mut card = DatasetCard {
title: Some("Annotated".into()),
..Default::default()
};
card.observe_quoted_triples(QuotedTriplesSignal {
present: true,
export_surfaces: vec![QUOTED_TRIPLE_SURFACE_RDF12.to_string()],
export_default: Some(QUOTED_TRIPLE_SURFACE_RDF12.to_string()),
});
let doc = serde_json::to_vec(&card).unwrap();
let back: DatasetCard = serde_json::from_slice(&doc).unwrap();
assert_eq!(back.signals.quoted_triples, card.signals.quoted_triples);
let mut none = DatasetCard::default();
none.observe_quoted_triples(QuotedTriplesSignal {
present: false,
export_surfaces: Vec::new(),
export_default: None,
});
let text = serde_json::to_string(&none).unwrap();
assert!(
text.contains(r#""quoted_triples":{"present":false}"#),
"{text}"
);
let absent = serde_json::to_string(&DatasetCard::default()).unwrap();
assert!(!absent.contains("quoted_triples"), "{absent}");
}
#[test]
fn split_namespace_handles_hash_slash_and_neither() {
assert_eq!(
split_namespace("<http://xmlns.com/foaf/0.1/name>").as_deref(),
Some("http://xmlns.com/foaf/0.1/")
);
assert_eq!(
split_namespace("<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>").as_deref(),
Some("http://www.w3.org/1999/02/22-rdf-syntax-ns#")
);
assert_eq!(split_namespace("<urn:isbn:12345>"), None);
assert_eq!(split_namespace("\"literal\""), None);
assert_eq!(split_namespace("_:b0"), None);
}
#[test]
fn derive_counts_and_orders() {
let quads = vec![
q("<http://ex/a>", TYPE, "<http://ex/Person>"),
q("<http://ex/b>", TYPE, "<http://ex/Person>"),
q("<http://ex/c>", TYPE, "<http://ex/City>"),
q("<http://ex/a>", "<http://ex/knows>", "<http://ex/b>"),
q("<http://ex/a>", "<http://ex/knows>", "<http://ex/c>"),
q("<http://ex/a>", "<http://ex/name>", "\"A\""),
];
let card = derive_card(&quads, 9, 0, CardInput::default());
assert_eq!(card.triple_count, 6);
assert_eq!(card.quad_count, 6);
assert_eq!(card.term_count, 9);
assert_eq!(card.predicates[0], (TYPE.to_string(), 3));
assert!(card
.predicates
.contains(&("<http://ex/knows>".to_string(), 2)));
assert_eq!(
card.classes,
vec![
("<http://ex/Person>".to_string(), 2),
("<http://ex/City>".to_string(), 1)
]
);
assert!(card.vocabularies.contains(&"http://ex/".to_string()));
assert!(card
.vocabularies
.iter()
.any(|v| v.ends_with("rdf-syntax-ns#")));
}
#[test]
fn named_graph_triples_excluded_from_default_stats() {
let quads = vec![
q("<http://ex/a>", "<http://ex/p>", "<http://ex/b>"),
(
"<http://ex/c>".into(),
"<http://ex/p>".into(),
"<http://ex/d>".into(),
Some("<http://ex/g1>".into()),
),
];
let card = derive_card(&quads, 5, 1, CardInput::default());
assert_eq!(card.triple_count, 1, "only the default-graph triple counts");
assert_eq!(card.quad_count, 2);
assert_eq!(card.named_graph_count, 1);
assert_eq!(card.predicates[0].1, 1);
}
const LABEL: &str = "<http://www.w3.org/2000/01/rdf-schema#label>";
const SAMEAS: &str = "<http://www.w3.org/2002/07/owl#sameAs>";
const XSD_INT: &str = "http://www.w3.org/2001/XMLSchema#integer";
const XSD_GYEAR: &str = "http://www.w3.org/2001/XMLSchema#gYear";
fn enriched_fixture() -> Vec<(String, String, String, Option<String>)> {
vec![
q("<http://ex/a>", TYPE, "<http://ex/Person>"),
q("<http://ex/b>", TYPE, "<http://ex/Person>"),
q("<http://ex/c>", TYPE, "<http://ex/City>"),
q("<http://ex/a>", LABEL, "\"Alice\"@en"),
q("<http://ex/b>", LABEL, "\"Bob\"@en"),
q(
"<http://ex/a>",
"<http://ex/age>",
&format!("\"30\"^^<{XSD_INT}>"),
),
q(
"<http://ex/a>",
"<http://ex/birthYear>",
&format!("\"1850\"^^<{XSD_GYEAR}>"),
),
q(
"<http://ex/b>",
"<http://ex/birthYear>",
&format!("\"1875\"^^<{XSD_GYEAR}>"),
),
q("<http://ex/a>", "<http://ex/knows>", "<http://ex/b>"),
q("<http://ex/a>", "<http://ex/livesIn>", "<http://ex/c>"),
q("<http://ex/a>", SAMEAS, "<http://other/a>"),
]
}
#[test]
fn enriched_profile_is_populated() {
let quads = enriched_fixture();
let card = derive_card(&quads, 12, 0, CardInput::default());
assert!(!card.datatypes.is_empty(), "datatypes derived");
assert!(!card.languages.is_empty(), "languages derived");
assert!(!card.class_links.is_empty(), "class_links derived");
assert!(!card.top_hubs.is_empty(), "out-degree hubs derived");
assert!(!card.in_hubs.is_empty(), "in-degree hubs derived");
assert!(card
.class_links
.iter()
.any(|l| l.s_class == "<http://ex/Person>"
&& l.predicate == "<http://ex/knows>"
&& l.o_class == "<http://ex/Person>"
&& l.count == 1));
assert!(card
.class_links
.iter()
.any(|l| l.o_class == "<http://ex/City>"));
assert!(card.class_links.iter().any(|l| l.o_class == "(untyped)"));
assert!(card.datatypes.iter().any(|(d, _)| d.contains("integer")));
assert!(card.datatypes.iter().any(|(d, _)| d.contains("gYear")));
assert!(card.datatypes.iter().any(|(d, _)| d.contains("langString")));
assert!(card.languages.iter().any(|(l, _)| l == "en"));
let s = &card.signals;
assert_eq!(s.label_predicate.as_deref(), Some(LABEL));
assert_eq!(s.default_lang.as_deref(), Some("en"));
assert_eq!(s.base_iri.as_deref(), Some("http://ex/"));
assert!(s.link_predicates.iter().any(|p| p.contains("sameAs")));
assert!(s.time_predicates.iter().any(|p| p.contains("birthYear")));
assert_eq!(
s.temporal_extent,
Some(("1850".to_string(), "1875".to_string())),
"extent over the dominant time predicate"
);
assert!(!s.geo_wkt && !s.geo_latlong);
assert!(s.spatial_bbox.is_none());
}
#[test]
fn signals_detect_geometry() {
let wkt = "http://www.opengis.net/ont/geosparql#wktLiteral";
let quads = vec![
q("<http://ex/p>", TYPE, "<http://ex/Place>"),
q(
"<http://ex/p>",
"<http://www.w3.org/2003/01/geo/wgs84_pos#lat>",
"\"41.9\"^^<http://www.w3.org/2001/XMLSchema#double>",
),
q(
"<http://ex/p>",
"<http://www.w3.org/2003/01/geo/wgs84_pos#long>",
"\"12.5\"^^<http://www.w3.org/2001/XMLSchema#double>",
),
q(
"<http://ex/p>",
"<http://www.opengis.net/ont/geosparql#asWKT>",
&format!("\"POINT(12.5 41.9)\"^^<{wkt}>"),
),
];
let card = derive_card(&quads, 6, 0, CardInput::default());
assert!(card.signals.geo_wkt, "asWKT/wktLiteral detected");
assert!(card.signals.geo_latlong, "wgs84 lat+long detected");
let bbox = card.signals.spatial_bbox.expect("bbox derived");
assert_eq!(
bbox,
[12.5, 41.9, 12.5, 41.9],
"[minLon,minLat,maxLon,maxLat]"
);
}
#[test]
fn class_links_match_schema_summary() {
use crate::{ingest, schema_summary, Rete};
let quads = enriched_fixture();
let card = derive_card(&quads, 12, 0, CardInput::default());
let (bytes, _) =
ingest::assemble_dataset_with_opts(quads, true, false, None, |_, _| Vec::new());
let rete = Rete::open(&bytes).unwrap();
let reference: BTreeSet<(String, String, String, u64)> = schema_summary(&rete)
.into_iter()
.map(|(s, p, o, c)| (s, p, o, c as u64))
.collect();
let derived: BTreeSet<(String, String, String, u64)> = card
.class_links
.iter()
.map(|l| {
(
l.s_class.clone(),
l.predicate.clone(),
l.o_class.clone(),
l.count,
)
})
.collect();
assert_eq!(
derived, reference,
"card class_links must equal schema_summary's quotient row-for-row"
);
}
#[test]
fn card_counts_are_the_deduplicated_counts() {
use crate::ingest::{self, DeferredMetadata};
use crate::Rete;
let g = |s: &str, p: &str, o: &str, graph: &str| {
(
s.to_string(),
p.to_string(),
o.to_string(),
Some(graph.to_string()),
)
};
let quads = vec![
q("<http://ex/a>", "<http://ex/p>", "<http://ex/1>"),
q("<http://ex/b>", "<http://ex/p>", "<http://ex/2>"),
q("<http://ex/a>", "<http://ex/p>", "<http://ex/1>"), g("<http://ex/n>", "<http://ex/q>", "\"v\"", "<http://ex/g>"),
g("<http://ex/n>", "<http://ex/q>", "\"v\"", "<http://ex/g>"), ];
let (bytes, stats) =
ingest::assemble_dataset_with_opts(quads, false, false, None, |stats, quads| {
let card = derive_card(
quads,
stats.terms as u64,
stats.named_graphs as u64,
CardInput::default(),
);
DeferredMetadata::new(move |counts| card.with_final_counts(counts).to_json_bytes())
});
let header = crate::Header::from_bytes(&bytes).unwrap();
let card = load_card(&bytes).unwrap().expect("card embedded");
assert_eq!(header.quad_count, 3, "header records the deduped quads");
assert_eq!(
card.quad_count, header.quad_count,
"card and header must not disagree about the size of the same file"
);
assert_eq!(card.triple_count, 2, "default-graph triples, deduped");
let rete = Rete::open(&bytes).unwrap();
let default_triples = rete.dump(None).len() as u64;
let named: u64 = rete
.graph_names()
.iter()
.map(|name| rete.dump(Some(name)).len() as u64)
.sum();
assert_eq!(default_triples + named, card.quad_count);
assert_eq!(stats.statements, 3);
assert_eq!(stats.default_triples, 2);
}
#[test]
fn derive_is_deterministic() {
let quads = enriched_fixture();
let a = derive_card(&quads, 12, 0, CardInput::default()).to_json_bytes();
let b = derive_card(&quads, 12, 0, CardInput::default()).to_json_bytes();
assert_eq!(a, b, "enriched card derivation must be deterministic");
assert!(!a.is_empty());
}
#[test]
fn old_plain_string_card_still_parses() {
let json = r#"{
"title":"Legacy",
"triple_count":3,"quad_count":3,"named_graph_count":0,"term_count":5,
"predicates":[["<http://ex/p>",2]],
"example_queries":["SELECT * WHERE { ?s ?p ?o }"],
"format_version":1
}"#;
let card = DatasetCard::from_json_bytes(json.as_bytes()).unwrap();
assert_eq!(card.title.as_deref(), Some("Legacy"));
assert_eq!(card.example_queries.len(), 1);
assert!(card.datatypes.is_empty());
assert!(card.languages.is_empty());
assert!(card.class_links.is_empty());
assert!(card.top_hubs.is_empty());
assert!(card.in_hubs.is_empty());
assert!(card.queries.is_empty());
assert!(card.signals.is_empty());
assert!(!card.truncated);
}
#[test]
fn curated_identity_fields_round_trip() {
let input: CardInput = serde_json::from_str(
r#"{
"title": "T",
"version": "2026-08-04",
"creators": [{"name":"Ada","orcid":"https://orcid.org/0000-0002-1825-0097"}],
"publisher": {"name":"EPFL","ror":"https://ror.org/02s376052"},
"canonical_url": "https://data.example.org/t.rete",
"sparql_endpoint": "https://example.org/sparql",
"source_date": "2026-07-01",
"derived_from": ["https://example.org/dump.nt"],
"doi": "https://doi.org/10.5281/zenodo.1",
"cite_as": "Ada (2026). T.",
"keywords": ["graphs", "citations"],
"theme": ["http://publications.europa.eu/resource/authority/data-theme/TECH"]
}"#,
)
.unwrap();
let card = curated_counts_card(5, 9, input);
let bytes = card.to_json_bytes();
let back = DatasetCard::from_json_bytes(&bytes).unwrap();
assert_eq!(back.version.as_deref(), Some("2026-08-04"));
assert_eq!(back.creators[0].name, "Ada");
assert_eq!(
back.creators[0].orcid.as_deref(),
Some("https://orcid.org/0000-0002-1825-0097")
);
assert_eq!(back.publisher.as_ref().unwrap().name, "EPFL");
assert_eq!(
back.canonical_url.as_deref(),
Some("https://data.example.org/t.rete")
);
assert_eq!(back.derived_from, vec!["https://example.org/dump.nt"]);
assert_eq!(
back.doi.as_deref(),
Some("https://doi.org/10.5281/zenodo.1")
);
assert_eq!(back.keywords, vec!["graphs", "citations"]);
assert_eq!(
back.theme,
vec!["http://publications.europa.eu/resource/authority/data-theme/TECH"]
);
let plain = curated_counts_card(1, 2, CardInput::default()).to_json_bytes();
let text = String::from_utf8(plain).unwrap();
for absent in [
"\"creators\"",
"\"publisher\"",
"\"canonical_url\"",
"\"sparql_endpoint\"",
"\"derived_from\"",
"\"doi\"",
"\"cite_as\"",
"\"source_date\"",
"\"version\"",
"\"top_n\"",
"\"keywords\"",
"\"theme\"",
"\"extra\"",
] {
assert!(
!text.contains(absent),
"{absent} must be omitted when unset"
);
}
}
#[test]
fn extra_round_trips_and_external_writer_cards_surface_it() {
let input: CardInput = serde_json::from_str(
r#"{"title":"T","extra":{"atlas:layer":84,"review":{"by":"dg","ok":true}}}"#,
)
.unwrap();
let card = curated_counts_card(5, 9, input);
let back = DatasetCard::from_json_bytes(&card.to_json_bytes()).unwrap();
assert_eq!(back.extra, card.extra);
assert_eq!(back.extra.get("atlas:layer"), Some(&serde_json::json!(84)));
let python_shaped = r#"{
"title":"From Python","extra":{"pipeline":"nightly"},
"triple_count":1,"quad_count":1,"named_graph_count":0,"term_count":3,
"format_version":5
}"#;
let card = DatasetCard::from_json_bytes(python_shaped.as_bytes()).unwrap();
assert_eq!(
card.extra.get("pipeline"),
Some(&serde_json::json!("nightly"))
);
}
#[test]
fn extra_limits_bite_exactly_at_the_boundary() {
use crate::card_input::normalize_extra;
use serde_json::{json, Map, Value};
let bag = |v: Vec<(String, Value)>| -> Map<String, Value> { v.into_iter().collect() };
let pad = |n: usize| bag(vec![("pad".into(), json!("x".repeat(n)))]);
let at = pad(CARD_EXTRA_MAX_BYTES - 10);
assert_eq!(serde_json::to_vec(&at).unwrap().len(), CARD_EXTRA_MAX_BYTES);
normalize_extra(at).expect("exactly at the byte cap is accepted");
let over = pad(CARD_EXTRA_MAX_BYTES - 9);
assert_eq!(
serde_json::to_vec(&over).unwrap().len(),
CARD_EXTRA_MAX_BYTES + 1
);
let err = normalize_extra(over).unwrap_err().to_string();
assert!(
err.contains(&format!("{} bytes", CARD_EXTRA_MAX_BYTES + 1)),
"the error states the actual size: {err}"
);
let keys = |n: usize| {
bag((0..n)
.map(|i| (format!("k{i:03}"), json!(1)))
.collect::<Vec<_>>())
};
normalize_extra(keys(CARD_EXTRA_MAX_KEYS)).expect("at the key cap");
assert!(normalize_extra(keys(CARD_EXTRA_MAX_KEYS + 1)).is_err());
normalize_extra(bag(vec![("k".repeat(CARD_EXTRA_MAX_KEY_BYTES), json!(1))]))
.expect("at the key-length cap");
assert!(normalize_extra(bag(vec![(
"k".repeat(CARD_EXTRA_MAX_KEY_BYTES + 1),
json!(1)
)]))
.is_err());
assert!(normalize_extra(bag(vec![(String::new(), json!(1))])).is_err());
normalize_extra(bag(vec![("d".into(), json!({"a": {"b": 1}}))])).expect("at the depth cap");
assert!(normalize_extra(bag(vec![("d".into(), json!({"a": {"b": {"c": 1}}}))])).is_err());
assert!(normalize_extra(bag(vec![("d".into(), json!([[[1]]]))])).is_err());
let err = normalize_extra(bag(vec![("@context".into(), json!({}))]))
.unwrap_err()
.to_string();
assert!(err.contains("reserved"), "loud reservation: {err}");
}
#[test]
fn extra_fields_keep_the_content_hash_reproducible() {
let build = |json: &str| {
let input: CardInput = serde_json::from_str::<CardInput>(json)
.unwrap()
.normalize()
.unwrap();
let quads = enriched_fixture();
let card = derive_card(&quads, 12, 0, input);
let (image, _) =
crate::ingest::assemble_dataset_with_opts(quads, true, false, None, |_, _| {
card.to_json_bytes()
});
image
};
let a = build(r#"{"title":"T","extra":{"b":{"y":2,"x":1},"a":[1,2]}}"#);
let b = build(r#"{"title":"T","extra":{"a":[1,2],"b":{"x":1,"y":2}}}"#);
assert_eq!(a, b, "reordered-input builds are byte-identical");
let ha = Header::from_bytes(&a).unwrap().content_hash;
assert_eq!(ha, Header::from_bytes(&b).unwrap().content_hash);
let plain = build(r#"{"title":"T"}"#);
assert_ne!(ha, Header::from_bytes(&plain).unwrap().content_hash);
}
#[test]
fn top_level_custom_keys_are_rejected_not_dropped() {
let err = serde_json::from_str::<CardInput>(r#"{"title":"T","my_field":1}"#)
.unwrap_err()
.to_string();
assert!(err.contains("my_field"), "names the stray key: {err}");
let ok: CardInput =
serde_json::from_str(r#"{"title":"T","extra":{"my_field":1}}"#).unwrap();
assert_eq!(ok.extra.get("my_field"), Some(&serde_json::json!(1)));
}
#[test]
fn description_reads_as_a_string_or_as_lines() {
let from_lines: CardInput = serde_json::from_value(serde_json::json!({
"description": ["## Contents", "", "- a", "- b"]
}))
.unwrap();
assert_eq!(
from_lines.description.as_deref(),
Some("## Contents\n\n- a\n- b")
);
let from_string: CardInput =
serde_json::from_str("{\"description\":\"## Contents\\n\\n- a\\n- b\"}").unwrap();
assert_eq!(from_string.description, from_lines.description);
let none: CardInput = serde_json::from_str(r#"{"title":"T"}"#).unwrap();
assert!(none.description.is_none());
let null: CardInput = serde_json::from_str(r#"{"description":null}"#).unwrap();
assert!(null.description.is_none());
let over = "x".repeat(crate::card::CARD_DESCRIPTION_MAX_BYTES + 1);
let err = serde_json::from_str::<CardInput>(&format!(
"{{\"description\":{}}}",
serde_json::to_string(&over).unwrap()
))
.unwrap_err()
.to_string();
assert!(err.contains("over the"), "{err}");
}
#[test]
fn curated_field_list_matches_the_deny_unknown_fields_struct() {
for f in crate::card::CURATED_CARD_FIELDS {
let doc = format!(r#"{{"{f}":null}}"#);
if let Err(e) = serde_json::from_str::<CardInput>(&doc) {
assert!(
!e.to_string().contains("unknown field"),
"CURATED_CARD_FIELDS lists `{f}`, but CardInput rejects it as unknown: {e}",
);
}
}
let err = serde_json::from_str::<CardInput>(r#"{"nope":1}"#)
.unwrap_err()
.to_string();
for name in err
.split("expected one of ")
.nth(1)
.expect("serde names the expected fields")
.split(" at line ")
.next()
.unwrap()
.split(", ")
.map(|s| s.trim().trim_matches('`'))
.filter(|s| !s.is_empty())
{
assert!(
crate::card::CURATED_CARD_FIELDS.contains(&name),
"CardInput accepts `{name}` but CURATED_CARD_FIELDS does not list it — \
the browser builder would reject a card the CLI accepts",
);
}
}
#[test]
fn a_browser_written_card_deserializes_here() {
let curated = crate::card::validate_curated_card(&serde_json::json!({
"title": "Built in a browser",
"keywords": ["b", "a"],
"theme": ["http://publications.europa.eu/resource/authority/data-theme/GOVE"],
"creators": [{"name": "Ada", "orcid": "https://orcid.org/0000-0002-1825-0097"}],
"extra": {"internal_id": "DS-1"},
}))
.expect("valid curated document");
let composed = crate::card::compose_curated_card(curated, 3, 4, 1, 9, 5);
let text = serde_json::to_string(&composed).unwrap();
let card: DatasetCard = serde_json::from_str(&text).expect("CLI reads a browser card");
assert_eq!(card.title.as_deref(), Some("Built in a browser"));
assert_eq!(card.keywords, vec!["a".to_string(), "b".to_string()]);
assert_eq!(card.creators.len(), 1);
assert_eq!(card.triple_count, 3);
assert_eq!(card.quad_count, 4);
assert_eq!(card.term_count, 9);
assert_eq!(card.format_version, 5);
assert!(card.predicates.is_empty() && card.queries.is_empty());
assert!(!text.contains("\"predicates\""), "{text}");
assert!(!text.contains("\"top_n\""), "{text}");
}
#[test]
fn old_reader_struct_accepts_new_card_json() {
#[derive(serde::Deserialize)]
struct OldCard {
title: Option<String>,
triple_count: u64,
#[serde(default)]
predicates: Vec<(String, u64)>,
format_version: u8,
}
let input: CardInput = serde_json::from_str(
r#"{"title":"New","version":"1","creators":[{"name":"A"}],
"canonical_url":"https://x/y.rete","derived_from":["https://x/d.nt"],
"extra":{"atlas:review":"approved"}}"#,
)
.unwrap();
let quads = vec![q("<http://ex/a>", "<http://ex/p>", "<http://ex/b>")];
let card = derive_card(&quads, 3, 0, input);
assert_eq!(card.top_n, CARD_TOP_N as u32);
let bytes = card.to_json_bytes();
let old: OldCard = serde_json::from_slice(&bytes).expect("old struct parses new JSON");
assert_eq!(old.title.as_deref(), Some("New"));
assert_eq!(old.triple_count, 1);
assert_eq!(old.format_version, card.format_version);
assert!(!old.predicates.is_empty());
}
#[test]
fn coherence_stamp_is_deterministic_and_additive() {
let dis = "<http://www.w3.org/2002/07/owl#disjointWith>";
let rt = "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>";
let base = vec![
("<http://ex/C>".into(), dis.into(), "<http://ex/D>".into()),
("<http://ex/x>".into(), rt.into(), "<http://ex/C>".into()),
("<http://ex/x>".into(), rt.into(), "<http://ex/D>".into()),
];
let r = crate::reason(&base);
let c1 = Coherence::from_reasoning(&r, false);
assert_eq!(c1, Coherence::from_reasoning(&r, false));
assert!(!c1.coherent);
assert_eq!(c1.inconsistency_count, 1);
assert_eq!(c1.by_kind, vec![("disjoint-classes".to_string(), 1)]);
assert_eq!(c1.rules, crate::REASON_RULESET);
let json = DatasetCard::default()
.with_coherence(&r, false)
.to_json_bytes();
let text = std::str::from_utf8(&json).unwrap();
assert!(text.contains("coherence"));
assert_eq!(DatasetCard::from_json_bytes(&json).unwrap().coherence, c1);
let plain = DatasetCard::default().to_json_bytes();
assert!(!std::str::from_utf8(&plain).unwrap().contains("coherence"));
assert!(DatasetCard::default().coherence.is_empty());
}
#[test]
fn base_iri_is_common_prefix_with_fallback() {
let mut ns = BTreeMap::new();
ns.insert("http://ex/person/".to_string(), 2000u64);
ns.insert("http://ex/place/".to_string(), 600);
ns.insert("http://ex/org/".to_string(), 120);
assert_eq!(derive_base_iri(&ns).as_deref(), Some("http://ex/"));
let mut one = BTreeMap::new();
one.insert("http://data.example.org/id/".to_string(), 10u64);
assert_eq!(
derive_base_iri(&one).as_deref(),
Some("http://data.example.org/id/")
);
let mut multi = BTreeMap::new();
multi.insert("http://a.example/".to_string(), 100u64);
multi.insert("http://b.other/".to_string(), 5);
assert_eq!(
derive_base_iri(&multi).as_deref(),
Some("http://a.example/")
);
assert_eq!(derive_base_iri(&BTreeMap::new()), None);
}
#[test]
fn parse_literal_decomposes_forms() {
let plain = parse_literal("\"hello\"").unwrap();
assert_eq!(plain.value, "hello");
assert!(plain.datatype.ends_with("XMLSchema#string"));
assert_eq!(plain.lang, "");
let tagged = parse_literal("\"bonjour\"@fr").unwrap();
assert_eq!(tagged.lang, "fr");
assert!(tagged.datatype.ends_with("#langString"));
let typed = parse_literal("\"42\"^^<http://www.w3.org/2001/XMLSchema#integer>").unwrap();
assert_eq!(typed.value, "42");
assert_eq!(typed.datatype, "http://www.w3.org/2001/XMLSchema#integer");
let esc = parse_literal("\"a\\\"b\"@en").unwrap();
assert_eq!(esc.value, "a\\\"b");
assert_eq!(esc.lang, "en");
assert!(parse_literal("<http://ex/x>").is_none());
assert!(parse_literal("_:b0").is_none());
}
}