1use std::collections::{BTreeMap, BTreeSet};
31
32use serde::{Deserialize, Serialize};
33
34use crate::{Header, RDF_TYPE};
35#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
37pub struct CardAnnexMarker {
38 pub fields: Vec<String>,
40 pub bytes: u64,
42}
43
44#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
48pub struct DatasetCard {
49 #[serde(default, skip_serializing_if = "Option::is_none")]
50 pub title: Option<String>,
51 #[serde(default, skip_serializing_if = "Option::is_none")]
52 pub description: Option<String>,
53 #[serde(default, skip_serializing_if = "Option::is_none")]
54 pub license: Option<String>,
55 #[serde(default, skip_serializing_if = "Option::is_none")]
56 pub source: Option<String>,
57 #[serde(default, skip_serializing_if = "Option::is_none")]
58 pub created: Option<String>,
59
60 #[serde(default, skip_serializing_if = "Option::is_none")]
71 pub version: Option<String>,
72 #[serde(default, skip_serializing_if = "Vec::is_empty")]
75 pub creators: Vec<Creator>,
76 #[serde(default, skip_serializing_if = "Option::is_none")]
78 pub publisher: Option<Publisher>,
79 #[serde(default, skip_serializing_if = "Option::is_none")]
82 pub canonical_url: Option<String>,
83 #[serde(default, skip_serializing_if = "Option::is_none")]
85 pub sparql_endpoint: Option<String>,
86 #[serde(default, skip_serializing_if = "Option::is_none")]
89 pub source_date: Option<String>,
90 #[serde(default, skip_serializing_if = "Vec::is_empty")]
93 pub derived_from: Vec<String>,
94 #[serde(default, skip_serializing_if = "Option::is_none")]
96 pub doi: Option<String>,
97 #[serde(default, skip_serializing_if = "Option::is_none")]
99 pub cite_as: Option<String>,
100 #[serde(default, skip_serializing_if = "Vec::is_empty")]
111 pub keywords: Vec<String>,
112 #[serde(default, skip_serializing_if = "Vec::is_empty")]
119 pub theme: Vec<String>,
120 #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
136 pub extra: BTreeMap<String, serde_json::Value>,
137 #[serde(default, skip_serializing_if = "Option::is_none")]
143 pub annex: Option<CardAnnexMarker>,
144
145 pub triple_count: u64,
146 pub quad_count: u64,
147 pub named_graph_count: u64,
148 pub term_count: u64,
149
150 #[serde(default, skip_serializing_if = "Vec::is_empty")]
151 pub predicates: Vec<(String, u64)>,
152 #[serde(default, skip_serializing_if = "Vec::is_empty")]
153 pub classes: Vec<(String, u64)>,
154 #[serde(default, skip_serializing_if = "Vec::is_empty")]
155 pub vocabularies: Vec<String>,
156 #[serde(default, skip_serializing_if = "Vec::is_empty")]
157 pub example_queries: Vec<String>,
158
159 #[serde(default, skip_serializing_if = "Vec::is_empty")]
166 pub datatypes: Vec<(String, u64)>,
167 #[serde(default, skip_serializing_if = "Vec::is_empty")]
170 pub languages: Vec<(String, u64)>,
171 #[serde(default, skip_serializing_if = "Vec::is_empty")]
175 pub class_links: Vec<ClassLink>,
176 #[serde(default, skip_serializing_if = "Vec::is_empty")]
178 pub top_hubs: Vec<(String, u64)>,
179 #[serde(default, skip_serializing_if = "Vec::is_empty")]
181 pub in_hubs: Vec<(String, u64)>,
182 #[serde(default, skip_serializing_if = "Signals::is_empty")]
185 pub signals: Signals,
186 #[serde(default, skip_serializing_if = "Vec::is_empty")]
190 pub queries: Vec<ExampleQuery>,
191 #[serde(default, skip_serializing_if = "Coherence::is_empty")]
195 pub coherence: Coherence,
196 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
198 pub truncated: bool,
199 #[serde(default, skip_serializing_if = "is_zero_u32")]
203 pub top_n: u32,
204
205 pub format_version: u8,
209}
210
211fn is_zero_u32(v: &u32) -> bool {
213 *v == 0
214}
215
216#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
218pub struct Creator {
219 pub name: String,
220 #[serde(default, skip_serializing_if = "Option::is_none")]
222 pub orcid: Option<String>,
223}
224
225#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
227pub struct Publisher {
228 pub name: String,
229 #[serde(default, skip_serializing_if = "Option::is_none")]
231 pub ror: Option<String>,
232}
233
234#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
238#[serde(rename_all = "lowercase")]
239pub enum Tier {
240 Card,
241 Summary,
242 Index,
243}
244
245#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
247pub struct ExampleQuery {
248 pub id: String,
250 pub title: String,
252 pub dimension: String,
254 pub question: String,
256 pub sparql: String,
258 pub tier: Tier,
260 #[serde(default, skip_serializing_if = "Vec::is_empty")]
262 pub requires: Vec<String>,
263}
264
265#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
269pub struct ClassLink {
270 pub s_class: String,
271 pub predicate: String,
272 pub o_class: String,
273 pub count: u64,
274}
275
276#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
281pub struct Signals {
282 #[serde(default, skip_serializing_if = "Option::is_none")]
284 pub label_predicate: Option<String>,
285 #[serde(default, skip_serializing_if = "Option::is_none")]
287 pub default_lang: Option<String>,
288 #[serde(default, skip_serializing_if = "Option::is_none")]
290 pub base_iri: Option<String>,
291 #[serde(default, skip_serializing_if = "Vec::is_empty")]
293 pub time_predicates: Vec<String>,
294 #[serde(default, skip_serializing_if = "Vec::is_empty")]
297 pub numeric_predicates: Vec<String>,
298 #[serde(default, skip_serializing_if = "Vec::is_empty")]
300 pub link_predicates: Vec<String>,
301 #[serde(default, skip_serializing_if = "Vec::is_empty")]
315 pub annotation_predicates: Vec<String>,
316 #[serde(default, skip_serializing_if = "Vec::is_empty")]
321 pub quoting_predicates: Vec<String>,
322 #[serde(default, skip_serializing_if = "Option::is_none")]
340 pub annotated_statement: Option<(String, String)>,
341 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
343 pub geo_wkt: bool,
344 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
346 pub geo_latlong: bool,
347 #[serde(default, skip_serializing_if = "Option::is_none")]
349 pub temporal_extent: Option<(String, String)>,
350 #[serde(default, skip_serializing_if = "Option::is_none")]
352 pub spatial_bbox: Option<[f64; 4]>,
353 #[serde(default, skip_serializing_if = "Option::is_none")]
363 pub text_index: Option<TextIndexSignal>,
364 #[serde(default, skip_serializing_if = "Option::is_none")]
369 pub label_index: Option<LabelIndexSignal>,
370 #[serde(default, skip_serializing_if = "Option::is_none")]
375 pub permutations: Option<PermutationsSignal>,
376 #[serde(default, skip_serializing_if = "Option::is_none")]
382 pub quoted_triples: Option<QuotedTriplesSignal>,
383}
384
385impl Signals {
386 pub fn is_empty(&self) -> bool {
388 *self == Signals::default()
389 }
390}
391
392#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
419pub struct TextIndexSignal {
420 pub present: bool,
423 #[serde(default, skip_serializing_if = "is_zero_u64")]
427 pub bytes: u64,
428 #[serde(default, skip_serializing_if = "Option::is_none")]
433 pub token_table_bytes: Option<u64>,
434}
435
436fn is_zero_u64(v: &u64) -> bool {
438 *v == 0
439}
440
441#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
461pub struct PermutationsSignal {
462 pub count: u8,
464 pub names: Vec<String>,
468 pub merge_join: bool,
472}
473
474impl PermutationsSignal {
475 pub fn probe(header: &Header) -> Self {
479 let perms = header.perms;
480 PermutationsSignal {
481 count: perms.len() as u8,
482 names: perms.names().into_iter().map(str::to_string).collect(),
483 merge_join: perms.has_merge_orders(),
484 }
485 }
486
487 pub fn describe(&self) -> String {
489 if self.merge_join {
490 format!(
491 "{} index permutations ({}) — sort-merge joins available",
492 self.count,
493 self.names.join("/")
494 )
495 } else {
496 format!(
497 "{} index permutations ({}) — same rows, same routing; \
498 no sort-merge join (hash/probe answers instead)",
499 self.count,
500 self.names.join("/")
501 )
502 }
503 }
504}
505
506pub const QUOTED_TRIPLE_SURFACE_RDF12: &str = "rdf12";
513
514pub const QUOTED_TRIPLE_SURFACE_RDF_STAR: &str = "rdf-star";
517
518#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
547pub struct QuotedTriplesSignal {
548 pub present: bool,
550 #[serde(default, skip_serializing_if = "Vec::is_empty")]
553 pub export_surfaces: Vec<String>,
554 #[serde(default, skip_serializing_if = "Option::is_none")]
557 pub export_default: Option<String>,
558}
559
560impl QuotedTriplesSignal {
561 pub fn probe(header: &Header) -> Self {
564 if !header.has_quoted_triples() {
565 return QuotedTriplesSignal {
566 present: false,
567 export_surfaces: Vec::new(),
568 export_default: None,
569 };
570 }
571 QuotedTriplesSignal {
572 present: true,
573 export_surfaces: vec![
574 QUOTED_TRIPLE_SURFACE_RDF12.to_string(),
575 QUOTED_TRIPLE_SURFACE_RDF_STAR.to_string(),
576 ],
577 export_default: Some(QUOTED_TRIPLE_SURFACE_RDF12.to_string()),
578 }
579 }
580
581 pub fn describe(&self) -> Option<String> {
585 if !self.present {
586 return None;
587 }
588 Some(format!(
589 "present — `rete export` writes RDF 1.2 triple terms `<<( s p o )>>` \
590 by default; `--quoted-triple-syntax {QUOTED_TRIPLE_SURFACE_RDF_STAR}` \
591 writes `<<s p o>>`"
592 ))
593 }
594}
595impl TextIndexSignal {
596 pub fn probe<R: crate::RangeReader + ?Sized>(reader: &R, header: &Header) -> Self {
601 if header.text_index_len == 0 {
602 return TextIndexSignal::default();
603 }
604 if crate::v06::reads(header.version) {
605 return match crate::v06::open::read_signals(reader) {
606 Ok((Some((bytes, tt)), _)) => TextIndexSignal {
607 present: true,
608 bytes,
609 token_table_bytes: Some(tt),
610 },
611 _ => TextIndexSignal {
612 present: true,
613 ..TextIndexSignal::default()
614 },
615 };
616 }
617 TextIndexSignal {
618 present: true,
619 bytes: header.text_index_len,
620 token_table_bytes: crate::read_text_index_token_table_len_ranged(reader, header),
621 }
622 }
623
624 pub fn describe(&self) -> String {
626 if !self.present {
627 return "no TEXT_INDEX section — CONTAINS/regex still answer, by full scan".to_string();
628 }
629 match self.token_table_bytes {
630 Some(tt) => format!(
631 "TEXT_INDEX present — {} bytes ({tt} of them the token table a first search reads)",
632 self.bytes
633 ),
634 None => format!("TEXT_INDEX present — {} bytes", self.bytes),
635 }
636 }
637}
638
639#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
650pub struct LabelIndexSignal {
651 pub present: bool,
654 #[serde(default, skip_serializing_if = "is_zero_u64")]
656 pub bytes: u64,
657 #[serde(default, skip_serializing_if = "Option::is_none")]
660 pub entries: Option<u64>,
661}
662
663impl LabelIndexSignal {
664 pub fn probe<R: crate::RangeReader + ?Sized>(reader: &R, header: &Header) -> Self {
668 let Some(s) = header.label_index_section() else {
669 return LabelIndexSignal::default();
670 };
671 if crate::v06::reads(header.version) {
672 return match crate::v06::open::read_signals(reader) {
673 Ok((_, Some((bytes, entries)))) => LabelIndexSignal {
674 present: true,
675 bytes,
676 entries,
677 },
678 _ => LabelIndexSignal {
679 present: true,
680 ..LabelIndexSignal::default()
681 },
682 };
683 }
684 LabelIndexSignal {
685 present: true,
686 bytes: s.length,
687 entries: crate::label_index::read_label_index_entries_ranged(reader, header),
688 }
689 }
690
691 pub fn describe(&self) -> String {
693 if !self.present {
694 return "no LABEL_INDEX section — a prefix search finds only what a pyramid's \
695 legacy label block holds, if any"
696 .to_string();
697 }
698 match self.entries {
699 Some(n) => format!("LABEL_INDEX present — {n} labels, {} bytes", self.bytes),
700 None => format!("LABEL_INDEX present — {} bytes", self.bytes),
701 }
702 }
703}
704
705#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
712pub struct Coherence {
713 pub coherent: bool,
715 pub inconsistency_count: u32,
717 #[serde(default, skip_serializing_if = "Vec::is_empty")]
719 pub by_kind: Vec<(String, u32)>,
720 pub scope: String,
722 pub rules: String,
724 pub materialized: bool,
726}
727
728impl Coherence {
729 pub fn is_empty(&self) -> bool {
731 *self == Coherence::default()
732 }
733
734 pub fn from_reasoning(r: &crate::Reasoning, materialized: bool) -> Self {
738 let mut hist: BTreeMap<&str, u32> = BTreeMap::new();
739 for inc in &r.inconsistencies {
740 *hist.entry(inc.kind).or_default() += 1;
741 }
742 Coherence {
743 coherent: r.inconsistencies.is_empty(),
744 inconsistency_count: r.inconsistencies.len() as u32,
745 by_kind: hist.into_iter().map(|(k, v)| (k.to_string(), v)).collect(),
746 scope: "default".to_string(),
747 rules: crate::REASON_RULESET.to_string(),
748 materialized,
749 }
750 }
751}
752
753#[derive(Debug, Clone, Default, Deserialize)]
765#[serde(deny_unknown_fields)]
766pub struct CardInput {
767 pub title: Option<String>,
768 #[serde(default, deserialize_with = "de_description")]
772 pub description: Option<String>,
773 pub license: Option<String>,
774 pub source: Option<String>,
775 pub created: Option<String>,
776 pub version: Option<String>,
777 #[serde(default)]
778 pub creators: Vec<Creator>,
779 pub publisher: Option<Publisher>,
780 pub canonical_url: Option<String>,
781 pub sparql_endpoint: Option<String>,
782 pub source_date: Option<String>,
783 #[serde(default)]
784 pub derived_from: Vec<String>,
785 pub doi: Option<String>,
786 pub cite_as: Option<String>,
787 #[serde(default)]
789 pub keywords: Vec<String>,
790 #[serde(default)]
792 pub theme: Vec<String>,
793 #[serde(default)]
795 pub extra: BTreeMap<String, serde_json::Value>,
796 #[serde(default)]
797 pub example_queries: Vec<String>,
798 pub base_iri: Option<String>,
806}
807
808fn de_description<'de, D>(d: D) -> Result<Option<String>, D::Error>
818where
819 D: serde::Deserializer<'de>,
820{
821 use serde::de::Error as _;
822 let v = Option::<serde_json::Value>::deserialize(d)?;
823 match v {
824 None | Some(serde_json::Value::Null) => Ok(None),
825 Some(v) => crate::card_input::normalize_description(&v)
826 .map(Some)
827 .map_err(D::Error::custom),
828 }
829}
830
831impl CardInput {
832 pub fn from_card(card: &DatasetCard) -> Self {
838 CardInput {
839 title: card.title.clone(),
840 description: card.description.clone(),
841 license: card.license.clone(),
842 source: card.source.clone(),
843 created: card.created.clone(),
844 version: card.version.clone(),
845 creators: card.creators.clone(),
846 publisher: card.publisher.clone(),
847 canonical_url: card.canonical_url.clone(),
848 sparql_endpoint: card.sparql_endpoint.clone(),
849 source_date: card.source_date.clone(),
850 derived_from: card.derived_from.clone(),
851 doi: card.doi.clone(),
852 cite_as: card.cite_as.clone(),
853 keywords: card.keywords.clone(),
854 theme: card.theme.clone(),
855 extra: card.extra.clone(),
856 example_queries: card.example_queries.clone(),
857 base_iri: None,
858 }
859 }
860
861 pub fn is_empty(&self) -> bool {
863 self.title.is_none()
864 && self.description.is_none()
865 && self.license.is_none()
866 && self.source.is_none()
867 && self.created.is_none()
868 && self.version.is_none()
869 && self.creators.is_empty()
870 && self.publisher.is_none()
871 && self.canonical_url.is_none()
872 && self.sparql_endpoint.is_none()
873 && self.source_date.is_none()
874 && self.derived_from.is_empty()
875 && self.doi.is_none()
876 && self.cite_as.is_none()
877 && self.keywords.is_empty()
878 && self.theme.is_empty()
879 && self.extra.is_empty()
880 && self.example_queries.is_empty()
881 && self.base_iri.is_none()
882 }
883
884 fn fill(self, card: &mut DatasetCard) {
887 card.title = self.title;
888 card.description = self.description;
889 card.license = self.license;
890 card.source = self.source;
891 card.created = self.created;
892 card.version = self.version;
893 card.creators = self.creators;
894 card.publisher = self.publisher;
895 card.canonical_url = self.canonical_url;
896 card.sparql_endpoint = self.sparql_endpoint;
897 card.source_date = self.source_date;
898 card.derived_from = self.derived_from;
899 card.doi = self.doi;
900 card.cite_as = self.cite_as;
901 card.keywords = self.keywords;
902 card.theme = self.theme;
903 card.extra = self.extra;
904 card.example_queries = self.example_queries;
905 if self.base_iri.is_some() {
906 card.signals.base_iri = self.base_iri;
907 }
908 }
909
910 pub fn from_json_str(text: &str) -> Result<Self, String> {
924 let parsed: CardInput = serde_json::from_str(text).map_err(|e| {
925 let hint = if e.to_string().contains("unknown field") {
928 crate::card_input::UNKNOWN_FIELD_HINT
929 } else {
930 ""
931 };
932 format!("{e}{hint}")
933 })?;
934 parsed.normalize()
935 }
936
937 pub fn normalize(self) -> Result<Self, String> {
946 self.normalize_max(
947 crate::card_input::CARD_DESCRIPTION_MAX_BYTES,
948 crate::card_input::CARD_EXTRA_MAX_BYTES,
949 )
950 }
951
952 pub fn normalize_with_annex(self) -> Result<Self, String> {
957 let max = crate::card_input::CARD_ANNEX_FIELD_MAX_BYTES;
958 self.normalize_max(max, max)
959 }
960
961 pub fn from_json_str_with_annex(text: &str) -> Result<Self, String> {
965 let mut doc: serde_json::Value = serde_json::from_str(text).map_err(|e| e.to_string())?;
966 let description = match doc.as_object_mut().and_then(|o| o.remove("description")) {
969 None | Some(serde_json::Value::Null) => None,
970 Some(v) => Some(crate::card_input::description_text(&v)?),
971 };
972 let mut parsed: CardInput = serde_json::from_value(doc).map_err(|e| {
973 let hint = if e.to_string().contains("unknown field") {
974 crate::card_input::UNKNOWN_FIELD_HINT
975 } else {
976 ""
977 };
978 format!("{e}{hint}")
979 })?;
980 parsed.description = description;
981 parsed.normalize_with_annex()
982 }
983
984 fn normalize_max(mut self, description_max: usize, extra_max: usize) -> Result<Self, String> {
985 if let Some(d) = &self.description {
986 crate::card_input::check_description_len_max(d, description_max)?;
987 }
988 self.keywords = crate::card_input::normalize_string_list(
989 "keywords",
990 std::mem::take(&mut self.keywords),
991 )?;
992 self.theme = crate::card_input::normalize_themes(std::mem::take(&mut self.theme))?;
993 let bag: serde_json::Map<String, serde_json::Value> =
994 std::mem::take(&mut self.extra).into_iter().collect();
995 self.extra = crate::card_input::normalize_extra_max(bag, extra_max)?
996 .into_iter()
997 .collect();
998 if let Some(b) = &self.base_iri {
999 crate::card_input::check_base_iri(b)?;
1000 }
1001 Ok(self)
1002 }
1003}
1004
1005impl DatasetCard {
1006 pub fn to_json_bytes(&self) -> Vec<u8> {
1017 if self.signals.text_index.is_none()
1018 && self.signals.label_index.is_none()
1019 && self.signals.permutations.is_none()
1020 && self.signals.quoted_triples.is_none()
1021 {
1022 return serde_json::to_vec(self).expect("DatasetCard serializes");
1023 }
1024 let mut stored = self.clone();
1025 stored.signals.text_index = None;
1026 stored.signals.label_index = None;
1027 stored.signals.permutations = None;
1028 stored.signals.quoted_triples = None;
1029 serde_json::to_vec(&stored).expect("DatasetCard serializes")
1030 }
1031
1032 pub fn observe_text_index(&mut self, measured: TextIndexSignal) -> Option<TextIndexSignal> {
1039 self.signals.text_index.replace(measured)
1040 }
1041
1042 pub fn observe_label_index(&mut self, measured: LabelIndexSignal) -> Option<LabelIndexSignal> {
1045 self.signals.label_index.replace(measured)
1046 }
1047
1048 pub fn observe_permutations(
1053 &mut self,
1054 measured: PermutationsSignal,
1055 ) -> Option<PermutationsSignal> {
1056 self.signals.permutations.replace(measured)
1057 }
1058
1059 pub fn observe_quoted_triples(
1064 &mut self,
1065 measured: QuotedTriplesSignal,
1066 ) -> Option<QuotedTriplesSignal> {
1067 self.signals.quoted_triples.replace(measured)
1068 }
1069
1070 pub fn from_json_bytes(b: &[u8]) -> Result<Self, String> {
1076 serde_json::from_slice(b).map_err(|e| format!("malformed dataset card: {e}"))
1077 }
1078
1079 pub fn with_coherence(mut self, r: &crate::Reasoning, materialized: bool) -> Self {
1081 self.coherence = Coherence::from_reasoning(r, materialized);
1082 self
1083 }
1084
1085 pub fn with_final_counts(mut self, counts: crate::ingest::FinalCounts) -> Self {
1099 self.triple_count = counts.default_triples;
1100 self.quad_count = counts.quads;
1101 self
1102 }
1103}
1104
1105pub const CARD_TOP_N: usize = 100;
1111
1112const ANNOTATION_WITNESS_SAMPLE: usize = 4096;
1128
1129const RDFS_LABEL: &str = "<http://www.w3.org/2000/01/rdf-schema#label>";
1131const SKOS_PREFLABEL: &str = "<http://www.w3.org/2004/02/skos/core#prefLabel>";
1132const SCHEMA_NAME: &str = "<http://schema.org/name>";
1133const FOAF_NAME: &str = "<http://xmlns.com/foaf/0.1/name>";
1134const DCT_TITLE: &str = "<http://purl.org/dc/terms/title>";
1135const OWL_SAMEAS: &str = "<http://www.w3.org/2002/07/owl#sameAs>";
1136const SKOS_EXACTMATCH: &str = "<http://www.w3.org/2004/02/skos/core#exactMatch>";
1137const SKOS_CLOSEMATCH: &str = "<http://www.w3.org/2004/02/skos/core#closeMatch>";
1138const RDFS_SEEALSO: &str = "<http://www.w3.org/2000/01/rdf-schema#seeAlso>";
1139pub(crate) const WGS_LAT: &str = "<http://www.w3.org/2003/01/geo/wgs84_pos#lat>";
1140pub(crate) const WGS_LONG: &str = "<http://www.w3.org/2003/01/geo/wgs84_pos#long>";
1141pub const GEO_ASWKT: &str = "<http://www.opengis.net/ont/geosparql#asWKT>";
1144pub const GEO_HASGEOMETRY: &str = "<http://www.opengis.net/ont/geosparql#hasGeometry>";
1145const GEO_WKTLITERAL: &str = "http://www.opengis.net/ont/geosparql#wktLiteral";
1147const XSD_STRING: &str = "http://www.w3.org/2001/XMLSchema#string";
1148const RDF_LANGSTRING: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#langString";
1149
1150pub const O_LITERAL: &str = "(literal)";
1155
1156const LABEL_PREDICATES: &[&str] = &[
1158 SKOS_PREFLABEL,
1159 RDFS_LABEL,
1160 SCHEMA_NAME,
1161 FOAF_NAME,
1162 DCT_TITLE,
1163];
1164pub(crate) const LINK_PREDICATES: &[&str] =
1166 &[OWL_SAMEAS, SKOS_EXACTMATCH, SKOS_CLOSEMATCH, RDFS_SEEALSO];
1167
1168pub trait CardTripleSource {
1175 fn replay(&self, f: &mut dyn FnMut(&str, &str, &str));
1176}
1177
1178pub fn curated_counts_card(statements: u64, term_count: u64, curated: CardInput) -> DatasetCard {
1187 let mut card = DatasetCard {
1188 triple_count: statements,
1189 quad_count: statements,
1190 named_graph_count: 0,
1191 term_count,
1192 format_version: crate::format::LEGACY_FORMAT_VERSION,
1193 ..DatasetCard::default()
1194 };
1195 curated.fill(&mut card);
1196 card
1197}
1198
1199pub fn derive_card(
1200 quads: &[(String, String, String, Option<String>)],
1201 term_count: u64,
1202 named_graph_count: u64,
1203 curated: CardInput,
1204) -> DatasetCard {
1205 if crate::build_progress::listening() {
1206 let default = quads.iter().filter(|q| q.3.is_none()).count() as u64;
1207 crate::build_progress::set_total(Some(2 * default), crate::build_progress::Unit::Records);
1208 }
1209 struct Quads<'a>(&'a [(String, String, String, Option<String>)]);
1210 impl CardTripleSource for Quads<'_> {
1211 fn replay(&self, f: &mut dyn FnMut(&str, &str, &str)) {
1212 for (s, p, o, g) in self.0 {
1213 if g.is_none() {
1214 f(s, p, o);
1215 }
1216 }
1217 }
1218 }
1219 derive_card_from(
1220 &Quads(quads),
1221 quads.len() as u64,
1222 term_count,
1223 named_graph_count,
1224 curated,
1225 )
1226}
1227
1228pub fn derive_card_encoded(
1234 dict: &crate::Dictionary,
1235 triples: &[(u32, u32, u32)],
1236 quad_count: u64,
1237 term_count: u64,
1238 named_graph_count: u64,
1239 curated: CardInput,
1240) -> DatasetCard {
1241 crate::build_progress::set_total(
1242 Some(2 * triples.len() as u64),
1243 crate::build_progress::Unit::Records,
1244 );
1245 struct Encoded<'a> {
1246 dict: &'a crate::Dictionary,
1247 triples: &'a [(u32, u32, u32)],
1248 }
1249 impl CardTripleSource for Encoded<'_> {
1250 fn replay(&self, f: &mut dyn FnMut(&str, &str, &str)) {
1251 for &(s, p, o) in self.triples {
1252 let st = self.dict.subject_term(s).unwrap_or_default();
1253 let pt = self.dict.predicate_term(p).unwrap_or_default();
1254 let ot = self.dict.object_term(o).unwrap_or_default();
1255 f(&st, &pt, &ot);
1256 }
1257 }
1258 }
1259 derive_card_from(
1260 &Encoded { dict, triples },
1261 quad_count,
1262 term_count,
1263 named_graph_count,
1264 curated,
1265 )
1266}
1267
1268pub fn derive_card_from(
1276 src: &dyn CardTripleSource,
1277 quad_count: u64,
1278 term_count: u64,
1279 named_graph_count: u64,
1280 curated: CardInput,
1281) -> DatasetCard {
1282 let mut class_of: std::collections::HashMap<String, String> = std::collections::HashMap::new();
1288 let mut quoted_subject_preds: BTreeSet<String> = BTreeSet::new();
1289 let mut annotated_tokens: BTreeMap<String, String> = BTreeMap::new();
1290 let mut ticker = crate::build_progress::Ticker::new();
1292 src.replay(&mut |s, p, o| {
1293 ticker.tick(1);
1294 if p == RDF_TYPE {
1295 class_of.insert(s.to_string(), o.to_string());
1296 }
1297 if crate::terms::is_quoted_triple(s) {
1298 if let Some((_, quoted_p, _)) = crate::ingest::quoted_triple_parts(s) {
1299 quoted_subject_preds.insert(quoted_p);
1300 }
1301 if annotated_tokens.len() < ANNOTATION_WITNESS_SAMPLE {
1302 annotated_tokens
1303 .entry(s.to_string())
1304 .or_insert_with(|| p.to_string());
1305 }
1306 }
1307 });
1308 let classify = |t: &str| -> String {
1309 if let Some(c) = class_of.get(t) {
1310 c.to_string()
1311 } else if t.starts_with('"') {
1312 O_LITERAL.to_string()
1313 } else {
1314 O_UNTYPED.to_string()
1315 }
1316 };
1317
1318 let mut prof = Profile::default();
1324 let mut out_degree: BTreeMap<String, u64> = BTreeMap::new();
1325 let mut in_degree: BTreeMap<String, u64> = BTreeMap::new();
1326
1327 src.replay(&mut |s, p, o| {
1328 ticker.tick(1);
1329 prof.triple_count += 1;
1330 *prof.pred_counts.entry(p.to_string()).or_default() += 1;
1331 *out_degree.entry(s.to_string()).or_default() += 1;
1332 if let Some(ns) = split_namespace(s) {
1333 prof.subject_ns.add(ns, 1);
1334 }
1335
1336 if !o.starts_with('"') {
1340 *in_degree.entry(o.to_string()).or_default() += 1;
1341 }
1342
1343 if crate::terms::is_quoted_triple(s) {
1347 *prof.annotation_hits.entry(p.to_string()).or_default() += 1;
1348 }
1349 if crate::terms::is_quoted_triple(o) {
1350 *prof.quoting_hits.entry(p.to_string()).or_default() += 1;
1351 }
1352 if !quoted_subject_preds.is_empty() && quoted_subject_preds.contains(p) {
1359 if let Some(about) = annotated_tokens.get(&format!("<<{s} {p} {o}>>")) {
1360 *prof
1361 .witnessed_pairs
1362 .entry((p.to_string(), about.clone()))
1363 .or_default() += 1;
1364 }
1365 }
1366
1367 if p == RDF_TYPE {
1368 *prof.class_counts.entry(o.to_string()).or_default() += 1;
1369 return; }
1371
1372 *prof
1374 .link_rows
1375 .entry((classify(s), p.to_string(), classify(o)))
1376 .or_default() += 1;
1377
1378 if p == GEO_ASWKT || p == GEO_HASGEOMETRY {
1379 prof.geo_wkt = true;
1380 }
1381
1382 if let Some(lit) = parse_literal(o) {
1384 prof.literal(p, &lit, 1);
1385 }
1386 });
1387
1388 let mut truncated = false;
1389 let top_hubs = cap(sort_desc_owned(out_degree), &mut truncated);
1390 let in_hubs = cap(sort_desc_owned(in_degree), &mut truncated);
1391 finish(
1392 prof,
1393 Hubs {
1394 top_hubs,
1395 in_hubs,
1396 truncated,
1397 },
1398 quad_count,
1399 term_count,
1400 named_graph_count,
1401 curated,
1402 )
1403}
1404
1405const O_UNTYPED: &str = "(untyped)";
1408
1409#[derive(Default)]
1420struct Profile {
1421 triple_count: u64,
1422 pred_counts: BTreeMap<String, u64>,
1423 class_counts: BTreeMap<String, u64>,
1424 datatype_counts: BTreeMap<String, u64>,
1425 lang_counts: BTreeMap<String, u64>,
1426 link_rows: BTreeMap<(String, String, String), u64>,
1427 subject_ns: NsTally,
1428 time_obj_hits: BTreeMap<String, u64>,
1430 num_obj_hits: BTreeMap<String, u64>,
1432 annotation_hits: BTreeMap<String, u64>,
1437 quoting_hits: BTreeMap<String, u64>,
1438 witnessed_pairs: BTreeMap<(String, String), u64>,
1439 time_extent: BTreeMap<String, (String, String)>,
1441 lat: Option<(f64, f64)>,
1443 lon: Option<(f64, f64)>,
1444 geo_wkt: bool,
1445}
1446
1447impl Profile {
1448 fn literal(&mut self, p: &str, lit: &Literal, weight: u64) {
1452 *self
1453 .datatype_counts
1454 .entry(format!("<{}>", lit.datatype))
1455 .or_default() += weight;
1456 *self.lang_counts.entry(lit.lang.clone()).or_default() += weight;
1457 if lit.datatype == GEO_WKTLITERAL {
1458 self.geo_wkt = true;
1459 }
1460 if is_temporal(p, lit) {
1461 *self.time_obj_hits.entry(p.to_string()).or_default() += weight;
1462 update_time_extent(p, &lit.value, &mut self.time_extent);
1463 } else if is_numeric_datatype(&lit.datatype) {
1464 *self.num_obj_hits.entry(p.to_string()).or_default() += weight;
1467 }
1468 if p == WGS_LAT {
1470 if let Ok(v) = lit.value.parse::<f64>() {
1471 widen(&mut self.lat, v);
1472 }
1473 } else if p == WGS_LONG {
1474 if let Ok(v) = lit.value.parse::<f64>() {
1475 widen(&mut self.lon, v);
1476 }
1477 }
1478 }
1479}
1480
1481fn widen(extent: &mut Option<(f64, f64)>, v: f64) {
1483 *extent = Some(match *extent {
1484 None => (v, v),
1485 Some((lo, hi)) => (lo.min(v), hi.max(v)),
1486 });
1487}
1488
1489struct Hubs {
1491 top_hubs: Vec<(String, u64)>,
1492 in_hubs: Vec<(String, u64)>,
1493 truncated: bool,
1494}
1495
1496fn finish(
1502 p: Profile,
1503 hubs: Hubs,
1504 quad_count: u64,
1505 term_count: u64,
1506 named_graph_count: u64,
1507 curated: CardInput,
1508) -> DatasetCard {
1509 let Profile {
1510 triple_count,
1511 pred_counts,
1512 class_counts,
1513 datatype_counts,
1514 lang_counts,
1515 link_rows,
1516 subject_ns,
1517 time_obj_hits,
1518 num_obj_hits,
1519 annotation_hits,
1520 quoting_hits,
1521 witnessed_pairs,
1522 time_extent,
1523 lat,
1524 lon,
1525 geo_wkt,
1526 } = p;
1527 let pred_count = |iri: &str| pred_counts.get(iri).copied().unwrap_or(0);
1530 let label_predicate = LABEL_PREDICATES
1531 .iter()
1532 .filter(|&&iri| pred_count(iri) > 0)
1533 .max_by_key(|&&iri| pred_count(iri))
1534 .map(|s| s.to_string());
1535 let link_predicates: Vec<String> = LINK_PREDICATES
1536 .iter()
1537 .filter(|&&iri| pred_count(iri) > 0)
1538 .map(|s| s.to_string())
1539 .collect();
1540
1541 let mut truncated = hubs.truncated || subject_ns.overflowed();
1542 let predicates = cap(sort_desc_owned(pred_counts), &mut truncated);
1543 let classes = cap(sort_desc_owned(class_counts), &mut truncated);
1544 let datatypes = cap(sort_desc_owned(datatype_counts), &mut truncated);
1545 let languages = cap(sort_desc_owned(lang_counts), &mut truncated);
1546 let class_links = cap(sort_links(link_rows), &mut truncated);
1547
1548 let mut vocab: BTreeSet<String> = BTreeSet::new();
1550 for (iri, _) in predicates.iter().chain(classes.iter()) {
1551 if let Some(ns) = split_namespace(iri) {
1552 vocab.insert(ns);
1553 }
1554 }
1555 let vocabularies = cap(vocab.into_iter().collect(), &mut truncated);
1556
1557 let default_lang = languages
1559 .iter()
1560 .find(|(l, _)| !l.is_empty())
1561 .map(|(l, _)| l.clone());
1562 let base_iri = derive_base_iri(&subject_ns);
1563 let mut time_ranked: Vec<(String, u64)> = time_obj_hits.into_iter().collect();
1564 time_ranked.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
1565 let time_predicates = cap(
1567 time_ranked.iter().map(|(p, _)| p.to_string()).collect(),
1568 &mut truncated,
1569 );
1570 let mut num_ranked: Vec<(String, u64)> = num_obj_hits.into_iter().collect();
1571 num_ranked.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
1572 let numeric_predicates = cap(
1573 num_ranked.iter().map(|(p, _)| p.to_string()).collect(),
1574 &mut truncated,
1575 );
1576 let annotation_predicates = cap(
1582 sort_desc_owned(annotation_hits)
1583 .into_iter()
1584 .map(|(p, _)| p)
1585 .collect(),
1586 &mut truncated,
1587 );
1588 let quoting_predicates = cap(
1589 sort_desc_owned(quoting_hits)
1590 .into_iter()
1591 .map(|(p, _)| p)
1592 .collect(),
1593 &mut truncated,
1594 );
1595 let annotated_statement = witnessed_pairs
1601 .into_iter()
1602 .max_by(|a, b| a.1.cmp(&b.1).then_with(|| b.0.cmp(&a.0)))
1603 .map(|(pair, _)| pair);
1604 let geo_latlong = lat.is_some() && lon.is_some();
1605 let temporal_extent = time_ranked
1606 .first()
1607 .and_then(|(p, _)| time_extent.get(p))
1608 .cloned();
1609 let spatial_bbox = match (lat, lon) {
1610 (Some((lat_lo, lat_hi)), Some((lon_lo, lon_hi)))
1611 if lon_lo.is_finite() && lat_lo.is_finite() =>
1612 {
1613 Some([lon_lo, lat_lo, lon_hi, lat_hi])
1614 }
1615 _ => None,
1616 };
1617
1618 let signals = Signals {
1619 label_predicate,
1620 default_lang,
1621 base_iri,
1622 time_predicates,
1623 numeric_predicates,
1624 link_predicates,
1625 annotation_predicates,
1626 quoting_predicates,
1627 annotated_statement,
1628 geo_wkt,
1629 geo_latlong,
1630 temporal_extent,
1631 spatial_bbox,
1632 text_index: None,
1636 label_index: None,
1637 permutations: None,
1638 quoted_triples: None,
1639 };
1640
1641 let mut card = DatasetCard {
1642 triple_count,
1643 quad_count,
1644 named_graph_count,
1645 term_count,
1646 predicates,
1647 classes,
1648 vocabularies,
1649 datatypes,
1650 languages,
1651 class_links,
1652 top_hubs: hubs.top_hubs,
1653 in_hubs: hubs.in_hubs,
1654 signals,
1655 queries: Vec::new(),
1656 coherence: Coherence::default(),
1659 truncated,
1660 top_n: CARD_TOP_N as u32,
1663 format_version: crate::LEGACY_FORMAT_VERSION,
1664 ..DatasetCard::default()
1665 };
1666 curated.fill(&mut card);
1667 card.queries = crate::card_queries::generate(&card);
1669 card
1670}
1671
1672fn cap<T>(mut v: Vec<T>, truncated: &mut bool) -> Vec<T> {
1674 if v.len() > CARD_TOP_N {
1675 v.truncate(CARD_TOP_N);
1676 *truncated = true;
1677 }
1678 v
1679}
1680
1681fn sort_desc_owned(counts: BTreeMap<String, u64>) -> Vec<(String, u64)> {
1683 let mut v: Vec<(String, u64)> = counts.into_iter().collect();
1684 v.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
1685 v
1686}
1687
1688fn sort_links(rows: BTreeMap<(String, String, String), u64>) -> Vec<ClassLink> {
1691 let mut v: Vec<ClassLink> = rows
1692 .into_iter()
1693 .map(|((s, p, o), count)| ClassLink {
1694 s_class: s,
1695 predicate: p,
1696 o_class: o,
1697 count,
1698 })
1699 .collect();
1700 v.sort_by(|a, b| {
1701 b.count
1702 .cmp(&a.count)
1703 .then_with(|| a.s_class.cmp(&b.s_class))
1704 .then_with(|| a.predicate.cmp(&b.predicate))
1705 .then_with(|| a.o_class.cmp(&b.o_class))
1706 });
1707 v
1708}
1709
1710fn derive_base_iri(ns: &NsTally) -> Option<String> {
1717 let dominant = ns.dominant()?;
1719 let lcp = ns.common_prefix().unwrap_or("");
1722 let lcp = &lcp[..lcp.len().min(dominant.len())];
1723 let base = match lcp.rfind(['/', '#']) {
1725 Some(i) => &lcp[..=i],
1726 None => "",
1727 };
1728 if base.matches('/').count() >= 3 {
1731 Some(base.to_string())
1732 } else {
1733 Some(dominant)
1734 }
1735}
1736
1737pub const NS_TALLY_CAP: usize = 65_536;
1742
1743pub const NS_TALLY_ENTRY_BYTES: u64 = 256;
1746
1747pub(crate) struct NsTally {
1755 cap: usize,
1756 counts: BTreeMap<String, u64>,
1757 order: Option<BTreeSet<(u64, String)>>,
1759 lcp: Option<String>,
1760 overflowed: bool,
1761}
1762
1763impl Default for NsTally {
1764 fn default() -> Self {
1765 Self::with_cap(NS_TALLY_CAP)
1766 }
1767}
1768
1769impl NsTally {
1770 pub(crate) fn with_cap(cap: usize) -> Self {
1771 NsTally {
1772 cap: cap.max(1),
1773 counts: BTreeMap::new(),
1774 order: None,
1775 lcp: None,
1776 overflowed: false,
1777 }
1778 }
1779
1780 pub(crate) fn len(&self) -> usize {
1781 self.counts.len()
1782 }
1783
1784 pub(crate) fn overflowed(&self) -> bool {
1786 self.overflowed
1787 }
1788
1789 pub(crate) fn common_prefix(&self) -> Option<&str> {
1790 self.lcp.as_deref()
1791 }
1792
1793 pub(crate) fn dominant(&self) -> Option<String> {
1794 self.counts
1795 .iter()
1796 .max_by(|a, b| a.1.cmp(b.1).then_with(|| b.0.cmp(a.0)))
1797 .map(|(k, _)| k.clone())
1798 }
1799
1800 pub(crate) fn add(&mut self, ns: String, n: u64) {
1801 match &mut self.lcp {
1802 None => self.lcp = Some(ns.clone()),
1803 Some(l) if !ns.starts_with(l.as_str()) => {
1804 let mut cut = l
1805 .bytes()
1806 .zip(ns.bytes())
1807 .take_while(|(a, b)| a == b)
1808 .count();
1809 while !l.is_char_boundary(cut) {
1810 cut -= 1;
1811 }
1812 l.truncate(cut);
1813 }
1814 Some(_) => {}
1815 }
1816 if let Some(c) = self.counts.get_mut(&ns) {
1817 let old = *c;
1818 *c += n;
1819 if let Some(order) = &mut self.order {
1820 order.remove(&(old, ns.clone()));
1821 order.insert((*c, ns));
1822 }
1823 return;
1824 }
1825 if self.counts.len() < self.cap {
1826 if let Some(order) = &mut self.order {
1827 order.insert((n, ns.clone()));
1828 }
1829 self.counts.insert(ns, n);
1830 return;
1831 }
1832 self.overflowed = true;
1834 let counts = &self.counts;
1835 let order = self
1836 .order
1837 .get_or_insert_with(|| counts.iter().map(|(k, &v)| (v, k.clone())).collect());
1838 let Some((min, victim)) = order.pop_first() else {
1839 return;
1840 };
1841 self.counts.remove(&victim);
1842 let c = min + n;
1843 order.insert((c, ns.clone()));
1844 self.counts.insert(ns, c);
1845 }
1846}
1847
1848struct Literal {
1850 value: String,
1852 datatype: String,
1855 lang: String,
1857}
1858
1859fn parse_literal(term: &str) -> Option<Literal> {
1862 let bytes = term.as_bytes();
1863 if bytes.first() != Some(&b'"') {
1864 return None;
1865 }
1866 let mut i = 1usize;
1868 let mut esc = false;
1869 let mut close = None;
1870 while i < bytes.len() {
1871 match bytes[i] {
1872 b'\\' => esc = !esc,
1873 b'"' if !esc => {
1874 close = Some(i);
1875 break;
1876 }
1877 _ => esc = false,
1878 }
1879 i += 1;
1880 }
1881 let close = close?;
1882 let value = term[1..close].to_string();
1883 let suffix = &term[close + 1..];
1884 if let Some(lang) = suffix.strip_prefix('@') {
1885 Some(Literal {
1886 value,
1887 datatype: RDF_LANGSTRING.to_string(),
1888 lang: lang.to_string(),
1889 })
1890 } else if let Some(dt) = suffix.strip_prefix("^^") {
1891 let dt = dt.trim_start_matches('<').trim_end_matches('>').to_string();
1892 Some(Literal {
1893 value,
1894 datatype: dt,
1895 lang: String::new(),
1896 })
1897 } else {
1898 Some(Literal {
1899 value,
1900 datatype: XSD_STRING.to_string(),
1901 lang: String::new(),
1902 })
1903 }
1904}
1905
1906fn is_temporal_datatype(dt: &str) -> bool {
1910 matches!(
1911 dt,
1912 "http://www.w3.org/2001/XMLSchema#date"
1913 | "http://www.w3.org/2001/XMLSchema#dateTime"
1914 | "http://www.w3.org/2001/XMLSchema#dateTimeStamp"
1915 | "http://www.w3.org/2001/XMLSchema#gYear"
1916 | "http://www.w3.org/2001/XMLSchema#gYearMonth"
1917 )
1918}
1919
1920const DATE_PREDICATES: &[&str] = &[
1925 "<http://purl.org/dc/terms/date>",
1926 "<http://purl.org/dc/terms/issued>",
1927 "<http://purl.org/dc/terms/created>",
1928 "<http://purl.org/dc/terms/modified>",
1929 "<http://purl.org/dc/terms/available>",
1930 "<http://purl.org/dc/terms/dateAccepted>",
1931 "<http://purl.org/dc/terms/dateCopyrighted>",
1932 "<http://purl.org/dc/terms/dateSubmitted>",
1933 "<http://purl.org/dc/elements/1.1/date>",
1934 "<http://schema.org/datePublished>",
1935 "<http://schema.org/dateCreated>",
1936 "<http://schema.org/dateModified>",
1937 "<http://schema.org/startDate>",
1938 "<http://schema.org/endDate>",
1939 "<http://schema.org/birthDate>",
1940 "<http://schema.org/deathDate>",
1941 "<http://schema.org/foundingDate>",
1942 "<http://schema.org/uploadDate>",
1943 "<https://schema.org/datePublished>",
1944 "<https://schema.org/dateCreated>",
1945 "<https://schema.org/dateModified>",
1946 "<https://schema.org/startDate>",
1947 "<https://schema.org/endDate>",
1948 "<https://schema.org/birthDate>",
1949 "<https://schema.org/deathDate>",
1950 "<http://prismstandard.org/namespaces/basic/2.0/publicationDate>",
1951 "<http://prismstandard.org/namespaces/basic/2.0/coverDate>",
1952 "<http://www.w3.org/ns/prov#generatedAtTime>",
1953 "<http://www.w3.org/ns/prov#invalidatedAtTime>",
1954 "<http://www.w3.org/ns/prov#startedAtTime>",
1955 "<http://www.w3.org/ns/prov#endedAtTime>",
1956 "<http://www.w3.org/ns/prov#atTime>",
1957 "<http://www.w3.org/2006/time#inXSDDate>",
1958 "<http://www.w3.org/2006/time#inXSDDateTime>",
1959 "<http://www.w3.org/2006/time#inXSDDateTimeStamp>",
1960 "<http://www.w3.org/2006/time#inXSDgYear>",
1961 "<http://www.w3.org/2006/time#inXSDgYearMonth>",
1962 "<http://dbpedia.org/ontology/birthDate>",
1963 "<http://dbpedia.org/ontology/deathDate>",
1964 "<http://dbpedia.org/ontology/releaseDate>",
1965 "<http://www.wikidata.org/prop/direct/P569>",
1966 "<http://www.wikidata.org/prop/direct/P570>",
1967 "<http://www.wikidata.org/prop/direct/P571>",
1968 "<http://www.wikidata.org/prop/direct/P577>",
1969 "<http://www.wikidata.org/prop/direct/P580>",
1970 "<http://www.wikidata.org/prop/direct/P582>",
1971 "<http://www.wikidata.org/prop/direct/P585>",
1972];
1973
1974fn is_temporal(p: &str, lit: &Literal) -> bool {
1988 if is_temporal_datatype(&lit.datatype) {
1989 return date_shaped(&lit.value);
1990 }
1991 DATE_PREDICATES.contains(&p) && date_shaped(&lit.value)
1992}
1993
1994fn date_shaped(v: &str) -> bool {
2002 let b = v.as_bytes();
2003 let digits =
2004 |at: usize, n: usize| b.len() >= at + n && b[at..at + n].iter().all(u8::is_ascii_digit);
2005 let two_in = |at: usize, lo: u8, hi: u8| {
2006 digits(at, 2) && {
2007 let n = (b[at] - b'0') * 10 + (b[at + 1] - b'0');
2008 (lo..=hi).contains(&n)
2009 }
2010 };
2011 let mut i = usize::from(b.first() == Some(&b'-'));
2012 if !digits(i, 4) {
2013 return false;
2014 }
2015 i += 4;
2016 if b.get(i) == Some(&b'-') {
2017 if !two_in(i + 1, 1, 12) {
2018 return false;
2019 }
2020 i += 3;
2021 if b.get(i) == Some(&b'-') {
2022 if !two_in(i + 1, 1, 31) {
2023 return false;
2024 }
2025 i += 3;
2026 if b.get(i) == Some(&b'T') {
2027 if !(two_in(i + 1, 0, 24) && b.get(i + 3) == Some(&b':') && two_in(i + 4, 0, 59)) {
2028 return false;
2029 }
2030 i += 6;
2031 if b.get(i) == Some(&b':') {
2032 if !two_in(i + 1, 0, 60) {
2033 return false;
2034 }
2035 i += 3;
2036 if b.get(i) == Some(&b'.') {
2037 let start = i + 1;
2038 i = start;
2039 while b.get(i).is_some_and(u8::is_ascii_digit) {
2040 i += 1;
2041 }
2042 if i == start {
2043 return false;
2044 }
2045 }
2046 }
2047 }
2048 }
2049 }
2050 match &b[i..] {
2051 [] | [b'Z'] => true,
2052 [sign, h1, h2, b':', m1, m2] => {
2053 (*sign == b'+' || *sign == b'-') && [h1, h2, m1, m2].iter().all(|c| c.is_ascii_digit())
2054 }
2055 _ => false,
2056 }
2057}
2058
2059fn is_numeric_datatype(dt: &str) -> bool {
2061 matches!(
2062 dt,
2063 "http://www.w3.org/2001/XMLSchema#integer"
2064 | "http://www.w3.org/2001/XMLSchema#decimal"
2065 | "http://www.w3.org/2001/XMLSchema#double"
2066 | "http://www.w3.org/2001/XMLSchema#float"
2067 | "http://www.w3.org/2001/XMLSchema#long"
2068 | "http://www.w3.org/2001/XMLSchema#int"
2069 | "http://www.w3.org/2001/XMLSchema#short"
2070 | "http://www.w3.org/2001/XMLSchema#nonNegativeInteger"
2071 | "http://www.w3.org/2001/XMLSchema#positiveInteger"
2072 )
2073}
2074
2075fn temporal_key(v: &str) -> (i64, &str) {
2080 let (neg, rest) = match v.strip_prefix('-') {
2081 Some(r) => (true, r),
2082 None => (false, v),
2083 };
2084 let n = rest.bytes().take_while(u8::is_ascii_digit).count();
2085 let year: i64 = rest[..n].parse().unwrap_or(0);
2086 (if neg { -year } else { year }, &rest[n..])
2087}
2088
2089fn update_time_extent(pred: &str, value: &str, extent: &mut BTreeMap<String, (String, String)>) {
2095 match extent.get_mut(pred) {
2096 None => {
2097 extent.insert(pred.to_string(), (value.to_string(), value.to_string()));
2098 }
2099 Some((lo, hi)) => {
2100 if temporal_key(value) < temporal_key(lo) {
2101 *lo = value.to_string();
2102 }
2103 if temporal_key(value) > temporal_key(hi) {
2104 *hi = value.to_string();
2105 }
2106 }
2107 }
2108}
2109
2110#[derive(Debug, Clone, Default)]
2138pub struct StreamReport {
2139 pub statements: u64,
2140 pub subjects: u64,
2141 pub literal_objects: u64,
2142 pub classes: u64,
2143 pub class_map_bytes: u64,
2144 pub namespaces_kept: u64,
2148 pub namespaces_overflowed: bool,
2149}
2150
2151struct ClassMap {
2164 data: Vec<u8>,
2165 width: usize,
2167}
2168
2169impl ClassMap {
2170 fn new(subjects: u64) -> Self {
2171 ClassMap {
2174 data: vec![0; subjects as usize + 1],
2175 width: 1,
2176 }
2177 }
2178
2179 fn len(&self) -> usize {
2180 self.data.len() / self.width
2181 }
2182
2183 fn read(data: &[u8], i: usize, width: usize) -> u32 {
2184 let at = i * width;
2185 match width {
2186 1 => data[at] as u32,
2187 2 => u16::from_le_bytes([data[at], data[at + 1]]) as u32,
2188 _ => u32::from_le_bytes([data[at], data[at + 1], data[at + 2], data[at + 3]]),
2189 }
2190 }
2191
2192 fn write(data: &mut [u8], i: usize, width: usize, code: u32) {
2193 let at = i * width;
2194 match width {
2195 1 => data[at] = code as u8,
2196 2 => data[at..at + 2].copy_from_slice(&(code as u16).to_le_bytes()),
2197 _ => data[at..at + 4].copy_from_slice(&code.to_le_bytes()),
2198 }
2199 }
2200
2201 fn get(&self, id: u32) -> u32 {
2202 let i = id as usize;
2203 if i < self.len() {
2204 Self::read(&self.data, i, self.width)
2205 } else {
2206 0
2207 }
2208 }
2209
2210 fn widen(&mut self, width: usize) {
2212 let (n, old) = (self.len(), self.width);
2213 if width <= old {
2214 return;
2215 }
2216 self.data.resize(n * width, 0);
2217 for i in (0..n).rev() {
2218 let v = Self::read(&self.data, i, old);
2219 Self::write(&mut self.data, i, width, v);
2220 }
2221 self.width = width;
2222 }
2223
2224 fn set(&mut self, id: u32, code: u32) {
2225 if code > u16::MAX as u32 {
2226 self.widen(4);
2227 } else if code > u8::MAX as u32 {
2228 self.widen(2);
2229 }
2230 let i = id as usize;
2231 if i < self.len() {
2232 Self::write(&mut self.data, i, self.width, code);
2233 }
2234 }
2235
2236 fn bytes(&self) -> u64 {
2237 self.data.len() as u64
2238 }
2239}
2240
2241struct TopN {
2247 heap: std::collections::BinaryHeap<std::cmp::Reverse<(u64, std::cmp::Reverse<u32>)>>,
2248 offered: u64,
2249}
2250
2251impl TopN {
2252 fn new() -> Self {
2253 TopN {
2254 heap: std::collections::BinaryHeap::with_capacity(CARD_TOP_N + 1),
2255 offered: 0,
2256 }
2257 }
2258
2259 fn offer(&mut self, id: u32, n: u64) {
2260 use std::cmp::Reverse;
2261 self.offered += 1;
2262 let key = Reverse((n, Reverse(id)));
2263 if self.heap.len() < CARD_TOP_N {
2264 self.heap.push(key);
2265 } else if let Some(worst) = self.heap.peek() {
2266 if (n, Reverse(id)) > worst.0 {
2268 self.heap.pop();
2269 self.heap.push(key);
2270 }
2271 }
2272 }
2273
2274 fn into_parts(self) -> (Vec<(u32, u64)>, bool) {
2276 let cut = self.offered > CARD_TOP_N as u64;
2277 let v = self
2278 .heap
2279 .into_iter()
2280 .map(|std::cmp::Reverse((n, std::cmp::Reverse(id)))| (id, n))
2281 .collect();
2282 (v, cut)
2283 }
2284}
2285
2286fn top_terms(
2290 tops: [TopN; 2],
2291 terms: &dyn Fn(&[u32]) -> Vec<Option<String>>,
2292) -> (Vec<(String, u64)>, bool) {
2293 let mut offered = 0u64;
2294 let mut ids: Vec<(u32, u64)> = Vec::new();
2295 for top in tops {
2296 offered += top.offered;
2297 ids.extend(top.into_parts().0);
2298 }
2299 let names = terms(&ids.iter().map(|&(id, _)| id).collect::<Vec<_>>());
2300 let mut v: Vec<(String, u64)> = ids
2301 .iter()
2302 .zip(names)
2303 .filter_map(|(&(_, n), t)| t.map(|t| (t, n)))
2304 .collect();
2305 v.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
2306 v.truncate(CARD_TOP_N);
2307 (v, offered > CARD_TOP_N as u64)
2308}
2309
2310#[derive(Default, Clone, Copy)]
2313struct IdHasher(u64);
2314
2315impl std::hash::Hasher for IdHasher {
2316 fn finish(&self) -> u64 {
2317 self.0
2318 }
2319 fn write(&mut self, bytes: &[u8]) {
2320 for &b in bytes {
2321 self.0 = (self.0.rotate_left(5) ^ b as u64).wrapping_mul(0x517c_c1b7_2722_0a95);
2322 }
2323 }
2324 fn write_u32(&mut self, n: u32) {
2325 self.0 = (self.0.rotate_left(5) ^ n as u64).wrapping_mul(0x517c_c1b7_2722_0a95);
2326 }
2327}
2328
2329type IdBuild = std::hash::BuildHasherDefault<IdHasher>;
2330
2331fn first_at_or_above(lo: u32, hi: u32, key: &str, term: &dyn Fn(u32) -> Option<String>) -> u32 {
2335 let (mut a, mut b) = (lo as u64, hi as u64 + 1);
2336 while a < b {
2337 let mid = a + (b - a) / 2;
2338 match term(mid as u32) {
2339 Some(t) if t.as_str() < key => a = mid + 1,
2340 _ => b = mid,
2341 }
2342 }
2343 a as u32
2344}
2345
2346fn id_range(
2351 lo: u32,
2352 hi: u32,
2353 from: &str,
2354 to: &str,
2355 term: &dyn Fn(u32) -> Option<String>,
2356) -> (u32, u32) {
2357 if hi < lo {
2358 return (lo, lo);
2359 }
2360 (
2361 first_at_or_above(lo, hi, from, term),
2362 first_at_or_above(lo, hi, to, term),
2363 )
2364}
2365
2366fn in_any(id: u32, ranges: &[(u32, u32)]) -> bool {
2367 ranges.iter().any(|&(a, b)| a <= id && id < b)
2368}
2369
2370pub fn derive_card_streaming(
2389 rete: &crate::Rete,
2390 curated: CardInput,
2391 progress: &mut dyn FnMut(&str),
2392) -> Result<(DatasetCard, StreamReport), String> {
2393 use crate::index::IndexPermutation;
2394
2395 let header = rete.header();
2396 let dict = rete.dictionary();
2397 let index = rete.default_index();
2398 let shared = dict.shared_count();
2399 let subjects_end = shared as u64 + dict.subject_only_count() as u64;
2400 let objects_end = shared as u64 + dict.object_only_count() as u64;
2401 let n_preds = dict.predicate_count() as usize;
2402 let pred_terms: Vec<String> = (0..=n_preds as u32)
2403 .map(|p| {
2404 if p == 0 {
2405 String::new()
2406 } else {
2407 dict.predicate_term(p).unwrap_or_default()
2408 }
2409 })
2410 .collect();
2411 let type_id = dict.predicate_id(RDF_TYPE);
2412
2413 let object_term = |id: u32| dict.object_term(id);
2415 let subject_term = |id: u32| dict.subject_term(id);
2416 let s_end = subjects_end.min(u32::MAX as u64) as u32;
2417 let o_end = objects_end.min(u32::MAX as u64) as u32;
2418 let literal_ranges = [
2419 id_range(1, shared, "\"", "#", &object_term),
2420 id_range(shared + 1, o_end, "\"", "#", &object_term),
2421 ];
2422 let is_literal = |o: u32| in_any(o, &literal_ranges);
2423 let quoted = header.has_quoted_triples();
2424 let (quoted_subjects, quoted_objects) = if quoted {
2425 (
2426 vec![
2427 id_range(1, shared, "<<", "<=", &subject_term),
2428 id_range(shared + 1, s_end, "<<", "<=", &subject_term),
2429 ],
2430 vec![
2431 id_range(1, shared, "<<", "<=", &object_term),
2432 id_range(shared + 1, o_end, "<<", "<=", &object_term),
2433 ],
2434 )
2435 } else {
2436 (Vec::new(), Vec::new())
2437 };
2438
2439 let mut report = StreamReport::default();
2440
2441 progress("pass 1/3: classes (the rdf:type range of POS)");
2443 let mut class_map = ClassMap::new(subjects_end);
2444 let mut class_code: std::collections::HashMap<u32, u32, IdBuild> = Default::default();
2445 let mut class_ids: Vec<u32> = vec![0];
2446 let mut class_counts: Vec<u64> = vec![0];
2447 if let Some(t) = type_id {
2448 for (s, _, o) in index.scan_iter((None, Some(t), None)) {
2449 let next = class_ids.len() as u32;
2450 let code = *class_code.entry(o).or_insert_with(|| {
2451 class_ids.push(o);
2452 class_counts.push(0);
2453 next
2454 });
2455 class_counts[code as usize] += 1;
2456 class_map.set(s, code);
2457 }
2458 }
2459 report.classes = class_ids.len() as u64 - 1;
2460 report.class_map_bytes = class_map.bytes();
2461
2462 progress("pass 2/3: subjects (SPO)");
2464 const LITERAL_CODE: u32 = u32::MAX;
2465 const WINDOW: usize = 1 << 16;
2466 let mut prof = Profile::default();
2467 let mut pred_counts = vec![0u64; n_preds + 1];
2468 let mut annotation_hits = vec![0u64; n_preds + 1];
2469 let mut quoting_hits = vec![0u64; n_preds + 1];
2470 let mut links: std::collections::HashMap<(u32, u32, u32), u64, IdBuild> = Default::default();
2471 let mut out_top = [TopN::new(), TopN::new()];
2473 let mut sample: Vec<(u32, u32)> = Vec::new();
2474 let mut window: Vec<(u32, u64)> = Vec::with_capacity(WINDOW);
2475 let mut subject_ns = NsTally::default();
2476 let flush_ns = |window: &mut Vec<(u32, u64)>, subject_ns: &mut NsTally| {
2477 let ids: Vec<u32> = window.iter().map(|&(s, _)| s).collect();
2478 let terms = dict.subject_terms(&ids);
2479 let mut run: Option<(String, u64)> = None;
2482 for (term, &(_, n)) in terms.iter().zip(window.iter()) {
2483 let Some(ns) = term.as_deref().and_then(split_namespace) else {
2484 continue;
2485 };
2486 match &mut run {
2487 Some((cur, total)) if *cur == ns => *total += n,
2488 _ => {
2489 if let Some((cur, total)) = run.take() {
2490 subject_ns.add(cur, total);
2491 }
2492 run = Some((ns, n));
2493 }
2494 }
2495 }
2496 if let Some((cur, total)) = run {
2497 subject_ns.add(cur, total);
2498 }
2499 window.clear();
2500 };
2501 let mut cur_s = 0u32;
2502 let mut cur_n = 0u64;
2503 let mut ticker = crate::build_progress::Ticker::new();
2505 let stored = index.for_each_in_order(IndexPermutation::Spo, &mut |s, p, o| {
2506 prof.triple_count += 1;
2507 ticker.tick(1);
2508 if let Some(c) = pred_counts.get_mut(p as usize) {
2509 *c += 1;
2510 }
2511 if s != cur_s {
2512 if cur_n > 0 {
2513 out_top[usize::from(cur_s > shared)].offer(cur_s, cur_n);
2514 window.push((cur_s, cur_n));
2515 if window.len() >= WINDOW {
2516 flush_ns(&mut window, &mut subject_ns);
2517 }
2518 }
2519 if quoted && in_any(s, "ed_subjects) && sample.len() < ANNOTATION_WITNESS_SAMPLE {
2520 sample.push((s, p));
2521 }
2522 cur_s = s;
2523 cur_n = 0;
2524 }
2525 cur_n += 1;
2526 if quoted {
2527 if in_any(s, "ed_subjects) {
2528 if let Some(c) = annotation_hits.get_mut(p as usize) {
2529 *c += 1;
2530 }
2531 }
2532 if in_any(o, "ed_objects) {
2533 if let Some(c) = quoting_hits.get_mut(p as usize) {
2534 *c += 1;
2535 }
2536 }
2537 }
2538 if Some(p) == type_id {
2539 return;
2540 }
2541 let sc = class_map.get(s);
2542 let oc = if is_literal(o) {
2545 LITERAL_CODE
2546 } else if o <= shared {
2547 class_map.get(o)
2548 } else {
2549 0
2550 };
2551 *links.entry((sc, p, oc)).or_default() += 1;
2552 });
2553 if !stored {
2554 return Err("the file stores no SPO permutation".to_string());
2555 }
2556 if cur_n > 0 {
2557 out_top[usize::from(cur_s > shared)].offer(cur_s, cur_n);
2558 window.push((cur_s, cur_n));
2559 }
2560 if !window.is_empty() {
2561 flush_ns(&mut window, &mut subject_ns);
2562 }
2563 report.statements = prof.triple_count;
2564 report.subjects = out_top[0].offered + out_top[1].offered;
2565 drop(class_map);
2566
2567 progress("pass 3/3: objects (OSP)");
2569 let mut in_top = [TopN::new(), TopN::new()];
2570 let mut lit_window: Vec<(u32, Vec<(u32, u64)>)> = Vec::with_capacity(WINDOW);
2571 let mut lit_objects = 0u64;
2572 let flush_lits = |lit_window: &mut Vec<(u32, Vec<(u32, u64)>)>, prof: &mut Profile| {
2573 let ids: Vec<u32> = lit_window.iter().map(|(o, _)| *o).collect();
2574 let terms = dict.object_terms(&ids);
2575 for (term, (_, preds)) in terms.iter().zip(lit_window.iter()) {
2576 let Some(lit) = term.as_deref().and_then(parse_literal) else {
2577 continue;
2578 };
2579 for &(p, n) in preds {
2580 if let Some(pt) = pred_terms.get(p as usize) {
2581 prof.literal(pt, &lit, n);
2582 }
2583 }
2584 }
2585 lit_window.clear();
2586 };
2587 let mut cur_o = 0u32;
2588 let mut cur_n = 0u64;
2589 let mut cur_preds: Vec<(u32, u64)> = Vec::new();
2590 index.for_each_in_order(IndexPermutation::Osp, &mut |o, _s, p| {
2591 ticker.tick(1);
2592 if o != cur_o {
2593 if cur_n > 0 {
2594 if is_literal(cur_o) {
2595 lit_objects += 1;
2596 lit_window.push((cur_o, std::mem::take(&mut cur_preds)));
2597 if lit_window.len() >= WINDOW {
2598 flush_lits(&mut lit_window, &mut prof);
2599 }
2600 } else {
2601 in_top[usize::from(cur_o > shared)].offer(cur_o, cur_n);
2602 }
2603 }
2604 cur_o = o;
2605 cur_n = 0;
2606 cur_preds.clear();
2607 }
2608 cur_n += 1;
2609 if is_literal(o) {
2610 match cur_preds.iter_mut().find(|(q, _)| *q == p) {
2611 Some((_, n)) => *n += 1,
2612 None => cur_preds.push((p, 1)),
2613 }
2614 }
2615 });
2616 if cur_n > 0 {
2617 if is_literal(cur_o) {
2618 lit_objects += 1;
2619 lit_window.push((cur_o, std::mem::take(&mut cur_preds)));
2620 } else {
2621 in_top[usize::from(cur_o > shared)].offer(cur_o, cur_n);
2622 }
2623 }
2624 if !lit_window.is_empty() {
2625 flush_lits(&mut lit_window, &mut prof);
2626 }
2627 report.literal_objects = lit_objects;
2628
2629 if rete.index_incomplete() {
2630 return Err("a read failed part-way through the file; the card would be incomplete".into());
2631 }
2632
2633 progress("resolving the profile's terms");
2635 let class_terms: Vec<String> = class_ids
2636 .iter()
2637 .map(|&o| {
2638 if o == 0 {
2639 O_UNTYPED.to_string()
2640 } else {
2641 dict.object_term(o).unwrap_or_default()
2642 }
2643 })
2644 .collect();
2645 let class_name = |code: u32| -> String {
2646 if code == LITERAL_CODE {
2647 O_LITERAL.to_string()
2648 } else {
2649 class_terms
2650 .get(code as usize)
2651 .cloned()
2652 .unwrap_or_else(|| O_UNTYPED.to_string())
2653 }
2654 };
2655 for (p, &n) in pred_counts.iter().enumerate() {
2656 if n > 0 {
2657 prof.pred_counts.insert(pred_terms[p].clone(), n);
2658 }
2659 }
2660 for (code, &n) in class_counts.iter().enumerate().skip(1) {
2661 if n > 0 {
2662 *prof
2663 .class_counts
2664 .entry(class_terms[code].clone())
2665 .or_default() += n;
2666 }
2667 }
2668 for ((sc, p, oc), n) in links {
2669 let Some(pt) = pred_terms.get(p as usize) else {
2670 continue;
2671 };
2672 *prof
2673 .link_rows
2674 .entry((class_name(sc), pt.clone(), class_name(oc)))
2675 .or_default() += n;
2676 }
2677 for (p, &n) in annotation_hits.iter().enumerate() {
2678 if n > 0 {
2679 prof.annotation_hits.insert(pred_terms[p].clone(), n);
2680 }
2681 }
2682 for (p, &n) in quoting_hits.iter().enumerate() {
2683 if n > 0 {
2684 prof.quoting_hits.insert(pred_terms[p].clone(), n);
2685 }
2686 }
2687 report.namespaces_kept = subject_ns.len() as u64;
2688 report.namespaces_overflowed = subject_ns.overflowed();
2689 prof.subject_ns = subject_ns;
2690 if pred_counts
2691 .iter()
2692 .enumerate()
2693 .any(|(p, &n)| n > 0 && (pred_terms[p] == GEO_ASWKT || pred_terms[p] == GEO_HASGEOMETRY))
2694 {
2695 prof.geo_wkt = true;
2696 }
2697
2698 let sample_ids: Vec<u32> = sample.iter().map(|&(s, _)| s).collect();
2701 for (term, &(_, about)) in dict.subject_terms(&sample_ids).iter().zip(sample.iter()) {
2702 let Some((a, qp, b)) = term.as_deref().and_then(crate::ingest::quoted_triple_parts) else {
2703 continue;
2704 };
2705 if let Some((sid, pid, oid)) = dict.encode(&a, &qp, &b) {
2706 if index
2707 .scan_iter((Some(sid), Some(pid), Some(oid)))
2708 .next()
2709 .is_some()
2710 {
2711 if let Some(about) = pred_terms.get(about as usize) {
2712 *prof.witnessed_pairs.entry((qp, about.clone())).or_default() += 1;
2713 }
2714 }
2715 }
2716 }
2717
2718 let (top_hubs, cut_out) = top_terms(out_top, &|ids| dict.subject_terms(ids));
2719 let (in_hubs, cut_in) = top_terms(in_top, &|ids| dict.object_terms(ids));
2720
2721 let card = finish(
2722 prof,
2723 Hubs {
2724 top_hubs,
2725 in_hubs,
2726 truncated: cut_out || cut_in,
2727 },
2728 header.quad_count,
2729 header.term_count,
2730 rete.named_graph_count() as u64,
2731 curated,
2732 );
2733 Ok((card, report))
2734}
2735
2736fn split_namespace(term: &str) -> Option<String> {
2740 let iri = term.strip_prefix('<')?.strip_suffix('>')?;
2741 let cut = iri.rfind(['#', '/'])?;
2742 Some(iri[..=cut].to_string())
2743}
2744
2745pub fn load_card(bytes: &[u8]) -> Result<Option<DatasetCard>, String> {
2753 if crate::v06::open::is_v06(bytes) {
2754 return match crate::file::read_card_full_ranged(&crate::reader::SliceReader::new(bytes))
2755 .map_err(|e| e.to_string())?
2756 {
2757 Some(b) => Ok(Some(DatasetCard::from_json_bytes(&b)?)),
2758 None => Ok(None),
2759 };
2760 }
2761 let header = Header::from_bytes(bytes).map_err(|e| e.to_string())?;
2762 if header.metadata_len == 0 {
2763 return Ok(None);
2764 }
2765 let start = header.metadata_offset as usize;
2766 let end = start
2767 .checked_add(header.metadata_len as usize)
2768 .filter(|&e| e <= bytes.len())
2769 .ok_or_else(|| "metadata section out of bounds".to_string())?;
2770 Ok(Some(DatasetCard::from_json_bytes(&bytes[start..end])?))
2771}
2772
2773#[cfg(test)]
2774mod tests {
2775 use super::*;
2776 use crate::card_input::{CARD_EXTRA_MAX_BYTES, CARD_EXTRA_MAX_KEYS, CARD_EXTRA_MAX_KEY_BYTES};
2779
2780 const TYPE: &str = RDF_TYPE;
2781
2782 fn q(s: &str, p: &str, o: &str) -> (String, String, String, Option<String>) {
2783 (s.into(), p.into(), o.into(), None)
2784 }
2785
2786 #[test]
2787 fn card_json_round_trips() {
2788 let card = DatasetCard {
2789 title: Some("Demo".into()),
2790 license: Some("CC0".into()),
2791 triple_count: 3,
2792 quad_count: 3,
2793 term_count: 5,
2794 predicates: vec![("<http://ex/p>".into(), 2)],
2795 vocabularies: vec!["http://ex/".into()],
2796 format_version: 1,
2797 ..Default::default()
2798 };
2799 let bytes = card.to_json_bytes();
2800 let back = DatasetCard::from_json_bytes(&bytes).unwrap();
2801 assert_eq!(card, back);
2802 let text = String::from_utf8(bytes).unwrap();
2804 assert!(!text.contains("description"));
2805 assert!(!text.contains("classes"));
2806 }
2807
2808 #[test]
2814 fn the_permutation_signal_is_stripped_from_the_stored_card() {
2815 let measured = PermutationsSignal {
2816 count: 3,
2817 names: vec!["SPO".into(), "POS".into(), "OSP".into()],
2818 merge_join: false,
2819 };
2820 let mut card = DatasetCard {
2821 title: Some("Lean".into()),
2822 triple_count: 3,
2823 ..Default::default()
2824 };
2825 assert_eq!(card.observe_permutations(measured.clone()), None);
2826 assert_eq!(card.signals.permutations, Some(measured.clone()));
2827
2828 let bytes = card.to_json_bytes();
2829 let text = String::from_utf8(bytes.clone()).unwrap();
2830 assert!(
2831 !text.contains("permutations"),
2832 "the signal must not reach the metadata section: {text}"
2833 );
2834 let back = DatasetCard::from_json_bytes(&bytes).unwrap();
2835 assert_eq!(back.signals.permutations, None, "unknown, not `six`");
2836 assert_eq!(back.title, card.title);
2837
2838 let authored = r#"{"title":"x","signals":{"permutations":{"count":6}}}"#;
2841 assert!(
2842 serde_json::from_str::<CardInput>(authored).is_err(),
2843 "a hand-written card must not be able to assert a permutation set"
2844 );
2845 }
2846
2847 #[test]
2851 fn the_permutation_signal_is_derived_from_the_header() {
2852 let quads = vec![
2853 q("<http://ex/a>", "<http://ex/p>", "<http://ex/b>"),
2854 q("<http://ex/b>", "<http://ex/p>", "<http://ex/c>"),
2855 ];
2856 let (six, _) = crate::ingest::assemble_dataset_with_perms(
2857 quads.clone(),
2858 false,
2859 false,
2860 None,
2861 crate::PyramidAlgo::Louvain,
2862 crate::PermSet::ALL,
2863 |_, _| Vec::new(),
2864 );
2865 let (three, _) = crate::ingest::assemble_dataset_with_perms(
2866 quads,
2867 false,
2868 false,
2869 None,
2870 crate::PyramidAlgo::Louvain,
2871 crate::PermSet::CORE,
2872 |_, _| Vec::new(),
2873 );
2874 assert_eq!(six[50], 0);
2877 assert_eq!(three[50], 0b0000_0111);
2878
2879 let h6 = crate::Header::from_bytes(&six).unwrap();
2880 let h3 = crate::Header::from_bytes(&three).unwrap();
2881 let s6 = PermutationsSignal::probe(&h6);
2882 let s3 = PermutationsSignal::probe(&h3);
2883 assert_eq!(s6.count, 6);
2884 assert!(s6.merge_join);
2885 assert_eq!(s3.count, 3);
2886 assert!(!s3.merge_join);
2887 assert_eq!(s3.names, vec!["SPO", "POS", "OSP"]);
2888 assert!(s3.describe().contains("no sort-merge join"));
2889 assert!(three.len() < six.len(), "{} vs {}", three.len(), six.len());
2891 assert_eq!(h3.dictionary_len, h6.dictionary_len);
2892 assert!(h3.root_dir_len < h6.root_dir_len);
2893 }
2894
2895 #[test]
2900 fn the_text_index_signal_is_stripped_from_the_stored_card() {
2901 let measured = TextIndexSignal {
2903 present: true,
2904 bytes: 1_879_287_762,
2905 token_table_bytes: Some(193_295_361),
2906 };
2907 let mut card = DatasetCard {
2908 title: Some("Indexed".into()),
2909 triple_count: 3,
2910 ..Default::default()
2911 };
2912 assert_eq!(card.observe_text_index(measured), None);
2913 assert_eq!(card.signals.text_index, Some(measured));
2914
2915 let bytes = card.to_json_bytes();
2916 let text = String::from_utf8(bytes.clone()).unwrap();
2917 assert!(
2918 !text.contains("text_index"),
2919 "the signal must not reach the metadata section: {text}"
2920 );
2921 let back = DatasetCard::from_json_bytes(&bytes).unwrap();
2924 assert_eq!(back.title, card.title);
2925 assert_eq!(back.signals.text_index, None);
2926
2927 assert_eq!(card.signals.text_index, Some(measured));
2930 }
2931
2932 #[test]
2938 fn the_text_index_signal_round_trips_through_a_card_document() {
2939 let mut card = DatasetCard {
2940 title: Some("Indexed".into()),
2941 ..Default::default()
2942 };
2943 card.observe_text_index(TextIndexSignal {
2944 present: true,
2945 bytes: 80,
2946 token_table_bytes: Some(68),
2947 });
2948 let doc = serde_json::to_vec(&card).unwrap();
2949 let back: DatasetCard = serde_json::from_slice(&doc).unwrap();
2950 assert_eq!(back.signals.text_index, card.signals.text_index);
2951
2952 let mut none = DatasetCard::default();
2956 none.observe_text_index(TextIndexSignal::default());
2957 let text = serde_json::to_string(&none).unwrap();
2958 assert!(text.contains(r#""text_index":{"present":false}"#), "{text}");
2959 let absent = serde_json::to_string(&DatasetCard::default()).unwrap();
2960 assert!(!absent.contains("text_index"), "{absent}");
2961 }
2962
2963 #[test]
2970 fn the_quoted_triple_signal_is_derived_from_the_header() {
2971 let plain = vec![
2972 q("<http://ex/a>", "<http://ex/p>", "<http://ex/b>"),
2973 q("<http://ex/b>", "<http://ex/p>", "<http://ex/c>"),
2974 ];
2975 let quoted = vec![
2976 q("<http://ex/a>", "<http://ex/p>", "<http://ex/b>"),
2977 q(
2978 "<http://ex/claim>",
2979 "<http://ex/states>",
2980 "<<<http://ex/a> <http://ex/p> <http://ex/b>>>",
2981 ),
2982 ];
2983 let (no, _) = crate::ingest::assemble_dataset(plain, &[]);
2984 let (yes, _) = crate::ingest::assemble_dataset(quoted, &[]);
2985
2986 let hn = crate::Header::from_bytes(&no).unwrap();
2987 let hy = crate::Header::from_bytes(&yes).unwrap();
2988 assert!(!hn.has_quoted_triples());
2989 assert!(hy.has_quoted_triples());
2990
2991 let sn = QuotedTriplesSignal::probe(&hn);
2992 let sy = QuotedTriplesSignal::probe(&hy);
2993 assert!(!sn.present);
2994 assert!(sy.present);
2995
2996 assert_eq!(sn.describe(), None);
2999 let line = sy.describe().expect("a file with quoted triples says so");
3003 assert!(line.contains("<<( s p o )>>"), "{line}");
3004 assert!(line.contains(QUOTED_TRIPLE_SURFACE_RDF_STAR), "{line}");
3005 assert_eq!(
3006 sy.export_surfaces,
3007 vec![QUOTED_TRIPLE_SURFACE_RDF12, QUOTED_TRIPLE_SURFACE_RDF_STAR]
3008 );
3009 assert_eq!(
3010 sy.export_default.as_deref(),
3011 Some(QUOTED_TRIPLE_SURFACE_RDF12)
3012 );
3013 assert!(sn.export_surfaces.is_empty());
3015 assert_eq!(sn.export_default, None);
3016 }
3017
3018 #[test]
3023 fn the_quoted_triple_signal_is_stripped_from_the_stored_card() {
3024 let measured = QuotedTriplesSignal {
3025 present: true,
3026 export_surfaces: vec![
3027 QUOTED_TRIPLE_SURFACE_RDF12.to_string(),
3028 QUOTED_TRIPLE_SURFACE_RDF_STAR.to_string(),
3029 ],
3030 export_default: Some(QUOTED_TRIPLE_SURFACE_RDF12.to_string()),
3031 };
3032 let mut card = DatasetCard {
3033 title: Some("Annotated".into()),
3034 triple_count: 3,
3035 ..Default::default()
3036 };
3037 assert_eq!(card.observe_quoted_triples(measured.clone()), None);
3038 assert_eq!(card.signals.quoted_triples, Some(measured.clone()));
3039
3040 let bytes = card.to_json_bytes();
3041 let text = String::from_utf8(bytes.clone()).unwrap();
3042 assert!(
3043 !text.contains("quoted_triples"),
3044 "the signal must not reach the metadata section: {text}"
3045 );
3046 let back = DatasetCard::from_json_bytes(&bytes).unwrap();
3047 assert_eq!(back.title, card.title);
3048 assert_eq!(back.signals.quoted_triples, None);
3049 assert_eq!(card.signals.quoted_triples, Some(measured));
3051
3052 let authored = r#"{"title":"x","signals":{"quoted_triples":{"present":true}}}"#;
3055 assert!(
3056 serde_json::from_str::<CardInput>(authored).is_err(),
3057 "a hand-written card must not be able to assert a quoted-triple set"
3058 );
3059 }
3060
3061 #[test]
3066 fn the_quoted_triple_signal_round_trips_through_a_card_document() {
3067 let mut card = DatasetCard {
3068 title: Some("Annotated".into()),
3069 ..Default::default()
3070 };
3071 card.observe_quoted_triples(QuotedTriplesSignal {
3072 present: true,
3073 export_surfaces: vec![QUOTED_TRIPLE_SURFACE_RDF12.to_string()],
3074 export_default: Some(QUOTED_TRIPLE_SURFACE_RDF12.to_string()),
3075 });
3076 let doc = serde_json::to_vec(&card).unwrap();
3077 let back: DatasetCard = serde_json::from_slice(&doc).unwrap();
3078 assert_eq!(back.signals.quoted_triples, card.signals.quoted_triples);
3079
3080 let mut none = DatasetCard::default();
3081 none.observe_quoted_triples(QuotedTriplesSignal {
3082 present: false,
3083 export_surfaces: Vec::new(),
3084 export_default: None,
3085 });
3086 let text = serde_json::to_string(&none).unwrap();
3087 assert!(
3088 text.contains(r#""quoted_triples":{"present":false}"#),
3089 "{text}"
3090 );
3091 let absent = serde_json::to_string(&DatasetCard::default()).unwrap();
3092 assert!(!absent.contains("quoted_triples"), "{absent}");
3093 }
3094 #[test]
3095 fn split_namespace_handles_hash_slash_and_neither() {
3096 assert_eq!(
3097 split_namespace("<http://xmlns.com/foaf/0.1/name>").as_deref(),
3098 Some("http://xmlns.com/foaf/0.1/")
3099 );
3100 assert_eq!(
3101 split_namespace("<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>").as_deref(),
3102 Some("http://www.w3.org/1999/02/22-rdf-syntax-ns#")
3103 );
3104 assert_eq!(split_namespace("<urn:isbn:12345>"), None);
3106 assert_eq!(split_namespace("\"literal\""), None);
3107 assert_eq!(split_namespace("_:b0"), None);
3108 }
3109
3110 #[test]
3111 fn derive_counts_and_orders() {
3112 let quads = vec![
3113 q("<http://ex/a>", TYPE, "<http://ex/Person>"),
3114 q("<http://ex/b>", TYPE, "<http://ex/Person>"),
3115 q("<http://ex/c>", TYPE, "<http://ex/City>"),
3116 q("<http://ex/a>", "<http://ex/knows>", "<http://ex/b>"),
3117 q("<http://ex/a>", "<http://ex/knows>", "<http://ex/c>"),
3118 q("<http://ex/a>", "<http://ex/name>", "\"A\""),
3119 ];
3120 let card = derive_card(&quads, 9, 0, CardInput::default());
3121 assert_eq!(card.triple_count, 6);
3122 assert_eq!(card.quad_count, 6);
3123 assert_eq!(card.term_count, 9);
3124 assert_eq!(card.predicates[0], (TYPE.to_string(), 3));
3126 assert!(card
3127 .predicates
3128 .contains(&("<http://ex/knows>".to_string(), 2)));
3129 assert_eq!(
3131 card.classes,
3132 vec![
3133 ("<http://ex/Person>".to_string(), 2),
3134 ("<http://ex/City>".to_string(), 1)
3135 ]
3136 );
3137 assert!(card.vocabularies.contains(&"http://ex/".to_string()));
3139 assert!(card
3140 .vocabularies
3141 .iter()
3142 .any(|v| v.ends_with("rdf-syntax-ns#")));
3143 }
3144
3145 #[test]
3146 fn named_graph_triples_excluded_from_default_stats() {
3147 let quads = vec![
3148 q("<http://ex/a>", "<http://ex/p>", "<http://ex/b>"),
3149 (
3150 "<http://ex/c>".into(),
3151 "<http://ex/p>".into(),
3152 "<http://ex/d>".into(),
3153 Some("<http://ex/g1>".into()),
3154 ),
3155 ];
3156 let card = derive_card(&quads, 5, 1, CardInput::default());
3157 assert_eq!(card.triple_count, 1, "only the default-graph triple counts");
3158 assert_eq!(card.quad_count, 2);
3159 assert_eq!(card.named_graph_count, 1);
3160 assert_eq!(card.predicates[0].1, 1);
3161 }
3162
3163 const LABEL: &str = "<http://www.w3.org/2000/01/rdf-schema#label>";
3165 const SAMEAS: &str = "<http://www.w3.org/2002/07/owl#sameAs>";
3166 const XSD_INT: &str = "http://www.w3.org/2001/XMLSchema#integer";
3167 const XSD_GYEAR: &str = "http://www.w3.org/2001/XMLSchema#gYear";
3168
3169 fn enriched_fixture() -> Vec<(String, String, String, Option<String>)> {
3170 vec![
3171 q("<http://ex/a>", TYPE, "<http://ex/Person>"),
3172 q("<http://ex/b>", TYPE, "<http://ex/Person>"),
3173 q("<http://ex/c>", TYPE, "<http://ex/City>"),
3174 q("<http://ex/a>", LABEL, "\"Alice\"@en"),
3175 q("<http://ex/b>", LABEL, "\"Bob\"@en"),
3176 q(
3177 "<http://ex/a>",
3178 "<http://ex/age>",
3179 &format!("\"30\"^^<{XSD_INT}>"),
3180 ),
3181 q(
3182 "<http://ex/a>",
3183 "<http://ex/birthYear>",
3184 &format!("\"1850\"^^<{XSD_GYEAR}>"),
3185 ),
3186 q(
3187 "<http://ex/b>",
3188 "<http://ex/birthYear>",
3189 &format!("\"1875\"^^<{XSD_GYEAR}>"),
3190 ),
3191 q("<http://ex/a>", "<http://ex/knows>", "<http://ex/b>"),
3192 q("<http://ex/a>", "<http://ex/livesIn>", "<http://ex/c>"),
3193 q("<http://ex/a>", SAMEAS, "<http://other/a>"),
3194 ]
3195 }
3196
3197 #[test]
3198 fn enriched_profile_is_populated() {
3199 let quads = enriched_fixture();
3200 let card = derive_card(&quads, 12, 0, CardInput::default());
3201
3202 assert!(!card.datatypes.is_empty(), "datatypes derived");
3203 assert!(!card.languages.is_empty(), "languages derived");
3204 assert!(!card.class_links.is_empty(), "class_links derived");
3205 assert!(!card.top_hubs.is_empty(), "out-degree hubs derived");
3206 assert!(!card.in_hubs.is_empty(), "in-degree hubs derived");
3207
3208 assert!(card
3210 .class_links
3211 .iter()
3212 .any(|l| l.s_class == "<http://ex/Person>"
3213 && l.predicate == "<http://ex/knows>"
3214 && l.o_class == "<http://ex/Person>"
3215 && l.count == 1));
3216 assert!(card
3217 .class_links
3218 .iter()
3219 .any(|l| l.o_class == "<http://ex/City>"));
3220 assert!(card.class_links.iter().any(|l| l.o_class == "(untyped)"));
3222
3223 assert!(card.datatypes.iter().any(|(d, _)| d.contains("integer")));
3225 assert!(card.datatypes.iter().any(|(d, _)| d.contains("gYear")));
3226 assert!(card.datatypes.iter().any(|(d, _)| d.contains("langString")));
3227 assert!(card.languages.iter().any(|(l, _)| l == "en"));
3229
3230 let s = &card.signals;
3232 assert_eq!(s.label_predicate.as_deref(), Some(LABEL));
3233 assert_eq!(s.default_lang.as_deref(), Some("en"));
3234 assert_eq!(s.base_iri.as_deref(), Some("http://ex/"));
3235 assert!(s.link_predicates.iter().any(|p| p.contains("sameAs")));
3236 assert!(s.time_predicates.iter().any(|p| p.contains("birthYear")));
3237 assert_eq!(
3238 s.temporal_extent,
3239 Some(("1850".to_string(), "1875".to_string())),
3240 "extent over the dominant time predicate"
3241 );
3242 assert!(!s.geo_wkt && !s.geo_latlong);
3244 assert!(s.spatial_bbox.is_none());
3245 }
3246
3247 #[test]
3248 fn signals_detect_geometry() {
3249 let wkt = "http://www.opengis.net/ont/geosparql#wktLiteral";
3250 let quads = vec![
3251 q("<http://ex/p>", TYPE, "<http://ex/Place>"),
3252 q(
3253 "<http://ex/p>",
3254 "<http://www.w3.org/2003/01/geo/wgs84_pos#lat>",
3255 "\"41.9\"^^<http://www.w3.org/2001/XMLSchema#double>",
3256 ),
3257 q(
3258 "<http://ex/p>",
3259 "<http://www.w3.org/2003/01/geo/wgs84_pos#long>",
3260 "\"12.5\"^^<http://www.w3.org/2001/XMLSchema#double>",
3261 ),
3262 q(
3263 "<http://ex/p>",
3264 "<http://www.opengis.net/ont/geosparql#asWKT>",
3265 &format!("\"POINT(12.5 41.9)\"^^<{wkt}>"),
3266 ),
3267 ];
3268 let card = derive_card(&quads, 6, 0, CardInput::default());
3269 assert!(card.signals.geo_wkt, "asWKT/wktLiteral detected");
3270 assert!(card.signals.geo_latlong, "wgs84 lat+long detected");
3271 let bbox = card.signals.spatial_bbox.expect("bbox derived");
3272 assert_eq!(
3273 bbox,
3274 [12.5, 41.9, 12.5, 41.9],
3275 "[minLon,minLat,maxLon,maxLat]"
3276 );
3277 }
3278
3279 #[test]
3280 fn class_links_match_schema_summary() {
3281 use crate::{ingest, schema_summary, Rete};
3282 let quads = enriched_fixture();
3283 let card = derive_card(&quads, 12, 0, CardInput::default());
3284
3285 let (bytes, _) =
3287 ingest::assemble_dataset_with_opts(quads, true, false, None, |_, _| Vec::new());
3288 let rete = Rete::open(&bytes).unwrap();
3289 let reference: BTreeSet<(String, String, String, u64)> = schema_summary(&rete)
3290 .into_iter()
3291 .map(|(s, p, o, c)| (s, p, o, c as u64))
3292 .collect();
3293
3294 let derived: BTreeSet<(String, String, String, u64)> = card
3295 .class_links
3296 .iter()
3297 .map(|l| {
3298 (
3299 l.s_class.clone(),
3300 l.predicate.clone(),
3301 l.o_class.clone(),
3302 l.count,
3303 )
3304 })
3305 .collect();
3306
3307 assert_eq!(
3308 derived, reference,
3309 "card class_links must equal schema_summary's quotient row-for-row"
3310 );
3311 }
3312
3313 #[test]
3323 fn card_counts_are_the_deduplicated_counts() {
3324 use crate::ingest::{self, DeferredMetadata};
3325 use crate::Rete;
3326
3327 let g = |s: &str, p: &str, o: &str, graph: &str| {
3328 (
3329 s.to_string(),
3330 p.to_string(),
3331 o.to_string(),
3332 Some(graph.to_string()),
3333 )
3334 };
3335 let quads = vec![
3336 q("<http://ex/a>", "<http://ex/p>", "<http://ex/1>"),
3337 q("<http://ex/b>", "<http://ex/p>", "<http://ex/2>"),
3338 q("<http://ex/a>", "<http://ex/p>", "<http://ex/1>"), g("<http://ex/n>", "<http://ex/q>", "\"v\"", "<http://ex/g>"),
3340 g("<http://ex/n>", "<http://ex/q>", "\"v\"", "<http://ex/g>"), ];
3342 let (bytes, stats) =
3344 ingest::assemble_dataset_with_opts(quads, false, false, None, |stats, quads| {
3345 let card = derive_card(
3346 quads,
3347 stats.terms as u64,
3348 stats.named_graphs as u64,
3349 CardInput::default(),
3350 );
3351 DeferredMetadata::new(move |counts| card.with_final_counts(counts).to_json_bytes())
3352 });
3353
3354 let header = crate::Header::from_bytes(&bytes).unwrap();
3355 let card = load_card(&bytes).unwrap().expect("card embedded");
3356 assert_eq!(header.quad_count, 3, "header records the deduped quads");
3357 assert_eq!(
3358 card.quad_count, header.quad_count,
3359 "card and header must not disagree about the size of the same file"
3360 );
3361 assert_eq!(card.triple_count, 2, "default-graph triples, deduped");
3362
3363 let rete = Rete::open(&bytes).unwrap();
3365 let default_triples = rete.dump(None).len() as u64;
3366 let named: u64 = rete
3367 .graph_names()
3368 .iter()
3369 .map(|name| rete.dump(Some(name)).len() as u64)
3370 .sum();
3371 assert_eq!(default_triples + named, card.quad_count);
3372
3373 assert_eq!(stats.statements, 3);
3376 assert_eq!(stats.default_triples, 2);
3377 }
3378
3379 #[test]
3380 fn derive_is_deterministic() {
3381 let quads = enriched_fixture();
3384 let a = derive_card(&quads, 12, 0, CardInput::default()).to_json_bytes();
3385 let b = derive_card(&quads, 12, 0, CardInput::default()).to_json_bytes();
3386 assert_eq!(a, b, "enriched card derivation must be deterministic");
3387 assert!(!a.is_empty());
3388 }
3389
3390 #[test]
3391 fn old_plain_string_card_still_parses() {
3392 let json = r#"{
3395 "title":"Legacy",
3396 "triple_count":3,"quad_count":3,"named_graph_count":0,"term_count":5,
3397 "predicates":[["<http://ex/p>",2]],
3398 "example_queries":["SELECT * WHERE { ?s ?p ?o }"],
3399 "format_version":1
3400 }"#;
3401 let card = DatasetCard::from_json_bytes(json.as_bytes()).unwrap();
3402 assert_eq!(card.title.as_deref(), Some("Legacy"));
3403 assert_eq!(card.example_queries.len(), 1);
3404 assert!(card.datatypes.is_empty());
3406 assert!(card.languages.is_empty());
3407 assert!(card.class_links.is_empty());
3408 assert!(card.top_hubs.is_empty());
3409 assert!(card.in_hubs.is_empty());
3410 assert!(card.queries.is_empty());
3411 assert!(card.signals.is_empty());
3412 assert!(!card.truncated);
3413 }
3414
3415 #[test]
3419 fn curated_base_iri_overrides_derived_and_is_validated() {
3420 let quads = vec![q(
3421 "<https://doi.org/10.1/a>",
3422 "<http://ex/p>",
3423 "<http://ex/o>",
3424 )];
3425 let derived = derive_card(&quads, 3, 0, CardInput::default());
3426 assert_eq!(
3427 derived.signals.base_iri.as_deref(),
3428 Some("https://doi.org/10.1/")
3429 );
3430 let input = CardInput::from_json_str(r#"{"base_iri": "https://doi.org/"}"#).unwrap();
3431 assert!(!input.is_empty());
3432 let card = derive_card(&quads, 3, 0, input.clone());
3433 assert_eq!(card.signals.base_iri.as_deref(), Some("https://doi.org/"));
3434 let counts = curated_counts_card(1, 3, input);
3435 assert_eq!(counts.signals.base_iri.as_deref(), Some("https://doi.org/"));
3436 for bad in [
3437 "doi.org/",
3438 "",
3439 "https:",
3440 "1http://x/",
3441 "http://a b/",
3442 "http://x/<y>",
3443 ] {
3444 let doc = serde_json::json!({ "base_iri": bad }).to_string();
3445 let err = CardInput::from_json_str(&doc).unwrap_err();
3446 assert!(err.contains("base_iri"), "{bad}: {err}");
3447 }
3448 }
3449
3450 #[test]
3451 fn curated_identity_fields_round_trip() {
3452 let input: CardInput = serde_json::from_str(
3453 r#"{
3454 "title": "T",
3455 "version": "2026-08-04",
3456 "creators": [{"name":"Ada","orcid":"https://orcid.org/0000-0002-1825-0097"}],
3457 "publisher": {"name":"EPFL","ror":"https://ror.org/02s376052"},
3458 "canonical_url": "https://data.example.org/t.rete",
3459 "sparql_endpoint": "https://example.org/sparql",
3460 "source_date": "2026-07-01",
3461 "derived_from": ["https://example.org/dump.nt"],
3462 "doi": "https://doi.org/10.5281/zenodo.1",
3463 "cite_as": "Ada (2026). T.",
3464 "keywords": ["graphs", "citations"],
3465 "theme": ["http://publications.europa.eu/resource/authority/data-theme/TECH"]
3466 }"#,
3467 )
3468 .unwrap();
3469 let card = curated_counts_card(5, 9, input);
3470 let bytes = card.to_json_bytes();
3471 let back = DatasetCard::from_json_bytes(&bytes).unwrap();
3472 assert_eq!(back.version.as_deref(), Some("2026-08-04"));
3473 assert_eq!(back.creators[0].name, "Ada");
3474 assert_eq!(
3475 back.creators[0].orcid.as_deref(),
3476 Some("https://orcid.org/0000-0002-1825-0097")
3477 );
3478 assert_eq!(back.publisher.as_ref().unwrap().name, "EPFL");
3479 assert_eq!(
3480 back.canonical_url.as_deref(),
3481 Some("https://data.example.org/t.rete")
3482 );
3483 assert_eq!(back.derived_from, vec!["https://example.org/dump.nt"]);
3484 assert_eq!(
3485 back.doi.as_deref(),
3486 Some("https://doi.org/10.5281/zenodo.1")
3487 );
3488 assert_eq!(back.keywords, vec!["graphs", "citations"]);
3491 assert_eq!(
3492 back.theme,
3493 vec!["http://publications.europa.eu/resource/authority/data-theme/TECH"]
3494 );
3495
3496 let plain = curated_counts_card(1, 2, CardInput::default()).to_json_bytes();
3498 let text = String::from_utf8(plain).unwrap();
3499 for absent in [
3500 "\"creators\"",
3501 "\"publisher\"",
3502 "\"canonical_url\"",
3503 "\"sparql_endpoint\"",
3504 "\"derived_from\"",
3505 "\"doi\"",
3506 "\"cite_as\"",
3507 "\"source_date\"",
3508 "\"version\"",
3509 "\"top_n\"",
3510 "\"keywords\"",
3511 "\"theme\"",
3512 "\"extra\"",
3513 ] {
3514 assert!(
3515 !text.contains(absent),
3516 "{absent} must be omitted when unset"
3517 );
3518 }
3519 }
3520
3521 #[test]
3526 fn extra_round_trips_and_external_writer_cards_surface_it() {
3527 let input: CardInput = serde_json::from_str(
3528 r#"{"title":"T","extra":{"atlas:layer":84,"review":{"by":"dg","ok":true}}}"#,
3529 )
3530 .unwrap();
3531 let card = curated_counts_card(5, 9, input);
3532 let back = DatasetCard::from_json_bytes(&card.to_json_bytes()).unwrap();
3533 assert_eq!(back.extra, card.extra);
3534 assert_eq!(back.extra.get("atlas:layer"), Some(&serde_json::json!(84)));
3535
3536 let python_shaped = r#"{
3539 "title":"From Python","extra":{"pipeline":"nightly"},
3540 "triple_count":1,"quad_count":1,"named_graph_count":0,"term_count":3,
3541 "format_version":5
3542 }"#;
3543 let card = DatasetCard::from_json_bytes(python_shaped.as_bytes()).unwrap();
3544 assert_eq!(
3545 card.extra.get("pipeline"),
3546 Some(&serde_json::json!("nightly"))
3547 );
3548 }
3549
3550 #[test]
3553 fn extra_limits_bite_exactly_at_the_boundary() {
3554 use crate::card_input::normalize_extra;
3555 use serde_json::{json, Map, Value};
3556 let bag = |v: Vec<(String, Value)>| -> Map<String, Value> { v.into_iter().collect() };
3559
3560 let pad = |n: usize| bag(vec![("pad".into(), json!("x".repeat(n)))]);
3562 let at = pad(CARD_EXTRA_MAX_BYTES - 10);
3563 assert_eq!(serde_json::to_vec(&at).unwrap().len(), CARD_EXTRA_MAX_BYTES);
3564 normalize_extra(at).expect("exactly at the byte cap is accepted");
3565 let over = pad(CARD_EXTRA_MAX_BYTES - 9);
3566 assert_eq!(
3567 serde_json::to_vec(&over).unwrap().len(),
3568 CARD_EXTRA_MAX_BYTES + 1
3569 );
3570 let err = normalize_extra(over).unwrap_err().to_string();
3571 assert!(
3572 err.contains(&format!("{} bytes", CARD_EXTRA_MAX_BYTES + 1)),
3573 "the error states the actual size: {err}"
3574 );
3575
3576 let keys = |n: usize| {
3578 bag((0..n)
3579 .map(|i| (format!("k{i:03}"), json!(1)))
3580 .collect::<Vec<_>>())
3581 };
3582 normalize_extra(keys(CARD_EXTRA_MAX_KEYS)).expect("at the key cap");
3583 assert!(normalize_extra(keys(CARD_EXTRA_MAX_KEYS + 1)).is_err());
3584
3585 normalize_extra(bag(vec![("k".repeat(CARD_EXTRA_MAX_KEY_BYTES), json!(1))]))
3587 .expect("at the key-length cap");
3588 assert!(normalize_extra(bag(vec![(
3589 "k".repeat(CARD_EXTRA_MAX_KEY_BYTES + 1),
3590 json!(1)
3591 )]))
3592 .is_err());
3593 assert!(normalize_extra(bag(vec![(String::new(), json!(1))])).is_err());
3594
3595 normalize_extra(bag(vec![("d".into(), json!({"a": {"b": 1}}))])).expect("under the cap");
3599 normalize_extra(bag(vec![("d".into(), json!([{"w": ["x", "y"]}]))]))
3600 .expect("at the depth cap");
3601 assert!(normalize_extra(bag(vec![("d".into(), json!({"a": {"b": {"c": [1]}}}))])).is_err());
3602 assert!(normalize_extra(bag(vec![("d".into(), json!([[[[1]]]]))])).is_err());
3603
3604 let err = normalize_extra(bag(vec![("@context".into(), json!({}))]))
3606 .unwrap_err()
3607 .to_string();
3608 assert!(err.contains("reserved"), "loud reservation: {err}");
3609 }
3610
3611 #[test]
3616 fn extra_fields_keep_the_content_hash_reproducible() {
3617 let build = |json: &str| {
3618 let input: CardInput = serde_json::from_str::<CardInput>(json)
3621 .unwrap()
3622 .normalize()
3623 .unwrap();
3624 let quads = enriched_fixture();
3625 let card = derive_card(&quads, 12, 0, input);
3626 let (image, _) =
3627 crate::ingest::assemble_dataset_with_opts(quads, true, false, None, |_, _| {
3628 card.to_json_bytes()
3629 });
3630 image
3631 };
3632 let a = build(r#"{"title":"T","extra":{"b":{"y":2,"x":1},"a":[1,2]}}"#);
3633 let b = build(r#"{"title":"T","extra":{"a":[1,2],"b":{"x":1,"y":2}}}"#);
3634 assert_eq!(a, b, "reordered-input builds are byte-identical");
3635 let ha = Header::from_bytes(&a).unwrap().content_hash;
3636 assert_eq!(ha, Header::from_bytes(&b).unwrap().content_hash);
3637
3638 let plain = build(r#"{"title":"T"}"#);
3641 assert_ne!(ha, Header::from_bytes(&plain).unwrap().content_hash);
3642 }
3643
3644 #[test]
3648 fn top_level_custom_keys_are_rejected_not_dropped() {
3649 let err = serde_json::from_str::<CardInput>(r#"{"title":"T","my_field":1}"#)
3650 .unwrap_err()
3651 .to_string();
3652 assert!(err.contains("my_field"), "names the stray key: {err}");
3653
3654 let ok: CardInput =
3656 serde_json::from_str(r#"{"title":"T","extra":{"my_field":1}}"#).unwrap();
3657 assert_eq!(ok.extra.get("my_field"), Some(&serde_json::json!(1)));
3658 }
3659
3660 #[test]
3665 fn description_reads_as_a_string_or_as_lines() {
3666 let from_lines: CardInput = serde_json::from_value(serde_json::json!({
3667 "description": ["## Contents", "", "- a", "- b"]
3668 }))
3669 .unwrap();
3670 assert_eq!(
3671 from_lines.description.as_deref(),
3672 Some("## Contents\n\n- a\n- b")
3673 );
3674 let from_string: CardInput =
3675 serde_json::from_str("{\"description\":\"## Contents\\n\\n- a\\n- b\"}").unwrap();
3676 assert_eq!(from_string.description, from_lines.description);
3677
3678 let none: CardInput = serde_json::from_str(r#"{"title":"T"}"#).unwrap();
3680 assert!(none.description.is_none());
3681 let null: CardInput = serde_json::from_str(r#"{"description":null}"#).unwrap();
3682 assert!(null.description.is_none());
3683
3684 let over = "x".repeat(crate::card::CARD_DESCRIPTION_MAX_BYTES + 1);
3687 let err = serde_json::from_str::<CardInput>(&format!(
3688 "{{\"description\":{}}}",
3689 serde_json::to_string(&over).unwrap()
3690 ))
3691 .unwrap_err()
3692 .to_string();
3693 assert!(err.contains("over the"), "{err}");
3694 }
3695
3696 #[test]
3704 fn curated_field_list_matches_the_deny_unknown_fields_struct() {
3705 for f in crate::card::CURATED_CARD_FIELDS {
3706 let doc = format!(r#"{{"{f}":null}}"#);
3710 if let Err(e) = serde_json::from_str::<CardInput>(&doc) {
3711 assert!(
3712 !e.to_string().contains("unknown field"),
3713 "CURATED_CARD_FIELDS lists `{f}`, but CardInput rejects it as unknown: {e}",
3714 );
3715 }
3716 }
3717 let err = serde_json::from_str::<CardInput>(r#"{"nope":1}"#)
3720 .unwrap_err()
3721 .to_string();
3722 for name in err
3723 .split("expected one of ")
3724 .nth(1)
3725 .expect("serde names the expected fields")
3726 .split(" at line ")
3728 .next()
3729 .unwrap()
3730 .split(", ")
3731 .map(|s| s.trim().trim_matches('`'))
3732 .filter(|s| !s.is_empty())
3733 {
3734 assert!(
3735 crate::card::CURATED_CARD_FIELDS.contains(&name),
3736 "CardInput accepts `{name}` but CURATED_CARD_FIELDS does not list it — \
3737 the browser builder would reject a card the CLI accepts",
3738 );
3739 }
3740 }
3741
3742 #[test]
3749 fn a_browser_written_card_deserializes_here() {
3750 let curated = crate::card::validate_curated_card(&serde_json::json!({
3751 "title": "Built in a browser",
3752 "keywords": ["b", "a"],
3753 "theme": ["http://publications.europa.eu/resource/authority/data-theme/GOVE"],
3754 "creators": [{"name": "Ada", "orcid": "https://orcid.org/0000-0002-1825-0097"}],
3755 "extra": {"internal_id": "DS-1"},
3756 }))
3757 .expect("valid curated document");
3758 let composed = crate::card::compose_curated_card(curated, 3, 4, 1, 9, 5);
3759 let text = serde_json::to_string(&composed).unwrap();
3760
3761 let card: DatasetCard = serde_json::from_str(&text).expect("CLI reads a browser card");
3762 assert_eq!(card.title.as_deref(), Some("Built in a browser"));
3763 assert_eq!(card.keywords, vec!["a".to_string(), "b".to_string()]);
3764 assert_eq!(card.creators.len(), 1);
3765 assert_eq!(card.triple_count, 3);
3766 assert_eq!(card.quad_count, 4);
3767 assert_eq!(card.term_count, 9);
3768 assert_eq!(card.format_version, 5);
3769 assert!(card.predicates.is_empty() && card.queries.is_empty());
3772 assert!(!text.contains("\"predicates\""), "{text}");
3773 assert!(!text.contains("\"top_n\""), "{text}");
3774 }
3775
3776 #[test]
3780 fn old_reader_struct_accepts_new_card_json() {
3781 #[derive(serde::Deserialize)]
3782 struct OldCard {
3783 title: Option<String>,
3784 triple_count: u64,
3785 #[serde(default)]
3786 predicates: Vec<(String, u64)>,
3787 format_version: u8,
3788 }
3789 let input: CardInput = serde_json::from_str(
3790 r#"{"title":"New","version":"1","creators":[{"name":"A"}],
3791 "canonical_url":"https://x/y.rete","derived_from":["https://x/d.nt"],
3792 "extra":{"atlas:review":"approved"}}"#,
3793 )
3794 .unwrap();
3795 let quads = vec![q("<http://ex/a>", "<http://ex/p>", "<http://ex/b>")];
3796 let card = derive_card(&quads, 3, 0, input);
3797 assert_eq!(card.top_n, CARD_TOP_N as u32);
3798 let bytes = card.to_json_bytes();
3799 let old: OldCard = serde_json::from_slice(&bytes).expect("old struct parses new JSON");
3800 assert_eq!(old.title.as_deref(), Some("New"));
3801 assert_eq!(old.triple_count, 1);
3802 assert_eq!(old.format_version, card.format_version);
3803 assert!(!old.predicates.is_empty());
3804 }
3805
3806 #[test]
3807 fn coherence_stamp_is_deterministic_and_additive() {
3808 let dis = "<http://www.w3.org/2002/07/owl#disjointWith>";
3809 let rt = "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>";
3810 let base = vec![
3811 ("<http://ex/C>".into(), dis.into(), "<http://ex/D>".into()),
3812 ("<http://ex/x>".into(), rt.into(), "<http://ex/C>".into()),
3813 ("<http://ex/x>".into(), rt.into(), "<http://ex/D>".into()),
3814 ];
3815 let r = crate::reason(&base);
3816 let c1 = Coherence::from_reasoning(&r, false);
3817 assert_eq!(c1, Coherence::from_reasoning(&r, false));
3819 assert!(!c1.coherent);
3820 assert_eq!(c1.inconsistency_count, 1);
3821 assert_eq!(c1.by_kind, vec![("disjoint-classes".to_string(), 1)]);
3822 assert_eq!(c1.rules, crate::REASON_RULESET);
3823
3824 let json = DatasetCard::default()
3826 .with_coherence(&r, false)
3827 .to_json_bytes();
3828 let text = std::str::from_utf8(&json).unwrap();
3829 assert!(text.contains("coherence"));
3830 assert_eq!(DatasetCard::from_json_bytes(&json).unwrap().coherence, c1);
3831
3832 let plain = DatasetCard::default().to_json_bytes();
3833 assert!(!std::str::from_utf8(&plain).unwrap().contains("coherence"));
3834 assert!(DatasetCard::default().coherence.is_empty());
3835 }
3836
3837 #[test]
3838 fn base_iri_is_common_prefix_with_fallback() {
3839 let mut ns = NsTally::default();
3841 ns.add("http://ex/person/".to_string(), 2000);
3842 ns.add("http://ex/place/".to_string(), 600);
3843 ns.add("http://ex/org/".to_string(), 120);
3844 assert_eq!(derive_base_iri(&ns).as_deref(), Some("http://ex/"));
3845
3846 let mut one = NsTally::default();
3848 one.add("http://data.example.org/id/".to_string(), 10);
3849 assert_eq!(
3850 derive_base_iri(&one).as_deref(),
3851 Some("http://data.example.org/id/")
3852 );
3853
3854 let mut multi = NsTally::default();
3857 multi.add("http://a.example/".to_string(), 100);
3858 multi.add("http://b.other/".to_string(), 5);
3859 assert_eq!(
3860 derive_base_iri(&multi).as_deref(),
3861 Some("http://a.example/")
3862 );
3863
3864 assert_eq!(derive_base_iri(&NsTally::default()), None);
3865 }
3866
3867 #[test]
3868 fn parse_literal_decomposes_forms() {
3869 let plain = parse_literal("\"hello\"").unwrap();
3870 assert_eq!(plain.value, "hello");
3871 assert!(plain.datatype.ends_with("XMLSchema#string"));
3872 assert_eq!(plain.lang, "");
3873
3874 let tagged = parse_literal("\"bonjour\"@fr").unwrap();
3875 assert_eq!(tagged.lang, "fr");
3876 assert!(tagged.datatype.ends_with("#langString"));
3877
3878 let typed = parse_literal("\"42\"^^<http://www.w3.org/2001/XMLSchema#integer>").unwrap();
3879 assert_eq!(typed.value, "42");
3880 assert_eq!(typed.datatype, "http://www.w3.org/2001/XMLSchema#integer");
3881
3882 let esc = parse_literal("\"a\\\"b\"@en").unwrap();
3884 assert_eq!(esc.value, "a\\\"b");
3885 assert_eq!(esc.lang, "en");
3886
3887 assert!(parse_literal("<http://ex/x>").is_none());
3889 assert!(parse_literal("_:b0").is_none());
3890 }
3891
3892 #[test]
3897 fn date_shapes_are_strict() {
3898 for ok in [
3899 "2019",
3900 "-0044",
3901 "2019-05",
3902 "2019-05-01",
3903 "2019-05-01Z",
3904 "2019-05-01+02:00",
3905 "2019-05-01T12:30",
3906 "2019-05-01T12:30:15",
3907 "2019-05-01T12:30:15.250Z",
3908 "1950Z",
3909 ] {
3910 assert!(date_shaped(ok), "{ok} is a date");
3911 }
3912 for bad in [
3913 "0000-0001-5001-3721", "9999880", "1234-1240", "123-456",
3917 "2019 annual report", "2019-13",
3919 "2019-05-32",
3920 "2019-05-01T25:00",
3921 "10.1000/182", "",
3923 "-",
3924 "2019-",
3925 ] {
3926 assert!(!date_shaped(bad), "{bad} is not a date");
3927 }
3928 }
3929
3930 #[test]
3936 fn temporal_extent_ignores_identifiers_pages_and_titles() {
3937 let value = "<http://www.essepuntato.it/2010/06/literalreification/hasLiteralValue>";
3938 let start = "<http://prismstandard.org/namespaces/basic/2.0/startingPage>";
3939 let end = "<http://prismstandard.org/namespaces/basic/2.0/endingPage>";
3940 let title = "<http://purl.org/dc/terms/title>";
3941 let pubdate = "<http://prismstandard.org/namespaces/basic/2.0/publicationDate>";
3942 let issued = "<http://purl.org/dc/terms/issued>";
3943 let xsd = |t: &str| format!("http://www.w3.org/2001/XMLSchema#{t}");
3944 let mut quads = Vec::new();
3945 for (i, id) in [
3946 "0000-0001-5001-3721",
3947 "0000-0002-1825-0097",
3948 "10.1000/182",
3949 "9999880",
3950 "2019",
3951 ]
3952 .iter()
3953 .enumerate()
3954 {
3955 quads.push(q(
3956 &format!("<http://ex/id/{i}>"),
3957 value,
3958 &format!("\"{id}\""),
3959 ));
3960 }
3961 quads.push(q("<http://ex/re/1>", start, "\"9999880\""));
3962 quads.push(q("<http://ex/re/1>", end, "\"9999899\""));
3963 quads.push(q("<http://ex/re/2>", start, "\"1234-1240\""));
3964 quads.push(q("<http://ex/br/1>", title, "\"2019 annual report\""));
3965 quads.push(q(
3966 "<http://ex/br/1>",
3967 pubdate,
3968 &format!("\"2019-05-01\"^^<{}>", xsd("date")),
3969 ));
3970 quads.push(q(
3971 "<http://ex/br/2>",
3972 pubdate,
3973 &format!("\"1950\"^^<{}>", xsd("gYear")),
3974 ));
3975 quads.push(q(
3976 "<http://ex/br/3>",
3977 pubdate,
3978 &format!("\"2021-03\"^^<{}>", xsd("gYearMonth")),
3979 ));
3980 quads.push(q(
3982 "<http://ex/br/4>",
3983 pubdate,
3984 &format!("\"0000-0001-5001-3721\"^^<{}>", xsd("date")),
3985 ));
3986 quads.push(q("<http://ex/br/5>", issued, "\"1999-12\""));
3988 let card = derive_card(&quads, 40, 0, CardInput::default());
3989 let s = &card.signals;
3990 assert_eq!(
3991 s.time_predicates,
3992 vec![pubdate.to_string(), issued.to_string()],
3993 "only the date predicates are time predicates"
3994 );
3995 assert_eq!(
3996 s.temporal_extent,
3997 Some(("1950".to_string(), "2021-03".to_string())),
3998 "calendar order across xsd:gYear, xsd:date and xsd:gYearMonth"
3999 );
4000 assert!(
4002 s.numeric_predicates.is_empty(),
4003 "{:?}",
4004 s.numeric_predicates
4005 );
4006 assert!(card
4008 .queries
4009 .iter()
4010 .filter(|q| q.dimension == "time")
4011 .all(|q| q.sparql.contains("publicationDate")));
4012 }
4013
4014 #[test]
4015 fn negative_years_order_by_value_not_by_bytes() {
4016 let mut extent = BTreeMap::new();
4017 for v in ["-0044", "-0100", "0001", "-0044-03-15"] {
4018 update_time_extent("<p>", v, &mut extent);
4019 }
4020 assert_eq!(
4021 extent.get("<p>"),
4022 Some(&("-0100".to_string(), "0001".to_string()))
4023 );
4024 }
4025
4026 #[test]
4029 fn an_integer_year_off_a_date_predicate_is_numeric() {
4030 let int = "http://www.w3.org/2001/XMLSchema#integer";
4031 let quads = vec![
4032 q(
4033 "<http://ex/a>",
4034 "<http://ex/year>",
4035 &format!("\"1492\"^^<{int}>"),
4036 ),
4037 q(
4038 "<http://ex/b>",
4039 "<http://ex/year>",
4040 &format!("\"1500\"^^<{int}>"),
4041 ),
4042 ];
4043 let card = derive_card(&quads, 6, 0, CardInput::default());
4044 assert!(card.signals.time_predicates.is_empty());
4045 assert_eq!(
4046 card.signals.numeric_predicates,
4047 vec!["<http://ex/year>".to_string()]
4048 );
4049 }
4050
4051 fn streaming_fixture() -> Vec<(String, String, String, Option<String>)> {
4060 let xsd = |t: &str| format!("http://www.w3.org/2001/XMLSchema#{t}");
4061 let mut v = enriched_fixture();
4062 for i in 0..5 {
4063 let s = format!("<http://ex/place/{i}>");
4064 v.push(q(&s, TYPE, "<http://ex/Place>"));
4065 v.push(q(&s, LABEL, &format!("\"Lieu {i}\"@fr")));
4066 v.push(q(
4067 &s,
4068 "<http://www.w3.org/2003/01/geo/wgs84_pos#lat>",
4069 &format!("\"4{i}.5\"^^<{}>", xsd("double")),
4070 ));
4071 v.push(q(
4072 &s,
4073 "<http://www.w3.org/2003/01/geo/wgs84_pos#long>",
4074 &format!("\"{i}.25\"^^<{}>", xsd("double")),
4075 ));
4076 v.push(q(
4077 &s,
4078 "<http://www.opengis.net/ont/geosparql#asWKT>",
4079 &format!(
4080 "\"POINT({i}.25 4{i}.5)\"^^<http://www.opengis.net/ont/geosparql#wktLiteral>"
4081 ),
4082 ));
4083 v.push(q(
4084 &s,
4085 "<http://purl.org/dc/terms/created>",
4086 &format!("\"20{i}0-01-0{}\"^^<{}>", i + 1, xsd("date")),
4087 ));
4088 v.push(q(&s, "<http://ex/near>", "<http://elsewhere/undescribed>"));
4089 }
4090 let knows = "<http://ex/a> <http://ex/knows> <http://ex/b>";
4093 let lives = "<http://ex/a> <http://ex/livesIn> <http://ex/c>";
4094 let ghost = "<http://ex/b> <http://ex/knows> <http://ex/c>";
4095 for (t, who) in [(knows, "alice"), (lives, "bob"), (ghost, "carol")] {
4096 v.push(q(
4097 &format!("<<{t}>>"),
4098 "<http://ex/recordedBy>",
4099 &format!("<http://ex/{who}>"),
4100 ));
4101 }
4102 v.push(q(
4103 "<http://ex/claim/1>",
4104 "<http://ex/states>",
4105 &format!("<<{knows}>>"),
4106 ));
4107 v.push((
4108 "<http://ex/note/1>".into(),
4109 "<http://ex/about>".into(),
4110 "<http://ex/a>".into(),
4111 Some("<http://ex/graph/notes>".into()),
4112 ));
4113 v
4114 }
4115
4116 fn both_ways(quads: &[(String, String, String, Option<String>)]) -> (DatasetCard, DatasetCard) {
4119 let (bytes, _) =
4120 crate::ingest::assemble_dataset_with_opts(quads.to_vec(), true, false, None, |_, _| {
4121 Vec::new()
4122 });
4123 let rete = crate::Rete::open(&bytes).unwrap();
4124 let term_count = rete.header().term_count;
4125 let (card, report) =
4126 derive_card_streaming(&rete, CardInput::default(), &mut |_| {}).unwrap();
4127 assert_eq!(report.statements, card.triple_count);
4128 let in_memory = derive_card(
4129 quads,
4130 term_count,
4131 rete.named_graph_count() as u64,
4132 CardInput::default(),
4133 );
4134 (card, in_memory)
4135 }
4136
4137 #[test]
4141 fn streaming_derivation_matches_the_in_memory_card() {
4142 for quads in [enriched_fixture(), streaming_fixture()] {
4143 let (streamed, in_memory) = both_ways(&quads);
4144 assert_eq!(
4145 String::from_utf8(streamed.to_json_bytes()).unwrap(),
4146 String::from_utf8(in_memory.to_json_bytes()).unwrap()
4147 );
4148 }
4149 let (card, _) = both_ways(&streaming_fixture());
4151 assert!(card.signals.annotated_statement.is_some());
4152 assert!(!card.signals.quoting_predicates.is_empty());
4153 assert!(card.signals.spatial_bbox.is_some());
4154 assert!(card.signals.temporal_extent.is_some());
4155 assert_eq!(card.named_graph_count, 1);
4156 }
4157
4158 #[test]
4161 fn streaming_derivation_widens_the_class_map() {
4162 let mut quads = Vec::new();
4163 for i in 0..300 {
4164 let s = format!("<http://ex/thing/{i:03}>");
4165 quads.push(q(&s, TYPE, &format!("<http://ex/Class{i:03}>")));
4166 quads.push(q(
4167 &s,
4168 "<http://ex/next>",
4169 &format!("<http://ex/thing/{:03}>", (i + 1) % 300),
4170 ));
4171 }
4172 let (streamed, in_memory) = both_ways(&quads);
4173 assert!(streamed.truncated);
4174 assert_eq!(streamed.classes, in_memory.classes);
4175 assert_eq!(streamed.class_links, in_memory.class_links);
4176 assert_eq!(
4177 String::from_utf8(streamed.to_json_bytes()).unwrap(),
4178 String::from_utf8(in_memory.to_json_bytes()).unwrap()
4179 );
4180 }
4181
4182 #[test]
4185 fn streaming_derivation_counts_the_deduplicated_file() {
4186 let mut quads = enriched_fixture();
4187 quads.push(quads[0].clone());
4188 let (streamed, in_memory) = both_ways(&quads);
4189 assert_eq!(streamed.triple_count, in_memory.triple_count - 1);
4190 let type_count = |c: &DatasetCard| {
4191 c.predicates
4192 .iter()
4193 .find(|(p, _)| p == TYPE)
4194 .map(|(_, n)| *n)
4195 };
4196 assert_eq!(type_count(&streamed), type_count(&in_memory).map(|n| n - 1));
4197 }
4198
4199 #[test]
4202 fn the_curated_half_of_a_card_round_trips() {
4203 let doc = r##"{"title":"T","description":["# Heading","line"],"license":"CC0-1.0",
4204 "source":"https://ex/src","created":"2026-01-01","version":"1",
4205 "creators":[{"name":"Ada","orcid":"https://orcid.org/0000-0002-1825-0097"}],
4206 "publisher":{"name":"P","ror":"https://ror.org/01ggx4157"},
4207 "canonical_url":"https://ex/f.rete","sparql_endpoint":"https://ex/sparql",
4208 "source_date":"2025-12-31","derived_from":["https://ex/dump"],
4209 "doi":"https://doi.org/10.1/x","cite_as":"Cite me","keywords":["b","a"],
4210 "theme":["http://publications.europa.eu/resource/authority/data-theme/EDUC"],
4211 "extra":{"k":[{"words":["x"]}]},"example_queries":["ASK {}"]}"##;
4212 let curated = CardInput::from_json_str(doc).unwrap();
4213 assert!(!curated.is_empty());
4214 let card = derive_card(&enriched_fixture(), 12, 0, curated);
4215 let again = derive_card(&enriched_fixture(), 12, 0, CardInput::from_card(&card));
4216 assert_eq!(again.to_json_bytes(), card.to_json_bytes());
4217 assert!(CardInput::from_card(&derive_card(
4218 &enriched_fixture(),
4219 12,
4220 0,
4221 CardInput::default()
4222 ))
4223 .is_empty());
4224 }
4225}
4226
4227#[cfg(test)]
4228mod ns_tally_tests {
4229 use super::*;
4230
4231 fn feed(t: &mut NsTally, n: usize) {
4232 for i in 0..n {
4233 let ns = if i % 7 == 0 {
4235 "https://doi.org/10.1/".to_string()
4236 } else {
4237 format!("https://doi.org/10.1/a{i}/")
4238 };
4239 t.add(ns, 1);
4240 }
4241 }
4242
4243 #[test]
4244 fn under_the_cap_the_tally_is_exact() {
4245 let mut t = NsTally::with_cap(100);
4246 feed(&mut t, 50);
4247 assert!(!t.overflowed());
4248 assert_eq!(t.len(), 43);
4249 assert_eq!(t.dominant().as_deref(), Some("https://doi.org/10.1/"));
4250 assert_eq!(
4251 derive_base_iri(&t).as_deref(),
4252 Some("https://doi.org/10.1/")
4253 );
4254 }
4255
4256 #[test]
4257 fn past_the_cap_the_tally_stays_bounded_and_deterministic() {
4258 let run = || {
4259 let mut t = NsTally::with_cap(64);
4260 feed(&mut t, 5_000);
4261 assert!(t.len() <= 64);
4262 assert!(t.overflowed());
4263 let snapshot: Vec<(String, u64)> =
4264 t.counts.iter().map(|(k, v)| (k.clone(), *v)).collect();
4265 (snapshot, t.dominant(), derive_base_iri(&t))
4266 };
4267 let (a, b) = (run(), run());
4268 assert_eq!(a, b);
4269 assert_eq!(a.1.as_deref(), Some("https://doi.org/10.1/"));
4272 assert_eq!(a.2.as_deref(), Some("https://doi.org/10.1/"));
4273 }
4274
4275 #[test]
4276 fn common_prefix_spans_all_hosts_even_when_entries_were_evicted() {
4277 let mut t = NsTally::with_cap(2);
4278 t.add("http://a.org/x/".into(), 10);
4279 t.add("http://a.org/y/".into(), 1);
4280 t.add("http://b.org/z/".into(), 1);
4281 assert!(t.overflowed());
4282 assert_eq!(t.common_prefix(), Some("http://"));
4283 assert_eq!(derive_base_iri(&t).as_deref(), Some("http://a.org/x/"));
4285 }
4286
4287 #[test]
4290 fn card_is_truncated_and_reproducible_past_the_cap() {
4291 let n = NS_TALLY_CAP + 2_000;
4292 let quads: Vec<_> = (0..n)
4293 .map(|i| {
4294 (
4295 format!("<https://doi.org/10.1/{i}/x>"),
4296 "<http://ex/p>".to_string(),
4297 "\"v\"".to_string(),
4298 None,
4299 )
4300 })
4301 .collect();
4302 let (bytes, _) =
4303 crate::ingest::assemble_dataset_with_opts(quads.clone(), true, false, None, |_, _| {
4304 Vec::new()
4305 });
4306 let rete = crate::Rete::open(&bytes).unwrap();
4307 let go = || derive_card_streaming(&rete, CardInput::default(), &mut |_| {}).unwrap();
4308 let (c1, r1) = go();
4309 let (c2, _) = go();
4310 assert!(r1.namespaces_overflowed);
4311 assert_eq!(r1.namespaces_kept as usize, NS_TALLY_CAP);
4312 assert!(c1.truncated);
4313 assert_eq!(c1.to_json_bytes(), c2.to_json_bytes());
4314 let mem = derive_card(&quads, rete.header().term_count, 0, CardInput::default());
4315 assert!(mem.truncated);
4316 assert_eq!(c1.signals.base_iri, mem.signals.base_iri);
4317 }
4318}
4319
4320#[cfg(test)]
4321mod class_map_tests {
4322 use super::ClassMap;
4323
4324 #[test]
4325 fn widening_keeps_every_code_and_never_holds_two_maps() {
4326 let mut m = ClassMap::new(999);
4327 for id in 0..1000u32 {
4328 m.set(id, id % 200);
4329 }
4330 assert_eq!(m.bytes(), 1000);
4331 m.set(5, 300);
4332 assert_eq!(m.bytes(), 2000);
4333 assert_eq!(m.get(5), 300);
4334 for id in (0..1000u32).filter(|&i| i != 5) {
4335 assert_eq!(m.get(id), id % 200, "{id}");
4336 }
4337 m.set(7, 70_000);
4338 assert_eq!(m.bytes(), 4000);
4339 assert_eq!((m.get(7), m.get(5), m.get(999)), (70_000, 300, 999 % 200));
4340 assert_eq!(m.get(5000), 0);
4342 m.set(5000, 1);
4343 assert_eq!(m.bytes(), 4000);
4344 }
4345}