1use std::collections::{BTreeMap, BTreeSet};
31
32use serde::{Deserialize, Serialize};
33
34use crate::{Header, RDF_TYPE};
35#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
37pub struct CardAnnexMarker {
38 pub fields: Vec<String>,
40 pub bytes: u64,
42}
43
44#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
48pub struct DatasetCard {
49 #[serde(default, skip_serializing_if = "Option::is_none")]
50 pub title: Option<String>,
51 #[serde(default, skip_serializing_if = "Option::is_none")]
52 pub description: Option<String>,
53 #[serde(default, skip_serializing_if = "Option::is_none")]
54 pub license: Option<String>,
55 #[serde(default, skip_serializing_if = "Option::is_none")]
56 pub source: Option<String>,
57 #[serde(default, skip_serializing_if = "Option::is_none")]
58 pub created: Option<String>,
59
60 #[serde(default, skip_serializing_if = "Option::is_none")]
71 pub version: Option<String>,
72 #[serde(default, skip_serializing_if = "Vec::is_empty")]
75 pub creators: Vec<Creator>,
76 #[serde(default, skip_serializing_if = "Option::is_none")]
78 pub publisher: Option<Publisher>,
79 #[serde(default, skip_serializing_if = "Option::is_none")]
82 pub canonical_url: Option<String>,
83 #[serde(default, skip_serializing_if = "Option::is_none")]
85 pub sparql_endpoint: Option<String>,
86 #[serde(default, skip_serializing_if = "Option::is_none")]
89 pub source_date: Option<String>,
90 #[serde(default, skip_serializing_if = "Vec::is_empty")]
93 pub derived_from: Vec<String>,
94 #[serde(default, skip_serializing_if = "Option::is_none")]
96 pub doi: Option<String>,
97 #[serde(default, skip_serializing_if = "Option::is_none")]
99 pub cite_as: Option<String>,
100 #[serde(default, skip_serializing_if = "Vec::is_empty")]
111 pub keywords: Vec<String>,
112 #[serde(default, skip_serializing_if = "Vec::is_empty")]
119 pub theme: Vec<String>,
120 #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
136 pub extra: BTreeMap<String, serde_json::Value>,
137 #[serde(default, skip_serializing_if = "Option::is_none")]
143 pub annex: Option<CardAnnexMarker>,
144
145 pub triple_count: u64,
146 pub quad_count: u64,
147 pub named_graph_count: u64,
148 pub term_count: u64,
149
150 #[serde(default, skip_serializing_if = "Vec::is_empty")]
151 pub predicates: Vec<(String, u64)>,
152 #[serde(default, skip_serializing_if = "Vec::is_empty")]
153 pub classes: Vec<(String, u64)>,
154 #[serde(default, skip_serializing_if = "Vec::is_empty")]
155 pub vocabularies: Vec<String>,
156 #[serde(default, skip_serializing_if = "Vec::is_empty")]
157 pub example_queries: Vec<String>,
158
159 #[serde(default, skip_serializing_if = "Vec::is_empty")]
166 pub datatypes: Vec<(String, u64)>,
167 #[serde(default, skip_serializing_if = "Vec::is_empty")]
170 pub languages: Vec<(String, u64)>,
171 #[serde(default, skip_serializing_if = "Vec::is_empty")]
175 pub class_links: Vec<ClassLink>,
176 #[serde(default, skip_serializing_if = "Vec::is_empty")]
178 pub top_hubs: Vec<(String, u64)>,
179 #[serde(default, skip_serializing_if = "Vec::is_empty")]
181 pub in_hubs: Vec<(String, u64)>,
182 #[serde(default, skip_serializing_if = "Signals::is_empty")]
185 pub signals: Signals,
186 #[serde(default, skip_serializing_if = "Vec::is_empty")]
190 pub queries: Vec<ExampleQuery>,
191 #[serde(default, skip_serializing_if = "Coherence::is_empty")]
195 pub coherence: Coherence,
196 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
198 pub truncated: bool,
199 #[serde(default, skip_serializing_if = "is_zero_u32")]
203 pub top_n: u32,
204
205 pub format_version: u8,
209}
210
211fn is_zero_u32(v: &u32) -> bool {
213 *v == 0
214}
215
216#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
218pub struct Creator {
219 pub name: String,
220 #[serde(default, skip_serializing_if = "Option::is_none")]
222 pub orcid: Option<String>,
223}
224
225#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
227pub struct Publisher {
228 pub name: String,
229 #[serde(default, skip_serializing_if = "Option::is_none")]
231 pub ror: Option<String>,
232}
233
234#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
238#[serde(rename_all = "lowercase")]
239pub enum Tier {
240 Card,
241 Summary,
242 Index,
243}
244
245#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
247pub struct ExampleQuery {
248 pub id: String,
250 pub title: String,
252 pub dimension: String,
254 pub question: String,
256 pub sparql: String,
258 pub tier: Tier,
260 #[serde(default, skip_serializing_if = "Vec::is_empty")]
262 pub requires: Vec<String>,
263}
264
265#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
269pub struct ClassLink {
270 pub s_class: String,
271 pub predicate: String,
272 pub o_class: String,
273 pub count: u64,
274}
275
276#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
281pub struct Signals {
282 #[serde(default, skip_serializing_if = "Option::is_none")]
284 pub label_predicate: Option<String>,
285 #[serde(default, skip_serializing_if = "Option::is_none")]
287 pub default_lang: Option<String>,
288 #[serde(default, skip_serializing_if = "Option::is_none")]
290 pub base_iri: Option<String>,
291 #[serde(default, skip_serializing_if = "Vec::is_empty")]
293 pub time_predicates: Vec<String>,
294 #[serde(default, skip_serializing_if = "Vec::is_empty")]
297 pub numeric_predicates: Vec<String>,
298 #[serde(default, skip_serializing_if = "Vec::is_empty")]
300 pub link_predicates: Vec<String>,
301 #[serde(default, skip_serializing_if = "Vec::is_empty")]
315 pub annotation_predicates: Vec<String>,
316 #[serde(default, skip_serializing_if = "Vec::is_empty")]
321 pub quoting_predicates: Vec<String>,
322 #[serde(default, skip_serializing_if = "Option::is_none")]
340 pub annotated_statement: Option<(String, String)>,
341 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
343 pub geo_wkt: bool,
344 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
346 pub geo_latlong: bool,
347 #[serde(default, skip_serializing_if = "Option::is_none")]
349 pub temporal_extent: Option<(String, String)>,
350 #[serde(default, skip_serializing_if = "Option::is_none")]
352 pub spatial_bbox: Option<[f64; 4]>,
353 #[serde(default, skip_serializing_if = "Option::is_none")]
363 pub text_index: Option<TextIndexSignal>,
364 #[serde(default, skip_serializing_if = "Option::is_none")]
369 pub label_index: Option<LabelIndexSignal>,
370 #[serde(default, skip_serializing_if = "Option::is_none")]
375 pub permutations: Option<PermutationsSignal>,
376 #[serde(default, skip_serializing_if = "Option::is_none")]
382 pub quoted_triples: Option<QuotedTriplesSignal>,
383}
384
385impl Signals {
386 pub fn is_empty(&self) -> bool {
388 *self == Signals::default()
389 }
390}
391
392#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
419pub struct TextIndexSignal {
420 pub present: bool,
423 #[serde(default, skip_serializing_if = "is_zero_u64")]
427 pub bytes: u64,
428 #[serde(default, skip_serializing_if = "Option::is_none")]
433 pub token_table_bytes: Option<u64>,
434}
435
436fn is_zero_u64(v: &u64) -> bool {
438 *v == 0
439}
440
441#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
461pub struct PermutationsSignal {
462 pub count: u8,
464 pub names: Vec<String>,
468 pub merge_join: bool,
472}
473
474impl PermutationsSignal {
475 pub fn probe(header: &Header) -> Self {
479 let perms = header.perms;
480 PermutationsSignal {
481 count: perms.len() as u8,
482 names: perms.names().into_iter().map(str::to_string).collect(),
483 merge_join: perms.has_merge_orders(),
484 }
485 }
486
487 pub fn describe(&self) -> String {
489 if self.merge_join {
490 format!(
491 "{} index permutations ({}) — sort-merge joins available",
492 self.count,
493 self.names.join("/")
494 )
495 } else {
496 format!(
497 "{} index permutations ({}) — same rows, same routing; \
498 no sort-merge join (hash/probe answers instead)",
499 self.count,
500 self.names.join("/")
501 )
502 }
503 }
504}
505
506pub const QUOTED_TRIPLE_SURFACE_RDF12: &str = "rdf12";
513
514pub const QUOTED_TRIPLE_SURFACE_RDF_STAR: &str = "rdf-star";
517
518#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
547pub struct QuotedTriplesSignal {
548 pub present: bool,
550 #[serde(default, skip_serializing_if = "Vec::is_empty")]
553 pub export_surfaces: Vec<String>,
554 #[serde(default, skip_serializing_if = "Option::is_none")]
557 pub export_default: Option<String>,
558}
559
560impl QuotedTriplesSignal {
561 pub fn probe(header: &Header) -> Self {
564 if !header.has_quoted_triples() {
565 return QuotedTriplesSignal {
566 present: false,
567 export_surfaces: Vec::new(),
568 export_default: None,
569 };
570 }
571 QuotedTriplesSignal {
572 present: true,
573 export_surfaces: vec![
574 QUOTED_TRIPLE_SURFACE_RDF12.to_string(),
575 QUOTED_TRIPLE_SURFACE_RDF_STAR.to_string(),
576 ],
577 export_default: Some(QUOTED_TRIPLE_SURFACE_RDF12.to_string()),
578 }
579 }
580
581 pub fn describe(&self) -> Option<String> {
585 if !self.present {
586 return None;
587 }
588 Some(format!(
589 "present — `rete export` writes RDF 1.2 triple terms `<<( s p o )>>` \
590 by default; `--quoted-triple-syntax {QUOTED_TRIPLE_SURFACE_RDF_STAR}` \
591 writes `<<s p o>>`"
592 ))
593 }
594}
595impl TextIndexSignal {
596 pub fn probe<R: crate::RangeReader + ?Sized>(reader: &R, header: &Header) -> Self {
601 if header.text_index_len == 0 {
602 return TextIndexSignal::default();
603 }
604 if crate::v06::reads(header.version) {
605 return match crate::v06::open::read_signals(reader) {
606 Ok((Some((bytes, tt)), _)) => TextIndexSignal {
607 present: true,
608 bytes,
609 token_table_bytes: Some(tt),
610 },
611 _ => TextIndexSignal {
612 present: true,
613 ..TextIndexSignal::default()
614 },
615 };
616 }
617 TextIndexSignal {
618 present: true,
619 bytes: header.text_index_len,
620 token_table_bytes: crate::read_text_index_token_table_len_ranged(reader, header),
621 }
622 }
623
624 pub fn describe(&self) -> String {
626 if !self.present {
627 return "no TEXT_INDEX section — CONTAINS/regex still answer, by full scan".to_string();
628 }
629 match self.token_table_bytes {
630 Some(tt) => format!(
631 "TEXT_INDEX present — {} bytes ({tt} of them the token table a first search reads)",
632 self.bytes
633 ),
634 None => format!("TEXT_INDEX present — {} bytes", self.bytes),
635 }
636 }
637}
638
639#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
650pub struct LabelIndexSignal {
651 pub present: bool,
654 #[serde(default, skip_serializing_if = "is_zero_u64")]
656 pub bytes: u64,
657 #[serde(default, skip_serializing_if = "Option::is_none")]
660 pub entries: Option<u64>,
661}
662
663impl LabelIndexSignal {
664 pub fn probe<R: crate::RangeReader + ?Sized>(reader: &R, header: &Header) -> Self {
668 let Some(s) = header.label_index_section() else {
669 return LabelIndexSignal::default();
670 };
671 if crate::v06::reads(header.version) {
672 return match crate::v06::open::read_signals(reader) {
673 Ok((_, Some((bytes, entries)))) => LabelIndexSignal {
674 present: true,
675 bytes,
676 entries,
677 },
678 _ => LabelIndexSignal {
679 present: true,
680 ..LabelIndexSignal::default()
681 },
682 };
683 }
684 LabelIndexSignal {
685 present: true,
686 bytes: s.length,
687 entries: crate::label_index::read_label_index_entries_ranged(reader, header),
688 }
689 }
690
691 pub fn describe(&self) -> String {
693 if !self.present {
694 return "no LABEL_INDEX section — a prefix search finds only what a pyramid's \
695 legacy label block holds, if any"
696 .to_string();
697 }
698 match self.entries {
699 Some(n) => format!("LABEL_INDEX present — {n} labels, {} bytes", self.bytes),
700 None => format!("LABEL_INDEX present — {} bytes", self.bytes),
701 }
702 }
703}
704
705#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
712pub struct Coherence {
713 pub coherent: bool,
715 pub inconsistency_count: u32,
717 #[serde(default, skip_serializing_if = "Vec::is_empty")]
719 pub by_kind: Vec<(String, u32)>,
720 pub scope: String,
722 pub rules: String,
724 pub materialized: bool,
726}
727
728impl Coherence {
729 pub fn is_empty(&self) -> bool {
731 *self == Coherence::default()
732 }
733
734 pub fn from_reasoning(r: &crate::Reasoning, materialized: bool) -> Self {
738 let mut hist: BTreeMap<&str, u32> = BTreeMap::new();
739 for inc in &r.inconsistencies {
740 *hist.entry(inc.kind).or_default() += 1;
741 }
742 Coherence {
743 coherent: r.inconsistencies.is_empty(),
744 inconsistency_count: r.inconsistencies.len() as u32,
745 by_kind: hist.into_iter().map(|(k, v)| (k.to_string(), v)).collect(),
746 scope: "default".to_string(),
747 rules: crate::REASON_RULESET.to_string(),
748 materialized,
749 }
750 }
751}
752
753#[derive(Debug, Clone, Default, Deserialize)]
765#[serde(deny_unknown_fields)]
766pub struct CardInput {
767 pub title: Option<String>,
768 #[serde(default, deserialize_with = "de_description")]
772 pub description: Option<String>,
773 pub license: Option<String>,
774 pub source: Option<String>,
775 pub created: Option<String>,
776 pub version: Option<String>,
777 #[serde(default)]
778 pub creators: Vec<Creator>,
779 pub publisher: Option<Publisher>,
780 pub canonical_url: Option<String>,
781 pub sparql_endpoint: Option<String>,
782 pub source_date: Option<String>,
783 #[serde(default)]
784 pub derived_from: Vec<String>,
785 pub doi: Option<String>,
786 pub cite_as: Option<String>,
787 #[serde(default)]
789 pub keywords: Vec<String>,
790 #[serde(default)]
792 pub theme: Vec<String>,
793 #[serde(default)]
795 pub extra: BTreeMap<String, serde_json::Value>,
796 #[serde(default)]
797 pub example_queries: Vec<String>,
798 pub base_iri: Option<String>,
806}
807
808fn de_description<'de, D>(d: D) -> Result<Option<String>, D::Error>
818where
819 D: serde::Deserializer<'de>,
820{
821 use serde::de::Error as _;
822 let v = Option::<serde_json::Value>::deserialize(d)?;
823 match v {
824 None | Some(serde_json::Value::Null) => Ok(None),
825 Some(v) => crate::card_input::normalize_description(&v)
826 .map(Some)
827 .map_err(D::Error::custom),
828 }
829}
830
831impl CardInput {
832 pub fn from_card(card: &DatasetCard) -> Self {
838 CardInput {
839 title: card.title.clone(),
840 description: card.description.clone(),
841 license: card.license.clone(),
842 source: card.source.clone(),
843 created: card.created.clone(),
844 version: card.version.clone(),
845 creators: card.creators.clone(),
846 publisher: card.publisher.clone(),
847 canonical_url: card.canonical_url.clone(),
848 sparql_endpoint: card.sparql_endpoint.clone(),
849 source_date: card.source_date.clone(),
850 derived_from: card.derived_from.clone(),
851 doi: card.doi.clone(),
852 cite_as: card.cite_as.clone(),
853 keywords: card.keywords.clone(),
854 theme: card.theme.clone(),
855 extra: card.extra.clone(),
856 example_queries: card.example_queries.clone(),
857 base_iri: None,
858 }
859 }
860
861 pub fn is_empty(&self) -> bool {
863 self.title.is_none()
864 && self.description.is_none()
865 && self.license.is_none()
866 && self.source.is_none()
867 && self.created.is_none()
868 && self.version.is_none()
869 && self.creators.is_empty()
870 && self.publisher.is_none()
871 && self.canonical_url.is_none()
872 && self.sparql_endpoint.is_none()
873 && self.source_date.is_none()
874 && self.derived_from.is_empty()
875 && self.doi.is_none()
876 && self.cite_as.is_none()
877 && self.keywords.is_empty()
878 && self.theme.is_empty()
879 && self.extra.is_empty()
880 && self.example_queries.is_empty()
881 && self.base_iri.is_none()
882 }
883
884 fn fill(self, card: &mut DatasetCard) {
887 card.title = self.title;
888 card.description = self.description;
889 card.license = self.license;
890 card.source = self.source;
891 card.created = self.created;
892 card.version = self.version;
893 card.creators = self.creators;
894 card.publisher = self.publisher;
895 card.canonical_url = self.canonical_url;
896 card.sparql_endpoint = self.sparql_endpoint;
897 card.source_date = self.source_date;
898 card.derived_from = self.derived_from;
899 card.doi = self.doi;
900 card.cite_as = self.cite_as;
901 card.keywords = self.keywords;
902 card.theme = self.theme;
903 card.extra = self.extra;
904 card.example_queries = self.example_queries;
905 if self.base_iri.is_some() {
906 card.signals.base_iri = self.base_iri;
907 }
908 }
909
910 pub fn from_json_str(text: &str) -> Result<Self, String> {
924 let parsed: CardInput = serde_json::from_str(text).map_err(|e| {
925 let hint = if e.to_string().contains("unknown field") {
928 crate::card_input::UNKNOWN_FIELD_HINT
929 } else {
930 ""
931 };
932 format!("{e}{hint}")
933 })?;
934 parsed.normalize()
935 }
936
937 pub fn normalize(self) -> Result<Self, String> {
946 self.normalize_max(
947 crate::card_input::CARD_DESCRIPTION_MAX_BYTES,
948 crate::card_input::CARD_EXTRA_MAX_BYTES,
949 )
950 }
951
952 pub fn normalize_with_annex(self) -> Result<Self, String> {
957 let max = crate::card_input::CARD_ANNEX_FIELD_MAX_BYTES;
958 self.normalize_max(max, max)
959 }
960
961 pub fn from_json_str_with_annex(text: &str) -> Result<Self, String> {
965 let mut doc: serde_json::Value = serde_json::from_str(text).map_err(|e| e.to_string())?;
966 let description = match doc.as_object_mut().and_then(|o| o.remove("description")) {
969 None | Some(serde_json::Value::Null) => None,
970 Some(v) => Some(crate::card_input::description_text(&v)?),
971 };
972 let mut parsed: CardInput = serde_json::from_value(doc).map_err(|e| {
973 let hint = if e.to_string().contains("unknown field") {
974 crate::card_input::UNKNOWN_FIELD_HINT
975 } else {
976 ""
977 };
978 format!("{e}{hint}")
979 })?;
980 parsed.description = description;
981 parsed.normalize_with_annex()
982 }
983
984 fn normalize_max(mut self, description_max: usize, extra_max: usize) -> Result<Self, String> {
985 if let Some(d) = &self.description {
986 crate::card_input::check_description_len_max(d, description_max)?;
987 }
988 self.keywords = crate::card_input::normalize_string_list(
989 "keywords",
990 std::mem::take(&mut self.keywords),
991 )?;
992 self.theme = crate::card_input::normalize_themes(std::mem::take(&mut self.theme))?;
993 let bag: serde_json::Map<String, serde_json::Value> =
994 std::mem::take(&mut self.extra).into_iter().collect();
995 self.extra = crate::card_input::normalize_extra_max(bag, extra_max)?
996 .into_iter()
997 .collect();
998 if let Some(b) = &self.base_iri {
999 crate::card_input::check_base_iri(b)?;
1000 }
1001 Ok(self)
1002 }
1003}
1004
1005impl DatasetCard {
1006 pub fn to_json_bytes(&self) -> Vec<u8> {
1017 if self.signals.text_index.is_none()
1018 && self.signals.label_index.is_none()
1019 && self.signals.permutations.is_none()
1020 && self.signals.quoted_triples.is_none()
1021 {
1022 return serde_json::to_vec(self).expect("DatasetCard serializes");
1023 }
1024 let mut stored = self.clone();
1025 stored.signals.text_index = None;
1026 stored.signals.label_index = None;
1027 stored.signals.permutations = None;
1028 stored.signals.quoted_triples = None;
1029 serde_json::to_vec(&stored).expect("DatasetCard serializes")
1030 }
1031
1032 pub fn observe_text_index(&mut self, measured: TextIndexSignal) -> Option<TextIndexSignal> {
1039 self.signals.text_index.replace(measured)
1040 }
1041
1042 pub fn observe_label_index(&mut self, measured: LabelIndexSignal) -> Option<LabelIndexSignal> {
1045 self.signals.label_index.replace(measured)
1046 }
1047
1048 pub fn observe_permutations(
1053 &mut self,
1054 measured: PermutationsSignal,
1055 ) -> Option<PermutationsSignal> {
1056 self.signals.permutations.replace(measured)
1057 }
1058
1059 pub fn observe_quoted_triples(
1064 &mut self,
1065 measured: QuotedTriplesSignal,
1066 ) -> Option<QuotedTriplesSignal> {
1067 self.signals.quoted_triples.replace(measured)
1068 }
1069
1070 pub fn from_json_bytes(b: &[u8]) -> Result<Self, String> {
1076 serde_json::from_slice(b).map_err(|e| format!("malformed dataset card: {e}"))
1077 }
1078
1079 pub fn with_coherence(mut self, r: &crate::Reasoning, materialized: bool) -> Self {
1081 self.coherence = Coherence::from_reasoning(r, materialized);
1082 self
1083 }
1084
1085 pub fn with_final_counts(mut self, counts: crate::ingest::FinalCounts) -> Self {
1099 self.triple_count = counts.default_triples;
1100 self.quad_count = counts.quads;
1101 self
1102 }
1103}
1104
1105pub const CARD_TOP_N: usize = 100;
1111
1112const ANNOTATION_WITNESS_SAMPLE: usize = 4096;
1128
1129const RDFS_LABEL: &str = "<http://www.w3.org/2000/01/rdf-schema#label>";
1131const SKOS_PREFLABEL: &str = "<http://www.w3.org/2004/02/skos/core#prefLabel>";
1132const SCHEMA_NAME: &str = "<http://schema.org/name>";
1133const FOAF_NAME: &str = "<http://xmlns.com/foaf/0.1/name>";
1134const DCT_TITLE: &str = "<http://purl.org/dc/terms/title>";
1135const OWL_SAMEAS: &str = "<http://www.w3.org/2002/07/owl#sameAs>";
1136const SKOS_EXACTMATCH: &str = "<http://www.w3.org/2004/02/skos/core#exactMatch>";
1137const SKOS_CLOSEMATCH: &str = "<http://www.w3.org/2004/02/skos/core#closeMatch>";
1138const RDFS_SEEALSO: &str = "<http://www.w3.org/2000/01/rdf-schema#seeAlso>";
1139pub(crate) const WGS_LAT: &str = "<http://www.w3.org/2003/01/geo/wgs84_pos#lat>";
1140pub(crate) const WGS_LONG: &str = "<http://www.w3.org/2003/01/geo/wgs84_pos#long>";
1141pub const GEO_ASWKT: &str = "<http://www.opengis.net/ont/geosparql#asWKT>";
1144pub const GEO_HASGEOMETRY: &str = "<http://www.opengis.net/ont/geosparql#hasGeometry>";
1145const GEO_WKTLITERAL: &str = "http://www.opengis.net/ont/geosparql#wktLiteral";
1147const XSD_STRING: &str = "http://www.w3.org/2001/XMLSchema#string";
1148const RDF_LANGSTRING: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#langString";
1149
1150pub const O_LITERAL: &str = "(literal)";
1155
1156const LABEL_PREDICATES: &[&str] = &[
1158 SKOS_PREFLABEL,
1159 RDFS_LABEL,
1160 SCHEMA_NAME,
1161 FOAF_NAME,
1162 DCT_TITLE,
1163];
1164pub(crate) const LINK_PREDICATES: &[&str] =
1166 &[OWL_SAMEAS, SKOS_EXACTMATCH, SKOS_CLOSEMATCH, RDFS_SEEALSO];
1167
1168pub trait CardTripleSource {
1175 fn replay(&self, f: &mut dyn FnMut(&str, &str, &str));
1176}
1177
1178pub fn curated_counts_card(statements: u64, term_count: u64, curated: CardInput) -> DatasetCard {
1187 let mut card = DatasetCard {
1188 triple_count: statements,
1189 quad_count: statements,
1190 named_graph_count: 0,
1191 term_count,
1192 format_version: crate::format::LEGACY_FORMAT_VERSION,
1193 ..DatasetCard::default()
1194 };
1195 curated.fill(&mut card);
1196 card
1197}
1198
1199pub fn derive_card(
1200 quads: &[(String, String, String, Option<String>)],
1201 term_count: u64,
1202 named_graph_count: u64,
1203 curated: CardInput,
1204) -> DatasetCard {
1205 if crate::build_progress::listening() {
1206 let default = quads.iter().filter(|q| q.3.is_none()).count() as u64;
1207 crate::build_progress::set_total(Some(2 * default), crate::build_progress::Unit::Records);
1208 }
1209 struct Quads<'a>(&'a [(String, String, String, Option<String>)]);
1210 impl CardTripleSource for Quads<'_> {
1211 fn replay(&self, f: &mut dyn FnMut(&str, &str, &str)) {
1212 for (s, p, o, g) in self.0 {
1213 if g.is_none() {
1214 f(s, p, o);
1215 }
1216 }
1217 }
1218 }
1219 derive_card_from(
1220 &Quads(quads),
1221 quads.len() as u64,
1222 term_count,
1223 named_graph_count,
1224 curated,
1225 )
1226}
1227
1228pub fn derive_card_encoded(
1234 dict: &crate::Dictionary,
1235 triples: &[(u32, u32, u32)],
1236 quad_count: u64,
1237 term_count: u64,
1238 named_graph_count: u64,
1239 curated: CardInput,
1240) -> DatasetCard {
1241 crate::build_progress::set_total(
1242 Some(2 * triples.len() as u64),
1243 crate::build_progress::Unit::Records,
1244 );
1245 struct Encoded<'a> {
1246 dict: &'a crate::Dictionary,
1247 triples: &'a [(u32, u32, u32)],
1248 }
1249 impl CardTripleSource for Encoded<'_> {
1250 fn replay(&self, f: &mut dyn FnMut(&str, &str, &str)) {
1251 for &(s, p, o) in self.triples {
1252 let st = self.dict.subject_term(s).unwrap_or_default();
1253 let pt = self.dict.predicate_term(p).unwrap_or_default();
1254 let ot = self.dict.object_term(o).unwrap_or_default();
1255 f(&st, &pt, &ot);
1256 }
1257 }
1258 }
1259 derive_card_from(
1260 &Encoded { dict, triples },
1261 quad_count,
1262 term_count,
1263 named_graph_count,
1264 curated,
1265 )
1266}
1267
1268pub fn derive_card_from(
1276 src: &dyn CardTripleSource,
1277 quad_count: u64,
1278 term_count: u64,
1279 named_graph_count: u64,
1280 curated: CardInput,
1281) -> DatasetCard {
1282 let mut class_of: std::collections::HashMap<String, String> = std::collections::HashMap::new();
1288 let mut quoted_subject_preds: BTreeSet<String> = BTreeSet::new();
1289 let mut annotated_tokens: BTreeMap<String, String> = BTreeMap::new();
1290 let mut ticker = crate::build_progress::Ticker::new();
1292 src.replay(&mut |s, p, o| {
1293 ticker.tick(1);
1294 if p == RDF_TYPE {
1295 class_of.insert(s.to_string(), o.to_string());
1296 }
1297 if crate::terms::is_quoted_triple(s) {
1298 if let Some((_, quoted_p, _)) = crate::ingest::quoted_triple_parts(s) {
1299 quoted_subject_preds.insert(quoted_p);
1300 }
1301 if annotated_tokens.len() < ANNOTATION_WITNESS_SAMPLE {
1302 annotated_tokens
1303 .entry(s.to_string())
1304 .or_insert_with(|| p.to_string());
1305 }
1306 }
1307 });
1308 let classify = |t: &str| -> String {
1309 if let Some(c) = class_of.get(t) {
1310 c.to_string()
1311 } else if t.starts_with('"') {
1312 O_LITERAL.to_string()
1313 } else {
1314 O_UNTYPED.to_string()
1315 }
1316 };
1317
1318 let mut prof = Profile::default();
1324 let mut out_degree: BTreeMap<String, u64> = BTreeMap::new();
1325 let mut in_degree: BTreeMap<String, u64> = BTreeMap::new();
1326
1327 src.replay(&mut |s, p, o| {
1328 ticker.tick(1);
1329 prof.triple_count += 1;
1330 *prof.pred_counts.entry(p.to_string()).or_default() += 1;
1331 *out_degree.entry(s.to_string()).or_default() += 1;
1332 if let Some(ns) = split_namespace(s) {
1333 prof.subject_ns.add(ns, 1);
1334 }
1335
1336 if !o.starts_with('"') {
1340 *in_degree.entry(o.to_string()).or_default() += 1;
1341 }
1342
1343 if crate::terms::is_quoted_triple(s) {
1347 *prof.annotation_hits.entry(p.to_string()).or_default() += 1;
1348 }
1349 if crate::terms::is_quoted_triple(o) {
1350 *prof.quoting_hits.entry(p.to_string()).or_default() += 1;
1351 }
1352 if !quoted_subject_preds.is_empty() && quoted_subject_preds.contains(p) {
1359 if let Some(about) = annotated_tokens.get(&format!("<<{s} {p} {o}>>")) {
1360 *prof
1361 .witnessed_pairs
1362 .entry((p.to_string(), about.clone()))
1363 .or_default() += 1;
1364 }
1365 }
1366
1367 if p == RDF_TYPE {
1368 *prof.class_counts.entry(o.to_string()).or_default() += 1;
1369 return; }
1371
1372 *prof
1374 .link_rows
1375 .entry((classify(s), p.to_string(), classify(o)))
1376 .or_default() += 1;
1377
1378 if p == GEO_ASWKT || p == GEO_HASGEOMETRY {
1379 prof.geo_wkt = true;
1380 }
1381
1382 if let Some(lit) = parse_literal(o) {
1384 prof.literal(p, &lit, 1);
1385 }
1386 });
1387
1388 let mut truncated = false;
1389 let top_hubs = cap(sort_desc_owned(out_degree), &mut truncated);
1390 let in_hubs = cap(sort_desc_owned(in_degree), &mut truncated);
1391 finish(
1392 prof,
1393 Hubs {
1394 top_hubs,
1395 in_hubs,
1396 truncated,
1397 },
1398 quad_count,
1399 term_count,
1400 named_graph_count,
1401 curated,
1402 )
1403}
1404
1405const O_UNTYPED: &str = "(untyped)";
1408
1409#[derive(Default)]
1420struct Profile {
1421 triple_count: u64,
1422 pred_counts: BTreeMap<String, u64>,
1423 class_counts: BTreeMap<String, u64>,
1424 datatype_counts: BTreeMap<String, u64>,
1425 lang_counts: BTreeMap<String, u64>,
1426 link_rows: BTreeMap<(String, String, String), u64>,
1427 subject_ns: NsTally,
1428 time_obj_hits: BTreeMap<String, u64>,
1430 num_obj_hits: BTreeMap<String, u64>,
1432 annotation_hits: BTreeMap<String, u64>,
1437 quoting_hits: BTreeMap<String, u64>,
1438 witnessed_pairs: BTreeMap<(String, String), u64>,
1439 time_extent: BTreeMap<String, (String, String)>,
1441 lat: Option<(f64, f64)>,
1443 lon: Option<(f64, f64)>,
1444 geo_wkt: bool,
1445}
1446
1447impl Profile {
1448 fn literal(&mut self, p: &str, lit: &Literal, weight: u64) {
1452 *self
1453 .datatype_counts
1454 .entry(format!("<{}>", lit.datatype))
1455 .or_default() += weight;
1456 *self.lang_counts.entry(lit.lang.clone()).or_default() += weight;
1457 if lit.datatype == GEO_WKTLITERAL {
1458 self.geo_wkt = true;
1459 }
1460 if is_temporal(p, lit) {
1461 *self.time_obj_hits.entry(p.to_string()).or_default() += weight;
1462 update_time_extent(p, &lit.value, &mut self.time_extent);
1463 } else if is_numeric_datatype(&lit.datatype) {
1464 *self.num_obj_hits.entry(p.to_string()).or_default() += weight;
1467 }
1468 if p == WGS_LAT {
1470 if let Ok(v) = lit.value.parse::<f64>() {
1471 widen(&mut self.lat, v);
1472 }
1473 } else if p == WGS_LONG {
1474 if let Ok(v) = lit.value.parse::<f64>() {
1475 widen(&mut self.lon, v);
1476 }
1477 }
1478 }
1479}
1480
1481fn widen(extent: &mut Option<(f64, f64)>, v: f64) {
1483 *extent = Some(match *extent {
1484 None => (v, v),
1485 Some((lo, hi)) => (lo.min(v), hi.max(v)),
1486 });
1487}
1488
1489struct Hubs {
1491 top_hubs: Vec<(String, u64)>,
1492 in_hubs: Vec<(String, u64)>,
1493 truncated: bool,
1494}
1495
1496fn finish(
1502 p: Profile,
1503 hubs: Hubs,
1504 quad_count: u64,
1505 term_count: u64,
1506 named_graph_count: u64,
1507 curated: CardInput,
1508) -> DatasetCard {
1509 let Profile {
1510 triple_count,
1511 pred_counts,
1512 class_counts,
1513 datatype_counts,
1514 lang_counts,
1515 link_rows,
1516 subject_ns,
1517 time_obj_hits,
1518 num_obj_hits,
1519 annotation_hits,
1520 quoting_hits,
1521 witnessed_pairs,
1522 time_extent,
1523 lat,
1524 lon,
1525 geo_wkt,
1526 } = p;
1527 let pred_count = |iri: &str| pred_counts.get(iri).copied().unwrap_or(0);
1530 let label_predicate = LABEL_PREDICATES
1531 .iter()
1532 .filter(|&&iri| pred_count(iri) > 0)
1533 .max_by_key(|&&iri| pred_count(iri))
1534 .map(|s| s.to_string());
1535 let link_predicates: Vec<String> = LINK_PREDICATES
1536 .iter()
1537 .filter(|&&iri| pred_count(iri) > 0)
1538 .map(|s| s.to_string())
1539 .collect();
1540
1541 let mut truncated = hubs.truncated || subject_ns.overflowed();
1542 let predicates = cap(sort_desc_owned(pred_counts), &mut truncated);
1543 let classes = cap(sort_desc_owned(class_counts), &mut truncated);
1544 let datatypes = cap(sort_desc_owned(datatype_counts), &mut truncated);
1545 let languages = cap(sort_desc_owned(lang_counts), &mut truncated);
1546 let class_links = cap(sort_links(link_rows), &mut truncated);
1547
1548 let mut vocab: BTreeSet<String> = BTreeSet::new();
1550 for (iri, _) in predicates.iter().chain(classes.iter()) {
1551 if let Some(ns) = split_namespace(iri) {
1552 vocab.insert(ns);
1553 }
1554 }
1555 let vocabularies = cap(vocab.into_iter().collect(), &mut truncated);
1556
1557 let default_lang = languages
1559 .iter()
1560 .find(|(l, _)| !l.is_empty())
1561 .map(|(l, _)| l.clone());
1562 let base_iri = derive_base_iri(&subject_ns);
1563 let mut time_ranked: Vec<(String, u64)> = time_obj_hits.into_iter().collect();
1564 time_ranked.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| term_cmp(&a.0, &b.0)));
1565 let time_predicates = cap(
1567 time_ranked.iter().map(|(p, _)| p.to_string()).collect(),
1568 &mut truncated,
1569 );
1570 let mut num_ranked: Vec<(String, u64)> = num_obj_hits.into_iter().collect();
1571 num_ranked.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| term_cmp(&a.0, &b.0)));
1572 let numeric_predicates = cap(
1573 num_ranked.iter().map(|(p, _)| p.to_string()).collect(),
1574 &mut truncated,
1575 );
1576 let annotation_predicates = cap(
1582 sort_desc_owned(annotation_hits)
1583 .into_iter()
1584 .map(|(p, _)| p)
1585 .collect(),
1586 &mut truncated,
1587 );
1588 let quoting_predicates = cap(
1589 sort_desc_owned(quoting_hits)
1590 .into_iter()
1591 .map(|(p, _)| p)
1592 .collect(),
1593 &mut truncated,
1594 );
1595 let annotated_statement = witnessed_pairs
1601 .into_iter()
1602 .max_by(|a, b| a.1.cmp(&b.1).then_with(|| b.0.cmp(&a.0)))
1603 .map(|(pair, _)| pair);
1604 let geo_latlong = lat.is_some() && lon.is_some();
1605 let temporal_extent = time_ranked
1606 .first()
1607 .and_then(|(p, _)| time_extent.get(p))
1608 .cloned();
1609 let spatial_bbox = match (lat, lon) {
1610 (Some((lat_lo, lat_hi)), Some((lon_lo, lon_hi)))
1611 if lon_lo.is_finite() && lat_lo.is_finite() =>
1612 {
1613 Some([lon_lo, lat_lo, lon_hi, lat_hi])
1614 }
1615 _ => None,
1616 };
1617
1618 let signals = Signals {
1619 label_predicate,
1620 default_lang,
1621 base_iri,
1622 time_predicates,
1623 numeric_predicates,
1624 link_predicates,
1625 annotation_predicates,
1626 quoting_predicates,
1627 annotated_statement,
1628 geo_wkt,
1629 geo_latlong,
1630 temporal_extent,
1631 spatial_bbox,
1632 text_index: None,
1636 label_index: None,
1637 permutations: None,
1638 quoted_triples: None,
1639 };
1640
1641 let mut card = DatasetCard {
1642 triple_count,
1643 quad_count,
1644 named_graph_count,
1645 term_count,
1646 predicates,
1647 classes,
1648 vocabularies,
1649 datatypes,
1650 languages,
1651 class_links,
1652 top_hubs: hubs.top_hubs,
1653 in_hubs: hubs.in_hubs,
1654 signals,
1655 queries: Vec::new(),
1656 coherence: Coherence::default(),
1659 truncated,
1660 top_n: CARD_TOP_N as u32,
1663 format_version: crate::LEGACY_FORMAT_VERSION,
1664 ..DatasetCard::default()
1665 };
1666 curated.fill(&mut card);
1667 card.queries = crate::card_queries::generate(&card);
1669 card
1670}
1671
1672fn cap<T>(mut v: Vec<T>, truncated: &mut bool) -> Vec<T> {
1674 if v.len() > CARD_TOP_N {
1675 v.truncate(CARD_TOP_N);
1676 *truncated = true;
1677 }
1678 v
1679}
1680
1681fn sort_desc_owned(counts: BTreeMap<String, u64>) -> Vec<(String, u64)> {
1683 let mut v: Vec<(String, u64)> = counts.into_iter().collect();
1684 v.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| term_cmp(&a.0, &b.0)));
1685 v
1686}
1687
1688fn term_cmp(a: &str, b: &str) -> std::cmp::Ordering {
1693 crate::term_order::TermOrder::Sparql.cmp_str(a, b)
1694}
1695
1696fn sort_links(rows: BTreeMap<(String, String, String), u64>) -> Vec<ClassLink> {
1699 let mut v: Vec<ClassLink> = rows
1700 .into_iter()
1701 .map(|((s, p, o), count)| ClassLink {
1702 s_class: s,
1703 predicate: p,
1704 o_class: o,
1705 count,
1706 })
1707 .collect();
1708 v.sort_by(|a, b| {
1709 b.count
1710 .cmp(&a.count)
1711 .then_with(|| term_cmp(&a.s_class, &b.s_class))
1712 .then_with(|| term_cmp(&a.predicate, &b.predicate))
1713 .then_with(|| term_cmp(&a.o_class, &b.o_class))
1714 });
1715 v
1716}
1717
1718fn derive_base_iri(ns: &NsTally) -> Option<String> {
1725 let dominant = ns.dominant()?;
1727 let lcp = ns.common_prefix().unwrap_or("");
1730 let lcp = &lcp[..lcp.len().min(dominant.len())];
1731 let base = match lcp.rfind(['/', '#']) {
1733 Some(i) => &lcp[..=i],
1734 None => "",
1735 };
1736 if base.matches('/').count() >= 3 {
1739 Some(base.to_string())
1740 } else {
1741 Some(dominant)
1742 }
1743}
1744
1745pub const NS_TALLY_CAP: usize = 65_536;
1750
1751pub const NS_TALLY_ENTRY_BYTES: u64 = 256;
1754
1755pub(crate) struct NsTally {
1763 cap: usize,
1764 counts: BTreeMap<String, u64>,
1765 order: Option<BTreeSet<(u64, String)>>,
1767 lcp: Option<String>,
1768 overflowed: bool,
1769}
1770
1771impl Default for NsTally {
1772 fn default() -> Self {
1773 Self::with_cap(NS_TALLY_CAP)
1774 }
1775}
1776
1777impl NsTally {
1778 pub(crate) fn with_cap(cap: usize) -> Self {
1779 NsTally {
1780 cap: cap.max(1),
1781 counts: BTreeMap::new(),
1782 order: None,
1783 lcp: None,
1784 overflowed: false,
1785 }
1786 }
1787
1788 pub(crate) fn len(&self) -> usize {
1789 self.counts.len()
1790 }
1791
1792 pub(crate) fn overflowed(&self) -> bool {
1794 self.overflowed
1795 }
1796
1797 pub(crate) fn common_prefix(&self) -> Option<&str> {
1798 self.lcp.as_deref()
1799 }
1800
1801 pub(crate) fn dominant(&self) -> Option<String> {
1802 self.counts
1803 .iter()
1804 .max_by(|a, b| a.1.cmp(b.1).then_with(|| b.0.cmp(a.0)))
1805 .map(|(k, _)| k.clone())
1806 }
1807
1808 pub(crate) fn add(&mut self, ns: String, n: u64) {
1809 match &mut self.lcp {
1810 None => self.lcp = Some(ns.clone()),
1811 Some(l) if !ns.starts_with(l.as_str()) => {
1812 let mut cut = l
1813 .bytes()
1814 .zip(ns.bytes())
1815 .take_while(|(a, b)| a == b)
1816 .count();
1817 while !l.is_char_boundary(cut) {
1818 cut -= 1;
1819 }
1820 l.truncate(cut);
1821 }
1822 Some(_) => {}
1823 }
1824 if let Some(c) = self.counts.get_mut(&ns) {
1825 let old = *c;
1826 *c += n;
1827 if let Some(order) = &mut self.order {
1828 order.remove(&(old, ns.clone()));
1829 order.insert((*c, ns));
1830 }
1831 return;
1832 }
1833 if self.counts.len() < self.cap {
1834 if let Some(order) = &mut self.order {
1835 order.insert((n, ns.clone()));
1836 }
1837 self.counts.insert(ns, n);
1838 return;
1839 }
1840 self.overflowed = true;
1842 let counts = &self.counts;
1843 let order = self
1844 .order
1845 .get_or_insert_with(|| counts.iter().map(|(k, &v)| (v, k.clone())).collect());
1846 let Some((min, victim)) = order.pop_first() else {
1847 return;
1848 };
1849 self.counts.remove(&victim);
1850 let c = min + n;
1851 order.insert((c, ns.clone()));
1852 self.counts.insert(ns, c);
1853 }
1854}
1855
1856struct Literal {
1858 value: String,
1860 datatype: String,
1863 lang: String,
1865}
1866
1867fn parse_literal(term: &str) -> Option<Literal> {
1870 let bytes = term.as_bytes();
1871 if bytes.first() != Some(&b'"') {
1872 return None;
1873 }
1874 let mut i = 1usize;
1876 let mut esc = false;
1877 let mut close = None;
1878 while i < bytes.len() {
1879 match bytes[i] {
1880 b'\\' => esc = !esc,
1881 b'"' if !esc => {
1882 close = Some(i);
1883 break;
1884 }
1885 _ => esc = false,
1886 }
1887 i += 1;
1888 }
1889 let close = close?;
1890 let value = term[1..close].to_string();
1891 let suffix = &term[close + 1..];
1892 if let Some(lang) = suffix.strip_prefix('@') {
1893 Some(Literal {
1894 value,
1895 datatype: RDF_LANGSTRING.to_string(),
1896 lang: lang.to_string(),
1897 })
1898 } else if let Some(dt) = suffix.strip_prefix("^^") {
1899 let dt = dt.trim_start_matches('<').trim_end_matches('>').to_string();
1900 Some(Literal {
1901 value,
1902 datatype: dt,
1903 lang: String::new(),
1904 })
1905 } else {
1906 Some(Literal {
1907 value,
1908 datatype: XSD_STRING.to_string(),
1909 lang: String::new(),
1910 })
1911 }
1912}
1913
1914fn is_temporal_datatype(dt: &str) -> bool {
1918 matches!(
1919 dt,
1920 "http://www.w3.org/2001/XMLSchema#date"
1921 | "http://www.w3.org/2001/XMLSchema#dateTime"
1922 | "http://www.w3.org/2001/XMLSchema#dateTimeStamp"
1923 | "http://www.w3.org/2001/XMLSchema#gYear"
1924 | "http://www.w3.org/2001/XMLSchema#gYearMonth"
1925 )
1926}
1927
1928const DATE_PREDICATES: &[&str] = &[
1933 "<http://purl.org/dc/terms/date>",
1934 "<http://purl.org/dc/terms/issued>",
1935 "<http://purl.org/dc/terms/created>",
1936 "<http://purl.org/dc/terms/modified>",
1937 "<http://purl.org/dc/terms/available>",
1938 "<http://purl.org/dc/terms/dateAccepted>",
1939 "<http://purl.org/dc/terms/dateCopyrighted>",
1940 "<http://purl.org/dc/terms/dateSubmitted>",
1941 "<http://purl.org/dc/elements/1.1/date>",
1942 "<http://schema.org/datePublished>",
1943 "<http://schema.org/dateCreated>",
1944 "<http://schema.org/dateModified>",
1945 "<http://schema.org/startDate>",
1946 "<http://schema.org/endDate>",
1947 "<http://schema.org/birthDate>",
1948 "<http://schema.org/deathDate>",
1949 "<http://schema.org/foundingDate>",
1950 "<http://schema.org/uploadDate>",
1951 "<https://schema.org/datePublished>",
1952 "<https://schema.org/dateCreated>",
1953 "<https://schema.org/dateModified>",
1954 "<https://schema.org/startDate>",
1955 "<https://schema.org/endDate>",
1956 "<https://schema.org/birthDate>",
1957 "<https://schema.org/deathDate>",
1958 "<http://prismstandard.org/namespaces/basic/2.0/publicationDate>",
1959 "<http://prismstandard.org/namespaces/basic/2.0/coverDate>",
1960 "<http://www.w3.org/ns/prov#generatedAtTime>",
1961 "<http://www.w3.org/ns/prov#invalidatedAtTime>",
1962 "<http://www.w3.org/ns/prov#startedAtTime>",
1963 "<http://www.w3.org/ns/prov#endedAtTime>",
1964 "<http://www.w3.org/ns/prov#atTime>",
1965 "<http://www.w3.org/2006/time#inXSDDate>",
1966 "<http://www.w3.org/2006/time#inXSDDateTime>",
1967 "<http://www.w3.org/2006/time#inXSDDateTimeStamp>",
1968 "<http://www.w3.org/2006/time#inXSDgYear>",
1969 "<http://www.w3.org/2006/time#inXSDgYearMonth>",
1970 "<http://dbpedia.org/ontology/birthDate>",
1971 "<http://dbpedia.org/ontology/deathDate>",
1972 "<http://dbpedia.org/ontology/releaseDate>",
1973 "<http://www.wikidata.org/prop/direct/P569>",
1974 "<http://www.wikidata.org/prop/direct/P570>",
1975 "<http://www.wikidata.org/prop/direct/P571>",
1976 "<http://www.wikidata.org/prop/direct/P577>",
1977 "<http://www.wikidata.org/prop/direct/P580>",
1978 "<http://www.wikidata.org/prop/direct/P582>",
1979 "<http://www.wikidata.org/prop/direct/P585>",
1980];
1981
1982fn is_temporal(p: &str, lit: &Literal) -> bool {
1996 if is_temporal_datatype(&lit.datatype) {
1997 return date_shaped(&lit.value);
1998 }
1999 DATE_PREDICATES.contains(&p) && date_shaped(&lit.value)
2000}
2001
2002fn date_shaped(v: &str) -> bool {
2010 let b = v.as_bytes();
2011 let digits =
2012 |at: usize, n: usize| b.len() >= at + n && b[at..at + n].iter().all(u8::is_ascii_digit);
2013 let two_in = |at: usize, lo: u8, hi: u8| {
2014 digits(at, 2) && {
2015 let n = (b[at] - b'0') * 10 + (b[at + 1] - b'0');
2016 (lo..=hi).contains(&n)
2017 }
2018 };
2019 let mut i = usize::from(b.first() == Some(&b'-'));
2020 if !digits(i, 4) {
2021 return false;
2022 }
2023 i += 4;
2024 if b.get(i) == Some(&b'-') {
2025 if !two_in(i + 1, 1, 12) {
2026 return false;
2027 }
2028 i += 3;
2029 if b.get(i) == Some(&b'-') {
2030 if !two_in(i + 1, 1, 31) {
2031 return false;
2032 }
2033 i += 3;
2034 if b.get(i) == Some(&b'T') {
2035 if !(two_in(i + 1, 0, 24) && b.get(i + 3) == Some(&b':') && two_in(i + 4, 0, 59)) {
2036 return false;
2037 }
2038 i += 6;
2039 if b.get(i) == Some(&b':') {
2040 if !two_in(i + 1, 0, 60) {
2041 return false;
2042 }
2043 i += 3;
2044 if b.get(i) == Some(&b'.') {
2045 let start = i + 1;
2046 i = start;
2047 while b.get(i).is_some_and(u8::is_ascii_digit) {
2048 i += 1;
2049 }
2050 if i == start {
2051 return false;
2052 }
2053 }
2054 }
2055 }
2056 }
2057 }
2058 match &b[i..] {
2059 [] | [b'Z'] => true,
2060 [sign, h1, h2, b':', m1, m2] => {
2061 (*sign == b'+' || *sign == b'-') && [h1, h2, m1, m2].iter().all(|c| c.is_ascii_digit())
2062 }
2063 _ => false,
2064 }
2065}
2066
2067fn is_numeric_datatype(dt: &str) -> bool {
2069 matches!(
2070 dt,
2071 "http://www.w3.org/2001/XMLSchema#integer"
2072 | "http://www.w3.org/2001/XMLSchema#decimal"
2073 | "http://www.w3.org/2001/XMLSchema#double"
2074 | "http://www.w3.org/2001/XMLSchema#float"
2075 | "http://www.w3.org/2001/XMLSchema#long"
2076 | "http://www.w3.org/2001/XMLSchema#int"
2077 | "http://www.w3.org/2001/XMLSchema#short"
2078 | "http://www.w3.org/2001/XMLSchema#nonNegativeInteger"
2079 | "http://www.w3.org/2001/XMLSchema#positiveInteger"
2080 )
2081}
2082
2083fn temporal_key(v: &str) -> (i64, &str) {
2088 let (neg, rest) = match v.strip_prefix('-') {
2089 Some(r) => (true, r),
2090 None => (false, v),
2091 };
2092 let n = rest.bytes().take_while(u8::is_ascii_digit).count();
2093 let year: i64 = rest[..n].parse().unwrap_or(0);
2094 (if neg { -year } else { year }, &rest[n..])
2095}
2096
2097fn update_time_extent(pred: &str, value: &str, extent: &mut BTreeMap<String, (String, String)>) {
2103 match extent.get_mut(pred) {
2104 None => {
2105 extent.insert(pred.to_string(), (value.to_string(), value.to_string()));
2106 }
2107 Some((lo, hi)) => {
2108 if temporal_key(value) < temporal_key(lo) {
2109 *lo = value.to_string();
2110 }
2111 if temporal_key(value) > temporal_key(hi) {
2112 *hi = value.to_string();
2113 }
2114 }
2115 }
2116}
2117
2118#[derive(Debug, Clone, Default)]
2146pub struct StreamReport {
2147 pub statements: u64,
2148 pub subjects: u64,
2149 pub literal_objects: u64,
2150 pub classes: u64,
2151 pub class_map_bytes: u64,
2152 pub namespaces_kept: u64,
2156 pub namespaces_overflowed: bool,
2157}
2158
2159struct ClassMap {
2172 data: Vec<u8>,
2173 width: usize,
2175}
2176
2177impl ClassMap {
2178 fn new(subjects: u64) -> Self {
2179 ClassMap {
2182 data: vec![0; subjects as usize + 1],
2183 width: 1,
2184 }
2185 }
2186
2187 fn len(&self) -> usize {
2188 self.data.len() / self.width
2189 }
2190
2191 fn read(data: &[u8], i: usize, width: usize) -> u32 {
2192 let at = i * width;
2193 match width {
2194 1 => data[at] as u32,
2195 2 => u16::from_le_bytes([data[at], data[at + 1]]) as u32,
2196 _ => u32::from_le_bytes([data[at], data[at + 1], data[at + 2], data[at + 3]]),
2197 }
2198 }
2199
2200 fn write(data: &mut [u8], i: usize, width: usize, code: u32) {
2201 let at = i * width;
2202 match width {
2203 1 => data[at] = code as u8,
2204 2 => data[at..at + 2].copy_from_slice(&(code as u16).to_le_bytes()),
2205 _ => data[at..at + 4].copy_from_slice(&code.to_le_bytes()),
2206 }
2207 }
2208
2209 fn get(&self, id: u32) -> u32 {
2210 let i = id as usize;
2211 if i < self.len() {
2212 Self::read(&self.data, i, self.width)
2213 } else {
2214 0
2215 }
2216 }
2217
2218 fn widen(&mut self, width: usize) {
2220 let (n, old) = (self.len(), self.width);
2221 if width <= old {
2222 return;
2223 }
2224 self.data.resize(n * width, 0);
2225 for i in (0..n).rev() {
2226 let v = Self::read(&self.data, i, old);
2227 Self::write(&mut self.data, i, width, v);
2228 }
2229 self.width = width;
2230 }
2231
2232 fn set(&mut self, id: u32, code: u32) {
2233 if code > u16::MAX as u32 {
2234 self.widen(4);
2235 } else if code > u8::MAX as u32 {
2236 self.widen(2);
2237 }
2238 let i = id as usize;
2239 if i < self.len() {
2240 Self::write(&mut self.data, i, self.width, code);
2241 }
2242 }
2243
2244 fn bytes(&self) -> u64 {
2245 self.data.len() as u64
2246 }
2247}
2248
2249struct TopN {
2255 heap: std::collections::BinaryHeap<std::cmp::Reverse<(u64, std::cmp::Reverse<u32>)>>,
2256 offered: u64,
2257}
2258
2259impl TopN {
2260 fn new() -> Self {
2261 TopN {
2262 heap: std::collections::BinaryHeap::with_capacity(CARD_TOP_N + 1),
2263 offered: 0,
2264 }
2265 }
2266
2267 fn offer(&mut self, id: u32, n: u64) {
2268 use std::cmp::Reverse;
2269 self.offered += 1;
2270 let key = Reverse((n, Reverse(id)));
2271 if self.heap.len() < CARD_TOP_N {
2272 self.heap.push(key);
2273 } else if let Some(worst) = self.heap.peek() {
2274 if (n, Reverse(id)) > worst.0 {
2276 self.heap.pop();
2277 self.heap.push(key);
2278 }
2279 }
2280 }
2281
2282 fn into_parts(self) -> (Vec<(u32, u64)>, bool) {
2284 let cut = self.offered > CARD_TOP_N as u64;
2285 let v = self
2286 .heap
2287 .into_iter()
2288 .map(|std::cmp::Reverse((n, std::cmp::Reverse(id)))| (id, n))
2289 .collect();
2290 (v, cut)
2291 }
2292}
2293
2294fn top_terms(
2298 tops: [TopN; 2],
2299 terms: &dyn Fn(&[u32]) -> Vec<Option<String>>,
2300) -> (Vec<(String, u64)>, bool) {
2301 let mut offered = 0u64;
2302 let mut ids: Vec<(u32, u64)> = Vec::new();
2303 for top in tops {
2304 offered += top.offered;
2305 ids.extend(top.into_parts().0);
2306 }
2307 let names = terms(&ids.iter().map(|&(id, _)| id).collect::<Vec<_>>());
2308 let mut v: Vec<(String, u64)> = ids
2309 .iter()
2310 .zip(names)
2311 .filter_map(|(&(_, n), t)| t.map(|t| (t, n)))
2312 .collect();
2313 v.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| term_cmp(&a.0, &b.0)));
2314 v.truncate(CARD_TOP_N);
2315 (v, offered > CARD_TOP_N as u64)
2316}
2317
2318#[derive(Default, Clone, Copy)]
2321struct IdHasher(u64);
2322
2323impl std::hash::Hasher for IdHasher {
2324 fn finish(&self) -> u64 {
2325 self.0
2326 }
2327 fn write(&mut self, bytes: &[u8]) {
2328 for &b in bytes {
2329 self.0 = (self.0.rotate_left(5) ^ b as u64).wrapping_mul(0x517c_c1b7_2722_0a95);
2330 }
2331 }
2332 fn write_u32(&mut self, n: u32) {
2333 self.0 = (self.0.rotate_left(5) ^ n as u64).wrapping_mul(0x517c_c1b7_2722_0a95);
2334 }
2335}
2336
2337type IdBuild = std::hash::BuildHasherDefault<IdHasher>;
2338
2339fn first_at_or_above(lo: u32, hi: u32, key: &str, term: &dyn Fn(u32) -> Option<String>) -> u32 {
2343 let (mut a, mut b) = (lo as u64, hi as u64 + 1);
2344 while a < b {
2345 let mid = a + (b - a) / 2;
2346 match term(mid as u32) {
2347 Some(t) if t.as_str() < key => a = mid + 1,
2348 _ => b = mid,
2349 }
2350 }
2351 a as u32
2352}
2353
2354fn id_range(
2359 lo: u32,
2360 hi: u32,
2361 from: &str,
2362 to: &str,
2363 term: &dyn Fn(u32) -> Option<String>,
2364) -> (u32, u32) {
2365 if hi < lo {
2366 return (lo, lo);
2367 }
2368 (
2369 first_at_or_above(lo, hi, from, term),
2370 first_at_or_above(lo, hi, to, term),
2371 )
2372}
2373
2374fn in_any(id: u32, ranges: &[(u32, u32)]) -> bool {
2375 ranges.iter().any(|&(a, b)| a <= id && id < b)
2376}
2377
2378pub fn derive_card_streaming(
2397 rete: &crate::Rete,
2398 curated: CardInput,
2399 progress: &mut dyn FnMut(&str),
2400) -> Result<(DatasetCard, StreamReport), String> {
2401 use crate::index::IndexPermutation;
2402
2403 let header = rete.header();
2404 let dict = rete.dictionary();
2405 let index = rete.default_index();
2406 let shared = dict.shared_count();
2407 let subjects_end = shared as u64 + dict.subject_only_count() as u64;
2408 let objects_end = shared as u64 + dict.object_only_count() as u64;
2409 let n_preds = dict.predicate_count() as usize;
2410 let pred_terms: Vec<String> = (0..=n_preds as u32)
2411 .map(|p| {
2412 if p == 0 {
2413 String::new()
2414 } else {
2415 dict.predicate_term(p).unwrap_or_default()
2416 }
2417 })
2418 .collect();
2419 let type_id = dict.predicate_id(RDF_TYPE);
2420
2421 let object_term = |id: u32| dict.object_term(id);
2423 let subject_term = |id: u32| dict.subject_term(id);
2424 let s_end = subjects_end.min(u32::MAX as u64) as u32;
2425 let o_end = objects_end.min(u32::MAX as u64) as u32;
2426 let sparql_order = dict.order() == crate::term_order::TermOrder::Sparql;
2431 let groups = |si: usize| {
2432 dict.section_groups(si)
2433 .ok_or_else(|| "the dictionary's term groups are unreadable".to_string())
2434 };
2435 let literal_ranges = if sparql_order {
2436 let (sh, ob) = (groups(0)?, groups(2)?);
2437 [
2438 (sh.lit_start() + 1, sh.lit_end() + 1),
2439 (shared + ob.lit_start() + 1, shared + ob.lit_end() + 1),
2440 ]
2441 } else {
2442 [
2443 id_range(1, shared, "\"", "#", &object_term),
2444 id_range(shared + 1, o_end, "\"", "#", &object_term),
2445 ]
2446 };
2447 let is_literal = |o: u32| in_any(o, &literal_ranges);
2448 let quoted = header.has_quoted_triples();
2449 let (quoted_subjects, quoted_objects) = if quoted && sparql_order {
2450 let (sh, su, ob) = (groups(0)?, groups(1)?, groups(2)?);
2451 let run = |base: u32, g: &crate::term_order::GroupCounts| {
2452 (base + g.lit_end() + 1, base + g.total() as u32 + 1)
2453 };
2454 (
2455 vec![run(0, &sh), run(shared, &su)],
2456 vec![run(0, &sh), run(shared, &ob)],
2457 )
2458 } else if quoted {
2459 (
2460 vec![
2461 id_range(1, shared, "<<", "<=", &subject_term),
2462 id_range(shared + 1, s_end, "<<", "<=", &subject_term),
2463 ],
2464 vec![
2465 id_range(1, shared, "<<", "<=", &object_term),
2466 id_range(shared + 1, o_end, "<<", "<=", &object_term),
2467 ],
2468 )
2469 } else {
2470 (Vec::new(), Vec::new())
2471 };
2472
2473 let mut report = StreamReport::default();
2474
2475 progress("pass 1/3: classes (the rdf:type range of POS)");
2477 let mut class_map = ClassMap::new(subjects_end);
2478 let mut class_code: std::collections::HashMap<u32, u32, IdBuild> = Default::default();
2479 let mut class_ids: Vec<u32> = vec![0];
2480 let mut class_counts: Vec<u64> = vec![0];
2481 if let Some(t) = type_id {
2482 for (s, _, o) in index.scan_iter((None, Some(t), None)) {
2483 let next = class_ids.len() as u32;
2484 let code = *class_code.entry(o).or_insert_with(|| {
2485 class_ids.push(o);
2486 class_counts.push(0);
2487 next
2488 });
2489 class_counts[code as usize] += 1;
2490 class_map.set(s, code);
2491 }
2492 }
2493 report.classes = class_ids.len() as u64 - 1;
2494 report.class_map_bytes = class_map.bytes();
2495
2496 progress("pass 2/3: subjects (SPO)");
2498 const LITERAL_CODE: u32 = u32::MAX;
2499 const WINDOW: usize = 1 << 16;
2500 let mut prof = Profile::default();
2501 let mut pred_counts = vec![0u64; n_preds + 1];
2502 let mut annotation_hits = vec![0u64; n_preds + 1];
2503 let mut quoting_hits = vec![0u64; n_preds + 1];
2504 let mut links: std::collections::HashMap<(u32, u32, u32), u64, IdBuild> = Default::default();
2505 let mut out_top = [TopN::new(), TopN::new()];
2507 let mut sample: Vec<(u32, u32)> = Vec::new();
2508 let mut window: Vec<(u32, u64)> = Vec::with_capacity(WINDOW);
2509 let mut subject_ns = NsTally::default();
2510 let flush_ns = |window: &mut Vec<(u32, u64)>, subject_ns: &mut NsTally| {
2511 let ids: Vec<u32> = window.iter().map(|&(s, _)| s).collect();
2512 let terms = dict.subject_terms(&ids);
2513 let mut run: Option<(String, u64)> = None;
2516 for (term, &(_, n)) in terms.iter().zip(window.iter()) {
2517 let Some(ns) = term.as_deref().and_then(split_namespace) else {
2518 continue;
2519 };
2520 match &mut run {
2521 Some((cur, total)) if *cur == ns => *total += n,
2522 _ => {
2523 if let Some((cur, total)) = run.take() {
2524 subject_ns.add(cur, total);
2525 }
2526 run = Some((ns, n));
2527 }
2528 }
2529 }
2530 if let Some((cur, total)) = run {
2531 subject_ns.add(cur, total);
2532 }
2533 window.clear();
2534 };
2535 let mut cur_s = 0u32;
2536 let mut cur_n = 0u64;
2537 let mut ticker = crate::build_progress::Ticker::new();
2539 let stored = index.for_each_in_order(IndexPermutation::Spo, &mut |s, p, o| {
2540 prof.triple_count += 1;
2541 ticker.tick(1);
2542 if let Some(c) = pred_counts.get_mut(p as usize) {
2543 *c += 1;
2544 }
2545 if s != cur_s {
2546 if cur_n > 0 {
2547 out_top[usize::from(cur_s > shared)].offer(cur_s, cur_n);
2548 window.push((cur_s, cur_n));
2549 if window.len() >= WINDOW {
2550 flush_ns(&mut window, &mut subject_ns);
2551 }
2552 }
2553 if quoted && in_any(s, "ed_subjects) && sample.len() < ANNOTATION_WITNESS_SAMPLE {
2554 sample.push((s, p));
2555 }
2556 cur_s = s;
2557 cur_n = 0;
2558 }
2559 cur_n += 1;
2560 if quoted {
2561 if in_any(s, "ed_subjects) {
2562 if let Some(c) = annotation_hits.get_mut(p as usize) {
2563 *c += 1;
2564 }
2565 }
2566 if in_any(o, "ed_objects) {
2567 if let Some(c) = quoting_hits.get_mut(p as usize) {
2568 *c += 1;
2569 }
2570 }
2571 }
2572 if Some(p) == type_id {
2573 return;
2574 }
2575 let sc = class_map.get(s);
2576 let oc = if is_literal(o) {
2579 LITERAL_CODE
2580 } else if o <= shared {
2581 class_map.get(o)
2582 } else {
2583 0
2584 };
2585 *links.entry((sc, p, oc)).or_default() += 1;
2586 });
2587 if !stored {
2588 return Err("the file stores no SPO permutation".to_string());
2589 }
2590 if cur_n > 0 {
2591 out_top[usize::from(cur_s > shared)].offer(cur_s, cur_n);
2592 window.push((cur_s, cur_n));
2593 }
2594 if !window.is_empty() {
2595 flush_ns(&mut window, &mut subject_ns);
2596 }
2597 report.statements = prof.triple_count;
2598 report.subjects = out_top[0].offered + out_top[1].offered;
2599 drop(class_map);
2600
2601 progress("pass 3/3: objects (OSP)");
2603 let mut in_top = [TopN::new(), TopN::new()];
2604 let mut lit_window: Vec<(u32, Vec<(u32, u64)>)> = Vec::with_capacity(WINDOW);
2605 let mut lit_objects = 0u64;
2606 let flush_lits = |lit_window: &mut Vec<(u32, Vec<(u32, u64)>)>, prof: &mut Profile| {
2607 let ids: Vec<u32> = lit_window.iter().map(|(o, _)| *o).collect();
2608 let terms = dict.object_terms(&ids);
2609 for (term, (_, preds)) in terms.iter().zip(lit_window.iter()) {
2610 let Some(lit) = term.as_deref().and_then(parse_literal) else {
2611 continue;
2612 };
2613 for &(p, n) in preds {
2614 if let Some(pt) = pred_terms.get(p as usize) {
2615 prof.literal(pt, &lit, n);
2616 }
2617 }
2618 }
2619 lit_window.clear();
2620 };
2621 let mut cur_o = 0u32;
2622 let mut cur_n = 0u64;
2623 let mut cur_preds: Vec<(u32, u64)> = Vec::new();
2624 index.for_each_in_order(IndexPermutation::Osp, &mut |o, _s, p| {
2625 ticker.tick(1);
2626 if o != cur_o {
2627 if cur_n > 0 {
2628 if is_literal(cur_o) {
2629 lit_objects += 1;
2630 lit_window.push((cur_o, std::mem::take(&mut cur_preds)));
2631 if lit_window.len() >= WINDOW {
2632 flush_lits(&mut lit_window, &mut prof);
2633 }
2634 } else {
2635 in_top[usize::from(cur_o > shared)].offer(cur_o, cur_n);
2636 }
2637 }
2638 cur_o = o;
2639 cur_n = 0;
2640 cur_preds.clear();
2641 }
2642 cur_n += 1;
2643 if is_literal(o) {
2644 match cur_preds.iter_mut().find(|(q, _)| *q == p) {
2645 Some((_, n)) => *n += 1,
2646 None => cur_preds.push((p, 1)),
2647 }
2648 }
2649 });
2650 if cur_n > 0 {
2651 if is_literal(cur_o) {
2652 lit_objects += 1;
2653 lit_window.push((cur_o, std::mem::take(&mut cur_preds)));
2654 } else {
2655 in_top[usize::from(cur_o > shared)].offer(cur_o, cur_n);
2656 }
2657 }
2658 if !lit_window.is_empty() {
2659 flush_lits(&mut lit_window, &mut prof);
2660 }
2661 report.literal_objects = lit_objects;
2662
2663 if rete.index_incomplete() {
2664 return Err("a read failed part-way through the file; the card would be incomplete".into());
2665 }
2666
2667 progress("resolving the profile's terms");
2669 let class_terms: Vec<String> = class_ids
2670 .iter()
2671 .map(|&o| {
2672 if o == 0 {
2673 O_UNTYPED.to_string()
2674 } else {
2675 dict.object_term(o).unwrap_or_default()
2676 }
2677 })
2678 .collect();
2679 let class_name = |code: u32| -> String {
2680 if code == LITERAL_CODE {
2681 O_LITERAL.to_string()
2682 } else {
2683 class_terms
2684 .get(code as usize)
2685 .cloned()
2686 .unwrap_or_else(|| O_UNTYPED.to_string())
2687 }
2688 };
2689 for (p, &n) in pred_counts.iter().enumerate() {
2690 if n > 0 {
2691 prof.pred_counts.insert(pred_terms[p].clone(), n);
2692 }
2693 }
2694 for (code, &n) in class_counts.iter().enumerate().skip(1) {
2695 if n > 0 {
2696 *prof
2697 .class_counts
2698 .entry(class_terms[code].clone())
2699 .or_default() += n;
2700 }
2701 }
2702 for ((sc, p, oc), n) in links {
2703 let Some(pt) = pred_terms.get(p as usize) else {
2704 continue;
2705 };
2706 *prof
2707 .link_rows
2708 .entry((class_name(sc), pt.clone(), class_name(oc)))
2709 .or_default() += n;
2710 }
2711 for (p, &n) in annotation_hits.iter().enumerate() {
2712 if n > 0 {
2713 prof.annotation_hits.insert(pred_terms[p].clone(), n);
2714 }
2715 }
2716 for (p, &n) in quoting_hits.iter().enumerate() {
2717 if n > 0 {
2718 prof.quoting_hits.insert(pred_terms[p].clone(), n);
2719 }
2720 }
2721 report.namespaces_kept = subject_ns.len() as u64;
2722 report.namespaces_overflowed = subject_ns.overflowed();
2723 prof.subject_ns = subject_ns;
2724 if pred_counts
2725 .iter()
2726 .enumerate()
2727 .any(|(p, &n)| n > 0 && (pred_terms[p] == GEO_ASWKT || pred_terms[p] == GEO_HASGEOMETRY))
2728 {
2729 prof.geo_wkt = true;
2730 }
2731
2732 let sample_ids: Vec<u32> = sample.iter().map(|&(s, _)| s).collect();
2735 for (term, &(_, about)) in dict.subject_terms(&sample_ids).iter().zip(sample.iter()) {
2736 let Some((a, qp, b)) = term.as_deref().and_then(crate::ingest::quoted_triple_parts) else {
2737 continue;
2738 };
2739 if let Some((sid, pid, oid)) = dict.encode(&a, &qp, &b) {
2740 if index
2741 .scan_iter((Some(sid), Some(pid), Some(oid)))
2742 .next()
2743 .is_some()
2744 {
2745 if let Some(about) = pred_terms.get(about as usize) {
2746 *prof.witnessed_pairs.entry((qp, about.clone())).or_default() += 1;
2747 }
2748 }
2749 }
2750 }
2751
2752 let (top_hubs, cut_out) = top_terms(out_top, &|ids| dict.subject_terms(ids));
2753 let (in_hubs, cut_in) = top_terms(in_top, &|ids| dict.object_terms(ids));
2754
2755 let card = finish(
2756 prof,
2757 Hubs {
2758 top_hubs,
2759 in_hubs,
2760 truncated: cut_out || cut_in,
2761 },
2762 header.quad_count,
2763 header.term_count,
2764 rete.named_graph_count() as u64,
2765 curated,
2766 );
2767 Ok((card, report))
2768}
2769
2770fn split_namespace(term: &str) -> Option<String> {
2774 let iri = term.strip_prefix('<')?.strip_suffix('>')?;
2775 let cut = iri.rfind(['#', '/'])?;
2776 Some(iri[..=cut].to_string())
2777}
2778
2779pub fn load_card(bytes: &[u8]) -> Result<Option<DatasetCard>, String> {
2787 if crate::v06::open::is_v06(bytes) {
2788 return match crate::file::read_card_full_ranged(&crate::reader::SliceReader::new(bytes))
2789 .map_err(|e| e.to_string())?
2790 {
2791 Some(b) => Ok(Some(DatasetCard::from_json_bytes(&b)?)),
2792 None => Ok(None),
2793 };
2794 }
2795 let header = Header::from_bytes(bytes).map_err(|e| e.to_string())?;
2796 if header.metadata_len == 0 {
2797 return Ok(None);
2798 }
2799 let start = header.metadata_offset as usize;
2800 let end = start
2801 .checked_add(header.metadata_len as usize)
2802 .filter(|&e| e <= bytes.len())
2803 .ok_or_else(|| "metadata section out of bounds".to_string())?;
2804 Ok(Some(DatasetCard::from_json_bytes(&bytes[start..end])?))
2805}
2806
2807#[cfg(test)]
2808mod tests {
2809 use super::*;
2810 use crate::card_input::{CARD_EXTRA_MAX_BYTES, CARD_EXTRA_MAX_KEYS, CARD_EXTRA_MAX_KEY_BYTES};
2813
2814 const TYPE: &str = RDF_TYPE;
2815
2816 fn q(s: &str, p: &str, o: &str) -> (String, String, String, Option<String>) {
2817 (s.into(), p.into(), o.into(), None)
2818 }
2819
2820 #[test]
2821 fn card_json_round_trips() {
2822 let card = DatasetCard {
2823 title: Some("Demo".into()),
2824 license: Some("CC0".into()),
2825 triple_count: 3,
2826 quad_count: 3,
2827 term_count: 5,
2828 predicates: vec![("<http://ex/p>".into(), 2)],
2829 vocabularies: vec!["http://ex/".into()],
2830 format_version: 1,
2831 ..Default::default()
2832 };
2833 let bytes = card.to_json_bytes();
2834 let back = DatasetCard::from_json_bytes(&bytes).unwrap();
2835 assert_eq!(card, back);
2836 let text = String::from_utf8(bytes).unwrap();
2838 assert!(!text.contains("description"));
2839 assert!(!text.contains("classes"));
2840 }
2841
2842 #[test]
2848 fn the_permutation_signal_is_stripped_from_the_stored_card() {
2849 let measured = PermutationsSignal {
2850 count: 3,
2851 names: vec!["SPO".into(), "POS".into(), "OSP".into()],
2852 merge_join: false,
2853 };
2854 let mut card = DatasetCard {
2855 title: Some("Lean".into()),
2856 triple_count: 3,
2857 ..Default::default()
2858 };
2859 assert_eq!(card.observe_permutations(measured.clone()), None);
2860 assert_eq!(card.signals.permutations, Some(measured.clone()));
2861
2862 let bytes = card.to_json_bytes();
2863 let text = String::from_utf8(bytes.clone()).unwrap();
2864 assert!(
2865 !text.contains("permutations"),
2866 "the signal must not reach the metadata section: {text}"
2867 );
2868 let back = DatasetCard::from_json_bytes(&bytes).unwrap();
2869 assert_eq!(back.signals.permutations, None, "unknown, not `six`");
2870 assert_eq!(back.title, card.title);
2871
2872 let authored = r#"{"title":"x","signals":{"permutations":{"count":6}}}"#;
2875 assert!(
2876 serde_json::from_str::<CardInput>(authored).is_err(),
2877 "a hand-written card must not be able to assert a permutation set"
2878 );
2879 }
2880
2881 #[test]
2885 fn the_permutation_signal_is_derived_from_the_header() {
2886 let quads = vec![
2887 q("<http://ex/a>", "<http://ex/p>", "<http://ex/b>"),
2888 q("<http://ex/b>", "<http://ex/p>", "<http://ex/c>"),
2889 ];
2890 let (six, _) = crate::ingest::assemble_dataset_with_perms(
2891 quads.clone(),
2892 false,
2893 false,
2894 None,
2895 crate::PyramidAlgo::Louvain,
2896 crate::PermSet::ALL,
2897 |_, _| Vec::new(),
2898 );
2899 let (three, _) = crate::ingest::assemble_dataset_with_perms(
2900 quads,
2901 false,
2902 false,
2903 None,
2904 crate::PyramidAlgo::Louvain,
2905 crate::PermSet::CORE,
2906 |_, _| Vec::new(),
2907 );
2908 assert_eq!(six[50], 0);
2911 assert_eq!(three[50], 0b0000_0111);
2912
2913 let h6 = crate::Header::from_bytes(&six).unwrap();
2914 let h3 = crate::Header::from_bytes(&three).unwrap();
2915 let s6 = PermutationsSignal::probe(&h6);
2916 let s3 = PermutationsSignal::probe(&h3);
2917 assert_eq!(s6.count, 6);
2918 assert!(s6.merge_join);
2919 assert_eq!(s3.count, 3);
2920 assert!(!s3.merge_join);
2921 assert_eq!(s3.names, vec!["SPO", "POS", "OSP"]);
2922 assert!(s3.describe().contains("no sort-merge join"));
2923 assert!(three.len() < six.len(), "{} vs {}", three.len(), six.len());
2925 assert_eq!(h3.dictionary_len, h6.dictionary_len);
2926 assert!(h3.root_dir_len < h6.root_dir_len);
2927 }
2928
2929 #[test]
2934 fn the_text_index_signal_is_stripped_from_the_stored_card() {
2935 let measured = TextIndexSignal {
2937 present: true,
2938 bytes: 1_879_287_762,
2939 token_table_bytes: Some(193_295_361),
2940 };
2941 let mut card = DatasetCard {
2942 title: Some("Indexed".into()),
2943 triple_count: 3,
2944 ..Default::default()
2945 };
2946 assert_eq!(card.observe_text_index(measured), None);
2947 assert_eq!(card.signals.text_index, Some(measured));
2948
2949 let bytes = card.to_json_bytes();
2950 let text = String::from_utf8(bytes.clone()).unwrap();
2951 assert!(
2952 !text.contains("text_index"),
2953 "the signal must not reach the metadata section: {text}"
2954 );
2955 let back = DatasetCard::from_json_bytes(&bytes).unwrap();
2958 assert_eq!(back.title, card.title);
2959 assert_eq!(back.signals.text_index, None);
2960
2961 assert_eq!(card.signals.text_index, Some(measured));
2964 }
2965
2966 #[test]
2972 fn the_text_index_signal_round_trips_through_a_card_document() {
2973 let mut card = DatasetCard {
2974 title: Some("Indexed".into()),
2975 ..Default::default()
2976 };
2977 card.observe_text_index(TextIndexSignal {
2978 present: true,
2979 bytes: 80,
2980 token_table_bytes: Some(68),
2981 });
2982 let doc = serde_json::to_vec(&card).unwrap();
2983 let back: DatasetCard = serde_json::from_slice(&doc).unwrap();
2984 assert_eq!(back.signals.text_index, card.signals.text_index);
2985
2986 let mut none = DatasetCard::default();
2990 none.observe_text_index(TextIndexSignal::default());
2991 let text = serde_json::to_string(&none).unwrap();
2992 assert!(text.contains(r#""text_index":{"present":false}"#), "{text}");
2993 let absent = serde_json::to_string(&DatasetCard::default()).unwrap();
2994 assert!(!absent.contains("text_index"), "{absent}");
2995 }
2996
2997 #[test]
3004 fn the_quoted_triple_signal_is_derived_from_the_header() {
3005 let plain = vec![
3006 q("<http://ex/a>", "<http://ex/p>", "<http://ex/b>"),
3007 q("<http://ex/b>", "<http://ex/p>", "<http://ex/c>"),
3008 ];
3009 let quoted = vec![
3010 q("<http://ex/a>", "<http://ex/p>", "<http://ex/b>"),
3011 q(
3012 "<http://ex/claim>",
3013 "<http://ex/states>",
3014 "<<<http://ex/a> <http://ex/p> <http://ex/b>>>",
3015 ),
3016 ];
3017 let (no, _) = crate::ingest::assemble_dataset(plain, &[]);
3018 let (yes, _) = crate::ingest::assemble_dataset(quoted, &[]);
3019
3020 let hn = crate::Header::from_bytes(&no).unwrap();
3021 let hy = crate::Header::from_bytes(&yes).unwrap();
3022 assert!(!hn.has_quoted_triples());
3023 assert!(hy.has_quoted_triples());
3024
3025 let sn = QuotedTriplesSignal::probe(&hn);
3026 let sy = QuotedTriplesSignal::probe(&hy);
3027 assert!(!sn.present);
3028 assert!(sy.present);
3029
3030 assert_eq!(sn.describe(), None);
3033 let line = sy.describe().expect("a file with quoted triples says so");
3037 assert!(line.contains("<<( s p o )>>"), "{line}");
3038 assert!(line.contains(QUOTED_TRIPLE_SURFACE_RDF_STAR), "{line}");
3039 assert_eq!(
3040 sy.export_surfaces,
3041 vec![QUOTED_TRIPLE_SURFACE_RDF12, QUOTED_TRIPLE_SURFACE_RDF_STAR]
3042 );
3043 assert_eq!(
3044 sy.export_default.as_deref(),
3045 Some(QUOTED_TRIPLE_SURFACE_RDF12)
3046 );
3047 assert!(sn.export_surfaces.is_empty());
3049 assert_eq!(sn.export_default, None);
3050 }
3051
3052 #[test]
3057 fn the_quoted_triple_signal_is_stripped_from_the_stored_card() {
3058 let measured = QuotedTriplesSignal {
3059 present: true,
3060 export_surfaces: vec![
3061 QUOTED_TRIPLE_SURFACE_RDF12.to_string(),
3062 QUOTED_TRIPLE_SURFACE_RDF_STAR.to_string(),
3063 ],
3064 export_default: Some(QUOTED_TRIPLE_SURFACE_RDF12.to_string()),
3065 };
3066 let mut card = DatasetCard {
3067 title: Some("Annotated".into()),
3068 triple_count: 3,
3069 ..Default::default()
3070 };
3071 assert_eq!(card.observe_quoted_triples(measured.clone()), None);
3072 assert_eq!(card.signals.quoted_triples, Some(measured.clone()));
3073
3074 let bytes = card.to_json_bytes();
3075 let text = String::from_utf8(bytes.clone()).unwrap();
3076 assert!(
3077 !text.contains("quoted_triples"),
3078 "the signal must not reach the metadata section: {text}"
3079 );
3080 let back = DatasetCard::from_json_bytes(&bytes).unwrap();
3081 assert_eq!(back.title, card.title);
3082 assert_eq!(back.signals.quoted_triples, None);
3083 assert_eq!(card.signals.quoted_triples, Some(measured));
3085
3086 let authored = r#"{"title":"x","signals":{"quoted_triples":{"present":true}}}"#;
3089 assert!(
3090 serde_json::from_str::<CardInput>(authored).is_err(),
3091 "a hand-written card must not be able to assert a quoted-triple set"
3092 );
3093 }
3094
3095 #[test]
3100 fn the_quoted_triple_signal_round_trips_through_a_card_document() {
3101 let mut card = DatasetCard {
3102 title: Some("Annotated".into()),
3103 ..Default::default()
3104 };
3105 card.observe_quoted_triples(QuotedTriplesSignal {
3106 present: true,
3107 export_surfaces: vec![QUOTED_TRIPLE_SURFACE_RDF12.to_string()],
3108 export_default: Some(QUOTED_TRIPLE_SURFACE_RDF12.to_string()),
3109 });
3110 let doc = serde_json::to_vec(&card).unwrap();
3111 let back: DatasetCard = serde_json::from_slice(&doc).unwrap();
3112 assert_eq!(back.signals.quoted_triples, card.signals.quoted_triples);
3113
3114 let mut none = DatasetCard::default();
3115 none.observe_quoted_triples(QuotedTriplesSignal {
3116 present: false,
3117 export_surfaces: Vec::new(),
3118 export_default: None,
3119 });
3120 let text = serde_json::to_string(&none).unwrap();
3121 assert!(
3122 text.contains(r#""quoted_triples":{"present":false}"#),
3123 "{text}"
3124 );
3125 let absent = serde_json::to_string(&DatasetCard::default()).unwrap();
3126 assert!(!absent.contains("quoted_triples"), "{absent}");
3127 }
3128 #[test]
3129 fn split_namespace_handles_hash_slash_and_neither() {
3130 assert_eq!(
3131 split_namespace("<http://xmlns.com/foaf/0.1/name>").as_deref(),
3132 Some("http://xmlns.com/foaf/0.1/")
3133 );
3134 assert_eq!(
3135 split_namespace("<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>").as_deref(),
3136 Some("http://www.w3.org/1999/02/22-rdf-syntax-ns#")
3137 );
3138 assert_eq!(split_namespace("<urn:isbn:12345>"), None);
3140 assert_eq!(split_namespace("\"literal\""), None);
3141 assert_eq!(split_namespace("_:b0"), None);
3142 }
3143
3144 #[test]
3145 fn derive_counts_and_orders() {
3146 let quads = vec![
3147 q("<http://ex/a>", TYPE, "<http://ex/Person>"),
3148 q("<http://ex/b>", TYPE, "<http://ex/Person>"),
3149 q("<http://ex/c>", TYPE, "<http://ex/City>"),
3150 q("<http://ex/a>", "<http://ex/knows>", "<http://ex/b>"),
3151 q("<http://ex/a>", "<http://ex/knows>", "<http://ex/c>"),
3152 q("<http://ex/a>", "<http://ex/name>", "\"A\""),
3153 ];
3154 let card = derive_card(&quads, 9, 0, CardInput::default());
3155 assert_eq!(card.triple_count, 6);
3156 assert_eq!(card.quad_count, 6);
3157 assert_eq!(card.term_count, 9);
3158 assert_eq!(card.predicates[0], (TYPE.to_string(), 3));
3160 assert!(card
3161 .predicates
3162 .contains(&("<http://ex/knows>".to_string(), 2)));
3163 assert_eq!(
3165 card.classes,
3166 vec![
3167 ("<http://ex/Person>".to_string(), 2),
3168 ("<http://ex/City>".to_string(), 1)
3169 ]
3170 );
3171 assert!(card.vocabularies.contains(&"http://ex/".to_string()));
3173 assert!(card
3174 .vocabularies
3175 .iter()
3176 .any(|v| v.ends_with("rdf-syntax-ns#")));
3177 }
3178
3179 #[test]
3180 fn named_graph_triples_excluded_from_default_stats() {
3181 let quads = vec![
3182 q("<http://ex/a>", "<http://ex/p>", "<http://ex/b>"),
3183 (
3184 "<http://ex/c>".into(),
3185 "<http://ex/p>".into(),
3186 "<http://ex/d>".into(),
3187 Some("<http://ex/g1>".into()),
3188 ),
3189 ];
3190 let card = derive_card(&quads, 5, 1, CardInput::default());
3191 assert_eq!(card.triple_count, 1, "only the default-graph triple counts");
3192 assert_eq!(card.quad_count, 2);
3193 assert_eq!(card.named_graph_count, 1);
3194 assert_eq!(card.predicates[0].1, 1);
3195 }
3196
3197 const LABEL: &str = "<http://www.w3.org/2000/01/rdf-schema#label>";
3199 const SAMEAS: &str = "<http://www.w3.org/2002/07/owl#sameAs>";
3200 const XSD_INT: &str = "http://www.w3.org/2001/XMLSchema#integer";
3201 const XSD_GYEAR: &str = "http://www.w3.org/2001/XMLSchema#gYear";
3202
3203 fn enriched_fixture() -> Vec<(String, String, String, Option<String>)> {
3204 vec![
3205 q("<http://ex/a>", TYPE, "<http://ex/Person>"),
3206 q("<http://ex/b>", TYPE, "<http://ex/Person>"),
3207 q("<http://ex/c>", TYPE, "<http://ex/City>"),
3208 q("<http://ex/a>", LABEL, "\"Alice\"@en"),
3209 q("<http://ex/b>", LABEL, "\"Bob\"@en"),
3210 q(
3211 "<http://ex/a>",
3212 "<http://ex/age>",
3213 &format!("\"30\"^^<{XSD_INT}>"),
3214 ),
3215 q(
3216 "<http://ex/a>",
3217 "<http://ex/birthYear>",
3218 &format!("\"1850\"^^<{XSD_GYEAR}>"),
3219 ),
3220 q(
3221 "<http://ex/b>",
3222 "<http://ex/birthYear>",
3223 &format!("\"1875\"^^<{XSD_GYEAR}>"),
3224 ),
3225 q("<http://ex/a>", "<http://ex/knows>", "<http://ex/b>"),
3226 q("<http://ex/a>", "<http://ex/livesIn>", "<http://ex/c>"),
3227 q("<http://ex/a>", SAMEAS, "<http://other/a>"),
3228 ]
3229 }
3230
3231 #[test]
3232 fn enriched_profile_is_populated() {
3233 let quads = enriched_fixture();
3234 let card = derive_card(&quads, 12, 0, CardInput::default());
3235
3236 assert!(!card.datatypes.is_empty(), "datatypes derived");
3237 assert!(!card.languages.is_empty(), "languages derived");
3238 assert!(!card.class_links.is_empty(), "class_links derived");
3239 assert!(!card.top_hubs.is_empty(), "out-degree hubs derived");
3240 assert!(!card.in_hubs.is_empty(), "in-degree hubs derived");
3241
3242 assert!(card
3244 .class_links
3245 .iter()
3246 .any(|l| l.s_class == "<http://ex/Person>"
3247 && l.predicate == "<http://ex/knows>"
3248 && l.o_class == "<http://ex/Person>"
3249 && l.count == 1));
3250 assert!(card
3251 .class_links
3252 .iter()
3253 .any(|l| l.o_class == "<http://ex/City>"));
3254 assert!(card.class_links.iter().any(|l| l.o_class == "(untyped)"));
3256
3257 assert!(card.datatypes.iter().any(|(d, _)| d.contains("integer")));
3259 assert!(card.datatypes.iter().any(|(d, _)| d.contains("gYear")));
3260 assert!(card.datatypes.iter().any(|(d, _)| d.contains("langString")));
3261 assert!(card.languages.iter().any(|(l, _)| l == "en"));
3263
3264 let s = &card.signals;
3266 assert_eq!(s.label_predicate.as_deref(), Some(LABEL));
3267 assert_eq!(s.default_lang.as_deref(), Some("en"));
3268 assert_eq!(s.base_iri.as_deref(), Some("http://ex/"));
3269 assert!(s.link_predicates.iter().any(|p| p.contains("sameAs")));
3270 assert!(s.time_predicates.iter().any(|p| p.contains("birthYear")));
3271 assert_eq!(
3272 s.temporal_extent,
3273 Some(("1850".to_string(), "1875".to_string())),
3274 "extent over the dominant time predicate"
3275 );
3276 assert!(!s.geo_wkt && !s.geo_latlong);
3278 assert!(s.spatial_bbox.is_none());
3279 }
3280
3281 #[test]
3282 fn signals_detect_geometry() {
3283 let wkt = "http://www.opengis.net/ont/geosparql#wktLiteral";
3284 let quads = vec![
3285 q("<http://ex/p>", TYPE, "<http://ex/Place>"),
3286 q(
3287 "<http://ex/p>",
3288 "<http://www.w3.org/2003/01/geo/wgs84_pos#lat>",
3289 "\"41.9\"^^<http://www.w3.org/2001/XMLSchema#double>",
3290 ),
3291 q(
3292 "<http://ex/p>",
3293 "<http://www.w3.org/2003/01/geo/wgs84_pos#long>",
3294 "\"12.5\"^^<http://www.w3.org/2001/XMLSchema#double>",
3295 ),
3296 q(
3297 "<http://ex/p>",
3298 "<http://www.opengis.net/ont/geosparql#asWKT>",
3299 &format!("\"POINT(12.5 41.9)\"^^<{wkt}>"),
3300 ),
3301 ];
3302 let card = derive_card(&quads, 6, 0, CardInput::default());
3303 assert!(card.signals.geo_wkt, "asWKT/wktLiteral detected");
3304 assert!(card.signals.geo_latlong, "wgs84 lat+long detected");
3305 let bbox = card.signals.spatial_bbox.expect("bbox derived");
3306 assert_eq!(
3307 bbox,
3308 [12.5, 41.9, 12.5, 41.9],
3309 "[minLon,minLat,maxLon,maxLat]"
3310 );
3311 }
3312
3313 #[test]
3314 fn class_links_match_schema_summary() {
3315 use crate::{ingest, schema_summary, Rete};
3316 let quads = enriched_fixture();
3317 let card = derive_card(&quads, 12, 0, CardInput::default());
3318
3319 let (bytes, _) =
3321 ingest::assemble_dataset_with_opts(quads, true, false, None, |_, _| Vec::new());
3322 let rete = Rete::open(&bytes).unwrap();
3323 let reference: BTreeSet<(String, String, String, u64)> = schema_summary(&rete)
3324 .into_iter()
3325 .map(|(s, p, o, c)| (s, p, o, c as u64))
3326 .collect();
3327
3328 let derived: BTreeSet<(String, String, String, u64)> = card
3329 .class_links
3330 .iter()
3331 .map(|l| {
3332 (
3333 l.s_class.clone(),
3334 l.predicate.clone(),
3335 l.o_class.clone(),
3336 l.count,
3337 )
3338 })
3339 .collect();
3340
3341 assert_eq!(
3342 derived, reference,
3343 "card class_links must equal schema_summary's quotient row-for-row"
3344 );
3345 }
3346
3347 #[test]
3357 fn card_counts_are_the_deduplicated_counts() {
3358 use crate::ingest::{self, DeferredMetadata};
3359 use crate::Rete;
3360
3361 let g = |s: &str, p: &str, o: &str, graph: &str| {
3362 (
3363 s.to_string(),
3364 p.to_string(),
3365 o.to_string(),
3366 Some(graph.to_string()),
3367 )
3368 };
3369 let quads = vec![
3370 q("<http://ex/a>", "<http://ex/p>", "<http://ex/1>"),
3371 q("<http://ex/b>", "<http://ex/p>", "<http://ex/2>"),
3372 q("<http://ex/a>", "<http://ex/p>", "<http://ex/1>"), g("<http://ex/n>", "<http://ex/q>", "\"v\"", "<http://ex/g>"),
3374 g("<http://ex/n>", "<http://ex/q>", "\"v\"", "<http://ex/g>"), ];
3376 let (bytes, stats) =
3378 ingest::assemble_dataset_with_opts(quads, false, false, None, |stats, quads| {
3379 let card = derive_card(
3380 quads,
3381 stats.terms as u64,
3382 stats.named_graphs as u64,
3383 CardInput::default(),
3384 );
3385 DeferredMetadata::new(move |counts| card.with_final_counts(counts).to_json_bytes())
3386 });
3387
3388 let header = crate::Header::from_bytes(&bytes).unwrap();
3389 let card = load_card(&bytes).unwrap().expect("card embedded");
3390 assert_eq!(header.quad_count, 3, "header records the deduped quads");
3391 assert_eq!(
3392 card.quad_count, header.quad_count,
3393 "card and header must not disagree about the size of the same file"
3394 );
3395 assert_eq!(card.triple_count, 2, "default-graph triples, deduped");
3396
3397 let rete = Rete::open(&bytes).unwrap();
3399 let default_triples = rete.dump(None).len() as u64;
3400 let named: u64 = rete
3401 .graph_names()
3402 .iter()
3403 .map(|name| rete.dump(Some(name)).len() as u64)
3404 .sum();
3405 assert_eq!(default_triples + named, card.quad_count);
3406
3407 assert_eq!(stats.statements, 3);
3410 assert_eq!(stats.default_triples, 2);
3411 }
3412
3413 #[test]
3414 fn derive_is_deterministic() {
3415 let quads = enriched_fixture();
3418 let a = derive_card(&quads, 12, 0, CardInput::default()).to_json_bytes();
3419 let b = derive_card(&quads, 12, 0, CardInput::default()).to_json_bytes();
3420 assert_eq!(a, b, "enriched card derivation must be deterministic");
3421 assert!(!a.is_empty());
3422 }
3423
3424 #[test]
3425 fn old_plain_string_card_still_parses() {
3426 let json = r#"{
3429 "title":"Legacy",
3430 "triple_count":3,"quad_count":3,"named_graph_count":0,"term_count":5,
3431 "predicates":[["<http://ex/p>",2]],
3432 "example_queries":["SELECT * WHERE { ?s ?p ?o }"],
3433 "format_version":1
3434 }"#;
3435 let card = DatasetCard::from_json_bytes(json.as_bytes()).unwrap();
3436 assert_eq!(card.title.as_deref(), Some("Legacy"));
3437 assert_eq!(card.example_queries.len(), 1);
3438 assert!(card.datatypes.is_empty());
3440 assert!(card.languages.is_empty());
3441 assert!(card.class_links.is_empty());
3442 assert!(card.top_hubs.is_empty());
3443 assert!(card.in_hubs.is_empty());
3444 assert!(card.queries.is_empty());
3445 assert!(card.signals.is_empty());
3446 assert!(!card.truncated);
3447 }
3448
3449 #[test]
3453 fn curated_base_iri_overrides_derived_and_is_validated() {
3454 let quads = vec![q(
3455 "<https://doi.org/10.1/a>",
3456 "<http://ex/p>",
3457 "<http://ex/o>",
3458 )];
3459 let derived = derive_card(&quads, 3, 0, CardInput::default());
3460 assert_eq!(
3461 derived.signals.base_iri.as_deref(),
3462 Some("https://doi.org/10.1/")
3463 );
3464 let input = CardInput::from_json_str(r#"{"base_iri": "https://doi.org/"}"#).unwrap();
3465 assert!(!input.is_empty());
3466 let card = derive_card(&quads, 3, 0, input.clone());
3467 assert_eq!(card.signals.base_iri.as_deref(), Some("https://doi.org/"));
3468 let counts = curated_counts_card(1, 3, input);
3469 assert_eq!(counts.signals.base_iri.as_deref(), Some("https://doi.org/"));
3470 for bad in [
3471 "doi.org/",
3472 "",
3473 "https:",
3474 "1http://x/",
3475 "http://a b/",
3476 "http://x/<y>",
3477 ] {
3478 let doc = serde_json::json!({ "base_iri": bad }).to_string();
3479 let err = CardInput::from_json_str(&doc).unwrap_err();
3480 assert!(err.contains("base_iri"), "{bad}: {err}");
3481 }
3482 }
3483
3484 #[test]
3485 fn curated_identity_fields_round_trip() {
3486 let input: CardInput = serde_json::from_str(
3487 r#"{
3488 "title": "T",
3489 "version": "2026-08-04",
3490 "creators": [{"name":"Ada","orcid":"https://orcid.org/0000-0002-1825-0097"}],
3491 "publisher": {"name":"EPFL","ror":"https://ror.org/02s376052"},
3492 "canonical_url": "https://data.example.org/t.rete",
3493 "sparql_endpoint": "https://example.org/sparql",
3494 "source_date": "2026-07-01",
3495 "derived_from": ["https://example.org/dump.nt"],
3496 "doi": "https://doi.org/10.5281/zenodo.1",
3497 "cite_as": "Ada (2026). T.",
3498 "keywords": ["graphs", "citations"],
3499 "theme": ["http://publications.europa.eu/resource/authority/data-theme/TECH"]
3500 }"#,
3501 )
3502 .unwrap();
3503 let card = curated_counts_card(5, 9, input);
3504 let bytes = card.to_json_bytes();
3505 let back = DatasetCard::from_json_bytes(&bytes).unwrap();
3506 assert_eq!(back.version.as_deref(), Some("2026-08-04"));
3507 assert_eq!(back.creators[0].name, "Ada");
3508 assert_eq!(
3509 back.creators[0].orcid.as_deref(),
3510 Some("https://orcid.org/0000-0002-1825-0097")
3511 );
3512 assert_eq!(back.publisher.as_ref().unwrap().name, "EPFL");
3513 assert_eq!(
3514 back.canonical_url.as_deref(),
3515 Some("https://data.example.org/t.rete")
3516 );
3517 assert_eq!(back.derived_from, vec!["https://example.org/dump.nt"]);
3518 assert_eq!(
3519 back.doi.as_deref(),
3520 Some("https://doi.org/10.5281/zenodo.1")
3521 );
3522 assert_eq!(back.keywords, vec!["graphs", "citations"]);
3525 assert_eq!(
3526 back.theme,
3527 vec!["http://publications.europa.eu/resource/authority/data-theme/TECH"]
3528 );
3529
3530 let plain = curated_counts_card(1, 2, CardInput::default()).to_json_bytes();
3532 let text = String::from_utf8(plain).unwrap();
3533 for absent in [
3534 "\"creators\"",
3535 "\"publisher\"",
3536 "\"canonical_url\"",
3537 "\"sparql_endpoint\"",
3538 "\"derived_from\"",
3539 "\"doi\"",
3540 "\"cite_as\"",
3541 "\"source_date\"",
3542 "\"version\"",
3543 "\"top_n\"",
3544 "\"keywords\"",
3545 "\"theme\"",
3546 "\"extra\"",
3547 ] {
3548 assert!(
3549 !text.contains(absent),
3550 "{absent} must be omitted when unset"
3551 );
3552 }
3553 }
3554
3555 #[test]
3560 fn extra_round_trips_and_external_writer_cards_surface_it() {
3561 let input: CardInput = serde_json::from_str(
3562 r#"{"title":"T","extra":{"atlas:layer":84,"review":{"by":"dg","ok":true}}}"#,
3563 )
3564 .unwrap();
3565 let card = curated_counts_card(5, 9, input);
3566 let back = DatasetCard::from_json_bytes(&card.to_json_bytes()).unwrap();
3567 assert_eq!(back.extra, card.extra);
3568 assert_eq!(back.extra.get("atlas:layer"), Some(&serde_json::json!(84)));
3569
3570 let python_shaped = r#"{
3573 "title":"From Python","extra":{"pipeline":"nightly"},
3574 "triple_count":1,"quad_count":1,"named_graph_count":0,"term_count":3,
3575 "format_version":5
3576 }"#;
3577 let card = DatasetCard::from_json_bytes(python_shaped.as_bytes()).unwrap();
3578 assert_eq!(
3579 card.extra.get("pipeline"),
3580 Some(&serde_json::json!("nightly"))
3581 );
3582 }
3583
3584 #[test]
3587 fn extra_limits_bite_exactly_at_the_boundary() {
3588 use crate::card_input::normalize_extra;
3589 use serde_json::{json, Map, Value};
3590 let bag = |v: Vec<(String, Value)>| -> Map<String, Value> { v.into_iter().collect() };
3593
3594 let pad = |n: usize| bag(vec![("pad".into(), json!("x".repeat(n)))]);
3596 let at = pad(CARD_EXTRA_MAX_BYTES - 10);
3597 assert_eq!(serde_json::to_vec(&at).unwrap().len(), CARD_EXTRA_MAX_BYTES);
3598 normalize_extra(at).expect("exactly at the byte cap is accepted");
3599 let over = pad(CARD_EXTRA_MAX_BYTES - 9);
3600 assert_eq!(
3601 serde_json::to_vec(&over).unwrap().len(),
3602 CARD_EXTRA_MAX_BYTES + 1
3603 );
3604 let err = normalize_extra(over).unwrap_err().to_string();
3605 assert!(
3606 err.contains(&format!("{} bytes", CARD_EXTRA_MAX_BYTES + 1)),
3607 "the error states the actual size: {err}"
3608 );
3609
3610 let keys = |n: usize| {
3612 bag((0..n)
3613 .map(|i| (format!("k{i:03}"), json!(1)))
3614 .collect::<Vec<_>>())
3615 };
3616 normalize_extra(keys(CARD_EXTRA_MAX_KEYS)).expect("at the key cap");
3617 assert!(normalize_extra(keys(CARD_EXTRA_MAX_KEYS + 1)).is_err());
3618
3619 normalize_extra(bag(vec![("k".repeat(CARD_EXTRA_MAX_KEY_BYTES), json!(1))]))
3621 .expect("at the key-length cap");
3622 assert!(normalize_extra(bag(vec![(
3623 "k".repeat(CARD_EXTRA_MAX_KEY_BYTES + 1),
3624 json!(1)
3625 )]))
3626 .is_err());
3627 assert!(normalize_extra(bag(vec![(String::new(), json!(1))])).is_err());
3628
3629 normalize_extra(bag(vec![("d".into(), json!({"a": {"b": 1}}))])).expect("under the cap");
3633 normalize_extra(bag(vec![("d".into(), json!([{"w": ["x", "y"]}]))]))
3634 .expect("at the depth cap");
3635 assert!(normalize_extra(bag(vec![("d".into(), json!({"a": {"b": {"c": [1]}}}))])).is_err());
3636 assert!(normalize_extra(bag(vec![("d".into(), json!([[[[1]]]]))])).is_err());
3637
3638 let err = normalize_extra(bag(vec![("@context".into(), json!({}))]))
3640 .unwrap_err()
3641 .to_string();
3642 assert!(err.contains("reserved"), "loud reservation: {err}");
3643 }
3644
3645 #[test]
3650 fn extra_fields_keep_the_content_hash_reproducible() {
3651 let build = |json: &str| {
3652 let input: CardInput = serde_json::from_str::<CardInput>(json)
3655 .unwrap()
3656 .normalize()
3657 .unwrap();
3658 let quads = enriched_fixture();
3659 let card = derive_card(&quads, 12, 0, input);
3660 let (image, _) =
3661 crate::ingest::assemble_dataset_with_opts(quads, true, false, None, |_, _| {
3662 card.to_json_bytes()
3663 });
3664 image
3665 };
3666 let a = build(r#"{"title":"T","extra":{"b":{"y":2,"x":1},"a":[1,2]}}"#);
3667 let b = build(r#"{"title":"T","extra":{"a":[1,2],"b":{"x":1,"y":2}}}"#);
3668 assert_eq!(a, b, "reordered-input builds are byte-identical");
3669 let ha = Header::from_bytes(&a).unwrap().content_hash;
3670 assert_eq!(ha, Header::from_bytes(&b).unwrap().content_hash);
3671
3672 let plain = build(r#"{"title":"T"}"#);
3675 assert_ne!(ha, Header::from_bytes(&plain).unwrap().content_hash);
3676 }
3677
3678 #[test]
3682 fn top_level_custom_keys_are_rejected_not_dropped() {
3683 let err = serde_json::from_str::<CardInput>(r#"{"title":"T","my_field":1}"#)
3684 .unwrap_err()
3685 .to_string();
3686 assert!(err.contains("my_field"), "names the stray key: {err}");
3687
3688 let ok: CardInput =
3690 serde_json::from_str(r#"{"title":"T","extra":{"my_field":1}}"#).unwrap();
3691 assert_eq!(ok.extra.get("my_field"), Some(&serde_json::json!(1)));
3692 }
3693
3694 #[test]
3699 fn description_reads_as_a_string_or_as_lines() {
3700 let from_lines: CardInput = serde_json::from_value(serde_json::json!({
3701 "description": ["## Contents", "", "- a", "- b"]
3702 }))
3703 .unwrap();
3704 assert_eq!(
3705 from_lines.description.as_deref(),
3706 Some("## Contents\n\n- a\n- b")
3707 );
3708 let from_string: CardInput =
3709 serde_json::from_str("{\"description\":\"## Contents\\n\\n- a\\n- b\"}").unwrap();
3710 assert_eq!(from_string.description, from_lines.description);
3711
3712 let none: CardInput = serde_json::from_str(r#"{"title":"T"}"#).unwrap();
3714 assert!(none.description.is_none());
3715 let null: CardInput = serde_json::from_str(r#"{"description":null}"#).unwrap();
3716 assert!(null.description.is_none());
3717
3718 let over = "x".repeat(crate::card::CARD_DESCRIPTION_MAX_BYTES + 1);
3721 let err = serde_json::from_str::<CardInput>(&format!(
3722 "{{\"description\":{}}}",
3723 serde_json::to_string(&over).unwrap()
3724 ))
3725 .unwrap_err()
3726 .to_string();
3727 assert!(err.contains("over the"), "{err}");
3728 }
3729
3730 #[test]
3738 fn curated_field_list_matches_the_deny_unknown_fields_struct() {
3739 for f in crate::card::CURATED_CARD_FIELDS {
3740 let doc = format!(r#"{{"{f}":null}}"#);
3744 if let Err(e) = serde_json::from_str::<CardInput>(&doc) {
3745 assert!(
3746 !e.to_string().contains("unknown field"),
3747 "CURATED_CARD_FIELDS lists `{f}`, but CardInput rejects it as unknown: {e}",
3748 );
3749 }
3750 }
3751 let err = serde_json::from_str::<CardInput>(r#"{"nope":1}"#)
3754 .unwrap_err()
3755 .to_string();
3756 for name in err
3757 .split("expected one of ")
3758 .nth(1)
3759 .expect("serde names the expected fields")
3760 .split(" at line ")
3762 .next()
3763 .unwrap()
3764 .split(", ")
3765 .map(|s| s.trim().trim_matches('`'))
3766 .filter(|s| !s.is_empty())
3767 {
3768 assert!(
3769 crate::card::CURATED_CARD_FIELDS.contains(&name),
3770 "CardInput accepts `{name}` but CURATED_CARD_FIELDS does not list it — \
3771 the browser builder would reject a card the CLI accepts",
3772 );
3773 }
3774 }
3775
3776 #[test]
3783 fn a_browser_written_card_deserializes_here() {
3784 let curated = crate::card::validate_curated_card(&serde_json::json!({
3785 "title": "Built in a browser",
3786 "keywords": ["b", "a"],
3787 "theme": ["http://publications.europa.eu/resource/authority/data-theme/GOVE"],
3788 "creators": [{"name": "Ada", "orcid": "https://orcid.org/0000-0002-1825-0097"}],
3789 "extra": {"internal_id": "DS-1"},
3790 }))
3791 .expect("valid curated document");
3792 let composed = crate::card::compose_curated_card(curated, 3, 4, 1, 9, 5);
3793 let text = serde_json::to_string(&composed).unwrap();
3794
3795 let card: DatasetCard = serde_json::from_str(&text).expect("CLI reads a browser card");
3796 assert_eq!(card.title.as_deref(), Some("Built in a browser"));
3797 assert_eq!(card.keywords, vec!["a".to_string(), "b".to_string()]);
3798 assert_eq!(card.creators.len(), 1);
3799 assert_eq!(card.triple_count, 3);
3800 assert_eq!(card.quad_count, 4);
3801 assert_eq!(card.term_count, 9);
3802 assert_eq!(card.format_version, 5);
3803 assert!(card.predicates.is_empty() && card.queries.is_empty());
3806 assert!(!text.contains("\"predicates\""), "{text}");
3807 assert!(!text.contains("\"top_n\""), "{text}");
3808 }
3809
3810 #[test]
3814 fn old_reader_struct_accepts_new_card_json() {
3815 #[derive(serde::Deserialize)]
3816 struct OldCard {
3817 title: Option<String>,
3818 triple_count: u64,
3819 #[serde(default)]
3820 predicates: Vec<(String, u64)>,
3821 format_version: u8,
3822 }
3823 let input: CardInput = serde_json::from_str(
3824 r#"{"title":"New","version":"1","creators":[{"name":"A"}],
3825 "canonical_url":"https://x/y.rete","derived_from":["https://x/d.nt"],
3826 "extra":{"atlas:review":"approved"}}"#,
3827 )
3828 .unwrap();
3829 let quads = vec![q("<http://ex/a>", "<http://ex/p>", "<http://ex/b>")];
3830 let card = derive_card(&quads, 3, 0, input);
3831 assert_eq!(card.top_n, CARD_TOP_N as u32);
3832 let bytes = card.to_json_bytes();
3833 let old: OldCard = serde_json::from_slice(&bytes).expect("old struct parses new JSON");
3834 assert_eq!(old.title.as_deref(), Some("New"));
3835 assert_eq!(old.triple_count, 1);
3836 assert_eq!(old.format_version, card.format_version);
3837 assert!(!old.predicates.is_empty());
3838 }
3839
3840 #[test]
3841 fn coherence_stamp_is_deterministic_and_additive() {
3842 let dis = "<http://www.w3.org/2002/07/owl#disjointWith>";
3843 let rt = "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>";
3844 let base = vec![
3845 ("<http://ex/C>".into(), dis.into(), "<http://ex/D>".into()),
3846 ("<http://ex/x>".into(), rt.into(), "<http://ex/C>".into()),
3847 ("<http://ex/x>".into(), rt.into(), "<http://ex/D>".into()),
3848 ];
3849 let r = crate::reason(&base);
3850 let c1 = Coherence::from_reasoning(&r, false);
3851 assert_eq!(c1, Coherence::from_reasoning(&r, false));
3853 assert!(!c1.coherent);
3854 assert_eq!(c1.inconsistency_count, 1);
3855 assert_eq!(c1.by_kind, vec![("disjoint-classes".to_string(), 1)]);
3856 assert_eq!(c1.rules, crate::REASON_RULESET);
3857
3858 let json = DatasetCard::default()
3860 .with_coherence(&r, false)
3861 .to_json_bytes();
3862 let text = std::str::from_utf8(&json).unwrap();
3863 assert!(text.contains("coherence"));
3864 assert_eq!(DatasetCard::from_json_bytes(&json).unwrap().coherence, c1);
3865
3866 let plain = DatasetCard::default().to_json_bytes();
3867 assert!(!std::str::from_utf8(&plain).unwrap().contains("coherence"));
3868 assert!(DatasetCard::default().coherence.is_empty());
3869 }
3870
3871 #[test]
3872 fn base_iri_is_common_prefix_with_fallback() {
3873 let mut ns = NsTally::default();
3875 ns.add("http://ex/person/".to_string(), 2000);
3876 ns.add("http://ex/place/".to_string(), 600);
3877 ns.add("http://ex/org/".to_string(), 120);
3878 assert_eq!(derive_base_iri(&ns).as_deref(), Some("http://ex/"));
3879
3880 let mut one = NsTally::default();
3882 one.add("http://data.example.org/id/".to_string(), 10);
3883 assert_eq!(
3884 derive_base_iri(&one).as_deref(),
3885 Some("http://data.example.org/id/")
3886 );
3887
3888 let mut multi = NsTally::default();
3891 multi.add("http://a.example/".to_string(), 100);
3892 multi.add("http://b.other/".to_string(), 5);
3893 assert_eq!(
3894 derive_base_iri(&multi).as_deref(),
3895 Some("http://a.example/")
3896 );
3897
3898 assert_eq!(derive_base_iri(&NsTally::default()), None);
3899 }
3900
3901 #[test]
3902 fn parse_literal_decomposes_forms() {
3903 let plain = parse_literal("\"hello\"").unwrap();
3904 assert_eq!(plain.value, "hello");
3905 assert!(plain.datatype.ends_with("XMLSchema#string"));
3906 assert_eq!(plain.lang, "");
3907
3908 let tagged = parse_literal("\"bonjour\"@fr").unwrap();
3909 assert_eq!(tagged.lang, "fr");
3910 assert!(tagged.datatype.ends_with("#langString"));
3911
3912 let typed = parse_literal("\"42\"^^<http://www.w3.org/2001/XMLSchema#integer>").unwrap();
3913 assert_eq!(typed.value, "42");
3914 assert_eq!(typed.datatype, "http://www.w3.org/2001/XMLSchema#integer");
3915
3916 let esc = parse_literal("\"a\\\"b\"@en").unwrap();
3918 assert_eq!(esc.value, "a\\\"b");
3919 assert_eq!(esc.lang, "en");
3920
3921 assert!(parse_literal("<http://ex/x>").is_none());
3923 assert!(parse_literal("_:b0").is_none());
3924 }
3925
3926 #[test]
3931 fn date_shapes_are_strict() {
3932 for ok in [
3933 "2019",
3934 "-0044",
3935 "2019-05",
3936 "2019-05-01",
3937 "2019-05-01Z",
3938 "2019-05-01+02:00",
3939 "2019-05-01T12:30",
3940 "2019-05-01T12:30:15",
3941 "2019-05-01T12:30:15.250Z",
3942 "1950Z",
3943 ] {
3944 assert!(date_shaped(ok), "{ok} is a date");
3945 }
3946 for bad in [
3947 "0000-0001-5001-3721", "9999880", "1234-1240", "123-456",
3951 "2019 annual report", "2019-13",
3953 "2019-05-32",
3954 "2019-05-01T25:00",
3955 "10.1000/182", "",
3957 "-",
3958 "2019-",
3959 ] {
3960 assert!(!date_shaped(bad), "{bad} is not a date");
3961 }
3962 }
3963
3964 #[test]
3970 fn temporal_extent_ignores_identifiers_pages_and_titles() {
3971 let value = "<http://www.essepuntato.it/2010/06/literalreification/hasLiteralValue>";
3972 let start = "<http://prismstandard.org/namespaces/basic/2.0/startingPage>";
3973 let end = "<http://prismstandard.org/namespaces/basic/2.0/endingPage>";
3974 let title = "<http://purl.org/dc/terms/title>";
3975 let pubdate = "<http://prismstandard.org/namespaces/basic/2.0/publicationDate>";
3976 let issued = "<http://purl.org/dc/terms/issued>";
3977 let xsd = |t: &str| format!("http://www.w3.org/2001/XMLSchema#{t}");
3978 let mut quads = Vec::new();
3979 for (i, id) in [
3980 "0000-0001-5001-3721",
3981 "0000-0002-1825-0097",
3982 "10.1000/182",
3983 "9999880",
3984 "2019",
3985 ]
3986 .iter()
3987 .enumerate()
3988 {
3989 quads.push(q(
3990 &format!("<http://ex/id/{i}>"),
3991 value,
3992 &format!("\"{id}\""),
3993 ));
3994 }
3995 quads.push(q("<http://ex/re/1>", start, "\"9999880\""));
3996 quads.push(q("<http://ex/re/1>", end, "\"9999899\""));
3997 quads.push(q("<http://ex/re/2>", start, "\"1234-1240\""));
3998 quads.push(q("<http://ex/br/1>", title, "\"2019 annual report\""));
3999 quads.push(q(
4000 "<http://ex/br/1>",
4001 pubdate,
4002 &format!("\"2019-05-01\"^^<{}>", xsd("date")),
4003 ));
4004 quads.push(q(
4005 "<http://ex/br/2>",
4006 pubdate,
4007 &format!("\"1950\"^^<{}>", xsd("gYear")),
4008 ));
4009 quads.push(q(
4010 "<http://ex/br/3>",
4011 pubdate,
4012 &format!("\"2021-03\"^^<{}>", xsd("gYearMonth")),
4013 ));
4014 quads.push(q(
4016 "<http://ex/br/4>",
4017 pubdate,
4018 &format!("\"0000-0001-5001-3721\"^^<{}>", xsd("date")),
4019 ));
4020 quads.push(q("<http://ex/br/5>", issued, "\"1999-12\""));
4022 let card = derive_card(&quads, 40, 0, CardInput::default());
4023 let s = &card.signals;
4024 assert_eq!(
4025 s.time_predicates,
4026 vec![pubdate.to_string(), issued.to_string()],
4027 "only the date predicates are time predicates"
4028 );
4029 assert_eq!(
4030 s.temporal_extent,
4031 Some(("1950".to_string(), "2021-03".to_string())),
4032 "calendar order across xsd:gYear, xsd:date and xsd:gYearMonth"
4033 );
4034 assert!(
4036 s.numeric_predicates.is_empty(),
4037 "{:?}",
4038 s.numeric_predicates
4039 );
4040 assert!(card
4042 .queries
4043 .iter()
4044 .filter(|q| q.dimension == "time")
4045 .all(|q| q.sparql.contains("publicationDate")));
4046 }
4047
4048 #[test]
4049 fn negative_years_order_by_value_not_by_bytes() {
4050 let mut extent = BTreeMap::new();
4051 for v in ["-0044", "-0100", "0001", "-0044-03-15"] {
4052 update_time_extent("<p>", v, &mut extent);
4053 }
4054 assert_eq!(
4055 extent.get("<p>"),
4056 Some(&("-0100".to_string(), "0001".to_string()))
4057 );
4058 }
4059
4060 #[test]
4063 fn an_integer_year_off_a_date_predicate_is_numeric() {
4064 let int = "http://www.w3.org/2001/XMLSchema#integer";
4065 let quads = vec![
4066 q(
4067 "<http://ex/a>",
4068 "<http://ex/year>",
4069 &format!("\"1492\"^^<{int}>"),
4070 ),
4071 q(
4072 "<http://ex/b>",
4073 "<http://ex/year>",
4074 &format!("\"1500\"^^<{int}>"),
4075 ),
4076 ];
4077 let card = derive_card(&quads, 6, 0, CardInput::default());
4078 assert!(card.signals.time_predicates.is_empty());
4079 assert_eq!(
4080 card.signals.numeric_predicates,
4081 vec!["<http://ex/year>".to_string()]
4082 );
4083 }
4084
4085 fn streaming_fixture() -> Vec<(String, String, String, Option<String>)> {
4094 let xsd = |t: &str| format!("http://www.w3.org/2001/XMLSchema#{t}");
4095 let mut v = enriched_fixture();
4096 for i in 0..5 {
4097 let s = format!("<http://ex/place/{i}>");
4098 v.push(q(&s, TYPE, "<http://ex/Place>"));
4099 v.push(q(&s, LABEL, &format!("\"Lieu {i}\"@fr")));
4100 v.push(q(
4101 &s,
4102 "<http://www.w3.org/2003/01/geo/wgs84_pos#lat>",
4103 &format!("\"4{i}.5\"^^<{}>", xsd("double")),
4104 ));
4105 v.push(q(
4106 &s,
4107 "<http://www.w3.org/2003/01/geo/wgs84_pos#long>",
4108 &format!("\"{i}.25\"^^<{}>", xsd("double")),
4109 ));
4110 v.push(q(
4111 &s,
4112 "<http://www.opengis.net/ont/geosparql#asWKT>",
4113 &format!(
4114 "\"POINT({i}.25 4{i}.5)\"^^<http://www.opengis.net/ont/geosparql#wktLiteral>"
4115 ),
4116 ));
4117 v.push(q(
4118 &s,
4119 "<http://purl.org/dc/terms/created>",
4120 &format!("\"20{i}0-01-0{}\"^^<{}>", i + 1, xsd("date")),
4121 ));
4122 v.push(q(&s, "<http://ex/near>", "<http://elsewhere/undescribed>"));
4123 }
4124 let knows = "<http://ex/a> <http://ex/knows> <http://ex/b>";
4127 let lives = "<http://ex/a> <http://ex/livesIn> <http://ex/c>";
4128 let ghost = "<http://ex/b> <http://ex/knows> <http://ex/c>";
4129 for (t, who) in [(knows, "alice"), (lives, "bob"), (ghost, "carol")] {
4130 v.push(q(
4131 &format!("<<{t}>>"),
4132 "<http://ex/recordedBy>",
4133 &format!("<http://ex/{who}>"),
4134 ));
4135 }
4136 v.push(q(
4137 "<http://ex/claim/1>",
4138 "<http://ex/states>",
4139 &format!("<<{knows}>>"),
4140 ));
4141 v.push((
4142 "<http://ex/note/1>".into(),
4143 "<http://ex/about>".into(),
4144 "<http://ex/a>".into(),
4145 Some("<http://ex/graph/notes>".into()),
4146 ));
4147 v
4148 }
4149
4150 fn both_ways(quads: &[(String, String, String, Option<String>)]) -> (DatasetCard, DatasetCard) {
4153 let (bytes, _) =
4154 crate::ingest::assemble_dataset_with_opts(quads.to_vec(), true, false, None, |_, _| {
4155 Vec::new()
4156 });
4157 let rete = crate::Rete::open(&bytes).unwrap();
4158 let term_count = rete.header().term_count;
4159 let (card, report) =
4160 derive_card_streaming(&rete, CardInput::default(), &mut |_| {}).unwrap();
4161 assert_eq!(report.statements, card.triple_count);
4162 let in_memory = derive_card(
4163 quads,
4164 term_count,
4165 rete.named_graph_count() as u64,
4166 CardInput::default(),
4167 );
4168 (card, in_memory)
4169 }
4170
4171 #[test]
4175 fn streaming_derivation_matches_the_in_memory_card() {
4176 for quads in [enriched_fixture(), streaming_fixture()] {
4177 let (streamed, in_memory) = both_ways(&quads);
4178 assert_eq!(
4179 String::from_utf8(streamed.to_json_bytes()).unwrap(),
4180 String::from_utf8(in_memory.to_json_bytes()).unwrap()
4181 );
4182 }
4183 let (card, _) = both_ways(&streaming_fixture());
4185 assert!(card.signals.annotated_statement.is_some());
4186 assert!(!card.signals.quoting_predicates.is_empty());
4187 assert!(card.signals.spatial_bbox.is_some());
4188 assert!(card.signals.temporal_extent.is_some());
4189 assert_eq!(card.named_graph_count, 1);
4190 }
4191
4192 #[test]
4195 fn streaming_derivation_widens_the_class_map() {
4196 let mut quads = Vec::new();
4197 for i in 0..300 {
4198 let s = format!("<http://ex/thing/{i:03}>");
4199 quads.push(q(&s, TYPE, &format!("<http://ex/Class{i:03}>")));
4200 quads.push(q(
4201 &s,
4202 "<http://ex/next>",
4203 &format!("<http://ex/thing/{:03}>", (i + 1) % 300),
4204 ));
4205 }
4206 let (streamed, in_memory) = both_ways(&quads);
4207 assert!(streamed.truncated);
4208 assert_eq!(streamed.classes, in_memory.classes);
4209 assert_eq!(streamed.class_links, in_memory.class_links);
4210 assert_eq!(
4211 String::from_utf8(streamed.to_json_bytes()).unwrap(),
4212 String::from_utf8(in_memory.to_json_bytes()).unwrap()
4213 );
4214 }
4215
4216 #[test]
4219 fn streaming_derivation_counts_the_deduplicated_file() {
4220 let mut quads = enriched_fixture();
4221 quads.push(quads[0].clone());
4222 let (streamed, in_memory) = both_ways(&quads);
4223 assert_eq!(streamed.triple_count, in_memory.triple_count - 1);
4224 let type_count = |c: &DatasetCard| {
4225 c.predicates
4226 .iter()
4227 .find(|(p, _)| p == TYPE)
4228 .map(|(_, n)| *n)
4229 };
4230 assert_eq!(type_count(&streamed), type_count(&in_memory).map(|n| n - 1));
4231 }
4232
4233 #[test]
4236 fn the_curated_half_of_a_card_round_trips() {
4237 let doc = r##"{"title":"T","description":["# Heading","line"],"license":"CC0-1.0",
4238 "source":"https://ex/src","created":"2026-01-01","version":"1",
4239 "creators":[{"name":"Ada","orcid":"https://orcid.org/0000-0002-1825-0097"}],
4240 "publisher":{"name":"P","ror":"https://ror.org/01ggx4157"},
4241 "canonical_url":"https://ex/f.rete","sparql_endpoint":"https://ex/sparql",
4242 "source_date":"2025-12-31","derived_from":["https://ex/dump"],
4243 "doi":"https://doi.org/10.1/x","cite_as":"Cite me","keywords":["b","a"],
4244 "theme":["http://publications.europa.eu/resource/authority/data-theme/EDUC"],
4245 "extra":{"k":[{"words":["x"]}]},"example_queries":["ASK {}"]}"##;
4246 let curated = CardInput::from_json_str(doc).unwrap();
4247 assert!(!curated.is_empty());
4248 let card = derive_card(&enriched_fixture(), 12, 0, curated);
4249 let again = derive_card(&enriched_fixture(), 12, 0, CardInput::from_card(&card));
4250 assert_eq!(again.to_json_bytes(), card.to_json_bytes());
4251 assert!(CardInput::from_card(&derive_card(
4252 &enriched_fixture(),
4253 12,
4254 0,
4255 CardInput::default()
4256 ))
4257 .is_empty());
4258 }
4259}
4260
4261#[cfg(test)]
4262mod ns_tally_tests {
4263 use super::*;
4264
4265 fn feed(t: &mut NsTally, n: usize) {
4266 for i in 0..n {
4267 let ns = if i % 7 == 0 {
4269 "https://doi.org/10.1/".to_string()
4270 } else {
4271 format!("https://doi.org/10.1/a{i}/")
4272 };
4273 t.add(ns, 1);
4274 }
4275 }
4276
4277 #[test]
4278 fn under_the_cap_the_tally_is_exact() {
4279 let mut t = NsTally::with_cap(100);
4280 feed(&mut t, 50);
4281 assert!(!t.overflowed());
4282 assert_eq!(t.len(), 43);
4283 assert_eq!(t.dominant().as_deref(), Some("https://doi.org/10.1/"));
4284 assert_eq!(
4285 derive_base_iri(&t).as_deref(),
4286 Some("https://doi.org/10.1/")
4287 );
4288 }
4289
4290 #[test]
4291 fn past_the_cap_the_tally_stays_bounded_and_deterministic() {
4292 let run = || {
4293 let mut t = NsTally::with_cap(64);
4294 feed(&mut t, 5_000);
4295 assert!(t.len() <= 64);
4296 assert!(t.overflowed());
4297 let snapshot: Vec<(String, u64)> =
4298 t.counts.iter().map(|(k, v)| (k.clone(), *v)).collect();
4299 (snapshot, t.dominant(), derive_base_iri(&t))
4300 };
4301 let (a, b) = (run(), run());
4302 assert_eq!(a, b);
4303 assert_eq!(a.1.as_deref(), Some("https://doi.org/10.1/"));
4306 assert_eq!(a.2.as_deref(), Some("https://doi.org/10.1/"));
4307 }
4308
4309 #[test]
4310 fn common_prefix_spans_all_hosts_even_when_entries_were_evicted() {
4311 let mut t = NsTally::with_cap(2);
4312 t.add("http://a.org/x/".into(), 10);
4313 t.add("http://a.org/y/".into(), 1);
4314 t.add("http://b.org/z/".into(), 1);
4315 assert!(t.overflowed());
4316 assert_eq!(t.common_prefix(), Some("http://"));
4317 assert_eq!(derive_base_iri(&t).as_deref(), Some("http://a.org/x/"));
4319 }
4320
4321 #[test]
4324 fn card_is_truncated_and_reproducible_past_the_cap() {
4325 let n = NS_TALLY_CAP + 2_000;
4326 let quads: Vec<_> = (0..n)
4327 .map(|i| {
4328 (
4329 format!("<https://doi.org/10.1/{i}/x>"),
4330 "<http://ex/p>".to_string(),
4331 "\"v\"".to_string(),
4332 None,
4333 )
4334 })
4335 .collect();
4336 let (bytes, _) =
4337 crate::ingest::assemble_dataset_with_opts(quads.clone(), true, false, None, |_, _| {
4338 Vec::new()
4339 });
4340 let rete = crate::Rete::open(&bytes).unwrap();
4341 let go = || derive_card_streaming(&rete, CardInput::default(), &mut |_| {}).unwrap();
4342 let (c1, r1) = go();
4343 let (c2, _) = go();
4344 assert!(r1.namespaces_overflowed);
4345 assert_eq!(r1.namespaces_kept as usize, NS_TALLY_CAP);
4346 assert!(c1.truncated);
4347 assert_eq!(c1.to_json_bytes(), c2.to_json_bytes());
4348 let mem = derive_card(&quads, rete.header().term_count, 0, CardInput::default());
4349 assert!(mem.truncated);
4350 assert_eq!(c1.signals.base_iri, mem.signals.base_iri);
4351 }
4352}
4353
4354#[cfg(test)]
4355mod class_map_tests {
4356 use super::ClassMap;
4357
4358 #[test]
4359 fn widening_keeps_every_code_and_never_holds_two_maps() {
4360 let mut m = ClassMap::new(999);
4361 for id in 0..1000u32 {
4362 m.set(id, id % 200);
4363 }
4364 assert_eq!(m.bytes(), 1000);
4365 m.set(5, 300);
4366 assert_eq!(m.bytes(), 2000);
4367 assert_eq!(m.get(5), 300);
4368 for id in (0..1000u32).filter(|&i| i != 5) {
4369 assert_eq!(m.get(id), id % 200, "{id}");
4370 }
4371 m.set(7, 70_000);
4372 assert_eq!(m.bytes(), 4000);
4373 assert_eq!((m.get(7), m.get(5), m.get(999)), (70_000, 300, 999 % 200));
4374 assert_eq!(m.get(5000), 0);
4376 m.set(5000, 1);
4377 assert_eq!(m.bytes(), 4000);
4378 }
4379}