Skip to main content

rete_core/
card_derive.rs

1//! Dataset Cards — the **derived** half: the card type itself and the profile
2//! computed from a graph's own statements.
3//!
4//! A card is an embeddable data-catalog record stored in a `.rete` file's
5//! metadata section. It carries **curated** metadata (title, license, source,
6//! description, created, keywords/theme, example queries, plus a bounded bag of
7//! publisher-defined custom fields under `extra` — see [`card_input`]) and an
8//! **auto-derived** profile (counts, top predicates and classes, vocabularies,
9//! datatypes, languages, class links, hubs, signals, and the tiered
10//! starter-query library of [`card_queries`]), serialized as JSON.
11//!
12//! # Why this lives in `rete-core` and not in the CLI
13//!
14//! It used to live in `rete-cli`, which is a **binary-only crate** — no `lib.rs`,
15//! so nothing can link it. A `.rete` built from Python, R, Java, JavaScript or
16//! the in-browser builder could therefore write the curated half of a card and
17//! *never* the derived half, no matter how the CLI was fixed (#152). The
18//! derivation touches nothing but `std::collections`, `serde` and this crate, so
19//! there was never a technical reason for the split: it is here now, and one
20//! correction holds for every surface.
21//!
22//! Everything that is *not* pure derivation stays in the CLI — reading a
23//! `--card-file` off disk, range-reading a card over HTTP, rendering the human
24//! catalog view, measuring query costs. Those are the parts that would have made
25//! this module unusable from wasm.
26//!
27//! [`card_input`]: crate::card_input
28//! [`card_queries`]: crate::card_queries
29
30use std::collections::{BTreeMap, BTreeSet};
31
32use serde::{Deserialize, Serialize};
33
34use crate::{Header, RDF_TYPE};
35/// What a hot card says about its annex (see [`DatasetCard::annex`]).
36#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
37pub struct CardAnnexMarker {
38    /// The card fields that live in the annex.
39    pub fields: Vec<String>,
40    /// The annex's size, uncompressed JSON bytes.
41    pub bytes: u64,
42}
43
44/// The embeddable dataset card. Curated fields come from `rete build` flags or a
45/// `--card-file` JSON document; the statistics are derived from the data at build
46/// time. Absent optional/empty fields are omitted from the JSON.
47#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
48pub struct DatasetCard {
49    #[serde(default, skip_serializing_if = "Option::is_none")]
50    pub title: Option<String>,
51    #[serde(default, skip_serializing_if = "Option::is_none")]
52    pub description: Option<String>,
53    #[serde(default, skip_serializing_if = "Option::is_none")]
54    pub license: Option<String>,
55    #[serde(default, skip_serializing_if = "Option::is_none")]
56    pub source: Option<String>,
57    #[serde(default, skip_serializing_if = "Option::is_none")]
58    pub created: Option<String>,
59
60    // --- Curated identity & provenance (issue #153). All deterministic
61    // (hand-supplied via `--card-file`), so they live INSIDE the content hash;
62    // the per-build volatile facts (timestamp, builder, timings) live in the
63    // separate unhashed build-info section instead. ---
64    /// The publisher's **dataset version** (e.g. a date or semver) — Croissant
65    /// requires one. One of three distinct versions a `.rete` carries, each
66    /// with its own owner, never merged: `version` (the *data*, set by the
67    /// publisher here), `format_version` (the *spec* the file conforms to,
68    /// stamped by the builder), and the builder's own identity
69    /// (`rete-cli 0.3.2`, in the build-info section).
70    #[serde(default, skip_serializing_if = "Option::is_none")]
71    pub version: Option<String>,
72    /// The people who made the dataset. ORCID as an IRI, not a string, so a
73    /// card's creator is joinable against the published ORCID graph.
74    #[serde(default, skip_serializing_if = "Vec::is_empty")]
75    pub creators: Vec<Creator>,
76    /// Publishing organisation, with its ROR IRI for the same joinability.
77    #[serde(default, skip_serializing_if = "Option::is_none")]
78    pub publisher: Option<Publisher>,
79    /// Where the authoritative copy of THIS file lives (`void:dataDump`). A
80    /// `.rete` found on a disk can then say where to verify against.
81    #[serde(default, skip_serializing_if = "Option::is_none")]
82    pub canonical_url: Option<String>,
83    /// A public SPARQL endpoint serving this dataset (`void:sparqlEndpoint`).
84    #[serde(default, skip_serializing_if = "Option::is_none")]
85    pub sparql_endpoint: Option<String>,
86    /// The SOURCE data's own date (harvest/snapshot), distinct from both the
87    /// curated `created` and the build-info timestamp.
88    #[serde(default, skip_serializing_if = "Option::is_none")]
89    pub source_date: Option<String>,
90    /// What this file was derived from (`prov:wasDerivedFrom`): source dumps,
91    /// upstream `.rete` shards a merge folded in, an endpoint harvested.
92    #[serde(default, skip_serializing_if = "Vec::is_empty")]
93    pub derived_from: Vec<String>,
94    /// DOI of the dataset, as an IRI (`https://doi.org/…`).
95    #[serde(default, skip_serializing_if = "Option::is_none")]
96    pub doi: Option<String>,
97    /// Preferred citation text.
98    #[serde(default, skip_serializing_if = "Option::is_none")]
99    pub cite_as: Option<String>,
100    /// Free-text keywords/tags describing the dataset. First-class rather
101    /// than an `extra` entry because keywords carry **agreed meaning**:
102    /// `dcat:keyword` and `schema:keywords` are long-standing terms (DCAT-AP
103    /// catalogs require the former; dataset-search harvesters read the
104    /// latter), and the bag's contract is precisely the *absence* of a term —
105    /// it projects as opaque values. Canonicalized by
106    /// [`crate::card_input::normalize_string_list`] at build time: trimmed, sorted, deduplicated
107    /// — `dcat:keyword` is an unordered repeated property, so sorting loses
108    /// nothing and keeps the card's bytes (hence the reproducible content
109    /// hash) independent of authoring order.
110    #[serde(default, skip_serializing_if = "Vec::is_empty")]
111    pub keywords: Vec<String>,
112    /// Curated **themes** — IRIs into a controlled vocabulary
113    /// (`dcat:theme`, e.g. the EU data-theme authority). IRIs are
114    /// **required** ([`crate::card_input::normalize_themes`]): a free-text theme is a keyword by
115    /// another name and belongs in `keywords` — the IRI into an agreed
116    /// scheme is exactly what makes `dcat:theme` worth a separate field.
117    /// Sorted/deduplicated like `keywords`.
118    #[serde(default, skip_serializing_if = "Vec::is_empty")]
119    pub theme: Vec<String>,
120    // There is deliberately NO curated language field next to these: in RDF
121    // the language rides on each literal (`@lang` / `rdf:langString`), so the
122    // dataset's languages are DERIVABLE — and derived below (`languages`,
123    // `signals.default_lang`). A curated duplicate would be a second source
124    // of truth that can drift from the data with no way to tell which is
125    // right. See "First-class field or the bag?" in docs/dataset-cards.md.
126    /// Publisher-defined **custom fields** — one bounded, reserved bag.
127    /// Anything under `extra` is by definition *not* a rete-defined field:
128    /// official card fields are only ever added at the top level (which
129    /// [`CardInput`] keeps reserved by rejecting unknown keys), so a future
130    /// release can never collide with a publisher's key. Curated input, so
131    /// the bag folds into the content hash; [`crate::card_input::normalize_extra`] canonicalizes
132    /// (sorts) nested object keys and enforces the `CARD_EXTRA_*` bounds at
133    /// build time. Readers accept whatever is present — the limits bite when
134    /// writing, never when reading.
135    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
136    pub extra: BTreeMap<String, serde_json::Value>,
137    /// Set by the format layer when the file's card has an **annex** (format
138    /// `0x07`): the fields named here were too big for the hot card and live in
139    /// a cold section; `rete card --full` and
140    /// [`crate::file::read_card_full_ranged`] bring them back. Never authored:
141    /// [`CardInput`] has no such field, and the writer states its own.
142    #[serde(default, skip_serializing_if = "Option::is_none")]
143    pub annex: Option<CardAnnexMarker>,
144
145    pub triple_count: u64,
146    pub quad_count: u64,
147    pub named_graph_count: u64,
148    pub term_count: u64,
149
150    #[serde(default, skip_serializing_if = "Vec::is_empty")]
151    pub predicates: Vec<(String, u64)>,
152    #[serde(default, skip_serializing_if = "Vec::is_empty")]
153    pub classes: Vec<(String, u64)>,
154    #[serde(default, skip_serializing_if = "Vec::is_empty")]
155    pub vocabularies: Vec<String>,
156    #[serde(default, skip_serializing_if = "Vec::is_empty")]
157    pub example_queries: Vec<String>,
158
159    // --- Enriched, auto-derived profile (all additive; `#[serde(default)]` so
160    // older cards without these fields still deserialize). Every list is capped
161    // (see `CARD_TOP_N`) and deterministically ordered so the card folds into a
162    // reproducible content hash; `truncated` records whether any list was cut. ---
163    /// `DATATYPE(o)` histogram over literal objects, descending by count. Keys are
164    /// bracketed datatype IRIs (`<…#integer>`, `<…#langString>` for `@lang`).
165    #[serde(default, skip_serializing_if = "Vec::is_empty")]
166    pub datatypes: Vec<(String, u64)>,
167    /// `LANG(o)` histogram over literal objects, descending by count. The empty
168    /// string `""` counts untagged/typed literals.
169    #[serde(default, skip_serializing_if = "Vec::is_empty")]
170    pub languages: Vec<(String, u64)>,
171    /// The effective schema: `(s_class, predicate, o_class, count)` over the
172    /// default graph, the same quotient as [`crate::schema_summary`]. Object
173    /// classes use the `(literal)`/`(untyped)` sentinels of that function.
174    #[serde(default, skip_serializing_if = "Vec::is_empty")]
175    pub class_links: Vec<ClassLink>,
176    /// Top subjects by out-degree (how many statements they make), descending.
177    #[serde(default, skip_serializing_if = "Vec::is_empty")]
178    pub top_hubs: Vec<(String, u64)>,
179    /// Top non-literal objects by in-degree (how often referenced), descending.
180    #[serde(default, skip_serializing_if = "Vec::is_empty")]
181    pub in_hubs: Vec<(String, u64)>,
182    /// Detected affordances — which exploration query families the data supports
183    /// and the vocabulary to instantiate them with.
184    #[serde(default, skip_serializing_if = "Signals::is_empty")]
185    pub signals: Signals,
186    /// Auto-generated, tiered starter-query library — vetted SPARQL with the
187    /// dataset's own vocabulary substituted in, each tagged with the cheapest
188    /// tier that can answer it. The cold-start "what do I ask?" fix.
189    #[serde(default, skip_serializing_if = "Vec::is_empty")]
190    pub queries: Vec<ExampleQuery>,
191    /// OWL RL / RDFS coherence verdict, stamped at build time by `--reason`. Lets a
192    /// remote reader learn the graph's coherence from the index-free card with zero
193    /// compute; `rete reason --verify-card` recomputes it to guard against drift.
194    #[serde(default, skip_serializing_if = "Coherence::is_empty")]
195    pub coherence: Coherence,
196    /// Set iff any capped list was actually truncated (the profile is partial).
197    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
198    pub truncated: bool,
199    /// The top-N cap the profile lists were derived under — the number
200    /// `truncated: true` was hinting at without stating. 0 (omitted) on cards
201    /// whose profile was never derived (external builds, pre-existing cards).
202    #[serde(default, skip_serializing_if = "is_zero_u32")]
203    pub top_n: u32,
204
205    /// The `.rete` **spec version** the file was written against (currently
206    /// 5). The format's version — not the dataset's (the curated `version`
207    /// above) and not the builder's (`builder` in build-info).
208    pub format_version: u8,
209}
210
211/// serde helper: omit a `0` cap (profile-less cards).
212fn is_zero_u32(v: &u32) -> bool {
213    *v == 0
214}
215
216/// A dataset creator: a person (or team) with an optional ORCID IRI.
217#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
218pub struct Creator {
219    pub name: String,
220    /// `https://orcid.org/0000-0000-0000-0000`
221    #[serde(default, skip_serializing_if = "Option::is_none")]
222    pub orcid: Option<String>,
223}
224
225/// The publishing organisation, with an optional ROR IRI.
226#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
227pub struct Publisher {
228    pub name: String,
229    /// `https://ror.org/…`
230    #[serde(default, skip_serializing_if = "Option::is_none")]
231    pub ror: Option<String>,
232}
233
234/// Which tier of the three-tier exploration model can answer a query cheapest:
235/// `Card` (precomputed in this metadata section, index-free), `Summary` (the
236/// pyramid superedge totals, index-free), or `Index` (needs the triple index).
237#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
238#[serde(rename_all = "lowercase")]
239pub enum Tier {
240    Card,
241    Summary,
242    Index,
243}
244
245/// One vetted starter query, instantiated with the dataset's real vocabulary.
246#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
247pub struct ExampleQuery {
248    /// Stable identifier (e.g. `ov-triples`).
249    pub id: String,
250    /// Short human title.
251    pub title: String,
252    /// Exploration dimension (overview/identity/labels/types/topology/…).
253    pub dimension: String,
254    /// The plain-language question a newcomer would ask.
255    pub question: String,
256    /// Full runnable SPARQL, PREFIX block included, placeholders substituted.
257    pub sparql: String,
258    /// The cheapest tier that can answer it.
259    pub tier: Tier,
260    /// Capability keys that had to be present to emit this query.
261    #[serde(default, skip_serializing_if = "Vec::is_empty")]
262    pub requires: Vec<String>,
263}
264
265/// One edge of the effective schema graph: a `predicate` connecting subjects of
266/// class `s_class` to objects of class `o_class`, with the instance `count`.
267/// Mirrors [`crate::schema_summary`] rows (sentinels `(literal)`/`(untyped)`).
268#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
269pub struct ClassLink {
270    pub s_class: String,
271    pub predicate: String,
272    pub o_class: String,
273    pub count: u64,
274}
275
276/// Detected affordances — the index-free hints a newcomer (or the query-library
277/// generator) needs to know what the data supports and how to address it. Every
278/// field is optional/empty when the corresponding signal is absent, so geo/time
279/// query families are emitted only when the data actually has geometry/time.
280#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
281pub struct Signals {
282    /// Most frequent labelling predicate (`rdfs:label`/`skos:prefLabel`/…).
283    #[serde(default, skip_serializing_if = "Option::is_none")]
284    pub label_predicate: Option<String>,
285    /// Dominant non-empty language tag over literal objects.
286    #[serde(default, skip_serializing_if = "Option::is_none")]
287    pub default_lang: Option<String>,
288    /// Dominant subject namespace — the dataset's own base IRI.
289    #[serde(default, skip_serializing_if = "Option::is_none")]
290    pub base_iri: Option<String>,
291    /// Temporal predicates, ranked by frequency (most-used first).
292    #[serde(default, skip_serializing_if = "Vec::is_empty")]
293    pub time_predicates: Vec<String>,
294    /// Predicates whose objects are numeric (xsd integer/decimal/double/float),
295    /// ranked by frequency — candidates for `MIN/AVG/MAX` value-range queries.
296    #[serde(default, skip_serializing_if = "Vec::is_empty")]
297    pub numeric_predicates: Vec<String>,
298    /// Cross-dataset link predicates present (`owl:sameAs`/`skos:exactMatch`/…).
299    #[serde(default, skip_serializing_if = "Vec::is_empty")]
300    pub link_predicates: Vec<String>,
301    /// Predicates that make a statement **about a statement** — the predicate
302    /// of a statement whose *subject* is a quoted triple
303    /// (`<< s p o >> :recordedBy :who`) — ranked by frequency.
304    ///
305    /// Derived from the triples, and therefore **stored**, unlike the
306    /// [`Signals::quoted_triples`] presence bit below, which is read off the
307    /// header at display time. The two answer different questions and neither
308    /// substitutes for the other: presence says *will I meet triple terms in
309    /// this file*, which every already-published file can answer for free;
310    /// this says *which vocabulary annotates statements here*, which only a
311    /// pass over the statements knows and which is what a starter query has to
312    /// be written in. Empty on the overwhelming majority of datasets, and the
313    /// whole `qt-*` query family is gated on it.
314    #[serde(default, skip_serializing_if = "Vec::is_empty")]
315    pub annotation_predicates: Vec<String>,
316    /// Predicates whose **object** is a quoted triple (`:claim :states
317    /// << s p o >>`) — the other place a triple term can stand — ranked by
318    /// frequency. A dataset can have one shape and not the other, and a query
319    /// written for the wrong one returns nothing, so they are counted apart.
320    #[serde(default, skip_serializing_if = "Vec::is_empty")]
321    pub quoting_predicates: Vec<String>,
322    /// The commonest **annotated statement shape**: `(the predicate inside the
323    /// quoted triple, the predicate that annotates it)`, for a statement this
324    /// graph was seen both to **assert** (`s p o`) and to **annotate**
325    /// (`<< s p o >> :about ?v`).
326    ///
327    /// The pair is the witness — the same role `class_links` plays for
328    /// `LABELED_CLASS`. `annotation_predicates[0]` and the commonest quoted
329    /// predicate are two independent maxima and need not describe the same
330    /// statement, so a body that pins both is only safe when the derivation
331    /// saw them together. Assertedness is part of the witness because RDF-star
332    /// does not entail it: `<< s p o >> :about ?v` says nothing about whether
333    /// `s p o` is in the graph, and a coverage query over a relation the graph
334    /// never asserts would report `0 / 0`.
335    ///
336    /// Matched over a sample of the star layer (`ANNOTATION_WITNESS_SAMPLE`),
337    /// so a miss can drop the query that needs it but can never ship a broken
338    /// one.
339    #[serde(default, skip_serializing_if = "Option::is_none")]
340    pub annotated_statement: Option<(String, String)>,
341    /// Any `geo:asWKT` / `geo:wktLiteral` geometry present.
342    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
343    pub geo_wkt: bool,
344    /// Both `wgs84:lat` and `wgs84:long` present.
345    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
346    pub geo_latlong: bool,
347    /// MIN/MAX lexical extent over the dominant time predicate (same datatype).
348    #[serde(default, skip_serializing_if = "Option::is_none")]
349    pub temporal_extent: Option<(String, String)>,
350    /// `[minLon, minLat, maxLon, maxLat]` over wgs84 lat/long objects.
351    #[serde(default, skip_serializing_if = "Option::is_none")]
352    pub spatial_bbox: Option<[f64; 4]>,
353    /// Whether the file carries a **full-text (TEXT_INDEX) section**.
354    ///
355    /// Unlike every other signal this one is **not derived from the triples and
356    /// not stored in the card**: it is measured from the file's section
357    /// directory by whoever reads the card, and [`DatasetCard::to_json_bytes`]
358    /// strips it before the bytes are written. See [`TextIndexSignal`] for why.
359    ///
360    /// `None` therefore means **unknown** — a card read out of a saved JSON
361    /// document, with no file to measure — and never "no index".
362    #[serde(default, skip_serializing_if = "Option::is_none")]
363    pub text_index: Option<TextIndexSignal>,
364    /// Whether the file carries a **label index (LABEL_INDEX section, kind 8)**,
365    /// and how large it is. Like [`Signals::text_index`], measured from the
366    /// file at read time and stripped before the card's bytes are written;
367    /// `None` means unknown, never "no index". See [`LabelIndexSignal`].
368    #[serde(default, skip_serializing_if = "Option::is_none")]
369    pub label_index: Option<LabelIndexSignal>,
370    /// Which **index permutations** the file stores. Like
371    /// [`Signals::text_index`], measured from the file at read time and
372    /// stripped before the card's bytes are written; `None` means unknown (a
373    /// card document with no file behind it), never "the default".
374    #[serde(default, skip_serializing_if = "Option::is_none")]
375    pub permutations: Option<PermutationsSignal>,
376    /// Whether the file holds any **quoted triple**, and which export surfaces
377    /// a consumer can ask for. Like the two signals above, measured from the
378    /// file at read time and stripped before the card's bytes are written;
379    /// `None` means unknown (a card document with no file behind it), never
380    /// "no quoted triples". See [`QuotedTriplesSignal`].
381    #[serde(default, skip_serializing_if = "Option::is_none")]
382    pub quoted_triples: Option<QuotedTriplesSignal>,
383}
384
385impl Signals {
386    /// True when no affordance was detected (lets the whole block be omitted).
387    pub fn is_empty(&self) -> bool {
388        *self == Signals::default()
389    }
390}
391
392/// The file's full-text-search affordance: does it carry a **TEXT_INDEX section
393/// (kind 6)**, and what does that index cost?
394///
395/// A `.rete` built with `--text-index` answers `FILTER(CONTAINS(…))` by word
396/// lookup; one built without it answers the *same query with the same rows* by
397/// full scan. The capability is invisible from the results, which is exactly how
398/// the playground catalog came to advertise an index two published files never
399/// carried (#189) — so a file has to be able to state it.
400///
401/// **Measured at read time, never stored.** The ground truth is the section
402/// directory in the 1 KiB header that every card read already fetches, so a
403/// stored copy would be a second source of truth for a fact the first source
404/// answers for free — the same reasoning that keeps a curated `language` field
405/// out of the card. Being a projection rather than a stamp also means every
406/// already-published file reports it today, with no re-card, and that
407/// `rete repyramid --text-index` (which rewrites the file's sections) can never
408/// leave it stale.
409///
410/// **What it deliberately does not carry: a token count.** The number of
411/// distinct indexed words is the first varint of the *decompressed* token
412/// table, so quoting it would mean fetching and inflating the whole table —
413/// 193 MB on the published `causenet-full-typed.rete`. [`token_table_bytes`] is
414/// the same question answered for ≤10 bytes, and is the more useful answer
415/// anyway: it is what a first search actually costs.
416///
417/// [`token_table_bytes`]: TextIndexSignal::token_table_bytes
418#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
419pub struct TextIndexSignal {
420    /// True iff the file has a kind-6 section. Always written, including as
421    /// `false`: "measured, and there is none" is the fact #189 was missing.
422    pub present: bool,
423    /// Byte length of the whole TEXT_INDEX section — free, straight from the
424    /// header. Worth stating because it can dominate the download:
425    /// `causenet-full-typed.rete` carries 1.88 GB of it, 29% of the file.
426    #[serde(default, skip_serializing_if = "is_zero_u64")]
427    pub bytes: u64,
428    /// Byte length of the section's leading **token table** — the prefix a
429    /// first search faults, several times smaller than `bytes` (which counts
430    /// the postings blob, fetched one posting at a time). One ≤10-byte range
431    /// read; `None` when there is no index, or its first bytes were unreadable.
432    #[serde(default, skip_serializing_if = "Option::is_none")]
433    pub token_table_bytes: Option<u64>,
434}
435
436/// serde helper: omit a `0` byte count (no section to measure).
437fn is_zero_u64(v: &u64) -> bool {
438    *v == 0
439}
440
441/// The file's **index permutation set**: how many orders of `(s, p, o)` it
442/// stores, and therefore whether its planner can run a sort-merge join.
443///
444/// `rete build --permutations 3` writes SPO, POS and OSP only. Those three tie
445/// the longest bound prefix on all eight triple-pattern shapes, so the file
446/// answers **every query with the same rows, routed to the same tiles** — the
447/// difference is invisible from the results, exactly like a missing
448/// full-text index (#189). What it gives up is the merge join: SOP/PSO/OPS
449/// exist only to hand a join two streams already sorted on the join key, and
450/// they are typically ~40% of a built file. A consumer choosing between two
451/// mirrors of one dataset, or deciding whether a join-heavy workload will hold
452/// up, has to be able to see which it has.
453///
454/// **Measured at read time, never stored**, for the reasons [`TextIndexSignal`]
455/// spells out and one more that is specific to this signal: which permutations
456/// a file carries is a fact about its *bytes*. A stored copy would be an
457/// authored claim about the file's own layout — the one class of statement the
458/// file can always check for itself, in the 1 KiB header every card read has
459/// already fetched. This costs **no extra read at all**.
460#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
461pub struct PermutationsSignal {
462    /// How many permutations are stored: 3 or 6 today.
463    pub count: u8,
464    /// Their names in section order, e.g. `["SPO", "POS", "OSP"]`. Written out
465    /// rather than implied by `count`, because the mask is a set and a future
466    /// build may keep a different three.
467    pub names: Vec<String>,
468    /// Whether the file carries the merge-join orders (SOP, PSO, OPS) — i.e.
469    /// whether a two-pattern join sharing a same-role variable can be answered
470    /// by a linear two-pointer merge instead of a hash join.
471    pub merge_join: bool,
472}
473
474impl PermutationsSignal {
475    /// Read it straight off the parsed header — the mask is one byte at `[50]`,
476    /// and `0` there means all six (every file written before the mask
477    /// existed). No range read of any kind.
478    pub fn probe(header: &Header) -> Self {
479        let perms = header.perms;
480        PermutationsSignal {
481            count: perms.len() as u8,
482            names: perms.names().into_iter().map(str::to_string).collect(),
483            merge_join: perms.has_merge_orders(),
484        }
485    }
486
487    /// One line for the human catalog view and the audit report.
488    pub fn describe(&self) -> String {
489        if self.merge_join {
490            format!(
491                "{} index permutations ({}) — sort-merge joins available",
492                self.count,
493                self.names.join("/")
494            )
495        } else {
496            format!(
497                "{} index permutations ({}) — same rows, same routing; \
498                 no sort-merge join (hash/probe answers instead)",
499                self.count,
500                self.names.join("/")
501            )
502        }
503    }
504}
505
506/// The `--quoted-triple-syntax` value that writes the RDF 1.2 triple term
507/// `<<( s p o )>>`, and `rete export`'s **default**.
508///
509/// Named here, not only in the CLI's flag table, because the dataset card
510/// states which surfaces a consumer can ask for — and a card that says `rdf12`
511/// while the flag has been renamed would be worse than one that says nothing.
512pub const QUOTED_TRIPLE_SURFACE_RDF12: &str = "rdf12";
513
514/// The `--quoted-triple-syntax` value that writes the RDF-star surface
515/// `<<s p o>>`, which is also the token rete stores.
516pub const QUOTED_TRIPLE_SURFACE_RDF_STAR: &str = "rdf-star";
517
518/// Whether the file contains **quoted triples** — statements standing inside
519/// other statements — and, if it does, what a consumer gets when they export it.
520///
521/// This is the question "will I meet triple terms in this dataset, and in which
522/// spelling?", which a consumer needs answered *before* they download a 50 GB
523/// file and run `rete export` on it. The two surfaces are not interchangeable
524/// downstream: an RDF 1.2 parser rejects `<<s p o>>` in N-Quads and silently
525/// reads it as a *reifier* in Turtle/TriG, and an RDF-star parser rejects
526/// `<<( s p o )>>` everywhere. Which one a dump carries decides whether it loads
527/// at all, so it belongs in the card rather than in the reader's memory of a CLI
528/// flag.
529///
530/// **Measured at read time from the header, never stored**, exactly like
531/// [`TextIndexSignal`] and [`PermutationsSignal`], and for the sharpest version
532/// of the same reason: header flag bit 2 (`FLAG_HAS_QUOTED_TRIPLES`) has been
533/// written by every build since quoted triples were supported, so this is an
534/// honest answer for **every file that already exists** — the 44 published
535/// datasets included — with no re-card and no rebuild. A stored copy would be
536/// `null` on all of them, which is worse than nothing: it would assert
537/// "unknown" about files whose own header knows.
538///
539/// `None` therefore means **unknown** — a card read out of a saved JSON
540/// document, with no file to measure — and never "no quoted triples".
541///
542/// There is deliberately **no count**. The header carries presence, not
543/// cardinality, and counting would mean decoding the dictionary — outside the
544/// CARD tier's budget (header + metadata, never the dictionary), and available
545/// only to files built after this existed, which is the failure mode above.
546#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
547pub struct QuotedTriplesSignal {
548    /// Does the file hold at least one quoted triple as a dictionary term?
549    pub present: bool,
550    /// The surfaces `rete export` can write them in, most interoperable first.
551    /// Empty when `present` is false: the question does not arise.
552    #[serde(default, skip_serializing_if = "Vec::is_empty")]
553    pub export_surfaces: Vec<String>,
554    /// Which of `export_surfaces` `rete export` writes with no flag. `None`
555    /// when `present` is false.
556    #[serde(default, skip_serializing_if = "Option::is_none")]
557    pub export_default: Option<String>,
558}
559
560impl QuotedTriplesSignal {
561    /// Read it straight off the parsed header — flag bit 2, in the 1 KiB every
562    /// card read has already fetched. No range read of any kind.
563    pub fn probe(header: &Header) -> Self {
564        if !header.has_quoted_triples() {
565            return QuotedTriplesSignal {
566                present: false,
567                export_surfaces: Vec::new(),
568                export_default: None,
569            };
570        }
571        QuotedTriplesSignal {
572            present: true,
573            export_surfaces: vec![
574                QUOTED_TRIPLE_SURFACE_RDF12.to_string(),
575                QUOTED_TRIPLE_SURFACE_RDF_STAR.to_string(),
576            ],
577            export_default: Some(QUOTED_TRIPLE_SURFACE_RDF12.to_string()),
578        }
579    }
580
581    /// One line for the human catalog view, or `None` when there is nothing to
582    /// say — most datasets have no quoted triples, and a card should not grow a
583    /// line per affordance they do not use.
584    pub fn describe(&self) -> Option<String> {
585        if !self.present {
586            return None;
587        }
588        Some(format!(
589            "present — `rete export` writes RDF 1.2 triple terms `<<( s p o )>>` \
590             by default; `--quoted-triple-syntax {QUOTED_TRIPLE_SURFACE_RDF_STAR}` \
591             writes `<<s p o>>`"
592        ))
593    }
594}
595impl TextIndexSignal {
596    /// Measure the signal from a file's header, through the same
597    /// [`crate::RangeReader`] the card was read with. Costs nothing beyond
598    /// the header already in hand when there is no index, and one ≤10-byte
599    /// range read when there is — never the index itself.
600    pub fn probe<R: crate::RangeReader + ?Sized>(reader: &R, header: &Header) -> Self {
601        if header.text_index_len == 0 {
602            return TextIndexSignal::default();
603        }
604        if crate::v06::reads(header.version) {
605            return match crate::v06::open::read_signals(reader) {
606                Ok((Some((bytes, tt)), _)) => TextIndexSignal {
607                    present: true,
608                    bytes,
609                    token_table_bytes: Some(tt),
610                },
611                _ => TextIndexSignal {
612                    present: true,
613                    ..TextIndexSignal::default()
614                },
615            };
616        }
617        TextIndexSignal {
618            present: true,
619            bytes: header.text_index_len,
620            token_table_bytes: crate::read_text_index_token_table_len_ranged(reader, header),
621        }
622    }
623
624    /// One line for the human catalog view and the audit report.
625    pub fn describe(&self) -> String {
626        if !self.present {
627            return "no TEXT_INDEX section — CONTAINS/regex still answer, by full scan".to_string();
628        }
629        match self.token_table_bytes {
630            Some(tt) => format!(
631                "TEXT_INDEX present — {} bytes ({tt} of them the token table a first search reads)",
632                self.bytes
633            ),
634            None => format!("TEXT_INDEX present — {} bytes", self.bytes),
635        }
636    }
637}
638
639/// The file's label-prefix-search affordance: does it carry a **LABEL_INDEX
640/// section (kind 8)**, how many labels, and how large is it?
641///
642/// **Measured at read time, never stored**, for the reasons
643/// [`TextIndexSignal`] gives: presence and size come free with the header,
644/// and `entries` is the u64 at bytes 20..28 of the section's fixed preamble —
645/// one 32-byte range read, never the index. A file without the section may
646/// still answer a prefix search from the legacy label block of its
647/// pyramid-meta (at most 8,192 labels); that block is not measured here,
648/// because reading it means fetching the whole pyramid-meta.
649#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
650pub struct LabelIndexSignal {
651    /// True iff the file has a kind-8 section. Always written, including as
652    /// `false`.
653    pub present: bool,
654    /// Byte length of the whole LABEL_INDEX section.
655    #[serde(default, skip_serializing_if = "is_zero_u64")]
656    pub bytes: u64,
657    /// Number of `(subject, label)` entries; `None` when there is no index or
658    /// its preamble was unreadable.
659    #[serde(default, skip_serializing_if = "Option::is_none")]
660    pub entries: Option<u64>,
661}
662
663impl LabelIndexSignal {
664    /// Measure the signal from a file's header, through the reader the card
665    /// was read with: nothing beyond the header when there is no index, one
666    /// 32-byte range read when there is.
667    pub fn probe<R: crate::RangeReader + ?Sized>(reader: &R, header: &Header) -> Self {
668        let Some(s) = header.label_index_section() else {
669            return LabelIndexSignal::default();
670        };
671        if crate::v06::reads(header.version) {
672            return match crate::v06::open::read_signals(reader) {
673                Ok((_, Some((bytes, entries)))) => LabelIndexSignal {
674                    present: true,
675                    bytes,
676                    entries,
677                },
678                _ => LabelIndexSignal {
679                    present: true,
680                    ..LabelIndexSignal::default()
681                },
682            };
683        }
684        LabelIndexSignal {
685            present: true,
686            bytes: s.length,
687            entries: crate::label_index::read_label_index_entries_ranged(reader, header),
688        }
689    }
690
691    /// One line for the human catalog view.
692    pub fn describe(&self) -> String {
693        if !self.present {
694            return "no LABEL_INDEX section — a prefix search finds only what a pyramid's \
695                    legacy label block holds, if any"
696                .to_string();
697        }
698        match self.entries {
699            Some(n) => format!("LABEL_INDEX present — {n} labels, {} bytes", self.bytes),
700            None => format!("LABEL_INDEX present — {} bytes", self.bytes),
701        }
702    }
703}
704
705/// The build-time OWL RL / RDFS coherence verdict, stamped into the card by
706/// `rete build --reason`. Deterministic and free of free-text detail (only a
707/// sorted `by_kind` histogram + short tags), so it folds into the file's content
708/// hash without destabilizing it. `scope` records what was checked and `rules` the
709/// ruleset version, so `coherent: true` can't be misread as a guarantee from a
710/// different scope or rule set.
711#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
712pub struct Coherence {
713    /// True iff the reasoner found no incoherent point.
714    pub coherent: bool,
715    /// Number of incoherent points found.
716    pub inconsistency_count: u32,
717    /// `(kind, count)` histogram of incoherent points, sorted by kind.
718    #[serde(default, skip_serializing_if = "Vec::is_empty")]
719    pub by_kind: Vec<(String, u32)>,
720    /// What was checked, e.g. `"default"` (the default graph).
721    pub scope: String,
722    /// The reasoner ruleset version (`crate::REASON_RULESET`).
723    pub rules: String,
724    /// True iff the inferred triples were also materialized into the file.
725    pub materialized: bool,
726}
727
728impl Coherence {
729    /// An unstamped card has the all-default block (omitted from the JSON).
730    pub fn is_empty(&self) -> bool {
731        *self == Coherence::default()
732    }
733
734    /// Build the verdict from a reasoning result over the default graph. The
735    /// `by_kind` histogram is `BTreeMap`-sourced (sorted) and carries no free-text
736    /// `Inconsistency::detail`, so the stamp is byte-stable across rebuilds.
737    pub fn from_reasoning(r: &crate::Reasoning, materialized: bool) -> Self {
738        let mut hist: BTreeMap<&str, u32> = BTreeMap::new();
739        for inc in &r.inconsistencies {
740            *hist.entry(inc.kind).or_default() += 1;
741        }
742        Coherence {
743            coherent: r.inconsistencies.is_empty(),
744            inconsistency_count: r.inconsistencies.len() as u32,
745            by_kind: hist.into_iter().map(|(k, v)| (k.to_string(), v)).collect(),
746            scope: "default".to_string(),
747            rules: crate::REASON_RULESET.to_string(),
748            materialized,
749        }
750    }
751}
752
753/// The curated subset, as supplied by a `--card-file` JSON document (every field
754/// optional). CLI flags override whatever the file provides. The identity/
755/// provenance fields (version, creators, publisher, canonical_url,
756/// sparql_endpoint, source_date, derived_from, doi, cite_as, keywords, theme)
757/// have no CLI flag — they come from the card file only.
758///
759/// `deny_unknown_fields` keeps the card file's top level **reserved for
760/// rete-defined fields**: a stray key is a loud error (usually a typo, or a
761/// custom field that belongs inside `extra`), never a silent drop — and it is
762/// what makes the collision guarantee real: a publisher's field can only ever
763/// live in the bag, so a future official top-level field cannot capture one.
764#[derive(Debug, Clone, Default, Deserialize)]
765#[serde(deny_unknown_fields)]
766pub struct CardInput {
767    pub title: Option<String>,
768    /// A string, or an array of lines joined with `\n` — see
769    /// [`crate::card_input::normalize_description`]. Markdown is allowed here (raw HTML is not); the
770    /// array shape exists because hand-writing `\n` escapes in JSON is awful.
771    #[serde(default, deserialize_with = "de_description")]
772    pub description: Option<String>,
773    pub license: Option<String>,
774    pub source: Option<String>,
775    pub created: Option<String>,
776    pub version: Option<String>,
777    #[serde(default)]
778    pub creators: Vec<Creator>,
779    pub publisher: Option<Publisher>,
780    pub canonical_url: Option<String>,
781    pub sparql_endpoint: Option<String>,
782    pub source_date: Option<String>,
783    #[serde(default)]
784    pub derived_from: Vec<String>,
785    pub doi: Option<String>,
786    pub cite_as: Option<String>,
787    /// Free-text keywords (see [`DatasetCard::keywords`]).
788    #[serde(default)]
789    pub keywords: Vec<String>,
790    /// Controlled-vocabulary theme IRIs (see [`DatasetCard::theme`]).
791    #[serde(default)]
792    pub theme: Vec<String>,
793    /// Publisher-defined custom fields (see [`DatasetCard::extra`]).
794    #[serde(default)]
795    pub extra: BTreeMap<String, serde_json::Value>,
796    #[serde(default)]
797    pub example_queries: Vec<String>,
798    /// A curated `signals.base_iri`, overriding the derived one: an absolute IRI
799    /// that prefixes the dataset's own identifiers. The derived value is the
800    /// longest common subject prefix, which past the namespace-tally cap
801    /// ([`NS_TALLY_CAP`]) degrades to an approximate dominant namespace (DataCite:
802    /// `https://doi.org/10.57451/` instead of `https://doi.org/`). Validated by
803    /// [`normalize`](Self::normalize). Not carried by [`from_card`](Self::from_card):
804    /// a card does not record whether its base was derived or curated.
805    pub base_iri: Option<String>,
806}
807
808/// Deserialize `description` from either a JSON string or an array of lines.
809///
810/// A description may be Markdown, and Markdown needs line breaks; in a JSON
811/// string those are `\n` escapes, which are miserable to write by hand and
812/// worse to review in a diff. An array of lines — joined with `\n` — reads as
813/// the Markdown it is. It is **input sugar only**: the card stores one string
814/// either way, so `rete card --json` output feeds straight back into
815/// `--card-file`. The shared rule lives in `crate::card` so the browser
816/// builder accepts exactly the same two shapes.
817fn de_description<'de, D>(d: D) -> Result<Option<String>, D::Error>
818where
819    D: serde::Deserializer<'de>,
820{
821    use serde::de::Error as _;
822    let v = Option::<serde_json::Value>::deserialize(d)?;
823    match v {
824        None | Some(serde_json::Value::Null) => Ok(None),
825        Some(v) => crate::card_input::normalize_description(&v)
826            .map(Some)
827            .map_err(D::Error::custom),
828    }
829}
830
831impl CardInput {
832    /// The curated half of an existing card, **verbatim** — the inverse of
833    /// `fill`. What `rete recard --keep-curated` hands the new
834    /// card, so a re-card carries a publisher's title, licence and prose
835    /// byte for byte instead of re-typing them (or, as a bare
836    /// `repyramid --card` did, silently dropping them).
837    pub fn from_card(card: &DatasetCard) -> Self {
838        CardInput {
839            title: card.title.clone(),
840            description: card.description.clone(),
841            license: card.license.clone(),
842            source: card.source.clone(),
843            created: card.created.clone(),
844            version: card.version.clone(),
845            creators: card.creators.clone(),
846            publisher: card.publisher.clone(),
847            canonical_url: card.canonical_url.clone(),
848            sparql_endpoint: card.sparql_endpoint.clone(),
849            source_date: card.source_date.clone(),
850            derived_from: card.derived_from.clone(),
851            doi: card.doi.clone(),
852            cite_as: card.cite_as.clone(),
853            keywords: card.keywords.clone(),
854            theme: card.theme.clone(),
855            extra: card.extra.clone(),
856            example_queries: card.example_queries.clone(),
857            base_iri: None,
858        }
859    }
860
861    /// True when no curated field is set — a card made of derived facts only.
862    pub fn is_empty(&self) -> bool {
863        self.title.is_none()
864            && self.description.is_none()
865            && self.license.is_none()
866            && self.source.is_none()
867            && self.created.is_none()
868            && self.version.is_none()
869            && self.creators.is_empty()
870            && self.publisher.is_none()
871            && self.canonical_url.is_none()
872            && self.sparql_endpoint.is_none()
873            && self.source_date.is_none()
874            && self.derived_from.is_empty()
875            && self.doi.is_none()
876            && self.cite_as.is_none()
877            && self.keywords.is_empty()
878            && self.theme.is_empty()
879            && self.extra.is_empty()
880            && self.example_queries.is_empty()
881            && self.base_iri.is_none()
882    }
883
884    /// Move every curated field into a card (the shared step of all derivation
885    /// paths, so a new curated field is wired exactly once).
886    fn fill(self, card: &mut DatasetCard) {
887        card.title = self.title;
888        card.description = self.description;
889        card.license = self.license;
890        card.source = self.source;
891        card.created = self.created;
892        card.version = self.version;
893        card.creators = self.creators;
894        card.publisher = self.publisher;
895        card.canonical_url = self.canonical_url;
896        card.sparql_endpoint = self.sparql_endpoint;
897        card.source_date = self.source_date;
898        card.derived_from = self.derived_from;
899        card.doi = self.doi;
900        card.cite_as = self.cite_as;
901        card.keywords = self.keywords;
902        card.theme = self.theme;
903        card.extra = self.extra;
904        card.example_queries = self.example_queries;
905        if self.base_iri.is_some() {
906            card.signals.base_iri = self.base_iri;
907        }
908    }
909
910    /// Parse a curated card **document** — the JSON a publisher writes for
911    /// `rete build --card-file`, or a client hands its builder — applying the
912    /// same write-time rules the CLI applies.
913    ///
914    /// This is the client-side twin of `rete-cli`'s `load_curated`, minus the
915    /// file read: same `deny_unknown_fields` top level, same
916    /// [`UNKNOWN_FIELD_HINT`] pointing a stray key at the `extra` bag, same
917    /// [`normalize`](Self::normalize) pass afterwards. A Python/R/JS caller
918    /// that parses its card this way is held to exactly the rules the CLI and
919    /// the browser builder enforce, instead of the "is it an object?" check
920    /// those bindings used to do on their own.
921    ///
922    /// [`UNKNOWN_FIELD_HINT`]: crate::card_input::UNKNOWN_FIELD_HINT
923    pub fn from_json_str(text: &str) -> Result<Self, String> {
924        let parsed: CardInput = serde_json::from_str(text).map_err(|e| {
925            // The top level is reserved for rete-defined fields; point a stray
926            // key at the bag instead of leaving a bare serde error.
927            let hint = if e.to_string().contains("unknown field") {
928                crate::card_input::UNKNOWN_FIELD_HINT
929            } else {
930                ""
931            };
932            format!("{e}{hint}")
933        })?;
934        parsed.normalize()
935    }
936
937    /// Canonicalize and bounds-check every curated field — the write-time gate
938    /// each card-writing path funnels through, applied **after** any per-field
939    /// override a caller layered on top of a card file.
940    ///
941    /// The rules themselves live in [`crate::card_input`]; this is the
942    /// serde-typed face of them, so `rete build --card-file`, the browser
943    /// builder and every language binding reject the same documents with the
944    /// same words.
945    pub fn normalize(self) -> Result<Self, String> {
946        self.normalize_max(
947            crate::card_input::CARD_DESCRIPTION_MAX_BYTES,
948            crate::card_input::CARD_EXTRA_MAX_BYTES,
949        )
950    }
951
952    /// [`normalize`](Self::normalize) for a card that goes into a format `0x07`
953    /// file: `extra` and `description` may be up to
954    /// [`CARD_ANNEX_FIELD_MAX_BYTES`](crate::card_input::CARD_ANNEX_FIELD_MAX_BYTES);
955    /// the writer moves a field over the 8 KiB hot cap to the card annex.
956    pub fn normalize_with_annex(self) -> Result<Self, String> {
957        let max = crate::card_input::CARD_ANNEX_FIELD_MAX_BYTES;
958        self.normalize_max(max, max)
959    }
960
961    /// [`from_json_str`](Self::from_json_str) for a card that may have an annex
962    /// (see [`normalize_with_annex`](Self::normalize_with_annex)): the
963    /// `description` is bounded by the annex cap instead of the hot one.
964    pub fn from_json_str_with_annex(text: &str) -> Result<Self, String> {
965        let mut doc: serde_json::Value = serde_json::from_str(text).map_err(|e| e.to_string())?;
966        // The description's own parse-time cap is the hot one; take it out and
967        // bound it by the annex cap here.
968        let description = match doc.as_object_mut().and_then(|o| o.remove("description")) {
969            None | Some(serde_json::Value::Null) => None,
970            Some(v) => Some(crate::card_input::description_text(&v)?),
971        };
972        let mut parsed: CardInput = serde_json::from_value(doc).map_err(|e| {
973            let hint = if e.to_string().contains("unknown field") {
974                crate::card_input::UNKNOWN_FIELD_HINT
975            } else {
976                ""
977            };
978            format!("{e}{hint}")
979        })?;
980        parsed.description = description;
981        parsed.normalize_with_annex()
982    }
983
984    fn normalize_max(mut self, description_max: usize, extra_max: usize) -> Result<Self, String> {
985        if let Some(d) = &self.description {
986            crate::card_input::check_description_len_max(d, description_max)?;
987        }
988        self.keywords = crate::card_input::normalize_string_list(
989            "keywords",
990            std::mem::take(&mut self.keywords),
991        )?;
992        self.theme = crate::card_input::normalize_themes(std::mem::take(&mut self.theme))?;
993        let bag: serde_json::Map<String, serde_json::Value> =
994            std::mem::take(&mut self.extra).into_iter().collect();
995        self.extra = crate::card_input::normalize_extra_max(bag, extra_max)?
996            .into_iter()
997            .collect();
998        if let Some(b) = &self.base_iri {
999            crate::card_input::check_base_iri(b)?;
1000        }
1001        Ok(self)
1002    }
1003}
1004
1005impl DatasetCard {
1006    /// Serialize to the JSON bytes stored in the metadata section (compact).
1007    ///
1008    /// [`Signals::text_index`] is **stripped** on the way out: it is measured
1009    /// from the file's section directory at read time, and a copy inside the
1010    /// hashed metadata section would be a claim that can outlive the sections it
1011    /// describes (`rete repyramid --text-index` rewrites them). Stripping here —
1012    /// the single choke point every card-writing command funnels through — is
1013    /// what makes "derived, never authored" true of the bytes and not only of
1014    /// the intent. (`CardInput` has no `signals` field and rejects unknown keys,
1015    /// so a hand-written card file cannot set it either.)
1016    pub fn to_json_bytes(&self) -> Vec<u8> {
1017        if self.signals.text_index.is_none()
1018            && self.signals.label_index.is_none()
1019            && self.signals.permutations.is_none()
1020            && self.signals.quoted_triples.is_none()
1021        {
1022            return serde_json::to_vec(self).expect("DatasetCard serializes");
1023        }
1024        let mut stored = self.clone();
1025        stored.signals.text_index = None;
1026        stored.signals.label_index = None;
1027        stored.signals.permutations = None;
1028        stored.signals.quoted_triples = None;
1029        serde_json::to_vec(&stored).expect("DatasetCard serializes")
1030    }
1031
1032    /// Attach the read-time [`TextIndexSignal`], returning whatever the card's
1033    /// own bytes claimed before it (normally `None` — the writers strip it).
1034    ///
1035    /// A `Some(_)` return is drift: a card asserting a full-text index that no
1036    /// longer has to be believed, because the file itself was just measured.
1037    /// `rete card-audit` reports it.
1038    pub fn observe_text_index(&mut self, measured: TextIndexSignal) -> Option<TextIndexSignal> {
1039        self.signals.text_index.replace(measured)
1040    }
1041
1042    /// Attach the read-time [`LabelIndexSignal`]. Same contract as
1043    /// [`observe_text_index`](Self::observe_text_index).
1044    pub fn observe_label_index(&mut self, measured: LabelIndexSignal) -> Option<LabelIndexSignal> {
1045        self.signals.label_index.replace(measured)
1046    }
1047
1048    /// Attach the read-time [`PermutationsSignal`]. Same contract as
1049    /// [`observe_text_index`](Self::observe_text_index): a `Some(_)` return is
1050    /// a card whose bytes claimed a permutation set, which the writers strip
1051    /// and `CardInput` cannot author.
1052    pub fn observe_permutations(
1053        &mut self,
1054        measured: PermutationsSignal,
1055    ) -> Option<PermutationsSignal> {
1056        self.signals.permutations.replace(measured)
1057    }
1058
1059    /// Attach the read-time [`QuotedTriplesSignal`]. Same contract as
1060    /// [`observe_permutations`](Self::observe_permutations): the writers strip
1061    /// it and `CardInput` cannot author it, so a `Some(_)` return is a card
1062    /// whose bytes claimed something about the file's own header.
1063    pub fn observe_quoted_triples(
1064        &mut self,
1065        measured: QuotedTriplesSignal,
1066    ) -> Option<QuotedTriplesSignal> {
1067        self.signals.quoted_triples.replace(measured)
1068    }
1069
1070    /// Parse a card from the metadata-section bytes.
1071    ///
1072    /// The error is a plain `String`, like every other message in the card
1073    /// modules: the CLI wraps it in `anyhow`, the wasm build hands it to
1074    /// JavaScript, and neither needs a dependency on the other's error type.
1075    pub fn from_json_bytes(b: &[u8]) -> Result<Self, String> {
1076        serde_json::from_slice(b).map_err(|e| format!("malformed dataset card: {e}"))
1077    }
1078
1079    /// Stamp the build-time coherence verdict (consuming builder).
1080    pub fn with_coherence(mut self, r: &crate::Reasoning, materialized: bool) -> Self {
1081        self.coherence = Coherence::from_reasoning(r, materialized);
1082        self
1083    }
1084
1085    /// Replace the derived sizes with the counts the file was actually written
1086    /// with (consuming builder).
1087    ///
1088    /// A card is derived from the ingested statements, which is the input
1089    /// multiset; the file holds that multiset **deduplicated**, because every
1090    /// permutation index sorts and dedups. The two agree for input with no
1091    /// duplicates — and only then, which is why `rete card` used to report the
1092    /// raw harvest size of any dataset paged with overlapping windows while
1093    /// `rete info` reported the real one. The header's count is authoritative;
1094    /// this is how it reaches the card. See `crate::FinalCounts`.
1095    ///
1096    /// The distributions (predicate/class histograms, hub degrees) stay over the
1097    /// derived multiset — they are shapes, not sizes.
1098    pub fn with_final_counts(mut self, counts: crate::ingest::FinalCounts) -> Self {
1099        self.triple_count = counts.default_triples;
1100        self.quad_count = counts.quads;
1101        self
1102    }
1103}
1104
1105/// Cap for every top-N list embedded in the card. The metadata section is
1106/// fetched on **every** overview (it is part of the index-free CARD tier), so an
1107/// unbounded `class_links` (O(classes × predicates × classes)) or predicate list
1108/// would bloat that fetch on a large schema (CIDOC-CRM/MMM). Capping keeps the
1109/// card small and bounded; `truncated` flags when a list was actually cut.
1110pub const CARD_TOP_N: usize = 100;
1111
1112/// How many distinct **annotated statements** the derivation remembers while
1113/// looking for the [`Signals::annotated_statement`] witness.
1114///
1115/// The witness is a statement this graph both *asserts* (`s p o`) and *annotates*
1116/// (`<< s p o >> :about ?v`) — the only thing that proves the coverage query
1117/// (`qt-coverage`) can answer in both of its columns. Finding it means matching
1118/// the two, which is why the star layer is learned in the pass that is already
1119/// walking every statement.
1120///
1121/// It is **sampled**, not exhaustive, because remembering every annotated
1122/// statement would make the derivation's memory scale with the star layer — on
1123/// a fully-annotated graph, three IRIs per statement — and one witness is all a
1124/// body needs. A graph whose first few thousand annotated statements happen to
1125/// be ones it never asserts loses `qt-coverage`; it can never gain a broken
1126/// one, which is the trade the geometry and label capabilities already make.
1127const ANNOTATION_WITNESS_SAMPLE: usize = 4096;
1128
1129// Well-known IRIs (bracketed N-Triples term form, as they appear in the quads).
1130const RDFS_LABEL: &str = "<http://www.w3.org/2000/01/rdf-schema#label>";
1131const SKOS_PREFLABEL: &str = "<http://www.w3.org/2004/02/skos/core#prefLabel>";
1132const SCHEMA_NAME: &str = "<http://schema.org/name>";
1133const FOAF_NAME: &str = "<http://xmlns.com/foaf/0.1/name>";
1134const DCT_TITLE: &str = "<http://purl.org/dc/terms/title>";
1135const OWL_SAMEAS: &str = "<http://www.w3.org/2002/07/owl#sameAs>";
1136const SKOS_EXACTMATCH: &str = "<http://www.w3.org/2004/02/skos/core#exactMatch>";
1137const SKOS_CLOSEMATCH: &str = "<http://www.w3.org/2004/02/skos/core#closeMatch>";
1138const RDFS_SEEALSO: &str = "<http://www.w3.org/2000/01/rdf-schema#seeAlso>";
1139pub(crate) const WGS_LAT: &str = "<http://www.w3.org/2003/01/geo/wgs84_pos#lat>";
1140pub(crate) const WGS_LONG: &str = "<http://www.w3.org/2003/01/geo/wgs84_pos#long>";
1141// `pub`: the query generator gates its geometry template on which of
1142// these two the card actually recorded, so both modules must name the same IRI.
1143pub const GEO_ASWKT: &str = "<http://www.opengis.net/ont/geosparql#asWKT>";
1144pub const GEO_HASGEOMETRY: &str = "<http://www.opengis.net/ont/geosparql#hasGeometry>";
1145// Datatype IRIs (unbracketed — as they appear after `^^` in a literal term).
1146const GEO_WKTLITERAL: &str = "http://www.opengis.net/ont/geosparql#wktLiteral";
1147const XSD_STRING: &str = "http://www.w3.org/2001/XMLSchema#string";
1148const RDF_LANGSTRING: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#langString";
1149
1150/// The [`ClassLink::o_class`] sentinel for a literal object (the object of that
1151/// row is a literal, so it has no class and no outgoing edges). `pub`:
1152/// the query generator reads it to tell an entity-to-entity relation — one a
1153/// path query can walk — from a literal-valued one.
1154pub const O_LITERAL: &str = "(literal)";
1155
1156/// Label predicates in priority order; the most frequent **present** one wins.
1157const LABEL_PREDICATES: &[&str] = &[
1158    SKOS_PREFLABEL,
1159    RDFS_LABEL,
1160    SCHEMA_NAME,
1161    FOAF_NAME,
1162    DCT_TITLE,
1163];
1164/// Cross-dataset linking predicates (emitted in `signals.link_predicates`).
1165pub(crate) const LINK_PREDICATES: &[&str] =
1166    &[OWL_SAMEAS, SKOS_EXACTMATCH, SKOS_CLOSEMATCH, RDFS_SEEALSO];
1167
1168/// A replayable source of a graph's **default-graph** triples for card
1169/// derivation. `replay` may be called more than once (the derivation makes two
1170/// passes); every call MUST yield the same triples in the same order. Terms are
1171/// passed as borrowed `&str` valid only for that call — so the derivation owns
1172/// (clones) any key it retains, letting one code path serve both an in-memory
1173/// quad slice and a streaming build's dictionary + id-triples.
1174pub trait CardTripleSource {
1175    fn replay(&self, f: &mut dyn FnMut(&str, &str, &str));
1176}
1177
1178/// Derive a full card from the parsed quads plus a few build-time counts — the
1179/// in-memory build path. Statistics are over the **default graph** only
1180/// (named-graph statistics are summarized by `quad_count`/`named_graph_count`),
1181/// matching `rete stats`/`rete predicates`.
1182/// A card holding curated fields + top-line counts only — for the external
1183/// (memory-bounded) build, where deriving the profile lists (predicates/classes/
1184/// hubs/links) would need unbounded RAM. Every derived list is left empty and
1185/// `truncated` is unset (the lists are absent, not cut).
1186pub fn curated_counts_card(statements: u64, term_count: u64, curated: CardInput) -> DatasetCard {
1187    let mut card = DatasetCard {
1188        triple_count: statements,
1189        quad_count: statements,
1190        named_graph_count: 0,
1191        term_count,
1192        format_version: crate::format::LEGACY_FORMAT_VERSION,
1193        ..DatasetCard::default()
1194    };
1195    curated.fill(&mut card);
1196    card
1197}
1198
1199pub fn derive_card(
1200    quads: &[(String, String, String, Option<String>)],
1201    term_count: u64,
1202    named_graph_count: u64,
1203    curated: CardInput,
1204) -> DatasetCard {
1205    if crate::build_progress::listening() {
1206        let default = quads.iter().filter(|q| q.3.is_none()).count() as u64;
1207        crate::build_progress::set_total(Some(2 * default), crate::build_progress::Unit::Records);
1208    }
1209    struct Quads<'a>(&'a [(String, String, String, Option<String>)]);
1210    impl CardTripleSource for Quads<'_> {
1211        fn replay(&self, f: &mut dyn FnMut(&str, &str, &str)) {
1212            for (s, p, o, g) in self.0 {
1213                if g.is_none() {
1214                    f(s, p, o);
1215                }
1216            }
1217        }
1218    }
1219    derive_card_from(
1220        &Quads(quads),
1221        quads.len() as u64,
1222        term_count,
1223        named_graph_count,
1224        curated,
1225    )
1226}
1227
1228/// Derive a card from a built dictionary + default-graph **id-triples** — the
1229/// streaming (low-RAM) build path, where the raw quads were never retained.
1230/// Resolves each id-triple back to its terms through the dictionary, yielding the
1231/// same `(s, p, o)` strings in file order, so the card is **byte-identical** to
1232/// the in-memory derivation on the same graph.
1233pub fn derive_card_encoded(
1234    dict: &crate::Dictionary,
1235    triples: &[(u32, u32, u32)],
1236    quad_count: u64,
1237    term_count: u64,
1238    named_graph_count: u64,
1239    curated: CardInput,
1240) -> DatasetCard {
1241    crate::build_progress::set_total(
1242        Some(2 * triples.len() as u64),
1243        crate::build_progress::Unit::Records,
1244    );
1245    struct Encoded<'a> {
1246        dict: &'a crate::Dictionary,
1247        triples: &'a [(u32, u32, u32)],
1248    }
1249    impl CardTripleSource for Encoded<'_> {
1250        fn replay(&self, f: &mut dyn FnMut(&str, &str, &str)) {
1251            for &(s, p, o) in self.triples {
1252                let st = self.dict.subject_term(s).unwrap_or_default();
1253                let pt = self.dict.predicate_term(p).unwrap_or_default();
1254                let ot = self.dict.object_term(o).unwrap_or_default();
1255                f(&st, &pt, &ot);
1256            }
1257        }
1258    }
1259    derive_card_from(
1260        &Encoded { dict, triples },
1261        quad_count,
1262        term_count,
1263        named_graph_count,
1264        curated,
1265    )
1266}
1267
1268/// The shared card derivation, over any [`CardTripleSource`]. Two passes: the
1269/// first builds the subject→class map needed to classify both endpoints of every
1270/// relation (the `class_links` quotient — the same logic as
1271/// [`crate::schema_summary`], folded in here to avoid a second full
1272/// materialization); the second tallies everything else. All counts are over the
1273/// raw (pre-dedup) default-graph multiset, matching the existing card stats and
1274/// `rete progressive`.
1275pub fn derive_card_from(
1276    src: &dyn CardTripleSource,
1277    quad_count: u64,
1278    term_count: u64,
1279    named_graph_count: u64,
1280    curated: CardInput,
1281) -> DatasetCard {
1282    // --- Pass 1: subject → class (last type wins, matching `schema_summary`),
1283    // plus the shape of the RDF-star layer, learned here because pass 2 needs
1284    // to recognize an *asserted* statement as one this graph also annotates
1285    // and the two can appear in either file order. See
1286    // `ANNOTATION_WITNESS_SAMPLE`. ---
1287    let mut class_of: std::collections::HashMap<String, String> = std::collections::HashMap::new();
1288    let mut quoted_subject_preds: BTreeSet<String> = BTreeSet::new();
1289    let mut annotated_tokens: BTreeMap<String, String> = BTreeMap::new();
1290    // `--progress`: two passes over the default graph.
1291    let mut ticker = crate::build_progress::Ticker::new();
1292    src.replay(&mut |s, p, o| {
1293        ticker.tick(1);
1294        if p == RDF_TYPE {
1295            class_of.insert(s.to_string(), o.to_string());
1296        }
1297        if crate::terms::is_quoted_triple(s) {
1298            if let Some((_, quoted_p, _)) = crate::ingest::quoted_triple_parts(s) {
1299                quoted_subject_preds.insert(quoted_p);
1300            }
1301            if annotated_tokens.len() < ANNOTATION_WITNESS_SAMPLE {
1302                annotated_tokens
1303                    .entry(s.to_string())
1304                    .or_insert_with(|| p.to_string());
1305            }
1306        }
1307    });
1308    let classify = |t: &str| -> String {
1309        if let Some(c) = class_of.get(t) {
1310            c.to_string()
1311        } else if t.starts_with('"') {
1312            O_LITERAL.to_string()
1313        } else {
1314            O_UNTYPED.to_string()
1315        }
1316    };
1317
1318    // --- Pass 2: every other statistic, in one sweep. ---
1319    // Keys are owned `String`s (not `&str` borrowed from a quad slice) so the
1320    // same derivation serves a streaming source whose term strings are transient;
1321    // a `BTreeMap<String, _>` sorts identically to the old `BTreeMap<&str, _>`, so
1322    // the output is byte-identical.
1323    let mut prof = Profile::default();
1324    let mut out_degree: BTreeMap<String, u64> = BTreeMap::new();
1325    let mut in_degree: BTreeMap<String, u64> = BTreeMap::new();
1326
1327    src.replay(&mut |s, p, o| {
1328        ticker.tick(1);
1329        prof.triple_count += 1;
1330        *prof.pred_counts.entry(p.to_string()).or_default() += 1;
1331        *out_degree.entry(s.to_string()).or_default() += 1;
1332        if let Some(ns) = split_namespace(s) {
1333            prof.subject_ns.add(ns, 1);
1334        }
1335
1336        // In-degree: every non-literal object referenced. Matches the
1337        // `top-in-hubs` query (`FILTER(!isLiteral(?o))`), which does not
1338        // special-case rdf:type, so the card precompute equals the query result.
1339        if !o.starts_with('"') {
1340            *in_degree.entry(o.to_string()).or_default() += 1;
1341        }
1342
1343        // RDF-star, before the `rdf:type` early return below: a quoted triple
1344        // may be typed (`<< s p o >> a :Claim`) like anything else, and that
1345        // statement is still one made about a statement.
1346        if crate::terms::is_quoted_triple(s) {
1347            *prof.annotation_hits.entry(p.to_string()).or_default() += 1;
1348        }
1349        if crate::terms::is_quoted_triple(o) {
1350            *prof.quoting_hits.entry(p.to_string()).or_default() += 1;
1351        }
1352        // The `annotated_statement` witness: THIS statement is asserted, and
1353        // pass 1 saw it quoted and annotated. `quoted_subject_preds` is empty
1354        // on every graph without an RDF-star layer — which is almost all of
1355        // them — so the common path is one set-emptiness check per statement,
1356        // and the token is only ever built for a predicate that really does
1357        // appear inside a quoted triple.
1358        if !quoted_subject_preds.is_empty() && quoted_subject_preds.contains(p) {
1359            if let Some(about) = annotated_tokens.get(&format!("<<{s} {p} {o}>>")) {
1360                *prof
1361                    .witnessed_pairs
1362                    .entry((p.to_string(), about.clone()))
1363                    .or_default() += 1;
1364            }
1365        }
1366
1367        if p == RDF_TYPE {
1368            *prof.class_counts.entry(o.to_string()).or_default() += 1;
1369            return; // type assertions define classes, not data relations
1370        }
1371
1372        // Class-to-class quotient (the effective schema).
1373        *prof
1374            .link_rows
1375            .entry((classify(s), p.to_string(), classify(o)))
1376            .or_default() += 1;
1377
1378        if p == GEO_ASWKT || p == GEO_HASGEOMETRY {
1379            prof.geo_wkt = true;
1380        }
1381
1382        // Literal objects: datatype / language / temporal / geo analysis.
1383        if let Some(lit) = parse_literal(o) {
1384            prof.literal(p, &lit, 1);
1385        }
1386    });
1387
1388    let mut truncated = false;
1389    let top_hubs = cap(sort_desc_owned(out_degree), &mut truncated);
1390    let in_hubs = cap(sort_desc_owned(in_degree), &mut truncated);
1391    finish(
1392        prof,
1393        Hubs {
1394            top_hubs,
1395            in_hubs,
1396            truncated,
1397        },
1398        quad_count,
1399        term_count,
1400        named_graph_count,
1401        curated,
1402    )
1403}
1404
1405/// The [`ClassLink::o_class`] / `s_class` sentinel for a term that carries no
1406/// `rdf:type` (an IRI or blank node nobody typed).
1407const O_UNTYPED: &str = "(untyped)";
1408
1409/// The bounded tallies a card profile is computed from — what both derivation
1410/// paths fill and [`finish`] turns into a card, so the signals, caps and
1411/// orderings have exactly one implementation.
1412///
1413/// "Bounded" is the point: every map here is keyed by *vocabulary* (predicates,
1414/// classes, datatypes, languages, namespaces), never by subject or object, so
1415/// its size does not grow with the graph. The two per-term tallies — out- and
1416/// in-degree — are the exception, and each path keeps them its own way: the
1417/// in-memory one in full maps, the streaming one in a top-N heap
1418/// ([`TopN`]), handing [`finish`] the capped lists through [`Hubs`].
1419#[derive(Default)]
1420struct Profile {
1421    triple_count: u64,
1422    pred_counts: BTreeMap<String, u64>,
1423    class_counts: BTreeMap<String, u64>,
1424    datatype_counts: BTreeMap<String, u64>,
1425    lang_counts: BTreeMap<String, u64>,
1426    link_rows: BTreeMap<(String, String, String), u64>,
1427    subject_ns: NsTally,
1428    /// Per-predicate temporal evidence (statements whose object is a date).
1429    time_obj_hits: BTreeMap<String, u64>,
1430    /// Per-predicate numeric-object hits (for value-range queries).
1431    num_obj_hits: BTreeMap<String, u64>,
1432    /// RDF-star: the predicate of a statement whose subject is a quoted triple
1433    /// (`annotation_hits`), or whose object is (`quoting_hits`), and the
1434    /// pairs matched against the sample — the witness a body pinning both
1435    /// needs (see `Signals::annotated_statement`).
1436    annotation_hits: BTreeMap<String, u64>,
1437    quoting_hits: BTreeMap<String, u64>,
1438    witnessed_pairs: BTreeMap<(String, String), u64>,
1439    /// Per-predicate earliest and latest date, in calendar order.
1440    time_extent: BTreeMap<String, (String, String)>,
1441    /// wgs84 extents, `(min, max)`.
1442    lat: Option<(f64, f64)>,
1443    lon: Option<(f64, f64)>,
1444    geo_wkt: bool,
1445}
1446
1447impl Profile {
1448    /// Fold one literal object of predicate `p` in, counted `weight` times —
1449    /// 1 per statement on the in-memory path; the streaming path sees each
1450    /// distinct `(object, predicate)` pair once, with its multiplicity.
1451    fn literal(&mut self, p: &str, lit: &Literal, weight: u64) {
1452        *self
1453            .datatype_counts
1454            .entry(format!("<{}>", lit.datatype))
1455            .or_default() += weight;
1456        *self.lang_counts.entry(lit.lang.clone()).or_default() += weight;
1457        if lit.datatype == GEO_WKTLITERAL {
1458            self.geo_wkt = true;
1459        }
1460        if is_temporal(p, lit) {
1461            *self.time_obj_hits.entry(p.to_string()).or_default() += weight;
1462            update_time_extent(p, &lit.value, &mut self.time_extent);
1463        } else if is_numeric_datatype(&lit.datatype) {
1464            // A date is temporal, not "numeric range" material, so this is an
1465            // `else if` — a year typed `xsd:gYear` is a time predicate.
1466            *self.num_obj_hits.entry(p.to_string()).or_default() += weight;
1467        }
1468        // wgs84 lat/long numeric extent.
1469        if p == WGS_LAT {
1470            if let Ok(v) = lit.value.parse::<f64>() {
1471                widen(&mut self.lat, v);
1472            }
1473        } else if p == WGS_LONG {
1474            if let Ok(v) = lit.value.parse::<f64>() {
1475                widen(&mut self.lon, v);
1476            }
1477        }
1478    }
1479}
1480
1481/// Grow a `(min, max)` extent to cover `v`.
1482fn widen(extent: &mut Option<(f64, f64)>, v: f64) {
1483    *extent = Some(match *extent {
1484        None => (v, v),
1485        Some((lo, hi)) => (lo.min(v), hi.max(v)),
1486    });
1487}
1488
1489/// The two per-term top-N lists, already capped, and whether either was cut.
1490struct Hubs {
1491    top_hubs: Vec<(String, u64)>,
1492    in_hubs: Vec<(String, u64)>,
1493    truncated: bool,
1494}
1495
1496/// Turn the tallies into a card: cap and order every list, derive the
1497/// signals, fill the curated half, and generate the starter queries. The one
1498/// place both derivation paths meet, so a card derived from quads and a card
1499/// derived from a finished `.rete` cannot disagree about anything but the
1500/// counts they were handed.
1501fn finish(
1502    p: Profile,
1503    hubs: Hubs,
1504    quad_count: u64,
1505    term_count: u64,
1506    named_graph_count: u64,
1507    curated: CardInput,
1508) -> DatasetCard {
1509    let Profile {
1510        triple_count,
1511        pred_counts,
1512        class_counts,
1513        datatype_counts,
1514        lang_counts,
1515        link_rows,
1516        subject_ns,
1517        time_obj_hits,
1518        num_obj_hits,
1519        annotation_hits,
1520        quoting_hits,
1521        witnessed_pairs,
1522        time_extent,
1523        lat,
1524        lon,
1525        geo_wkt,
1526    } = p;
1527    // Signal lookups that need the raw per-predicate counts — computed before
1528    // `pred_counts` is consumed by the sort below.
1529    let pred_count = |iri: &str| pred_counts.get(iri).copied().unwrap_or(0);
1530    let label_predicate = LABEL_PREDICATES
1531        .iter()
1532        .filter(|&&iri| pred_count(iri) > 0)
1533        .max_by_key(|&&iri| pred_count(iri))
1534        .map(|s| s.to_string());
1535    let link_predicates: Vec<String> = LINK_PREDICATES
1536        .iter()
1537        .filter(|&&iri| pred_count(iri) > 0)
1538        .map(|s| s.to_string())
1539        .collect();
1540
1541    let mut truncated = hubs.truncated || subject_ns.overflowed();
1542    let predicates = cap(sort_desc_owned(pred_counts), &mut truncated);
1543    let classes = cap(sort_desc_owned(class_counts), &mut truncated);
1544    let datatypes = cap(sort_desc_owned(datatype_counts), &mut truncated);
1545    let languages = cap(sort_desc_owned(lang_counts), &mut truncated);
1546    let class_links = cap(sort_links(link_rows), &mut truncated);
1547
1548    // Vocabularies: distinct namespaces of the predicate and class IRIs.
1549    let mut vocab: BTreeSet<String> = BTreeSet::new();
1550    for (iri, _) in predicates.iter().chain(classes.iter()) {
1551        if let Some(ns) = split_namespace(iri) {
1552            vocab.insert(ns);
1553        }
1554    }
1555    let vocabularies = cap(vocab.into_iter().collect(), &mut truncated);
1556
1557    // --- Signals (index-free affordances for the query-library generator). ---
1558    let default_lang = languages
1559        .iter()
1560        .find(|(l, _)| !l.is_empty())
1561        .map(|(l, _)| l.clone());
1562    let base_iri = derive_base_iri(&subject_ns);
1563    let mut time_ranked: Vec<(String, u64)> = time_obj_hits.into_iter().collect();
1564    time_ranked.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
1565    // Cap through the shared helper so an over-long list also flips `truncated`.
1566    let time_predicates = cap(
1567        time_ranked.iter().map(|(p, _)| p.to_string()).collect(),
1568        &mut truncated,
1569    );
1570    let mut num_ranked: Vec<(String, u64)> = num_obj_hits.into_iter().collect();
1571    num_ranked.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
1572    let numeric_predicates = cap(
1573        num_ranked.iter().map(|(p, _)| p.to_string()).collect(),
1574        &mut truncated,
1575    );
1576    // RDF-star vocabulary, ranked the same way (most statements first, ties by
1577    // term) and capped through the same helper, so an over-long list also
1578    // flips `truncated`. All three are empty/`None` on a graph with no quoted
1579    // triples — which is almost every graph — and the whole `qt-*` family then
1580    // does not exist, leaving those cards byte-identical.
1581    let annotation_predicates = cap(
1582        sort_desc_owned(annotation_hits)
1583            .into_iter()
1584            .map(|(p, _)| p)
1585            .collect(),
1586        &mut truncated,
1587    );
1588    let quoting_predicates = cap(
1589        sort_desc_owned(quoting_hits)
1590            .into_iter()
1591            .map(|(p, _)| p)
1592            .collect(),
1593        &mut truncated,
1594    );
1595    // The single most-witnessed pair, not a list: one witness is all a body
1596    // needs, and shipping the whole cross-product would cost the metadata
1597    // section for nothing. Ties break on the key so the choice is
1598    // deterministic (`max_by` keeps the last maximum, so the key comparison is
1599    // reversed to make it the smallest one).
1600    let annotated_statement = witnessed_pairs
1601        .into_iter()
1602        .max_by(|a, b| a.1.cmp(&b.1).then_with(|| b.0.cmp(&a.0)))
1603        .map(|(pair, _)| pair);
1604    let geo_latlong = lat.is_some() && lon.is_some();
1605    let temporal_extent = time_ranked
1606        .first()
1607        .and_then(|(p, _)| time_extent.get(p))
1608        .cloned();
1609    let spatial_bbox = match (lat, lon) {
1610        (Some((lat_lo, lat_hi)), Some((lon_lo, lon_hi)))
1611            if lon_lo.is_finite() && lat_lo.is_finite() =>
1612        {
1613            Some([lon_lo, lat_lo, lon_hi, lat_hi])
1614        }
1615        _ => None,
1616    };
1617
1618    let signals = Signals {
1619        label_predicate,
1620        default_lang,
1621        base_iri,
1622        time_predicates,
1623        numeric_predicates,
1624        link_predicates,
1625        annotation_predicates,
1626        quoting_predicates,
1627        annotated_statement,
1628        geo_wkt,
1629        geo_latlong,
1630        temporal_extent,
1631        spatial_bbox,
1632        // Measured by the READER from the file's header/section directory,
1633        // never derived from the triples and never written — see
1634        // `TextIndexSignal`, `PermutationsSignal` and `QuotedTriplesSignal`.
1635        text_index: None,
1636        label_index: None,
1637        permutations: None,
1638        quoted_triples: None,
1639    };
1640
1641    let mut card = DatasetCard {
1642        triple_count,
1643        quad_count,
1644        named_graph_count,
1645        term_count,
1646        predicates,
1647        classes,
1648        vocabularies,
1649        datatypes,
1650        languages,
1651        class_links,
1652        top_hubs: hubs.top_hubs,
1653        in_hubs: hubs.in_hubs,
1654        signals,
1655        queries: Vec::new(),
1656        // Stamped later by the build pipeline via `with_coherence` when `--reason`
1657        // (or `--materialize`) ran; derive_card itself must not run the reasoner.
1658        coherence: Coherence::default(),
1659        truncated,
1660        // The cap the profile lists above were derived under (deterministic:
1661        // a compile-time constant of this builder, not a per-build fact).
1662        top_n: CARD_TOP_N as u32,
1663        format_version: crate::LEGACY_FORMAT_VERSION,
1664        ..DatasetCard::default()
1665    };
1666    curated.fill(&mut card);
1667    // The tiered starter-query library, instantiated from the profile above.
1668    card.queries = crate::card_queries::generate(&card);
1669    card
1670}
1671
1672/// Cap a top-N list, flagging `truncated` if anything was dropped.
1673fn cap<T>(mut v: Vec<T>, truncated: &mut bool) -> Vec<T> {
1674    if v.len() > CARD_TOP_N {
1675        v.truncate(CARD_TOP_N);
1676        *truncated = true;
1677    }
1678    v
1679}
1680
1681/// Sort an owned-`String`-keyed count map descending by count, then by term.
1682fn sort_desc_owned(counts: BTreeMap<String, u64>) -> Vec<(String, u64)> {
1683    let mut v: Vec<(String, u64)> = counts.into_iter().collect();
1684    v.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
1685    v
1686}
1687
1688/// Sort the class-link quotient deterministically: descending by count, then by
1689/// the `(s_class, predicate, o_class)` key — a total order for a stable hash.
1690fn sort_links(rows: BTreeMap<(String, String, String), u64>) -> Vec<ClassLink> {
1691    let mut v: Vec<ClassLink> = rows
1692        .into_iter()
1693        .map(|((s, p, o), count)| ClassLink {
1694            s_class: s,
1695            predicate: p,
1696            o_class: o,
1697            count,
1698        })
1699        .collect();
1700    v.sort_by(|a, b| {
1701        b.count
1702            .cmp(&a.count)
1703            .then_with(|| a.s_class.cmp(&b.s_class))
1704            .then_with(|| a.predicate.cmp(&b.predicate))
1705            .then_with(|| a.o_class.cmp(&b.o_class))
1706    });
1707    v
1708}
1709
1710/// The dataset's own base IRI, for the `{{BASE_IRI}}` substitution (and the
1711/// "links pointing outside the dataset" query). The longest common prefix of all
1712/// subject namespaces, truncated to a `/`/`#` boundary — `http://ex/` for
1713/// subjects under `http://ex/person/`, `http://ex/place/`, … When the LCP is
1714/// coarser than `scheme://host/` (subjects span unrelated hosts), fall back to
1715/// the single most-frequent namespace so the value stays a usable prefix.
1716fn derive_base_iri(ns: &NsTally) -> Option<String> {
1717    // Dominant namespace: highest count, ties broken by the smaller string.
1718    let dominant = ns.dominant()?;
1719    // Longest common prefix across every subject namespace, tracked
1720    // incrementally over all of them (exact even past the tally cap).
1721    let lcp = ns.common_prefix().unwrap_or("");
1722    let lcp = &lcp[..lcp.len().min(dominant.len())];
1723    // Truncate to the last namespace delimiter so we never cut mid-segment.
1724    let base = match lcp.rfind(['/', '#']) {
1725        Some(i) => &lcp[..=i],
1726        None => "",
1727    };
1728    // Require at least `scheme://host/` (three slashes); else the LCP collapsed
1729    // across hosts — the dominant namespace is the more useful prefix.
1730    if base.matches('/').count() >= 3 {
1731        Some(base.to_string())
1732    } else {
1733        Some(dominant)
1734    }
1735}
1736
1737/// How many distinct subject namespaces the tally keeps. A subject namespace is
1738/// the IRI up to its last `/` or `#`, so on a graph whose identifiers contain
1739/// slashes (DataCite's `https://doi.org/<prefix>/<suffix>`) the number of
1740/// distinct namespaces grows with the record count; the tally must not.
1741pub const NS_TALLY_CAP: usize = 65_536;
1742
1743/// Budgeted size of one tally entry (key string, map and order-set nodes), used
1744/// for the memory accounting a build prints. Namespaces are short in practice.
1745pub const NS_TALLY_ENTRY_BYTES: u64 = 256;
1746
1747/// Bounded subject-namespace counts: exact up to `cap` distinct namespaces, a
1748/// deterministic space-saving summary past it (the entry with the lowest count,
1749/// smaller string first, is replaced by the newcomer, which inherits its count).
1750/// Counts are then upper bounds and the card is marked `truncated`.
1751///
1752/// The common prefix of every namespace seen is kept separately and stays exact,
1753/// so `derive_base_iri` needs nothing else.
1754pub(crate) struct NsTally {
1755    cap: usize,
1756    counts: BTreeMap<String, u64>,
1757    /// `(count, namespace)` ascending; built on the first overflow only.
1758    order: Option<BTreeSet<(u64, String)>>,
1759    lcp: Option<String>,
1760    overflowed: bool,
1761}
1762
1763impl Default for NsTally {
1764    fn default() -> Self {
1765        Self::with_cap(NS_TALLY_CAP)
1766    }
1767}
1768
1769impl NsTally {
1770    pub(crate) fn with_cap(cap: usize) -> Self {
1771        NsTally {
1772            cap: cap.max(1),
1773            counts: BTreeMap::new(),
1774            order: None,
1775            lcp: None,
1776            overflowed: false,
1777        }
1778    }
1779
1780    pub(crate) fn len(&self) -> usize {
1781        self.counts.len()
1782    }
1783
1784    /// True once a namespace had to be dropped: the counts are approximate.
1785    pub(crate) fn overflowed(&self) -> bool {
1786        self.overflowed
1787    }
1788
1789    pub(crate) fn common_prefix(&self) -> Option<&str> {
1790        self.lcp.as_deref()
1791    }
1792
1793    pub(crate) fn dominant(&self) -> Option<String> {
1794        self.counts
1795            .iter()
1796            .max_by(|a, b| a.1.cmp(b.1).then_with(|| b.0.cmp(a.0)))
1797            .map(|(k, _)| k.clone())
1798    }
1799
1800    pub(crate) fn add(&mut self, ns: String, n: u64) {
1801        match &mut self.lcp {
1802            None => self.lcp = Some(ns.clone()),
1803            Some(l) if !ns.starts_with(l.as_str()) => {
1804                let mut cut = l
1805                    .bytes()
1806                    .zip(ns.bytes())
1807                    .take_while(|(a, b)| a == b)
1808                    .count();
1809                while !l.is_char_boundary(cut) {
1810                    cut -= 1;
1811                }
1812                l.truncate(cut);
1813            }
1814            Some(_) => {}
1815        }
1816        if let Some(c) = self.counts.get_mut(&ns) {
1817            let old = *c;
1818            *c += n;
1819            if let Some(order) = &mut self.order {
1820                order.remove(&(old, ns.clone()));
1821                order.insert((*c, ns));
1822            }
1823            return;
1824        }
1825        if self.counts.len() < self.cap {
1826            if let Some(order) = &mut self.order {
1827                order.insert((n, ns.clone()));
1828            }
1829            self.counts.insert(ns, n);
1830            return;
1831        }
1832        // Full: replace the least-counted entry (smaller string on a tie).
1833        self.overflowed = true;
1834        let counts = &self.counts;
1835        let order = self
1836            .order
1837            .get_or_insert_with(|| counts.iter().map(|(k, &v)| (v, k.clone())).collect());
1838        let Some((min, victim)) = order.pop_first() else {
1839            return;
1840        };
1841        self.counts.remove(&victim);
1842        let c = min + n;
1843        order.insert((c, ns.clone()));
1844        self.counts.insert(ns, c);
1845    }
1846}
1847
1848/// The decomposed parts of an RDF literal term.
1849struct Literal {
1850    /// Lexical value with the surrounding quotes and escapes removed.
1851    value: String,
1852    /// Datatype IRI (unbracketed): `xsd:string` for plain, `rdf:langString` for
1853    /// language-tagged, else the explicit `^^<…>` type.
1854    datatype: String,
1855    /// Language tag (`""` when none).
1856    lang: String,
1857}
1858
1859/// Decompose an N-Triples literal term (`"…"`, `"…"@lang`, `"…"^^<dt>`) into its
1860/// value, datatype, and language. Returns `None` for non-literal terms.
1861fn parse_literal(term: &str) -> Option<Literal> {
1862    let bytes = term.as_bytes();
1863    if bytes.first() != Some(&b'"') {
1864        return None;
1865    }
1866    // Find the closing unescaped quote.
1867    let mut i = 1usize;
1868    let mut esc = false;
1869    let mut close = None;
1870    while i < bytes.len() {
1871        match bytes[i] {
1872            b'\\' => esc = !esc,
1873            b'"' if !esc => {
1874                close = Some(i);
1875                break;
1876            }
1877            _ => esc = false,
1878        }
1879        i += 1;
1880    }
1881    let close = close?;
1882    let value = term[1..close].to_string();
1883    let suffix = &term[close + 1..];
1884    if let Some(lang) = suffix.strip_prefix('@') {
1885        Some(Literal {
1886            value,
1887            datatype: RDF_LANGSTRING.to_string(),
1888            lang: lang.to_string(),
1889        })
1890    } else if let Some(dt) = suffix.strip_prefix("^^") {
1891        let dt = dt.trim_start_matches('<').trim_end_matches('>').to_string();
1892        Some(Literal {
1893            value,
1894            datatype: dt,
1895            lang: String::new(),
1896        })
1897    } else {
1898        Some(Literal {
1899            value,
1900            datatype: XSD_STRING.to_string(),
1901            lang: String::new(),
1902        })
1903    }
1904}
1905
1906/// True for the XSD datatypes whose values are points on a calendar — the
1907/// strong temporal signal. `gMonthDay` and `gDay` are recurring dates with no
1908/// year, so they have no extent and are not counted.
1909fn is_temporal_datatype(dt: &str) -> bool {
1910    matches!(
1911        dt,
1912        "http://www.w3.org/2001/XMLSchema#date"
1913            | "http://www.w3.org/2001/XMLSchema#dateTime"
1914            | "http://www.w3.org/2001/XMLSchema#dateTimeStamp"
1915            | "http://www.w3.org/2001/XMLSchema#gYear"
1916            | "http://www.w3.org/2001/XMLSchema#gYearMonth"
1917    )
1918}
1919
1920/// Predicates whose values are dates by definition, so a date-shaped value
1921/// that arrives as a plain string (or a bare integer year) still counts as
1922/// time. Deliberately a closed list: guessing from a local name (`…Year`,
1923/// `…date…`) is how ORCIDs and page numbers became "years".
1924const DATE_PREDICATES: &[&str] = &[
1925    "<http://purl.org/dc/terms/date>",
1926    "<http://purl.org/dc/terms/issued>",
1927    "<http://purl.org/dc/terms/created>",
1928    "<http://purl.org/dc/terms/modified>",
1929    "<http://purl.org/dc/terms/available>",
1930    "<http://purl.org/dc/terms/dateAccepted>",
1931    "<http://purl.org/dc/terms/dateCopyrighted>",
1932    "<http://purl.org/dc/terms/dateSubmitted>",
1933    "<http://purl.org/dc/elements/1.1/date>",
1934    "<http://schema.org/datePublished>",
1935    "<http://schema.org/dateCreated>",
1936    "<http://schema.org/dateModified>",
1937    "<http://schema.org/startDate>",
1938    "<http://schema.org/endDate>",
1939    "<http://schema.org/birthDate>",
1940    "<http://schema.org/deathDate>",
1941    "<http://schema.org/foundingDate>",
1942    "<http://schema.org/uploadDate>",
1943    "<https://schema.org/datePublished>",
1944    "<https://schema.org/dateCreated>",
1945    "<https://schema.org/dateModified>",
1946    "<https://schema.org/startDate>",
1947    "<https://schema.org/endDate>",
1948    "<https://schema.org/birthDate>",
1949    "<https://schema.org/deathDate>",
1950    "<http://prismstandard.org/namespaces/basic/2.0/publicationDate>",
1951    "<http://prismstandard.org/namespaces/basic/2.0/coverDate>",
1952    "<http://www.w3.org/ns/prov#generatedAtTime>",
1953    "<http://www.w3.org/ns/prov#invalidatedAtTime>",
1954    "<http://www.w3.org/ns/prov#startedAtTime>",
1955    "<http://www.w3.org/ns/prov#endedAtTime>",
1956    "<http://www.w3.org/ns/prov#atTime>",
1957    "<http://www.w3.org/2006/time#inXSDDate>",
1958    "<http://www.w3.org/2006/time#inXSDDateTime>",
1959    "<http://www.w3.org/2006/time#inXSDDateTimeStamp>",
1960    "<http://www.w3.org/2006/time#inXSDgYear>",
1961    "<http://www.w3.org/2006/time#inXSDgYearMonth>",
1962    "<http://dbpedia.org/ontology/birthDate>",
1963    "<http://dbpedia.org/ontology/deathDate>",
1964    "<http://dbpedia.org/ontology/releaseDate>",
1965    "<http://www.wikidata.org/prop/direct/P569>",
1966    "<http://www.wikidata.org/prop/direct/P570>",
1967    "<http://www.wikidata.org/prop/direct/P571>",
1968    "<http://www.wikidata.org/prop/direct/P577>",
1969    "<http://www.wikidata.org/prop/direct/P580>",
1970    "<http://www.wikidata.org/prop/direct/P582>",
1971    "<http://www.wikidata.org/prop/direct/P585>",
1972];
1973
1974/// Is this literal, on this predicate, evidence of time?
1975///
1976/// Two ways in, and both demand a value that is **shaped like a date**:
1977///
1978/// * a calendar datatype ([`is_temporal_datatype`]) — a typed value that is
1979///   not date-shaped is malformed, and malformed values are no evidence;
1980/// * a known date predicate ([`DATE_PREDICATES`]) with an untyped, string or
1981///   integer value.
1982///
1983/// What it no longer accepts is the old weak signal, "any value whose first
1984/// four characters are digits", which read an ORCID (`0000-0001-5001-3721`),
1985/// a page number (`9999880`) and a title starting with a year as years and
1986/// gave the OpenCitations card a "temporal extent" from one to the other.
1987fn is_temporal(p: &str, lit: &Literal) -> bool {
1988    if is_temporal_datatype(&lit.datatype) {
1989        return date_shaped(&lit.value);
1990    }
1991    DATE_PREDICATES.contains(&p) && date_shaped(&lit.value)
1992}
1993
1994/// A strict ISO 8601 / XSD date shape: `[-]YYYY`, `YYYY-MM`, `YYYY-MM-DD` or
1995/// `YYYY-MM-DDThh:mm[:ss[.f+]]`, each with an optional `Z` / `±hh:mm` zone —
1996/// exactly four year digits, month 01–12, day 01–31, and nothing after.
1997///
1998/// Identifier-shaped and page-shaped strings fail it by construction: a fifth
1999/// digit makes a number, not a year (`9999880`); a second four-digit group
2000/// cannot be a month (`0000-0001-5001-3721`, `1234-1240`).
2001fn date_shaped(v: &str) -> bool {
2002    let b = v.as_bytes();
2003    let digits =
2004        |at: usize, n: usize| b.len() >= at + n && b[at..at + n].iter().all(u8::is_ascii_digit);
2005    let two_in = |at: usize, lo: u8, hi: u8| {
2006        digits(at, 2) && {
2007            let n = (b[at] - b'0') * 10 + (b[at + 1] - b'0');
2008            (lo..=hi).contains(&n)
2009        }
2010    };
2011    let mut i = usize::from(b.first() == Some(&b'-'));
2012    if !digits(i, 4) {
2013        return false;
2014    }
2015    i += 4;
2016    if b.get(i) == Some(&b'-') {
2017        if !two_in(i + 1, 1, 12) {
2018            return false;
2019        }
2020        i += 3;
2021        if b.get(i) == Some(&b'-') {
2022            if !two_in(i + 1, 1, 31) {
2023                return false;
2024            }
2025            i += 3;
2026            if b.get(i) == Some(&b'T') {
2027                if !(two_in(i + 1, 0, 24) && b.get(i + 3) == Some(&b':') && two_in(i + 4, 0, 59)) {
2028                    return false;
2029                }
2030                i += 6;
2031                if b.get(i) == Some(&b':') {
2032                    if !two_in(i + 1, 0, 60) {
2033                        return false;
2034                    }
2035                    i += 3;
2036                    if b.get(i) == Some(&b'.') {
2037                        let start = i + 1;
2038                        i = start;
2039                        while b.get(i).is_some_and(u8::is_ascii_digit) {
2040                            i += 1;
2041                        }
2042                        if i == start {
2043                            return false;
2044                        }
2045                    }
2046                }
2047            }
2048        }
2049    }
2050    match &b[i..] {
2051        [] | [b'Z'] => true,
2052        [sign, h1, h2, b':', m1, m2] => {
2053            (*sign == b'+' || *sign == b'-') && [h1, h2, m1, m2].iter().all(|c| c.is_ascii_digit())
2054        }
2055        _ => false,
2056    }
2057}
2058
2059/// True for XSD numeric datatypes — candidates for value-range aggregates.
2060fn is_numeric_datatype(dt: &str) -> bool {
2061    matches!(
2062        dt,
2063        "http://www.w3.org/2001/XMLSchema#integer"
2064            | "http://www.w3.org/2001/XMLSchema#decimal"
2065            | "http://www.w3.org/2001/XMLSchema#double"
2066            | "http://www.w3.org/2001/XMLSchema#float"
2067            | "http://www.w3.org/2001/XMLSchema#long"
2068            | "http://www.w3.org/2001/XMLSchema#int"
2069            | "http://www.w3.org/2001/XMLSchema#short"
2070            | "http://www.w3.org/2001/XMLSchema#nonNegativeInteger"
2071            | "http://www.w3.org/2001/XMLSchema#positiveInteger"
2072    )
2073}
2074
2075/// Calendar order over date-shaped lexical values of mixed precision: the
2076/// signed year first (so `-0100` sorts before `-0044`, which a byte compare
2077/// gets backwards), then the rest of the value lexically (`2019` < `2019-05` <
2078/// `2019-05-01`, the order a reader expects of an extent).
2079fn temporal_key(v: &str) -> (i64, &str) {
2080    let (neg, rest) = match v.strip_prefix('-') {
2081        Some(r) => (true, r),
2082        None => (false, v),
2083    };
2084    let n = rest.bytes().take_while(u8::is_ascii_digit).count();
2085    let year: i64 = rest[..n].parse().unwrap_or(0);
2086    (if neg { -year } else { year }, &rest[n..])
2087}
2088
2089/// Fold one temporal value into its predicate's earliest/latest. Every value
2090/// reaching here passed [`date_shaped`], so the calendar order of
2091/// [`temporal_key`] is meaningful across `xsd:date`, `xsd:gYear` and
2092/// `xsd:gYearMonth` alike — the mix OpenCitations publishes on one
2093/// predicate, which the old per-datatype extent split three ways.
2094fn update_time_extent(pred: &str, value: &str, extent: &mut BTreeMap<String, (String, String)>) {
2095    match extent.get_mut(pred) {
2096        None => {
2097            extent.insert(pred.to_string(), (value.to_string(), value.to_string()));
2098        }
2099        Some((lo, hi)) => {
2100            if temporal_key(value) < temporal_key(lo) {
2101                *lo = value.to_string();
2102            }
2103            if temporal_key(value) > temporal_key(hi) {
2104                *hi = value.to_string();
2105            }
2106        }
2107    }
2108}
2109
2110// ===========================================================================
2111// Deriving a card from a `.rete` that already exists (`rete recard`)
2112// ===========================================================================
2113//
2114// The in-memory derivation above replays the statements as strings twice and
2115// keeps a map from every typed subject to its class, and a degree for every
2116// subject and every object: hundreds of bytes per statement, which is why
2117// `repyramid --card` tops out near 80 M statements. A finished `.rete` already
2118// holds the same statements **sorted six ways and deduplicated**, and sorted
2119// input is what turns every per-term tally into a running one:
2120//
2121//   * SPO groups each subject's statements together — its out-degree is known
2122//     the moment the subject changes, so the top-N hubs need a heap of N, not a
2123//     map of every subject; its namespace is read once per subject;
2124//   * OSP does the same for objects — in-degrees, and each literal object is
2125//     decoded once with the multiplicity of every predicate that uses it;
2126//   * the `rdf:type` range of POS gives every typed subject its class before
2127//     the SPO pass needs it.
2128//
2129// What remains per-term is the subject→class map behind `class_links`, kept as
2130// one small integer per subject id ([`ClassMap`], 1 byte on a graph with fewer
2131// than 255 classes). Every other map is keyed by vocabulary. On the 2.9 B-
2132// statement OpenCitations parts that is the difference between a 3 h rebuild
2133// and a re-card inside a 4 GiB container.
2134
2135/// How many statements a streamed card was derived from, and what the
2136/// per-subject class map cost — for the caller's progress report.
2137#[derive(Debug, Clone, Default)]
2138pub struct StreamReport {
2139    pub statements: u64,
2140    pub subjects: u64,
2141    pub literal_objects: u64,
2142    pub classes: u64,
2143    pub class_map_bytes: u64,
2144    /// Distinct subject namespaces the bounded tally ended with, and whether it
2145    /// overflowed its cap (the namespace counts, and so the card, are then
2146    /// approximate and `truncated`).
2147    pub namespaces_kept: u64,
2148    pub namespaces_overflowed: bool,
2149}
2150
2151/// One subject's class code, for every subject id: `0` = untyped, `1..` the
2152/// classes in order of first appearance. Starts one byte wide and widens the
2153/// first time a code does not fit, so the common case (tens of classes) costs
2154/// one byte per subject.
2155///
2156/// **Widening happens in place.** The codes live in one byte buffer with a
2157/// width of 1, 2 or 4 bytes per subject. Widening grows that buffer (a
2158/// `realloc`, which for a buffer this size moves pages instead of copying
2159/// them) and spreads the codes out from the last subject backwards, so the old
2160/// and the new map are never both alive. Allocating a second, wider vector
2161/// beside the first would have cost 3 bytes per subject for the moment of the
2162/// copy: 7.7 GB at 2.56 B subjects, which no budget split accounted for.
2163struct ClassMap {
2164    data: Vec<u8>,
2165    /// Bytes per subject: 1, 2 or 4.
2166    width: usize,
2167}
2168
2169impl ClassMap {
2170    fn new(subjects: u64) -> Self {
2171        // `vec![0; n]` is a zeroed allocation: pages that never get a class
2172        // are never touched, so an untyped region costs address space only.
2173        ClassMap {
2174            data: vec![0; subjects as usize + 1],
2175            width: 1,
2176        }
2177    }
2178
2179    fn len(&self) -> usize {
2180        self.data.len() / self.width
2181    }
2182
2183    fn read(data: &[u8], i: usize, width: usize) -> u32 {
2184        let at = i * width;
2185        match width {
2186            1 => data[at] as u32,
2187            2 => u16::from_le_bytes([data[at], data[at + 1]]) as u32,
2188            _ => u32::from_le_bytes([data[at], data[at + 1], data[at + 2], data[at + 3]]),
2189        }
2190    }
2191
2192    fn write(data: &mut [u8], i: usize, width: usize, code: u32) {
2193        let at = i * width;
2194        match width {
2195            1 => data[at] = code as u8,
2196            2 => data[at..at + 2].copy_from_slice(&(code as u16).to_le_bytes()),
2197            _ => data[at..at + 4].copy_from_slice(&code.to_le_bytes()),
2198        }
2199    }
2200
2201    fn get(&self, id: u32) -> u32 {
2202        let i = id as usize;
2203        if i < self.len() {
2204            Self::read(&self.data, i, self.width)
2205        } else {
2206            0
2207        }
2208    }
2209
2210    /// Widen to `width` bytes per subject, in place.
2211    fn widen(&mut self, width: usize) {
2212        let (n, old) = (self.len(), self.width);
2213        if width <= old {
2214            return;
2215        }
2216        self.data.resize(n * width, 0);
2217        for i in (0..n).rev() {
2218            let v = Self::read(&self.data, i, old);
2219            Self::write(&mut self.data, i, width, v);
2220        }
2221        self.width = width;
2222    }
2223
2224    fn set(&mut self, id: u32, code: u32) {
2225        if code > u16::MAX as u32 {
2226            self.widen(4);
2227        } else if code > u8::MAX as u32 {
2228            self.widen(2);
2229        }
2230        let i = id as usize;
2231        if i < self.len() {
2232            Self::write(&mut self.data, i, self.width, code);
2233        }
2234    }
2235
2236    fn bytes(&self) -> u64 {
2237        self.data.len() as u64
2238    }
2239}
2240
2241/// The N terms with the highest counts, kept in a heap of N while a sorted
2242/// pass offers every term once. Ties at the cut go to the lower dictionary id,
2243/// which is the lower **term** within one dictionary section (each is sorted),
2244/// so one `TopN` per section, merged by term afterwards ([`top_terms`]), keeps
2245/// exactly the list the in-memory derivation's sort-then-cap keeps.
2246struct TopN {
2247    heap: std::collections::BinaryHeap<std::cmp::Reverse<(u64, std::cmp::Reverse<u32>)>>,
2248    offered: u64,
2249}
2250
2251impl TopN {
2252    fn new() -> Self {
2253        TopN {
2254            heap: std::collections::BinaryHeap::with_capacity(CARD_TOP_N + 1),
2255            offered: 0,
2256        }
2257    }
2258
2259    fn offer(&mut self, id: u32, n: u64) {
2260        use std::cmp::Reverse;
2261        self.offered += 1;
2262        let key = Reverse((n, Reverse(id)));
2263        if self.heap.len() < CARD_TOP_N {
2264            self.heap.push(key);
2265        } else if let Some(worst) = self.heap.peek() {
2266            // `worst` is the smallest (count, then highest id) kept so far.
2267            if (n, Reverse(id)) > worst.0 {
2268                self.heap.pop();
2269                self.heap.push(key);
2270            }
2271        }
2272    }
2273
2274    /// The kept `(id, count)` pairs, and whether anything was cut.
2275    fn into_parts(self) -> (Vec<(u32, u64)>, bool) {
2276        let cut = self.offered > CARD_TOP_N as u64;
2277        let v = self
2278            .heap
2279            .into_iter()
2280            .map(|std::cmp::Reverse((n, std::cmp::Reverse(id)))| (id, n))
2281            .collect();
2282        (v, cut)
2283    }
2284}
2285
2286/// Merge per-section [`TopN`]s into the card's list: resolve the survivors'
2287/// terms, order by count then term, cut at [`CARD_TOP_N`] — the in-memory
2288/// `cap(sort_desc_owned(…))`, over the only candidates that could reach it.
2289fn top_terms(
2290    tops: [TopN; 2],
2291    terms: &dyn Fn(&[u32]) -> Vec<Option<String>>,
2292) -> (Vec<(String, u64)>, bool) {
2293    let mut offered = 0u64;
2294    let mut ids: Vec<(u32, u64)> = Vec::new();
2295    for top in tops {
2296        offered += top.offered;
2297        ids.extend(top.into_parts().0);
2298    }
2299    let names = terms(&ids.iter().map(|&(id, _)| id).collect::<Vec<_>>());
2300    let mut v: Vec<(String, u64)> = ids
2301        .iter()
2302        .zip(names)
2303        .filter_map(|(&(_, n), t)| t.map(|t| (t, n)))
2304        .collect();
2305    v.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
2306    v.truncate(CARD_TOP_N);
2307    (v, offered > CARD_TOP_N as u64)
2308}
2309
2310/// A fast, deterministic hasher for small integer keys — `class_links` is
2311/// tallied once per statement, and SipHash there is measurable at 2.9 B.
2312#[derive(Default, Clone, Copy)]
2313struct IdHasher(u64);
2314
2315impl std::hash::Hasher for IdHasher {
2316    fn finish(&self) -> u64 {
2317        self.0
2318    }
2319    fn write(&mut self, bytes: &[u8]) {
2320        for &b in bytes {
2321            self.0 = (self.0.rotate_left(5) ^ b as u64).wrapping_mul(0x517c_c1b7_2722_0a95);
2322        }
2323    }
2324    fn write_u32(&mut self, n: u32) {
2325        self.0 = (self.0.rotate_left(5) ^ n as u64).wrapping_mul(0x517c_c1b7_2722_0a95);
2326    }
2327}
2328
2329type IdBuild = std::hash::BuildHasherDefault<IdHasher>;
2330
2331/// The smallest id in `lo..=hi` whose term is `>= key`, or `hi + 1` — a binary
2332/// search over one dictionary section, which is sorted bytewise. `O(log n)`
2333/// term decodes; what finds the literal and quoted-triple ranges below.
2334fn first_at_or_above(lo: u32, hi: u32, key: &str, term: &dyn Fn(u32) -> Option<String>) -> u32 {
2335    let (mut a, mut b) = (lo as u64, hi as u64 + 1);
2336    while a < b {
2337        let mid = a + (b - a) / 2;
2338        match term(mid as u32) {
2339            Some(t) if t.as_str() < key => a = mid + 1,
2340            _ => b = mid,
2341        }
2342    }
2343    a as u32
2344}
2345
2346/// The ids in each section whose terms start with `prefix`'s first byte(s),
2347/// as half-open ranges in the role's id space — sorted sections make every
2348/// term kind a contiguous run (`"…` literals before `<<…` quoted triples
2349/// before `<…` IRIs before `_:…` blank nodes).
2350fn id_range(
2351    lo: u32,
2352    hi: u32,
2353    from: &str,
2354    to: &str,
2355    term: &dyn Fn(u32) -> Option<String>,
2356) -> (u32, u32) {
2357    if hi < lo {
2358        return (lo, lo);
2359    }
2360    (
2361        first_at_or_above(lo, hi, from, term),
2362        first_at_or_above(lo, hi, to, term),
2363    )
2364}
2365
2366fn in_any(id: u32, ranges: &[(u32, u32)]) -> bool {
2367    ranges.iter().any(|&(a, b)| a <= id && id < b)
2368}
2369
2370/// Derive a full card — profile, signals and starter queries — from a `.rete`
2371/// that already exists, **streaming** its sections in sorted order with memory
2372/// that does not grow with the statement count (see the section notes above).
2373///
2374/// The result is the card [`derive_card`] would derive from the same
2375/// statements, with two differences that come from reading a finished file
2376/// rather than its input, both documented in `docs/dataset-cards.md`:
2377///
2378/// * counts are over the **deduplicated** statements the file holds (the
2379///   in-memory card counts its input multiset, then corrects only the totals);
2380/// * a subject typed more than once is classified by the class with the
2381///   highest dictionary id ("last type wins" in index order, as
2382///   [`crate::schema_summary`] reads it), not by the last type in input order.
2383///
2384/// On a graph with no duplicates and no multi-typed subjects the two cards are
2385/// equal, which is what the tests hold it to. Named graphs are summarized by
2386/// their counts, as on every other path. `progress` receives one line per
2387/// phase.
2388pub fn derive_card_streaming(
2389    rete: &crate::Rete,
2390    curated: CardInput,
2391    progress: &mut dyn FnMut(&str),
2392) -> Result<(DatasetCard, StreamReport), String> {
2393    use crate::index::IndexPermutation;
2394
2395    let header = rete.header();
2396    let dict = rete.dictionary();
2397    let index = rete.default_index();
2398    let shared = dict.shared_count();
2399    let subjects_end = shared as u64 + dict.subject_only_count() as u64;
2400    let objects_end = shared as u64 + dict.object_only_count() as u64;
2401    let n_preds = dict.predicate_count() as usize;
2402    let pred_terms: Vec<String> = (0..=n_preds as u32)
2403        .map(|p| {
2404            if p == 0 {
2405                String::new()
2406            } else {
2407                dict.predicate_term(p).unwrap_or_default()
2408            }
2409        })
2410        .collect();
2411    let type_id = dict.predicate_id(RDF_TYPE);
2412
2413    // Term-kind ranges, found by binary search instead of decoding every term.
2414    let object_term = |id: u32| dict.object_term(id);
2415    let subject_term = |id: u32| dict.subject_term(id);
2416    let s_end = subjects_end.min(u32::MAX as u64) as u32;
2417    let o_end = objects_end.min(u32::MAX as u64) as u32;
2418    let literal_ranges = [
2419        id_range(1, shared, "\"", "#", &object_term),
2420        id_range(shared + 1, o_end, "\"", "#", &object_term),
2421    ];
2422    let is_literal = |o: u32| in_any(o, &literal_ranges);
2423    let quoted = header.has_quoted_triples();
2424    let (quoted_subjects, quoted_objects) = if quoted {
2425        (
2426            vec![
2427                id_range(1, shared, "<<", "<=", &subject_term),
2428                id_range(shared + 1, s_end, "<<", "<=", &subject_term),
2429            ],
2430            vec![
2431                id_range(1, shared, "<<", "<=", &object_term),
2432                id_range(shared + 1, o_end, "<<", "<=", &object_term),
2433            ],
2434        )
2435    } else {
2436        (Vec::new(), Vec::new())
2437    };
2438
2439    let mut report = StreamReport::default();
2440
2441    // --- Pass 1: the `rdf:type` range of POS → class counts and the class map.
2442    progress("pass 1/3: classes (the rdf:type range of POS)");
2443    let mut class_map = ClassMap::new(subjects_end);
2444    let mut class_code: std::collections::HashMap<u32, u32, IdBuild> = Default::default();
2445    let mut class_ids: Vec<u32> = vec![0];
2446    let mut class_counts: Vec<u64> = vec![0];
2447    if let Some(t) = type_id {
2448        for (s, _, o) in index.scan_iter((None, Some(t), None)) {
2449            let next = class_ids.len() as u32;
2450            let code = *class_code.entry(o).or_insert_with(|| {
2451                class_ids.push(o);
2452                class_counts.push(0);
2453                next
2454            });
2455            class_counts[code as usize] += 1;
2456            class_map.set(s, code);
2457        }
2458    }
2459    report.classes = class_ids.len() as u64 - 1;
2460    report.class_map_bytes = class_map.bytes();
2461
2462    // --- Pass 2: SPO → totals, out-degrees, namespaces, the quotient, RDF-star.
2463    progress("pass 2/3: subjects (SPO)");
2464    const LITERAL_CODE: u32 = u32::MAX;
2465    const WINDOW: usize = 1 << 16;
2466    let mut prof = Profile::default();
2467    let mut pred_counts = vec![0u64; n_preds + 1];
2468    let mut annotation_hits = vec![0u64; n_preds + 1];
2469    let mut quoting_hits = vec![0u64; n_preds + 1];
2470    let mut links: std::collections::HashMap<(u32, u32, u32), u64, IdBuild> = Default::default();
2471    // One per dictionary section: shared ids, then the role's own ids.
2472    let mut out_top = [TopN::new(), TopN::new()];
2473    let mut sample: Vec<(u32, u32)> = Vec::new();
2474    let mut window: Vec<(u32, u64)> = Vec::with_capacity(WINDOW);
2475    let mut subject_ns = NsTally::default();
2476    let flush_ns = |window: &mut Vec<(u32, u64)>, subject_ns: &mut NsTally| {
2477        let ids: Vec<u32> = window.iter().map(|&(s, _)| s).collect();
2478        let terms = dict.subject_terms(&ids);
2479        // Sorted subjects mostly share a namespace with their neighbour, so
2480        // one map write per run of equal namespaces, not per subject.
2481        let mut run: Option<(String, u64)> = None;
2482        for (term, &(_, n)) in terms.iter().zip(window.iter()) {
2483            let Some(ns) = term.as_deref().and_then(split_namespace) else {
2484                continue;
2485            };
2486            match &mut run {
2487                Some((cur, total)) if *cur == ns => *total += n,
2488                _ => {
2489                    if let Some((cur, total)) = run.take() {
2490                        subject_ns.add(cur, total);
2491                    }
2492                    run = Some((ns, n));
2493                }
2494            }
2495        }
2496        if let Some((cur, total)) = run {
2497            subject_ns.add(cur, total);
2498        }
2499        window.clear();
2500    };
2501    let mut cur_s = 0u32;
2502    let mut cur_n = 0u64;
2503    // `--progress`: passes 2 and 3 read the default graph once each.
2504    let mut ticker = crate::build_progress::Ticker::new();
2505    let stored = index.for_each_in_order(IndexPermutation::Spo, &mut |s, p, o| {
2506        prof.triple_count += 1;
2507        ticker.tick(1);
2508        if let Some(c) = pred_counts.get_mut(p as usize) {
2509            *c += 1;
2510        }
2511        if s != cur_s {
2512            if cur_n > 0 {
2513                out_top[usize::from(cur_s > shared)].offer(cur_s, cur_n);
2514                window.push((cur_s, cur_n));
2515                if window.len() >= WINDOW {
2516                    flush_ns(&mut window, &mut subject_ns);
2517                }
2518            }
2519            if quoted && in_any(s, &quoted_subjects) && sample.len() < ANNOTATION_WITNESS_SAMPLE {
2520                sample.push((s, p));
2521            }
2522            cur_s = s;
2523            cur_n = 0;
2524        }
2525        cur_n += 1;
2526        if quoted {
2527            if in_any(s, &quoted_subjects) {
2528                if let Some(c) = annotation_hits.get_mut(p as usize) {
2529                    *c += 1;
2530                }
2531            }
2532            if in_any(o, &quoted_objects) {
2533                if let Some(c) = quoting_hits.get_mut(p as usize) {
2534                    *c += 1;
2535                }
2536            }
2537        }
2538        if Some(p) == type_id {
2539            return;
2540        }
2541        let sc = class_map.get(s);
2542        // Literal first: a writer may file a frequent literal in the shared
2543        // section (`--frequent-objects`), so a shared id is not a subject.
2544        let oc = if is_literal(o) {
2545            LITERAL_CODE
2546        } else if o <= shared {
2547            class_map.get(o)
2548        } else {
2549            0
2550        };
2551        *links.entry((sc, p, oc)).or_default() += 1;
2552    });
2553    if !stored {
2554        return Err("the file stores no SPO permutation".to_string());
2555    }
2556    if cur_n > 0 {
2557        out_top[usize::from(cur_s > shared)].offer(cur_s, cur_n);
2558        window.push((cur_s, cur_n));
2559    }
2560    if !window.is_empty() {
2561        flush_ns(&mut window, &mut subject_ns);
2562    }
2563    report.statements = prof.triple_count;
2564    report.subjects = out_top[0].offered + out_top[1].offered;
2565    drop(class_map);
2566
2567    // --- Pass 3: OSP → in-degrees and every literal object, decoded once.
2568    progress("pass 3/3: objects (OSP)");
2569    let mut in_top = [TopN::new(), TopN::new()];
2570    let mut lit_window: Vec<(u32, Vec<(u32, u64)>)> = Vec::with_capacity(WINDOW);
2571    let mut lit_objects = 0u64;
2572    let flush_lits = |lit_window: &mut Vec<(u32, Vec<(u32, u64)>)>, prof: &mut Profile| {
2573        let ids: Vec<u32> = lit_window.iter().map(|(o, _)| *o).collect();
2574        let terms = dict.object_terms(&ids);
2575        for (term, (_, preds)) in terms.iter().zip(lit_window.iter()) {
2576            let Some(lit) = term.as_deref().and_then(parse_literal) else {
2577                continue;
2578            };
2579            for &(p, n) in preds {
2580                if let Some(pt) = pred_terms.get(p as usize) {
2581                    prof.literal(pt, &lit, n);
2582                }
2583            }
2584        }
2585        lit_window.clear();
2586    };
2587    let mut cur_o = 0u32;
2588    let mut cur_n = 0u64;
2589    let mut cur_preds: Vec<(u32, u64)> = Vec::new();
2590    index.for_each_in_order(IndexPermutation::Osp, &mut |o, _s, p| {
2591        ticker.tick(1);
2592        if o != cur_o {
2593            if cur_n > 0 {
2594                if is_literal(cur_o) {
2595                    lit_objects += 1;
2596                    lit_window.push((cur_o, std::mem::take(&mut cur_preds)));
2597                    if lit_window.len() >= WINDOW {
2598                        flush_lits(&mut lit_window, &mut prof);
2599                    }
2600                } else {
2601                    in_top[usize::from(cur_o > shared)].offer(cur_o, cur_n);
2602                }
2603            }
2604            cur_o = o;
2605            cur_n = 0;
2606            cur_preds.clear();
2607        }
2608        cur_n += 1;
2609        if is_literal(o) {
2610            match cur_preds.iter_mut().find(|(q, _)| *q == p) {
2611                Some((_, n)) => *n += 1,
2612                None => cur_preds.push((p, 1)),
2613            }
2614        }
2615    });
2616    if cur_n > 0 {
2617        if is_literal(cur_o) {
2618            lit_objects += 1;
2619            lit_window.push((cur_o, std::mem::take(&mut cur_preds)));
2620        } else {
2621            in_top[usize::from(cur_o > shared)].offer(cur_o, cur_n);
2622        }
2623    }
2624    if !lit_window.is_empty() {
2625        flush_lits(&mut lit_window, &mut prof);
2626    }
2627    report.literal_objects = lit_objects;
2628
2629    if rete.index_incomplete() {
2630        return Err("a read failed part-way through the file; the card would be incomplete".into());
2631    }
2632
2633    // --- Resolve the handful of ids the profile keeps back to terms.
2634    progress("resolving the profile's terms");
2635    let class_terms: Vec<String> = class_ids
2636        .iter()
2637        .map(|&o| {
2638            if o == 0 {
2639                O_UNTYPED.to_string()
2640            } else {
2641                dict.object_term(o).unwrap_or_default()
2642            }
2643        })
2644        .collect();
2645    let class_name = |code: u32| -> String {
2646        if code == LITERAL_CODE {
2647            O_LITERAL.to_string()
2648        } else {
2649            class_terms
2650                .get(code as usize)
2651                .cloned()
2652                .unwrap_or_else(|| O_UNTYPED.to_string())
2653        }
2654    };
2655    for (p, &n) in pred_counts.iter().enumerate() {
2656        if n > 0 {
2657            prof.pred_counts.insert(pred_terms[p].clone(), n);
2658        }
2659    }
2660    for (code, &n) in class_counts.iter().enumerate().skip(1) {
2661        if n > 0 {
2662            *prof
2663                .class_counts
2664                .entry(class_terms[code].clone())
2665                .or_default() += n;
2666        }
2667    }
2668    for ((sc, p, oc), n) in links {
2669        let Some(pt) = pred_terms.get(p as usize) else {
2670            continue;
2671        };
2672        *prof
2673            .link_rows
2674            .entry((class_name(sc), pt.clone(), class_name(oc)))
2675            .or_default() += n;
2676    }
2677    for (p, &n) in annotation_hits.iter().enumerate() {
2678        if n > 0 {
2679            prof.annotation_hits.insert(pred_terms[p].clone(), n);
2680        }
2681    }
2682    for (p, &n) in quoting_hits.iter().enumerate() {
2683        if n > 0 {
2684            prof.quoting_hits.insert(pred_terms[p].clone(), n);
2685        }
2686    }
2687    report.namespaces_kept = subject_ns.len() as u64;
2688    report.namespaces_overflowed = subject_ns.overflowed();
2689    prof.subject_ns = subject_ns;
2690    if pred_counts
2691        .iter()
2692        .enumerate()
2693        .any(|(p, &n)| n > 0 && (pred_terms[p] == GEO_ASWKT || pred_terms[p] == GEO_HASGEOMETRY))
2694    {
2695        prof.geo_wkt = true;
2696    }
2697
2698    // The RDF-star witness: a sampled annotated statement that the graph also
2699    // asserts — one point lookup each.
2700    let sample_ids: Vec<u32> = sample.iter().map(|&(s, _)| s).collect();
2701    for (term, &(_, about)) in dict.subject_terms(&sample_ids).iter().zip(sample.iter()) {
2702        let Some((a, qp, b)) = term.as_deref().and_then(crate::ingest::quoted_triple_parts) else {
2703            continue;
2704        };
2705        if let Some((sid, pid, oid)) = dict.encode(&a, &qp, &b) {
2706            if index
2707                .scan_iter((Some(sid), Some(pid), Some(oid)))
2708                .next()
2709                .is_some()
2710            {
2711                if let Some(about) = pred_terms.get(about as usize) {
2712                    *prof.witnessed_pairs.entry((qp, about.clone())).or_default() += 1;
2713                }
2714            }
2715        }
2716    }
2717
2718    let (top_hubs, cut_out) = top_terms(out_top, &|ids| dict.subject_terms(ids));
2719    let (in_hubs, cut_in) = top_terms(in_top, &|ids| dict.object_terms(ids));
2720
2721    let card = finish(
2722        prof,
2723        Hubs {
2724            top_hubs,
2725            in_hubs,
2726            truncated: cut_out || cut_in,
2727        },
2728        header.quad_count,
2729        header.term_count,
2730        rete.named_graph_count() as u64,
2731        curated,
2732    );
2733    Ok((card, report))
2734}
2735
2736/// The namespace of an IRI term: the prefix up to and including the last `#` or
2737/// `/`. Returns `None` for non-IRI terms (literals/bnodes) or IRIs with neither
2738/// delimiter. Best-effort — adequate for a descriptive vocabulary list.
2739fn split_namespace(term: &str) -> Option<String> {
2740    let iri = term.strip_prefix('<')?.strip_suffix('>')?;
2741    let cut = iri.rfind(['#', '/'])?;
2742    Some(iri[..=cut].to_string())
2743}
2744
2745/// Read the dataset card embedded in a `.rete` file image, or `None` if it has
2746/// no metadata section. Parses only the header and the metadata range — it
2747/// never decodes the dictionary, index or pyramid, so a client that just built a
2748/// file can read back the card it wrote for the price of a slice.
2749/// A card with an annex (format `0x07`) comes back whole: the annex is merged in.
2750///
2751/// The ranged (remote) companion lives in `rete-cli`, which has the HTTP reader.
2752pub fn load_card(bytes: &[u8]) -> Result<Option<DatasetCard>, String> {
2753    if crate::v06::open::is_v06(bytes) {
2754        return match crate::file::read_card_full_ranged(&crate::reader::SliceReader::new(bytes))
2755            .map_err(|e| e.to_string())?
2756        {
2757            Some(b) => Ok(Some(DatasetCard::from_json_bytes(&b)?)),
2758            None => Ok(None),
2759        };
2760    }
2761    let header = Header::from_bytes(bytes).map_err(|e| e.to_string())?;
2762    if header.metadata_len == 0 {
2763        return Ok(None);
2764    }
2765    let start = header.metadata_offset as usize;
2766    let end = start
2767        .checked_add(header.metadata_len as usize)
2768        .filter(|&e| e <= bytes.len())
2769        .ok_or_else(|| "metadata section out of bounds".to_string())?;
2770    Ok(Some(DatasetCard::from_json_bytes(&bytes[start..end])?))
2771}
2772
2773#[cfg(test)]
2774mod tests {
2775    use super::*;
2776    // The bag's bounds are the format's, enforced in `card_input`; the tests
2777    // below assert the derived half honours exactly those numbers.
2778    use crate::card_input::{CARD_EXTRA_MAX_BYTES, CARD_EXTRA_MAX_KEYS, CARD_EXTRA_MAX_KEY_BYTES};
2779
2780    const TYPE: &str = RDF_TYPE;
2781
2782    fn q(s: &str, p: &str, o: &str) -> (String, String, String, Option<String>) {
2783        (s.into(), p.into(), o.into(), None)
2784    }
2785
2786    #[test]
2787    fn card_json_round_trips() {
2788        let card = DatasetCard {
2789            title: Some("Demo".into()),
2790            license: Some("CC0".into()),
2791            triple_count: 3,
2792            quad_count: 3,
2793            term_count: 5,
2794            predicates: vec![("<http://ex/p>".into(), 2)],
2795            vocabularies: vec!["http://ex/".into()],
2796            format_version: 1,
2797            ..Default::default()
2798        };
2799        let bytes = card.to_json_bytes();
2800        let back = DatasetCard::from_json_bytes(&bytes).unwrap();
2801        assert_eq!(card, back);
2802        // Absent optionals/empties are omitted from the JSON.
2803        let text = String::from_utf8(bytes).unwrap();
2804        assert!(!text.contains("description"));
2805        assert!(!text.contains("classes"));
2806    }
2807
2808    /// `signals.permutations` is measured from the header's permutation mask at
2809    /// READ time. It must never reach the metadata section: a card that
2810    /// *claimed* a permutation set would be an authored statement about the
2811    /// file's own bytes — the one class of claim the file always answers for
2812    /// itself, for free, in the 1 KiB header every card read already fetched.
2813    #[test]
2814    fn the_permutation_signal_is_stripped_from_the_stored_card() {
2815        let measured = PermutationsSignal {
2816            count: 3,
2817            names: vec!["SPO".into(), "POS".into(), "OSP".into()],
2818            merge_join: false,
2819        };
2820        let mut card = DatasetCard {
2821            title: Some("Lean".into()),
2822            triple_count: 3,
2823            ..Default::default()
2824        };
2825        assert_eq!(card.observe_permutations(measured.clone()), None);
2826        assert_eq!(card.signals.permutations, Some(measured.clone()));
2827
2828        let bytes = card.to_json_bytes();
2829        let text = String::from_utf8(bytes.clone()).unwrap();
2830        assert!(
2831            !text.contains("permutations"),
2832            "the signal must not reach the metadata section: {text}"
2833        );
2834        let back = DatasetCard::from_json_bytes(&bytes).unwrap();
2835        assert_eq!(back.signals.permutations, None, "unknown, not `six`");
2836        assert_eq!(back.title, card.title);
2837
2838        // And a curated card document cannot author it: `CardInput` is
2839        // `deny_unknown_fields` and has no `signals` field at all.
2840        let authored = r#"{"title":"x","signals":{"permutations":{"count":6}}}"#;
2841        assert!(
2842            serde_json::from_str::<CardInput>(authored).is_err(),
2843            "a hand-written card must not be able to assert a permutation set"
2844        );
2845    }
2846
2847    /// The signal is derived from the header, so it answers for a file with no
2848    /// card at all — and says six for every file written before the mask
2849    /// existed, whose byte 50 is zero.
2850    #[test]
2851    fn the_permutation_signal_is_derived_from_the_header() {
2852        let quads = vec![
2853            q("<http://ex/a>", "<http://ex/p>", "<http://ex/b>"),
2854            q("<http://ex/b>", "<http://ex/p>", "<http://ex/c>"),
2855        ];
2856        let (six, _) = crate::ingest::assemble_dataset_with_perms(
2857            quads.clone(),
2858            false,
2859            false,
2860            None,
2861            crate::PyramidAlgo::Louvain,
2862            crate::PermSet::ALL,
2863            |_, _| Vec::new(),
2864        );
2865        let (three, _) = crate::ingest::assemble_dataset_with_perms(
2866            quads,
2867            false,
2868            false,
2869            None,
2870            crate::PyramidAlgo::Louvain,
2871            crate::PermSet::CORE,
2872            |_, _| Vec::new(),
2873        );
2874        // Byte 50 is the mask; `0` is the canonical spelling of "all six", so a
2875        // default build stays byte-identical to every file written before it.
2876        assert_eq!(six[50], 0);
2877        assert_eq!(three[50], 0b0000_0111);
2878
2879        let h6 = crate::Header::from_bytes(&six).unwrap();
2880        let h3 = crate::Header::from_bytes(&three).unwrap();
2881        let s6 = PermutationsSignal::probe(&h6);
2882        let s3 = PermutationsSignal::probe(&h3);
2883        assert_eq!(s6.count, 6);
2884        assert!(s6.merge_join);
2885        assert_eq!(s3.count, 3);
2886        assert!(!s3.merge_join);
2887        assert_eq!(s3.names, vec!["SPO", "POS", "OSP"]);
2888        assert!(s3.describe().contains("no sort-merge join"));
2889        // The lean file is smaller, and it is the index that shrank.
2890        assert!(three.len() < six.len(), "{} vs {}", three.len(), six.len());
2891        assert_eq!(h3.dictionary_len, h6.dictionary_len);
2892        assert!(h3.root_dir_len < h6.root_dir_len);
2893    }
2894
2895    /// `signals.text_index` is measured from the file's sections at READ time,
2896    /// so the write path must drop it: a stored copy would be a claim that
2897    /// survives the sections it describes (a `repyramid --text-index` rewrites
2898    /// them), which is the class of drift #189 had to repair by hand.
2899    #[test]
2900    fn the_text_index_signal_is_stripped_from_the_stored_card() {
2901        // `causenet-full-typed.rete`'s real figures.
2902        let measured = TextIndexSignal {
2903            present: true,
2904            bytes: 1_879_287_762,
2905            token_table_bytes: Some(193_295_361),
2906        };
2907        let mut card = DatasetCard {
2908            title: Some("Indexed".into()),
2909            triple_count: 3,
2910            ..Default::default()
2911        };
2912        assert_eq!(card.observe_text_index(measured), None);
2913        assert_eq!(card.signals.text_index, Some(measured));
2914
2915        let bytes = card.to_json_bytes();
2916        let text = String::from_utf8(bytes.clone()).unwrap();
2917        assert!(
2918            !text.contains("text_index"),
2919            "the signal must not reach the metadata section: {text}"
2920        );
2921        // Everything else still round-trips, and the reader gets `None` =
2922        // "unknown" rather than a fabricated `false`.
2923        let back = DatasetCard::from_json_bytes(&bytes).unwrap();
2924        assert_eq!(back.title, card.title);
2925        assert_eq!(back.signals.text_index, None);
2926
2927        // Stripping is not mutation: the in-memory card keeps what was measured
2928        // so the SAME card can be rendered and serialized in either order.
2929        assert_eq!(card.signals.text_index, Some(measured));
2930    }
2931
2932    /// The signal survives the card's own JSON envelope — which is what
2933    /// `rete card --json`, the wasm reader and `card-audit`'s document input all
2934    /// speak — even though it never reaches the file. (Plain `serde` here, not
2935    /// the CLI's `card_json` wrapper: that only adds a `schemaVersion` key, and
2936    /// the serialization under test is this struct's.)
2937    #[test]
2938    fn the_text_index_signal_round_trips_through_a_card_document() {
2939        let mut card = DatasetCard {
2940            title: Some("Indexed".into()),
2941            ..Default::default()
2942        };
2943        card.observe_text_index(TextIndexSignal {
2944            present: true,
2945            bytes: 80,
2946            token_table_bytes: Some(68),
2947        });
2948        let doc = serde_json::to_vec(&card).unwrap();
2949        let back: DatasetCard = serde_json::from_slice(&doc).unwrap();
2950        assert_eq!(back.signals.text_index, card.signals.text_index);
2951
2952        // A measured negative serializes as `{"present": false}` — nothing else
2953        // is asserted alongside it, and it is NOT the same as the field's
2954        // absence, which means "nobody measured".
2955        let mut none = DatasetCard::default();
2956        none.observe_text_index(TextIndexSignal::default());
2957        let text = serde_json::to_string(&none).unwrap();
2958        assert!(text.contains(r#""text_index":{"present":false}"#), "{text}");
2959        let absent = serde_json::to_string(&DatasetCard::default()).unwrap();
2960        assert!(!absent.contains("text_index"), "{absent}");
2961    }
2962
2963    /// The whole point of deriving this from the header rather than storing it:
2964    /// it is an honest answer for a file built long before the signal existed.
2965    /// `FLAG_HAS_QUOTED_TRIPLES` has been written by every build since quoted
2966    /// triples were supported, so the 44 published datasets answer it today,
2967    /// with no re-card and no rebuild — and a file with none says `false`, not
2968    /// "unknown".
2969    #[test]
2970    fn the_quoted_triple_signal_is_derived_from_the_header() {
2971        let plain = vec![
2972            q("<http://ex/a>", "<http://ex/p>", "<http://ex/b>"),
2973            q("<http://ex/b>", "<http://ex/p>", "<http://ex/c>"),
2974        ];
2975        let quoted = vec![
2976            q("<http://ex/a>", "<http://ex/p>", "<http://ex/b>"),
2977            q(
2978                "<http://ex/claim>",
2979                "<http://ex/states>",
2980                "<<<http://ex/a> <http://ex/p> <http://ex/b>>>",
2981            ),
2982        ];
2983        let (no, _) = crate::ingest::assemble_dataset(plain, &[]);
2984        let (yes, _) = crate::ingest::assemble_dataset(quoted, &[]);
2985
2986        let hn = crate::Header::from_bytes(&no).unwrap();
2987        let hy = crate::Header::from_bytes(&yes).unwrap();
2988        assert!(!hn.has_quoted_triples());
2989        assert!(hy.has_quoted_triples());
2990
2991        let sn = QuotedTriplesSignal::probe(&hn);
2992        let sy = QuotedTriplesSignal::probe(&hy);
2993        assert!(!sn.present);
2994        assert!(sy.present);
2995
2996        // A file with none says nothing in the human view — most datasets have
2997        // none and a line per unused affordance is noise.
2998        assert_eq!(sn.describe(), None);
2999        // One that has them says which surfaces are on offer, and which is the
3000        // default, so the card answers "what will I get if I export this?"
3001        // without the reader consulting the CLI docs.
3002        let line = sy.describe().expect("a file with quoted triples says so");
3003        assert!(line.contains("<<( s p o )>>"), "{line}");
3004        assert!(line.contains(QUOTED_TRIPLE_SURFACE_RDF_STAR), "{line}");
3005        assert_eq!(
3006            sy.export_surfaces,
3007            vec![QUOTED_TRIPLE_SURFACE_RDF12, QUOTED_TRIPLE_SURFACE_RDF_STAR]
3008        );
3009        assert_eq!(
3010            sy.export_default.as_deref(),
3011            Some(QUOTED_TRIPLE_SURFACE_RDF12)
3012        );
3013        // Nothing to offer when there is nothing to export.
3014        assert!(sn.export_surfaces.is_empty());
3015        assert_eq!(sn.export_default, None);
3016    }
3017
3018    /// Measured at read time, never written — the same contract as
3019    /// `text_index` and `permutations`, and for the sharper reason: a stored
3020    /// copy would be a claim about the file's own header, which the header
3021    /// answers for free.
3022    #[test]
3023    fn the_quoted_triple_signal_is_stripped_from_the_stored_card() {
3024        let measured = QuotedTriplesSignal {
3025            present: true,
3026            export_surfaces: vec![
3027                QUOTED_TRIPLE_SURFACE_RDF12.to_string(),
3028                QUOTED_TRIPLE_SURFACE_RDF_STAR.to_string(),
3029            ],
3030            export_default: Some(QUOTED_TRIPLE_SURFACE_RDF12.to_string()),
3031        };
3032        let mut card = DatasetCard {
3033            title: Some("Annotated".into()),
3034            triple_count: 3,
3035            ..Default::default()
3036        };
3037        assert_eq!(card.observe_quoted_triples(measured.clone()), None);
3038        assert_eq!(card.signals.quoted_triples, Some(measured.clone()));
3039
3040        let bytes = card.to_json_bytes();
3041        let text = String::from_utf8(bytes.clone()).unwrap();
3042        assert!(
3043            !text.contains("quoted_triples"),
3044            "the signal must not reach the metadata section: {text}"
3045        );
3046        let back = DatasetCard::from_json_bytes(&bytes).unwrap();
3047        assert_eq!(back.title, card.title);
3048        assert_eq!(back.signals.quoted_triples, None);
3049        // Stripping is not mutation.
3050        assert_eq!(card.signals.quoted_triples, Some(measured));
3051
3052        // And a hand-written card file cannot author it: `CardInput` has no
3053        // `signals` field and rejects unknown keys.
3054        let authored = r#"{"title":"x","signals":{"quoted_triples":{"present":true}}}"#;
3055        assert!(
3056            serde_json::from_str::<CardInput>(authored).is_err(),
3057            "a hand-written card must not be able to assert a quoted-triple set"
3058        );
3059    }
3060
3061    /// It survives the card's own JSON envelope — what `rete card --json`, the
3062    /// wasm reader and `card-audit`'s document input all speak — even though it
3063    /// never reaches the file. A measured negative is `{"present":false}`,
3064    /// which is NOT the same as the field being absent.
3065    #[test]
3066    fn the_quoted_triple_signal_round_trips_through_a_card_document() {
3067        let mut card = DatasetCard {
3068            title: Some("Annotated".into()),
3069            ..Default::default()
3070        };
3071        card.observe_quoted_triples(QuotedTriplesSignal {
3072            present: true,
3073            export_surfaces: vec![QUOTED_TRIPLE_SURFACE_RDF12.to_string()],
3074            export_default: Some(QUOTED_TRIPLE_SURFACE_RDF12.to_string()),
3075        });
3076        let doc = serde_json::to_vec(&card).unwrap();
3077        let back: DatasetCard = serde_json::from_slice(&doc).unwrap();
3078        assert_eq!(back.signals.quoted_triples, card.signals.quoted_triples);
3079
3080        let mut none = DatasetCard::default();
3081        none.observe_quoted_triples(QuotedTriplesSignal {
3082            present: false,
3083            export_surfaces: Vec::new(),
3084            export_default: None,
3085        });
3086        let text = serde_json::to_string(&none).unwrap();
3087        assert!(
3088            text.contains(r#""quoted_triples":{"present":false}"#),
3089            "{text}"
3090        );
3091        let absent = serde_json::to_string(&DatasetCard::default()).unwrap();
3092        assert!(!absent.contains("quoted_triples"), "{absent}");
3093    }
3094    #[test]
3095    fn split_namespace_handles_hash_slash_and_neither() {
3096        assert_eq!(
3097            split_namespace("<http://xmlns.com/foaf/0.1/name>").as_deref(),
3098            Some("http://xmlns.com/foaf/0.1/")
3099        );
3100        assert_eq!(
3101            split_namespace("<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>").as_deref(),
3102            Some("http://www.w3.org/1999/02/22-rdf-syntax-ns#")
3103        );
3104        // No delimiter, a literal, and a bnode all yield None.
3105        assert_eq!(split_namespace("<urn:isbn:12345>"), None);
3106        assert_eq!(split_namespace("\"literal\""), None);
3107        assert_eq!(split_namespace("_:b0"), None);
3108    }
3109
3110    #[test]
3111    fn derive_counts_and_orders() {
3112        let quads = vec![
3113            q("<http://ex/a>", TYPE, "<http://ex/Person>"),
3114            q("<http://ex/b>", TYPE, "<http://ex/Person>"),
3115            q("<http://ex/c>", TYPE, "<http://ex/City>"),
3116            q("<http://ex/a>", "<http://ex/knows>", "<http://ex/b>"),
3117            q("<http://ex/a>", "<http://ex/knows>", "<http://ex/c>"),
3118            q("<http://ex/a>", "<http://ex/name>", "\"A\""),
3119        ];
3120        let card = derive_card(&quads, 9, 0, CardInput::default());
3121        assert_eq!(card.triple_count, 6);
3122        assert_eq!(card.quad_count, 6);
3123        assert_eq!(card.term_count, 9);
3124        // Predicates desc by count: knows(2) and rdf:type(3) and name(1).
3125        assert_eq!(card.predicates[0], (TYPE.to_string(), 3));
3126        assert!(card
3127            .predicates
3128            .contains(&("<http://ex/knows>".to_string(), 2)));
3129        // Classes from rdf:type objects, desc: Person(2) before City(1).
3130        assert_eq!(
3131            card.classes,
3132            vec![
3133                ("<http://ex/Person>".to_string(), 2),
3134                ("<http://ex/City>".to_string(), 1)
3135            ]
3136        );
3137        // Vocabularies: ex/ namespace + the rdf-syntax-ns# namespace.
3138        assert!(card.vocabularies.contains(&"http://ex/".to_string()));
3139        assert!(card
3140            .vocabularies
3141            .iter()
3142            .any(|v| v.ends_with("rdf-syntax-ns#")));
3143    }
3144
3145    #[test]
3146    fn named_graph_triples_excluded_from_default_stats() {
3147        let quads = vec![
3148            q("<http://ex/a>", "<http://ex/p>", "<http://ex/b>"),
3149            (
3150                "<http://ex/c>".into(),
3151                "<http://ex/p>".into(),
3152                "<http://ex/d>".into(),
3153                Some("<http://ex/g1>".into()),
3154            ),
3155        ];
3156        let card = derive_card(&quads, 5, 1, CardInput::default());
3157        assert_eq!(card.triple_count, 1, "only the default-graph triple counts");
3158        assert_eq!(card.quad_count, 2);
3159        assert_eq!(card.named_graph_count, 1);
3160        assert_eq!(card.predicates[0].1, 1);
3161    }
3162
3163    /// A literal-bearing triple helper for the enriched-profile tests.
3164    const LABEL: &str = "<http://www.w3.org/2000/01/rdf-schema#label>";
3165    const SAMEAS: &str = "<http://www.w3.org/2002/07/owl#sameAs>";
3166    const XSD_INT: &str = "http://www.w3.org/2001/XMLSchema#integer";
3167    const XSD_GYEAR: &str = "http://www.w3.org/2001/XMLSchema#gYear";
3168
3169    fn enriched_fixture() -> Vec<(String, String, String, Option<String>)> {
3170        vec![
3171            q("<http://ex/a>", TYPE, "<http://ex/Person>"),
3172            q("<http://ex/b>", TYPE, "<http://ex/Person>"),
3173            q("<http://ex/c>", TYPE, "<http://ex/City>"),
3174            q("<http://ex/a>", LABEL, "\"Alice\"@en"),
3175            q("<http://ex/b>", LABEL, "\"Bob\"@en"),
3176            q(
3177                "<http://ex/a>",
3178                "<http://ex/age>",
3179                &format!("\"30\"^^<{XSD_INT}>"),
3180            ),
3181            q(
3182                "<http://ex/a>",
3183                "<http://ex/birthYear>",
3184                &format!("\"1850\"^^<{XSD_GYEAR}>"),
3185            ),
3186            q(
3187                "<http://ex/b>",
3188                "<http://ex/birthYear>",
3189                &format!("\"1875\"^^<{XSD_GYEAR}>"),
3190            ),
3191            q("<http://ex/a>", "<http://ex/knows>", "<http://ex/b>"),
3192            q("<http://ex/a>", "<http://ex/livesIn>", "<http://ex/c>"),
3193            q("<http://ex/a>", SAMEAS, "<http://other/a>"),
3194        ]
3195    }
3196
3197    #[test]
3198    fn enriched_profile_is_populated() {
3199        let quads = enriched_fixture();
3200        let card = derive_card(&quads, 12, 0, CardInput::default());
3201
3202        assert!(!card.datatypes.is_empty(), "datatypes derived");
3203        assert!(!card.languages.is_empty(), "languages derived");
3204        assert!(!card.class_links.is_empty(), "class_links derived");
3205        assert!(!card.top_hubs.is_empty(), "out-degree hubs derived");
3206        assert!(!card.in_hubs.is_empty(), "in-degree hubs derived");
3207
3208        // Effective schema: Person --knows--> Person, Person --livesIn--> City.
3209        assert!(card
3210            .class_links
3211            .iter()
3212            .any(|l| l.s_class == "<http://ex/Person>"
3213                && l.predicate == "<http://ex/knows>"
3214                && l.o_class == "<http://ex/Person>"
3215                && l.count == 1));
3216        assert!(card
3217            .class_links
3218            .iter()
3219            .any(|l| l.o_class == "<http://ex/City>"));
3220        // Untyped external object is the `(untyped)` sentinel.
3221        assert!(card.class_links.iter().any(|l| l.o_class == "(untyped)"));
3222
3223        // Datatypes: integer, gYear, and langString from the @en labels.
3224        assert!(card.datatypes.iter().any(|(d, _)| d.contains("integer")));
3225        assert!(card.datatypes.iter().any(|(d, _)| d.contains("gYear")));
3226        assert!(card.datatypes.iter().any(|(d, _)| d.contains("langString")));
3227        // Languages: "en" present, plus untagged "".
3228        assert!(card.languages.iter().any(|(l, _)| l == "en"));
3229
3230        // Signals.
3231        let s = &card.signals;
3232        assert_eq!(s.label_predicate.as_deref(), Some(LABEL));
3233        assert_eq!(s.default_lang.as_deref(), Some("en"));
3234        assert_eq!(s.base_iri.as_deref(), Some("http://ex/"));
3235        assert!(s.link_predicates.iter().any(|p| p.contains("sameAs")));
3236        assert!(s.time_predicates.iter().any(|p| p.contains("birthYear")));
3237        assert_eq!(
3238            s.temporal_extent,
3239            Some(("1850".to_string(), "1875".to_string())),
3240            "extent over the dominant time predicate"
3241        );
3242        // No geometry in this fixture.
3243        assert!(!s.geo_wkt && !s.geo_latlong);
3244        assert!(s.spatial_bbox.is_none());
3245    }
3246
3247    #[test]
3248    fn signals_detect_geometry() {
3249        let wkt = "http://www.opengis.net/ont/geosparql#wktLiteral";
3250        let quads = vec![
3251            q("<http://ex/p>", TYPE, "<http://ex/Place>"),
3252            q(
3253                "<http://ex/p>",
3254                "<http://www.w3.org/2003/01/geo/wgs84_pos#lat>",
3255                "\"41.9\"^^<http://www.w3.org/2001/XMLSchema#double>",
3256            ),
3257            q(
3258                "<http://ex/p>",
3259                "<http://www.w3.org/2003/01/geo/wgs84_pos#long>",
3260                "\"12.5\"^^<http://www.w3.org/2001/XMLSchema#double>",
3261            ),
3262            q(
3263                "<http://ex/p>",
3264                "<http://www.opengis.net/ont/geosparql#asWKT>",
3265                &format!("\"POINT(12.5 41.9)\"^^<{wkt}>"),
3266            ),
3267        ];
3268        let card = derive_card(&quads, 6, 0, CardInput::default());
3269        assert!(card.signals.geo_wkt, "asWKT/wktLiteral detected");
3270        assert!(card.signals.geo_latlong, "wgs84 lat+long detected");
3271        let bbox = card.signals.spatial_bbox.expect("bbox derived");
3272        assert_eq!(
3273            bbox,
3274            [12.5, 41.9, 12.5, 41.9],
3275            "[minLon,minLat,maxLon,maxLat]"
3276        );
3277    }
3278
3279    #[test]
3280    fn class_links_match_schema_summary() {
3281        use crate::{ingest, schema_summary, Rete};
3282        let quads = enriched_fixture();
3283        let card = derive_card(&quads, 12, 0, CardInput::default());
3284
3285        // Build the same data and compute the reference quotient from the index.
3286        let (bytes, _) =
3287            ingest::assemble_dataset_with_opts(quads, true, false, None, |_, _| Vec::new());
3288        let rete = Rete::open(&bytes).unwrap();
3289        let reference: BTreeSet<(String, String, String, u64)> = schema_summary(&rete)
3290            .into_iter()
3291            .map(|(s, p, o, c)| (s, p, o, c as u64))
3292            .collect();
3293
3294        let derived: BTreeSet<(String, String, String, u64)> = card
3295            .class_links
3296            .iter()
3297            .map(|l| {
3298                (
3299                    l.s_class.clone(),
3300                    l.predicate.clone(),
3301                    l.o_class.clone(),
3302                    l.count,
3303                )
3304            })
3305            .collect();
3306
3307        assert_eq!(
3308            derived, reference,
3309            "card class_links must equal schema_summary's quotient row-for-row"
3310        );
3311    }
3312
3313    /// The card must report the size of the FILE, not the size of the input.
3314    ///
3315    /// Regression for #128: the card was derived from the ingested statement
3316    /// count while the header records the count the index kept after dedup, so
3317    /// any dataset built from input containing duplicate statements — which is
3318    /// every harvest that pages with overlapping windows — published an inflated
3319    /// size. The two agree for duplicate-free input, which is why it went
3320    /// unnoticed. Here the input has duplicates in both the default graph and a
3321    /// named graph, so a card taken from the ingest count cannot pass.
3322    #[test]
3323    fn card_counts_are_the_deduplicated_counts() {
3324        use crate::ingest::{self, DeferredMetadata};
3325        use crate::Rete;
3326
3327        let g = |s: &str, p: &str, o: &str, graph: &str| {
3328            (
3329                s.to_string(),
3330                p.to_string(),
3331                o.to_string(),
3332                Some(graph.to_string()),
3333            )
3334        };
3335        let quads = vec![
3336            q("<http://ex/a>", "<http://ex/p>", "<http://ex/1>"),
3337            q("<http://ex/b>", "<http://ex/p>", "<http://ex/2>"),
3338            q("<http://ex/a>", "<http://ex/p>", "<http://ex/1>"), // duplicate
3339            g("<http://ex/n>", "<http://ex/q>", "\"v\"", "<http://ex/g>"),
3340            g("<http://ex/n>", "<http://ex/q>", "\"v\"", "<http://ex/g>"), // duplicate
3341        ];
3342        // 5 statements ingested; 3 unique (2 default + 1 named).
3343        let (bytes, stats) =
3344            ingest::assemble_dataset_with_opts(quads, false, false, None, |stats, quads| {
3345                let card = derive_card(
3346                    quads,
3347                    stats.terms as u64,
3348                    stats.named_graphs as u64,
3349                    CardInput::default(),
3350                );
3351                DeferredMetadata::new(move |counts| card.with_final_counts(counts).to_json_bytes())
3352            });
3353
3354        let header = crate::Header::from_bytes(&bytes).unwrap();
3355        let card = load_card(&bytes).unwrap().expect("card embedded");
3356        assert_eq!(header.quad_count, 3, "header records the deduped quads");
3357        assert_eq!(
3358            card.quad_count, header.quad_count,
3359            "card and header must not disagree about the size of the same file"
3360        );
3361        assert_eq!(card.triple_count, 2, "default-graph triples, deduped");
3362
3363        // And the index agrees with both.
3364        let rete = Rete::open(&bytes).unwrap();
3365        let default_triples = rete.dump(None).len() as u64;
3366        let named: u64 = rete
3367            .graph_names()
3368            .iter()
3369            .map(|name| rete.dump(Some(name)).len() as u64)
3370            .sum();
3371        assert_eq!(default_triples + named, card.quad_count);
3372
3373        // The returned stats describe the file too — they print the `wrote …`
3374        // line, which used to report the raw input size.
3375        assert_eq!(stats.statements, 3);
3376        assert_eq!(stats.default_triples, 2);
3377    }
3378
3379    #[test]
3380    fn derive_is_deterministic() {
3381        // The card folds into the content hash, so the same input must yield
3382        // byte-identical JSON across rebuilds (no map-iteration nondeterminism).
3383        let quads = enriched_fixture();
3384        let a = derive_card(&quads, 12, 0, CardInput::default()).to_json_bytes();
3385        let b = derive_card(&quads, 12, 0, CardInput::default()).to_json_bytes();
3386        assert_eq!(a, b, "enriched card derivation must be deterministic");
3387        assert!(!a.is_empty());
3388    }
3389
3390    #[test]
3391    fn old_plain_string_card_still_parses() {
3392        // A pre-enrichment card: example_queries as plain strings, none of the
3393        // new fields present. Serde `default`s must keep it deserializing.
3394        let json = r#"{
3395            "title":"Legacy",
3396            "triple_count":3,"quad_count":3,"named_graph_count":0,"term_count":5,
3397            "predicates":[["<http://ex/p>",2]],
3398            "example_queries":["SELECT * WHERE { ?s ?p ?o }"],
3399            "format_version":1
3400        }"#;
3401        let card = DatasetCard::from_json_bytes(json.as_bytes()).unwrap();
3402        assert_eq!(card.title.as_deref(), Some("Legacy"));
3403        assert_eq!(card.example_queries.len(), 1);
3404        // Every enriched field defaults to empty/false when absent from the JSON.
3405        assert!(card.datatypes.is_empty());
3406        assert!(card.languages.is_empty());
3407        assert!(card.class_links.is_empty());
3408        assert!(card.top_hubs.is_empty());
3409        assert!(card.in_hubs.is_empty());
3410        assert!(card.queries.is_empty());
3411        assert!(card.signals.is_empty());
3412        assert!(!card.truncated);
3413    }
3414
3415    /// New curated identity/provenance fields round-trip through a card-file
3416    /// JSON document and into the card, and absent ones are omitted from the
3417    /// stored bytes (size discipline).
3418    #[test]
3419    fn curated_base_iri_overrides_derived_and_is_validated() {
3420        let quads = vec![q(
3421            "<https://doi.org/10.1/a>",
3422            "<http://ex/p>",
3423            "<http://ex/o>",
3424        )];
3425        let derived = derive_card(&quads, 3, 0, CardInput::default());
3426        assert_eq!(
3427            derived.signals.base_iri.as_deref(),
3428            Some("https://doi.org/10.1/")
3429        );
3430        let input = CardInput::from_json_str(r#"{"base_iri": "https://doi.org/"}"#).unwrap();
3431        assert!(!input.is_empty());
3432        let card = derive_card(&quads, 3, 0, input.clone());
3433        assert_eq!(card.signals.base_iri.as_deref(), Some("https://doi.org/"));
3434        let counts = curated_counts_card(1, 3, input);
3435        assert_eq!(counts.signals.base_iri.as_deref(), Some("https://doi.org/"));
3436        for bad in [
3437            "doi.org/",
3438            "",
3439            "https:",
3440            "1http://x/",
3441            "http://a b/",
3442            "http://x/<y>",
3443        ] {
3444            let doc = serde_json::json!({ "base_iri": bad }).to_string();
3445            let err = CardInput::from_json_str(&doc).unwrap_err();
3446            assert!(err.contains("base_iri"), "{bad}: {err}");
3447        }
3448    }
3449
3450    #[test]
3451    fn curated_identity_fields_round_trip() {
3452        let input: CardInput = serde_json::from_str(
3453            r#"{
3454                "title": "T",
3455                "version": "2026-08-04",
3456                "creators": [{"name":"Ada","orcid":"https://orcid.org/0000-0002-1825-0097"}],
3457                "publisher": {"name":"EPFL","ror":"https://ror.org/02s376052"},
3458                "canonical_url": "https://data.example.org/t.rete",
3459                "sparql_endpoint": "https://example.org/sparql",
3460                "source_date": "2026-07-01",
3461                "derived_from": ["https://example.org/dump.nt"],
3462                "doi": "https://doi.org/10.5281/zenodo.1",
3463                "cite_as": "Ada (2026). T.",
3464                "keywords": ["graphs", "citations"],
3465                "theme": ["http://publications.europa.eu/resource/authority/data-theme/TECH"]
3466            }"#,
3467        )
3468        .unwrap();
3469        let card = curated_counts_card(5, 9, input);
3470        let bytes = card.to_json_bytes();
3471        let back = DatasetCard::from_json_bytes(&bytes).unwrap();
3472        assert_eq!(back.version.as_deref(), Some("2026-08-04"));
3473        assert_eq!(back.creators[0].name, "Ada");
3474        assert_eq!(
3475            back.creators[0].orcid.as_deref(),
3476            Some("https://orcid.org/0000-0002-1825-0097")
3477        );
3478        assert_eq!(back.publisher.as_ref().unwrap().name, "EPFL");
3479        assert_eq!(
3480            back.canonical_url.as_deref(),
3481            Some("https://data.example.org/t.rete")
3482        );
3483        assert_eq!(back.derived_from, vec!["https://example.org/dump.nt"]);
3484        assert_eq!(
3485            back.doi.as_deref(),
3486            Some("https://doi.org/10.5281/zenodo.1")
3487        );
3488        // Round-trip preserves what was stored (normalization is
3489        // `load_curated`'s job, covered by its own test below).
3490        assert_eq!(back.keywords, vec!["graphs", "citations"]);
3491        assert_eq!(
3492            back.theme,
3493            vec!["http://publications.europa.eu/resource/authority/data-theme/TECH"]
3494        );
3495
3496        // A minimal card omits every absent identity field from its bytes.
3497        let plain = curated_counts_card(1, 2, CardInput::default()).to_json_bytes();
3498        let text = String::from_utf8(plain).unwrap();
3499        for absent in [
3500            "\"creators\"",
3501            "\"publisher\"",
3502            "\"canonical_url\"",
3503            "\"sparql_endpoint\"",
3504            "\"derived_from\"",
3505            "\"doi\"",
3506            "\"cite_as\"",
3507            "\"source_date\"",
3508            "\"version\"",
3509            "\"top_n\"",
3510            "\"keywords\"",
3511            "\"theme\"",
3512            "\"extra\"",
3513        ] {
3514            assert!(
3515                !text.contains(absent),
3516                "{absent} must be omitted when unset"
3517            );
3518        }
3519    }
3520
3521    /// The `extra` bag round-trips through the stored bytes, is omitted when
3522    /// empty, and a card written by an EXTERNAL writer (the Python client
3523    /// serializes the caller's dict verbatim, counts spliced in) surfaces its
3524    /// custom fields in this reader — the pass-through the client relies on.
3525    #[test]
3526    fn extra_round_trips_and_external_writer_cards_surface_it() {
3527        let input: CardInput = serde_json::from_str(
3528            r#"{"title":"T","extra":{"atlas:layer":84,"review":{"by":"dg","ok":true}}}"#,
3529        )
3530        .unwrap();
3531        let card = curated_counts_card(5, 9, input);
3532        let back = DatasetCard::from_json_bytes(&card.to_json_bytes()).unwrap();
3533        assert_eq!(back.extra, card.extra);
3534        assert_eq!(back.extra.get("atlas:layer"), Some(&serde_json::json!(84)));
3535
3536        // The exact JSON shape `clients/python` `card_bytes` writes: curated
3537        // fields + the count fields + format_version, `extra` included.
3538        let python_shaped = r#"{
3539            "title":"From Python","extra":{"pipeline":"nightly"},
3540            "triple_count":1,"quad_count":1,"named_graph_count":0,"term_count":3,
3541            "format_version":5
3542        }"#;
3543        let card = DatasetCard::from_json_bytes(python_shaped.as_bytes()).unwrap();
3544        assert_eq!(
3545            card.extra.get("pipeline"),
3546            Some(&serde_json::json!("nightly"))
3547        );
3548    }
3549
3550    /// The bounds bite exactly at the boundary — one byte / key / level over
3551    /// is a loud error, at the limit is accepted.
3552    #[test]
3553    fn extra_limits_bite_exactly_at_the_boundary() {
3554        use crate::card_input::normalize_extra;
3555        use serde_json::{json, Map, Value};
3556        // The gate takes serde_json's `Map` (BTree-backed, so it serializes in
3557        // the same key order the card's `BTreeMap` does).
3558        let bag = |v: Vec<(String, Value)>| -> Map<String, Value> { v.into_iter().collect() };
3559
3560        // Byte cap: `{"pad":"…"}` serializes to 10 + n bytes.
3561        let pad = |n: usize| bag(vec![("pad".into(), json!("x".repeat(n)))]);
3562        let at = pad(CARD_EXTRA_MAX_BYTES - 10);
3563        assert_eq!(serde_json::to_vec(&at).unwrap().len(), CARD_EXTRA_MAX_BYTES);
3564        normalize_extra(at).expect("exactly at the byte cap is accepted");
3565        let over = pad(CARD_EXTRA_MAX_BYTES - 9);
3566        assert_eq!(
3567            serde_json::to_vec(&over).unwrap().len(),
3568            CARD_EXTRA_MAX_BYTES + 1
3569        );
3570        let err = normalize_extra(over).unwrap_err().to_string();
3571        assert!(
3572            err.contains(&format!("{} bytes", CARD_EXTRA_MAX_BYTES + 1)),
3573            "the error states the actual size: {err}"
3574        );
3575
3576        // Key-count cap.
3577        let keys = |n: usize| {
3578            bag((0..n)
3579                .map(|i| (format!("k{i:03}"), json!(1)))
3580                .collect::<Vec<_>>())
3581        };
3582        normalize_extra(keys(CARD_EXTRA_MAX_KEYS)).expect("at the key cap");
3583        assert!(normalize_extra(keys(CARD_EXTRA_MAX_KEYS + 1)).is_err());
3584
3585        // Key-length cap, and the empty key.
3586        normalize_extra(bag(vec![("k".repeat(CARD_EXTRA_MAX_KEY_BYTES), json!(1))]))
3587            .expect("at the key-length cap");
3588        assert!(normalize_extra(bag(vec![(
3589            "k".repeat(CARD_EXTRA_MAX_KEY_BYTES + 1),
3590            json!(1)
3591        )]))
3592        .is_err());
3593        assert!(normalize_extra(bag(vec![(String::new(), json!(1))])).is_err());
3594
3595        // Depth cap: three container levels ok (a list of objects holding a
3596        // list), four rejected — records belong in Parquet companions, not
3597        // the card.
3598        normalize_extra(bag(vec![("d".into(), json!({"a": {"b": 1}}))])).expect("under the cap");
3599        normalize_extra(bag(vec![("d".into(), json!([{"w": ["x", "y"]}]))]))
3600            .expect("at the depth cap");
3601        assert!(normalize_extra(bag(vec![("d".into(), json!({"a": {"b": {"c": [1]}}}))])).is_err());
3602        assert!(normalize_extra(bag(vec![("d".into(), json!([[[[1]]]]))])).is_err());
3603
3604        // "@context" is reserved for a future author-supplied mapping.
3605        let err = normalize_extra(bag(vec![("@context".into(), json!({}))]))
3606            .unwrap_err()
3607            .to_string();
3608        assert!(err.contains("reserved"), "loud reservation: {err}");
3609    }
3610
3611    /// Two builds of identical input — the `extra` bag authored in different
3612    /// (semantically equal) key order — produce byte-identical images, hence
3613    /// equal blake3 content hashes. The bag is curated, so it sits INSIDE the
3614    /// hash; this reproducibility is what makes that placement safe.
3615    #[test]
3616    fn extra_fields_keep_the_content_hash_reproducible() {
3617        let build = |json: &str| {
3618            // The same normalization `rete build --card-file` applies on every
3619            // path — it is literally the call `load_curated` ends with.
3620            let input: CardInput = serde_json::from_str::<CardInput>(json)
3621                .unwrap()
3622                .normalize()
3623                .unwrap();
3624            let quads = enriched_fixture();
3625            let card = derive_card(&quads, 12, 0, input);
3626            let (image, _) =
3627                crate::ingest::assemble_dataset_with_opts(quads, true, false, None, |_, _| {
3628                    card.to_json_bytes()
3629                });
3630            image
3631        };
3632        let a = build(r#"{"title":"T","extra":{"b":{"y":2,"x":1},"a":[1,2]}}"#);
3633        let b = build(r#"{"title":"T","extra":{"a":[1,2],"b":{"x":1,"y":2}}}"#);
3634        assert_eq!(a, b, "reordered-input builds are byte-identical");
3635        let ha = Header::from_bytes(&a).unwrap().content_hash;
3636        assert_eq!(ha, Header::from_bytes(&b).unwrap().content_hash);
3637
3638        // …and the bag IS hashed: dropping it changes the content hash
3639        // (tamper-evident, not cosmetic).
3640        let plain = build(r#"{"title":"T"}"#);
3641        assert_ne!(ha, Header::from_bytes(&plain).unwrap().content_hash);
3642    }
3643
3644    /// A stray top-level key in a card file is a LOUD error naming the key —
3645    /// the enforcement that keeps the top level rete's namespace, so a future
3646    /// official field can never capture (or be shadowed by) a publisher's.
3647    #[test]
3648    fn top_level_custom_keys_are_rejected_not_dropped() {
3649        let err = serde_json::from_str::<CardInput>(r#"{"title":"T","my_field":1}"#)
3650            .unwrap_err()
3651            .to_string();
3652        assert!(err.contains("my_field"), "names the stray key: {err}");
3653
3654        // The same content is accepted once the key moves into the bag.
3655        let ok: CardInput =
3656            serde_json::from_str(r#"{"title":"T","extra":{"my_field":1}}"#).unwrap();
3657        assert_eq!(ok.extra.get("my_field"), Some(&serde_json::json!(1)));
3658    }
3659
3660    /// A card file may write `description` as an array of lines, because a
3661    /// Markdown description in a JSON string means hand-writing `\n` escapes.
3662    /// Both shapes have to land as the same single string — the card stores one
3663    /// string either way, so `rete card --json` feeds straight back in.
3664    #[test]
3665    fn description_reads_as_a_string_or_as_lines() {
3666        let from_lines: CardInput = serde_json::from_value(serde_json::json!({
3667            "description": ["## Contents", "", "- a", "- b"]
3668        }))
3669        .unwrap();
3670        assert_eq!(
3671            from_lines.description.as_deref(),
3672            Some("## Contents\n\n- a\n- b")
3673        );
3674        let from_string: CardInput =
3675            serde_json::from_str("{\"description\":\"## Contents\\n\\n- a\\n- b\"}").unwrap();
3676        assert_eq!(from_string.description, from_lines.description);
3677
3678        // An absent description stays absent (the field is `default`ed).
3679        let none: CardInput = serde_json::from_str(r#"{"title":"T"}"#).unwrap();
3680        assert!(none.description.is_none());
3681        let null: CardInput = serde_json::from_str(r#"{"description":null}"#).unwrap();
3682        assert!(null.description.is_none());
3683
3684        // The cap is the shared one, and it is reported by serde, not silently
3685        // truncated on the way in.
3686        let over = "x".repeat(crate::card::CARD_DESCRIPTION_MAX_BYTES + 1);
3687        let err = serde_json::from_str::<CardInput>(&format!(
3688            "{{\"description\":{}}}",
3689            serde_json::to_string(&over).unwrap()
3690        ))
3691        .unwrap_err()
3692        .to_string();
3693        assert!(err.contains("over the"), "{err}");
3694    }
3695
3696    /// The browser builder validates a card document against
3697    /// `crate::card::CURATED_CARD_FIELDS` — a list, not this struct, because
3698    /// wasm has no serde derive here. A list that drifted from the struct would
3699    /// let one writer accept what the other refuses, which is the whole failure
3700    /// this shared module exists to prevent. So pin them to each other in BOTH
3701    /// directions: every listed field must be accepted by `CardInput`, and every
3702    /// field `CardInput` accepts must be listed.
3703    #[test]
3704    fn curated_field_list_matches_the_deny_unknown_fields_struct() {
3705        for f in crate::card::CURATED_CARD_FIELDS {
3706            // Probe membership, not types: feed the key a `null` and require that
3707            // whatever serde complains about is a TYPE problem, never "unknown
3708            // field" — that is the only verdict this test is about.
3709            let doc = format!(r#"{{"{f}":null}}"#);
3710            if let Err(e) = serde_json::from_str::<CardInput>(&doc) {
3711                assert!(
3712                    !e.to_string().contains("unknown field"),
3713                    "CURATED_CARD_FIELDS lists `{f}`, but CardInput rejects it as unknown: {e}",
3714                );
3715            }
3716        }
3717        // The other direction: serde names the accepted set in its error, so a
3718        // field the struct gained without being listed shows up here.
3719        let err = serde_json::from_str::<CardInput>(r#"{"nope":1}"#)
3720            .unwrap_err()
3721            .to_string();
3722        for name in err
3723            .split("expected one of ")
3724            .nth(1)
3725            .expect("serde names the expected fields")
3726            // serde appends " at line L column C" — not a field name.
3727            .split(" at line ")
3728            .next()
3729            .unwrap()
3730            .split(", ")
3731            .map(|s| s.trim().trim_matches('`'))
3732            .filter(|s| !s.is_empty())
3733        {
3734            assert!(
3735                crate::card::CURATED_CARD_FIELDS.contains(&name),
3736                "CardInput accepts `{name}` but CURATED_CARD_FIELDS does not list it — \
3737                 the browser builder would reject a card the CLI accepts",
3738            );
3739        }
3740    }
3741
3742    /// The playground's in-browser builder writes a card with no access to
3743    /// this crate: curated fields validated by `rete_core::card`, plus the four
3744    /// counts its build measured. Whatever it writes has to be a card THIS
3745    /// reader accepts — otherwise `rete card` on a downloaded browser build
3746    /// fails, and the round trip the builder promises is broken. Pin the shape
3747    /// by deserializing exactly what `rete_wasm::build_with_card` composes.
3748    #[test]
3749    fn a_browser_written_card_deserializes_here() {
3750        let curated = crate::card::validate_curated_card(&serde_json::json!({
3751            "title": "Built in a browser",
3752            "keywords": ["b", "a"],
3753            "theme": ["http://publications.europa.eu/resource/authority/data-theme/GOVE"],
3754            "creators": [{"name": "Ada", "orcid": "https://orcid.org/0000-0002-1825-0097"}],
3755            "extra": {"internal_id": "DS-1"},
3756        }))
3757        .expect("valid curated document");
3758        let composed = crate::card::compose_curated_card(curated, 3, 4, 1, 9, 5);
3759        let text = serde_json::to_string(&composed).unwrap();
3760
3761        let card: DatasetCard = serde_json::from_str(&text).expect("CLI reads a browser card");
3762        assert_eq!(card.title.as_deref(), Some("Built in a browser"));
3763        assert_eq!(card.keywords, vec!["a".to_string(), "b".to_string()]);
3764        assert_eq!(card.creators.len(), 1);
3765        assert_eq!(card.triple_count, 3);
3766        assert_eq!(card.quad_count, 4);
3767        assert_eq!(card.term_count, 9);
3768        assert_eq!(card.format_version, 5);
3769        // The derived half is ABSENT, not zeroed — a browser build never
3770        // measured it and must not look as though it had.
3771        assert!(card.predicates.is_empty() && card.queries.is_empty());
3772        assert!(!text.contains("\"predicates\""), "{text}");
3773        assert!(!text.contains("\"top_n\""), "{text}");
3774    }
3775
3776    /// Forward-compat: an OLD reader's struct (the pre-#153 field set, no
3777    /// `deny_unknown_fields` — mirrored here) must still deserialize a card
3778    /// written by THIS build, new fields and all.
3779    #[test]
3780    fn old_reader_struct_accepts_new_card_json() {
3781        #[derive(serde::Deserialize)]
3782        struct OldCard {
3783            title: Option<String>,
3784            triple_count: u64,
3785            #[serde(default)]
3786            predicates: Vec<(String, u64)>,
3787            format_version: u8,
3788        }
3789        let input: CardInput = serde_json::from_str(
3790            r#"{"title":"New","version":"1","creators":[{"name":"A"}],
3791                "canonical_url":"https://x/y.rete","derived_from":["https://x/d.nt"],
3792                "extra":{"atlas:review":"approved"}}"#,
3793        )
3794        .unwrap();
3795        let quads = vec![q("<http://ex/a>", "<http://ex/p>", "<http://ex/b>")];
3796        let card = derive_card(&quads, 3, 0, input);
3797        assert_eq!(card.top_n, CARD_TOP_N as u32);
3798        let bytes = card.to_json_bytes();
3799        let old: OldCard = serde_json::from_slice(&bytes).expect("old struct parses new JSON");
3800        assert_eq!(old.title.as_deref(), Some("New"));
3801        assert_eq!(old.triple_count, 1);
3802        assert_eq!(old.format_version, card.format_version);
3803        assert!(!old.predicates.is_empty());
3804    }
3805
3806    #[test]
3807    fn coherence_stamp_is_deterministic_and_additive() {
3808        let dis = "<http://www.w3.org/2002/07/owl#disjointWith>";
3809        let rt = "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>";
3810        let base = vec![
3811            ("<http://ex/C>".into(), dis.into(), "<http://ex/D>".into()),
3812            ("<http://ex/x>".into(), rt.into(), "<http://ex/C>".into()),
3813            ("<http://ex/x>".into(), rt.into(), "<http://ex/D>".into()),
3814        ];
3815        let r = crate::reason(&base);
3816        let c1 = Coherence::from_reasoning(&r, false);
3817        // Deterministic + the right verdict (no free-text detail in the stamp).
3818        assert_eq!(c1, Coherence::from_reasoning(&r, false));
3819        assert!(!c1.coherent);
3820        assert_eq!(c1.inconsistency_count, 1);
3821        assert_eq!(c1.by_kind, vec![("disjoint-classes".to_string(), 1)]);
3822        assert_eq!(c1.rules, crate::REASON_RULESET);
3823
3824        // A stamped card round-trips; an unstamped one omits the block entirely.
3825        let json = DatasetCard::default()
3826            .with_coherence(&r, false)
3827            .to_json_bytes();
3828        let text = std::str::from_utf8(&json).unwrap();
3829        assert!(text.contains("coherence"));
3830        assert_eq!(DatasetCard::from_json_bytes(&json).unwrap().coherence, c1);
3831
3832        let plain = DatasetCard::default().to_json_bytes();
3833        assert!(!std::str::from_utf8(&plain).unwrap().contains("coherence"));
3834        assert!(DatasetCard::default().coherence.is_empty());
3835    }
3836
3837    #[test]
3838    fn base_iri_is_common_prefix_with_fallback() {
3839        // Subjects under several sibling namespaces → their common root.
3840        let mut ns = NsTally::default();
3841        ns.add("http://ex/person/".to_string(), 2000);
3842        ns.add("http://ex/place/".to_string(), 600);
3843        ns.add("http://ex/org/".to_string(), 120);
3844        assert_eq!(derive_base_iri(&ns).as_deref(), Some("http://ex/"));
3845
3846        // A single namespace is its own base.
3847        let mut one = NsTally::default();
3848        one.add("http://data.example.org/id/".to_string(), 10);
3849        assert_eq!(
3850            derive_base_iri(&one).as_deref(),
3851            Some("http://data.example.org/id/")
3852        );
3853
3854        // Subjects across unrelated hosts: the LCP collapses to `http://`, so we
3855        // fall back to the dominant namespace rather than ship a useless prefix.
3856        let mut multi = NsTally::default();
3857        multi.add("http://a.example/".to_string(), 100);
3858        multi.add("http://b.other/".to_string(), 5);
3859        assert_eq!(
3860            derive_base_iri(&multi).as_deref(),
3861            Some("http://a.example/")
3862        );
3863
3864        assert_eq!(derive_base_iri(&NsTally::default()), None);
3865    }
3866
3867    #[test]
3868    fn parse_literal_decomposes_forms() {
3869        let plain = parse_literal("\"hello\"").unwrap();
3870        assert_eq!(plain.value, "hello");
3871        assert!(plain.datatype.ends_with("XMLSchema#string"));
3872        assert_eq!(plain.lang, "");
3873
3874        let tagged = parse_literal("\"bonjour\"@fr").unwrap();
3875        assert_eq!(tagged.lang, "fr");
3876        assert!(tagged.datatype.ends_with("#langString"));
3877
3878        let typed = parse_literal("\"42\"^^<http://www.w3.org/2001/XMLSchema#integer>").unwrap();
3879        assert_eq!(typed.value, "42");
3880        assert_eq!(typed.datatype, "http://www.w3.org/2001/XMLSchema#integer");
3881
3882        // Escaped quote inside the value is not mistaken for the terminator.
3883        let esc = parse_literal("\"a\\\"b\"@en").unwrap();
3884        assert_eq!(esc.value, "a\\\"b");
3885        assert_eq!(esc.lang, "en");
3886
3887        // Non-literals decline.
3888        assert!(parse_literal("<http://ex/x>").is_none());
3889        assert!(parse_literal("_:b0").is_none());
3890    }
3891
3892    // ---------------------------------------------------------------------
3893    // D2: the temporal detector reads dates, not digits.
3894    // ---------------------------------------------------------------------
3895
3896    #[test]
3897    fn date_shapes_are_strict() {
3898        for ok in [
3899            "2019",
3900            "-0044",
3901            "2019-05",
3902            "2019-05-01",
3903            "2019-05-01Z",
3904            "2019-05-01+02:00",
3905            "2019-05-01T12:30",
3906            "2019-05-01T12:30:15",
3907            "2019-05-01T12:30:15.250Z",
3908            "1950Z",
3909        ] {
3910            assert!(date_shaped(ok), "{ok} is a date");
3911        }
3912        for bad in [
3913            "0000-0001-5001-3721", // an ORCID
3914            "9999880",             // a page number
3915            "1234-1240",           // a page range
3916            "123-456",
3917            "2019 annual report", // a title that starts with a year
3918            "2019-13",
3919            "2019-05-32",
3920            "2019-05-01T25:00",
3921            "10.1000/182", // a DOI
3922            "",
3923            "-",
3924            "2019-",
3925        ] {
3926            assert!(!date_shaped(bad), "{bad} is not a date");
3927        }
3928    }
3929
3930    /// OpenCitations-shaped literals: identifiers through
3931    /// `literal:hasLiteralValue`, pages as strings, titles that start with a
3932    /// year, and publication dates in three precisions on one predicate. Only
3933    /// the dates may count — and their extent is calendar-ordered across the
3934    /// three datatypes.
3935    #[test]
3936    fn temporal_extent_ignores_identifiers_pages_and_titles() {
3937        let value = "<http://www.essepuntato.it/2010/06/literalreification/hasLiteralValue>";
3938        let start = "<http://prismstandard.org/namespaces/basic/2.0/startingPage>";
3939        let end = "<http://prismstandard.org/namespaces/basic/2.0/endingPage>";
3940        let title = "<http://purl.org/dc/terms/title>";
3941        let pubdate = "<http://prismstandard.org/namespaces/basic/2.0/publicationDate>";
3942        let issued = "<http://purl.org/dc/terms/issued>";
3943        let xsd = |t: &str| format!("http://www.w3.org/2001/XMLSchema#{t}");
3944        let mut quads = Vec::new();
3945        for (i, id) in [
3946            "0000-0001-5001-3721",
3947            "0000-0002-1825-0097",
3948            "10.1000/182",
3949            "9999880",
3950            "2019",
3951        ]
3952        .iter()
3953        .enumerate()
3954        {
3955            quads.push(q(
3956                &format!("<http://ex/id/{i}>"),
3957                value,
3958                &format!("\"{id}\""),
3959            ));
3960        }
3961        quads.push(q("<http://ex/re/1>", start, "\"9999880\""));
3962        quads.push(q("<http://ex/re/1>", end, "\"9999899\""));
3963        quads.push(q("<http://ex/re/2>", start, "\"1234-1240\""));
3964        quads.push(q("<http://ex/br/1>", title, "\"2019 annual report\""));
3965        quads.push(q(
3966            "<http://ex/br/1>",
3967            pubdate,
3968            &format!("\"2019-05-01\"^^<{}>", xsd("date")),
3969        ));
3970        quads.push(q(
3971            "<http://ex/br/2>",
3972            pubdate,
3973            &format!("\"1950\"^^<{}>", xsd("gYear")),
3974        ));
3975        quads.push(q(
3976            "<http://ex/br/3>",
3977            pubdate,
3978            &format!("\"2021-03\"^^<{}>", xsd("gYearMonth")),
3979        ));
3980        // A malformed typed date is no evidence.
3981        quads.push(q(
3982            "<http://ex/br/4>",
3983            pubdate,
3984            &format!("\"0000-0001-5001-3721\"^^<{}>", xsd("date")),
3985        ));
3986        // A plain-string date on a known date predicate counts.
3987        quads.push(q("<http://ex/br/5>", issued, "\"1999-12\""));
3988        let card = derive_card(&quads, 40, 0, CardInput::default());
3989        let s = &card.signals;
3990        assert_eq!(
3991            s.time_predicates,
3992            vec![pubdate.to_string(), issued.to_string()],
3993            "only the date predicates are time predicates"
3994        );
3995        assert_eq!(
3996            s.temporal_extent,
3997            Some(("1950".to_string(), "2021-03".to_string())),
3998            "calendar order across xsd:gYear, xsd:date and xsd:gYearMonth"
3999        );
4000        // Pages and identifiers are strings, not numbers either.
4001        assert!(
4002            s.numeric_predicates.is_empty(),
4003            "{:?}",
4004            s.numeric_predicates
4005        );
4006        // And no time query is written against them.
4007        assert!(card
4008            .queries
4009            .iter()
4010            .filter(|q| q.dimension == "time")
4011            .all(|q| q.sparql.contains("publicationDate")));
4012    }
4013
4014    #[test]
4015    fn negative_years_order_by_value_not_by_bytes() {
4016        let mut extent = BTreeMap::new();
4017        for v in ["-0044", "-0100", "0001", "-0044-03-15"] {
4018            update_time_extent("<p>", v, &mut extent);
4019        }
4020        assert_eq!(
4021            extent.get("<p>"),
4022            Some(&("-0100".to_string(), "0001".to_string()))
4023        );
4024    }
4025
4026    /// A year typed as an integer on a predicate nobody declared temporal is a
4027    /// number — the value-range family takes it, the time family does not.
4028    #[test]
4029    fn an_integer_year_off_a_date_predicate_is_numeric() {
4030        let int = "http://www.w3.org/2001/XMLSchema#integer";
4031        let quads = vec![
4032            q(
4033                "<http://ex/a>",
4034                "<http://ex/year>",
4035                &format!("\"1492\"^^<{int}>"),
4036            ),
4037            q(
4038                "<http://ex/b>",
4039                "<http://ex/year>",
4040                &format!("\"1500\"^^<{int}>"),
4041            ),
4042        ];
4043        let card = derive_card(&quads, 6, 0, CardInput::default());
4044        assert!(card.signals.time_predicates.is_empty());
4045        assert_eq!(
4046            card.signals.numeric_predicates,
4047            vec!["<http://ex/year>".to_string()]
4048        );
4049    }
4050
4051    // ---------------------------------------------------------------------
4052    // B3: deriving the card from the finished file, streaming.
4053    // ---------------------------------------------------------------------
4054
4055    /// A fixture that exercises every tally the profile keeps: types, labels
4056    /// in two languages, numbers, dates, geometry, an alignment link, an
4057    /// undescribed IRI, statements about statements (asserted and not, subject
4058    /// and object position), and a named graph.
4059    fn streaming_fixture() -> Vec<(String, String, String, Option<String>)> {
4060        let xsd = |t: &str| format!("http://www.w3.org/2001/XMLSchema#{t}");
4061        let mut v = enriched_fixture();
4062        for i in 0..5 {
4063            let s = format!("<http://ex/place/{i}>");
4064            v.push(q(&s, TYPE, "<http://ex/Place>"));
4065            v.push(q(&s, LABEL, &format!("\"Lieu {i}\"@fr")));
4066            v.push(q(
4067                &s,
4068                "<http://www.w3.org/2003/01/geo/wgs84_pos#lat>",
4069                &format!("\"4{i}.5\"^^<{}>", xsd("double")),
4070            ));
4071            v.push(q(
4072                &s,
4073                "<http://www.w3.org/2003/01/geo/wgs84_pos#long>",
4074                &format!("\"{i}.25\"^^<{}>", xsd("double")),
4075            ));
4076            v.push(q(
4077                &s,
4078                "<http://www.opengis.net/ont/geosparql#asWKT>",
4079                &format!(
4080                    "\"POINT({i}.25 4{i}.5)\"^^<http://www.opengis.net/ont/geosparql#wktLiteral>"
4081                ),
4082            ));
4083            v.push(q(
4084                &s,
4085                "<http://purl.org/dc/terms/created>",
4086                &format!("\"20{i}0-01-0{}\"^^<{}>", i + 1, xsd("date")),
4087            ));
4088            v.push(q(&s, "<http://ex/near>", "<http://elsewhere/undescribed>"));
4089        }
4090        // RDF-star: two asserted, annotated statements, one annotated but not
4091        // asserted, and one held as a value.
4092        let knows = "<http://ex/a> <http://ex/knows> <http://ex/b>";
4093        let lives = "<http://ex/a> <http://ex/livesIn> <http://ex/c>";
4094        let ghost = "<http://ex/b> <http://ex/knows> <http://ex/c>";
4095        for (t, who) in [(knows, "alice"), (lives, "bob"), (ghost, "carol")] {
4096            v.push(q(
4097                &format!("<<{t}>>"),
4098                "<http://ex/recordedBy>",
4099                &format!("<http://ex/{who}>"),
4100            ));
4101        }
4102        v.push(q(
4103            "<http://ex/claim/1>",
4104            "<http://ex/states>",
4105            &format!("<<{knows}>>"),
4106        ));
4107        v.push((
4108            "<http://ex/note/1>".into(),
4109            "<http://ex/about>".into(),
4110            "<http://ex/a>".into(),
4111            Some("<http://ex/graph/notes>".into()),
4112        ));
4113        v
4114    }
4115
4116    /// Build the fixture, then derive its card both ways: `(streamed,
4117    /// in_memory)`.
4118    fn both_ways(quads: &[(String, String, String, Option<String>)]) -> (DatasetCard, DatasetCard) {
4119        let (bytes, _) =
4120            crate::ingest::assemble_dataset_with_opts(quads.to_vec(), true, false, None, |_, _| {
4121                Vec::new()
4122            });
4123        let rete = crate::Rete::open(&bytes).unwrap();
4124        let term_count = rete.header().term_count;
4125        let (card, report) =
4126            derive_card_streaming(&rete, CardInput::default(), &mut |_| {}).unwrap();
4127        assert_eq!(report.statements, card.triple_count);
4128        let in_memory = derive_card(
4129            quads,
4130            term_count,
4131            rete.named_graph_count() as u64,
4132            CardInput::default(),
4133        );
4134        (card, in_memory)
4135    }
4136
4137    /// On a graph with no duplicate statements and no multi-typed subject, the
4138    /// card derived by streaming the finished file is the card derived from the
4139    /// quads — byte for byte, starter queries included.
4140    #[test]
4141    fn streaming_derivation_matches_the_in_memory_card() {
4142        for quads in [enriched_fixture(), streaming_fixture()] {
4143            let (streamed, in_memory) = both_ways(&quads);
4144            assert_eq!(
4145                String::from_utf8(streamed.to_json_bytes()).unwrap(),
4146                String::from_utf8(in_memory.to_json_bytes()).unwrap()
4147            );
4148        }
4149        // The fixture really exercised the RDF-star and geometry halves.
4150        let (card, _) = both_ways(&streaming_fixture());
4151        assert!(card.signals.annotated_statement.is_some());
4152        assert!(!card.signals.quoting_predicates.is_empty());
4153        assert!(card.signals.spatial_bbox.is_some());
4154        assert!(card.signals.temporal_extent.is_some());
4155        assert_eq!(card.named_graph_count, 1);
4156    }
4157
4158    /// More than 255 classes: the per-subject class map widens from one byte
4159    /// to two without losing a class, and the capped lists still agree.
4160    #[test]
4161    fn streaming_derivation_widens_the_class_map() {
4162        let mut quads = Vec::new();
4163        for i in 0..300 {
4164            let s = format!("<http://ex/thing/{i:03}>");
4165            quads.push(q(&s, TYPE, &format!("<http://ex/Class{i:03}>")));
4166            quads.push(q(
4167                &s,
4168                "<http://ex/next>",
4169                &format!("<http://ex/thing/{:03}>", (i + 1) % 300),
4170            ));
4171        }
4172        let (streamed, in_memory) = both_ways(&quads);
4173        assert!(streamed.truncated);
4174        assert_eq!(streamed.classes, in_memory.classes);
4175        assert_eq!(streamed.class_links, in_memory.class_links);
4176        assert_eq!(
4177            String::from_utf8(streamed.to_json_bytes()).unwrap(),
4178            String::from_utf8(in_memory.to_json_bytes()).unwrap()
4179        );
4180    }
4181
4182    /// Duplicates are where the two paths are allowed to differ: the streamed
4183    /// card counts the statements the file holds, the in-memory card its input.
4184    #[test]
4185    fn streaming_derivation_counts_the_deduplicated_file() {
4186        let mut quads = enriched_fixture();
4187        quads.push(quads[0].clone());
4188        let (streamed, in_memory) = both_ways(&quads);
4189        assert_eq!(streamed.triple_count, in_memory.triple_count - 1);
4190        let type_count = |c: &DatasetCard| {
4191            c.predicates
4192                .iter()
4193                .find(|(p, _)| p == TYPE)
4194                .map(|(_, n)| *n)
4195        };
4196        assert_eq!(type_count(&streamed), type_count(&in_memory).map(|n| n - 1));
4197    }
4198
4199    /// `--keep-curated` is the inverse of `fill`: every curated field of a card
4200    /// comes back out verbatim — and only those, never a derived one.
4201    #[test]
4202    fn the_curated_half_of_a_card_round_trips() {
4203        let doc = r##"{"title":"T","description":["# Heading","line"],"license":"CC0-1.0",
4204            "source":"https://ex/src","created":"2026-01-01","version":"1",
4205            "creators":[{"name":"Ada","orcid":"https://orcid.org/0000-0002-1825-0097"}],
4206            "publisher":{"name":"P","ror":"https://ror.org/01ggx4157"},
4207            "canonical_url":"https://ex/f.rete","sparql_endpoint":"https://ex/sparql",
4208            "source_date":"2025-12-31","derived_from":["https://ex/dump"],
4209            "doi":"https://doi.org/10.1/x","cite_as":"Cite me","keywords":["b","a"],
4210            "theme":["http://publications.europa.eu/resource/authority/data-theme/EDUC"],
4211            "extra":{"k":[{"words":["x"]}]},"example_queries":["ASK {}"]}"##;
4212        let curated = CardInput::from_json_str(doc).unwrap();
4213        assert!(!curated.is_empty());
4214        let card = derive_card(&enriched_fixture(), 12, 0, curated);
4215        let again = derive_card(&enriched_fixture(), 12, 0, CardInput::from_card(&card));
4216        assert_eq!(again.to_json_bytes(), card.to_json_bytes());
4217        assert!(CardInput::from_card(&derive_card(
4218            &enriched_fixture(),
4219            12,
4220            0,
4221            CardInput::default()
4222        ))
4223        .is_empty());
4224    }
4225}
4226
4227#[cfg(test)]
4228mod ns_tally_tests {
4229    use super::*;
4230
4231    fn feed(t: &mut NsTally, n: usize) {
4232        for i in 0..n {
4233            // Distinct namespaces, with a heavy dominant one every 7th record.
4234            let ns = if i % 7 == 0 {
4235                "https://doi.org/10.1/".to_string()
4236            } else {
4237                format!("https://doi.org/10.1/a{i}/")
4238            };
4239            t.add(ns, 1);
4240        }
4241    }
4242
4243    #[test]
4244    fn under_the_cap_the_tally_is_exact() {
4245        let mut t = NsTally::with_cap(100);
4246        feed(&mut t, 50);
4247        assert!(!t.overflowed());
4248        assert_eq!(t.len(), 43);
4249        assert_eq!(t.dominant().as_deref(), Some("https://doi.org/10.1/"));
4250        assert_eq!(
4251            derive_base_iri(&t).as_deref(),
4252            Some("https://doi.org/10.1/")
4253        );
4254    }
4255
4256    #[test]
4257    fn past_the_cap_the_tally_stays_bounded_and_deterministic() {
4258        let run = || {
4259            let mut t = NsTally::with_cap(64);
4260            feed(&mut t, 5_000);
4261            assert!(t.len() <= 64);
4262            assert!(t.overflowed());
4263            let snapshot: Vec<(String, u64)> =
4264                t.counts.iter().map(|(k, v)| (k.clone(), *v)).collect();
4265            (snapshot, t.dominant(), derive_base_iri(&t))
4266        };
4267        let (a, b) = (run(), run());
4268        assert_eq!(a, b);
4269        // The heavy namespace survives eviction and the exact common prefix
4270        // still gives the dataset's base IRI.
4271        assert_eq!(a.1.as_deref(), Some("https://doi.org/10.1/"));
4272        assert_eq!(a.2.as_deref(), Some("https://doi.org/10.1/"));
4273    }
4274
4275    #[test]
4276    fn common_prefix_spans_all_hosts_even_when_entries_were_evicted() {
4277        let mut t = NsTally::with_cap(2);
4278        t.add("http://a.org/x/".into(), 10);
4279        t.add("http://a.org/y/".into(), 1);
4280        t.add("http://b.org/z/".into(), 1);
4281        assert!(t.overflowed());
4282        assert_eq!(t.common_prefix(), Some("http://"));
4283        // LCP collapsed across hosts: fall back to the dominant namespace.
4284        assert_eq!(derive_base_iri(&t).as_deref(), Some("http://a.org/x/"));
4285    }
4286
4287    /// End to end: more distinct namespaces than the cap. Both derivations
4288    /// finish, mark the card `truncated`, and agree with themselves.
4289    #[test]
4290    fn card_is_truncated_and_reproducible_past_the_cap() {
4291        let n = NS_TALLY_CAP + 2_000;
4292        let quads: Vec<_> = (0..n)
4293            .map(|i| {
4294                (
4295                    format!("<https://doi.org/10.1/{i}/x>"),
4296                    "<http://ex/p>".to_string(),
4297                    "\"v\"".to_string(),
4298                    None,
4299                )
4300            })
4301            .collect();
4302        let (bytes, _) =
4303            crate::ingest::assemble_dataset_with_opts(quads.clone(), true, false, None, |_, _| {
4304                Vec::new()
4305            });
4306        let rete = crate::Rete::open(&bytes).unwrap();
4307        let go = || derive_card_streaming(&rete, CardInput::default(), &mut |_| {}).unwrap();
4308        let (c1, r1) = go();
4309        let (c2, _) = go();
4310        assert!(r1.namespaces_overflowed);
4311        assert_eq!(r1.namespaces_kept as usize, NS_TALLY_CAP);
4312        assert!(c1.truncated);
4313        assert_eq!(c1.to_json_bytes(), c2.to_json_bytes());
4314        let mem = derive_card(&quads, rete.header().term_count, 0, CardInput::default());
4315        assert!(mem.truncated);
4316        assert_eq!(c1.signals.base_iri, mem.signals.base_iri);
4317    }
4318}
4319
4320#[cfg(test)]
4321mod class_map_tests {
4322    use super::ClassMap;
4323
4324    #[test]
4325    fn widening_keeps_every_code_and_never_holds_two_maps() {
4326        let mut m = ClassMap::new(999);
4327        for id in 0..1000u32 {
4328            m.set(id, id % 200);
4329        }
4330        assert_eq!(m.bytes(), 1000);
4331        m.set(5, 300);
4332        assert_eq!(m.bytes(), 2000);
4333        assert_eq!(m.get(5), 300);
4334        for id in (0..1000u32).filter(|&i| i != 5) {
4335            assert_eq!(m.get(id), id % 200, "{id}");
4336        }
4337        m.set(7, 70_000);
4338        assert_eq!(m.bytes(), 4000);
4339        assert_eq!((m.get(7), m.get(5), m.get(999)), (70_000, 300, 999 % 200));
4340        // Out of range reads are untyped and writes are ignored, as before.
4341        assert_eq!(m.get(5000), 0);
4342        m.set(5000, 1);
4343        assert_eq!(m.bytes(), 4000);
4344    }
4345}