rete_core/
build_record.rs

1//! The build record every writer leaves in its file (the kind-7 `BuildInfo`
2//! section; in `0x06`, beside the card), and the one step that turns an in-RAM
3//! build into the file every front end writes.
4//!
5//! The CLI keeps its own, richer structures for the record (it measures phases,
6//! spill and starter-query costs). What is here is what a *binding or the
7//! browser* knows: the engine (version, commit, target, profile, features), the
8//! host's OS and architecture, the options of the build, the inputs where it
9//! knows them, the counts, the writer knobs, and **which front end built it**
10//! (`cli`, `python`, `r`, `java`, `wasm`). The JSON has the CLI's schema: `rete
11//! card` and every reader of the record read both.
12//!
13//! `RETE_BUILD_RECORD=reproducible` (the CLI's switch, honoured here too where
14//! there is an environment) leaves out the facts that change from run to run:
15//! the timestamp, unless `SOURCE_DATE_EPOCH` pins it.
16//!
17//! A browser engine is a tracked, byte-compared artifact, so its `engine.commit`
18//! is `unknown` (see `build.rs`); `docs/wasm-build.json` carries the commit.
19
20use serde_json::{json, Map, Value};
21
22use crate::file::FileError;
23use crate::ingest::BuildStats;
24
25/// The front ends that write files. The record says which one did.
26pub mod front_end {
27    /// `rete build`, `rete merge`, `rete convert`, `rete recard`.
28    pub const CLI: &str = "cli";
29    /// The Python client (`rete-graph`).
30    pub const PYTHON: &str = "python";
31    /// The R client.
32    pub const R: &str = "r";
33    /// The Java client.
34    pub const JAVA: &str = "java";
35    /// The browser engine (the playground's builder, the npm and MCPB packages).
36    pub const WASM: &str = "wasm";
37}
38
39/// What a build asked for, as the record states it.
40#[derive(Debug, Clone, Default)]
41pub struct Options {
42    /// The entry point: `build`, `build_dataset`, `from_text`, …
43    pub command: String,
44    /// Input syntax, when one syntax was read.
45    pub format: Option<String>,
46    /// No community pyramid.
47    pub no_pyramid: bool,
48    /// A full-text index was written.
49    pub text_index: bool,
50    /// A label index was written.
51    pub label_index: bool,
52    /// `louvain` or `types`, when a pyramid was built.
53    pub pyramid_algo: Option<String>,
54    /// 6 or 3 stored permutations.
55    pub permutations: Option<u8>,
56}
57
58/// One thing the build read.
59#[derive(Debug, Clone, Default)]
60pub struct Input {
61    /// A file name (never a path) or a label like `string`.
62    pub name: String,
63    /// Syntax: `nt`, `nq`, `ttl`, `trig`, `rdfxml`, `rete`.
64    pub format: String,
65    /// Bytes read.
66    pub bytes: Option<u64>,
67    /// BLAKE3 (hex) of exactly those bytes.
68    pub blake3: Option<String>,
69}
70
71impl Input {
72    /// An input held in memory: its length and digest are known.
73    pub fn from_bytes(name: &str, format: &str, data: &[u8]) -> Self {
74        Input {
75            name: name.to_string(),
76            format: format.to_string(),
77            bytes: Some(data.len() as u64),
78            blake3: Some(blake3::hash(data).to_hex().to_string()),
79        }
80    }
81}
82
83/// Everything a record is made of.
84#[derive(Debug, Clone, Default)]
85pub struct Record {
86    /// One of [`front_end`].
87    pub front_end: String,
88    /// The front end's own package version (`rete-python 0.4.2`).
89    pub front_end_version: String,
90    /// When the file was written, RFC 3339 UTC, whole seconds. `None` to leave
91    /// it out (a browser with no clock to read, or a reproducible record).
92    pub built_at: Option<String>,
93    /// The options of the build.
94    pub options: Options,
95    /// What it read.
96    pub inputs: Vec<Input>,
97    /// The statements the parser handed over, when known (else the unique count).
98    pub statements_read: Option<u64>,
99}
100
101/// `RETE_BUILD_RECORD=reproducible`.
102pub fn reproducible() -> bool {
103    #[cfg(not(target_arch = "wasm32"))]
104    {
105        std::env::var("RETE_BUILD_RECORD").is_ok_and(|v| v.trim() == "reproducible")
106    }
107    #[cfg(target_arch = "wasm32")]
108    {
109        false
110    }
111}
112
113/// `RETE_WRITE_FORMAT=5`: write the legacy `0x05` layout — the test hook the
114/// identity tests use, not a user option.
115pub fn writes_legacy() -> bool {
116    #[cfg(not(target_arch = "wasm32"))]
117    {
118        std::env::var("RETE_WRITE_FORMAT").is_ok_and(|v| v.trim() == "5")
119    }
120    #[cfg(target_arch = "wasm32")]
121    {
122        false
123    }
124}
125
126/// The timestamp a record stamps now: the clock (whole seconds, UTC), or
127/// `SOURCE_DATE_EPOCH` when set; `None` for a reproducible record without one,
128/// and always `None` where there is no clock to read (the browser passes its
129/// own through [`Record::built_at`]).
130pub fn timestamp() -> Option<String> {
131    #[cfg(not(target_arch = "wasm32"))]
132    {
133        let epoch = std::env::var("SOURCE_DATE_EPOCH")
134            .ok()
135            .and_then(|v| v.parse::<i64>().ok());
136        if reproducible() && epoch.is_none() {
137            return None;
138        }
139        let secs = epoch.unwrap_or_else(|| {
140            std::time::SystemTime::now()
141                .duration_since(std::time::UNIX_EPOCH)
142                .map(|d| d.as_secs() as i64)
143                .unwrap_or(0)
144        });
145        Some(rfc3339(secs))
146    }
147    #[cfg(target_arch = "wasm32")]
148    {
149        None
150    }
151}
152
153/// RFC 3339 UTC for an epoch second (Howard Hinnant's `civil_from_days`).
154pub fn rfc3339(secs: i64) -> String {
155    let days = secs.div_euclid(86_400);
156    let rem = secs.rem_euclid(86_400);
157    let z = days + 719_468;
158    let era = z.div_euclid(146_097);
159    let doe = z.rem_euclid(146_097);
160    let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
161    let y = yoe + era * 400;
162    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
163    let mp = (5 * doy + 2) / 153;
164    let d = doy - (153 * mp + 2) / 5 + 1;
165    let m = if mp < 10 { mp + 3 } else { mp - 9 };
166    let y = if m <= 2 { y + 1 } else { y };
167    format!(
168        "{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}Z",
169        rem / 3600,
170        rem % 3600 / 60,
171        rem % 60
172    )
173}
174
175/// The engine this `rete-core` is: enough to rebuild it.
176pub fn engine() -> Value {
177    json!({
178        "name": "rete-core",
179        "version": env!("CARGO_PKG_VERSION"),
180        "commit": env!("RETE_GIT_COMMIT"),
181        "target": env!("RETE_TARGET"),
182        "profile": env!("RETE_PROFILE"),
183        "features": crate::build_trace::enabled_features(),
184    })
185}
186
187/// The record as JSON bytes, in the CLI's schema. `stats` are the build's
188/// counts (absent: no `counts` block).
189pub fn to_json(r: &Record, stats: Option<&BuildStats>) -> Vec<u8> {
190    let mut o = Map::new();
191    o.insert("schema".into(), json!(1));
192    if let Some(t) = &r.built_at {
193        o.insert("built_at".into(), json!(t));
194    }
195    o.insert(
196        "builder".into(),
197        json!(format!("rete-{} {}", r.front_end, r.front_end_version)),
198    );
199    o.insert("front_end".into(), json!(r.front_end));
200    o.insert("engine".into(), engine());
201    #[cfg(not(target_arch = "wasm32"))]
202    let host = json!({"os": std::env::consts::OS, "arch": std::env::consts::ARCH});
203    #[cfg(target_arch = "wasm32")]
204    let host = json!({"os": "browser", "arch": "wasm32"});
205    o.insert("host".into(), host);
206
207    let mut p = Map::new();
208    p.insert("command".into(), json!(r.options.command));
209    if let Some(f) = &r.options.format {
210        p.insert("format".into(), json!(f));
211    }
212    for (k, v) in [
213        ("no_pyramid", r.options.no_pyramid),
214        ("text_index", r.options.text_index),
215        (
216            "label_index",
217            r.options.label_index || stats.is_some_and(|s| s.label_index),
218        ),
219    ] {
220        if v {
221            p.insert(k.into(), json!(true));
222        }
223    }
224    if let Some(a) = &r.options.pyramid_algo {
225        p.insert("pyramid_algo".into(), json!(a));
226    }
227    p.insert(
228        "codec".into(),
229        json!(if crate::file::writer_codec() == crate::CODEC_ZSTD {
230            "zstd"
231        } else {
232            "none"
233        }),
234    );
235    p.insert(
236        "permutations".into(),
237        json!(r.options.permutations.unwrap_or(6)),
238    );
239    p.insert("named_graphs".into(), json!("quad"));
240    let w = crate::write_opts::write_options();
241    p.insert(
242        "write".into(),
243        json!({
244            "tile_kib": w.tile_budget / 1024,
245            "dict_chunk_kib": w.dict_chunk_budget / 1024,
246            "zstd_level": w.zstd_level,
247            "zstd_frame": match w.frame {
248                crate::write_opts::FrameShape::Sized => "sized",
249                crate::write_opts::FrameShape::Stream => "stream",
250            },
251        }),
252    );
253    o.insert("params".into(), Value::Object(p));
254
255    if !r.inputs.is_empty() {
256        let inputs: Vec<Value> = r
257            .inputs
258            .iter()
259            .map(|i| {
260                let mut m = Map::new();
261                m.insert("name".into(), json!(i.name));
262                m.insert("format".into(), json!(i.format));
263                if let Some(b) = i.bytes {
264                    m.insert("bytes".into(), json!(b));
265                }
266                if let Some(h) = &i.blake3 {
267                    m.insert("blake3".into(), json!(h));
268                }
269                Value::Object(m)
270            })
271            .collect();
272        o.insert("inputs".into(), Value::Array(inputs));
273    }
274    if let Some(s) = stats {
275        let unique = s.statements as u64;
276        let default = s.default_triples as u64;
277        let read = r.statements_read.unwrap_or(unique);
278        o.insert(
279            "counts".into(),
280            json!({
281                "statements_read": read,
282                "statements_unique": unique,
283                "duplicates_dropped": read.saturating_sub(unique),
284                "default_graph_triples": default,
285                "named_graphs": s.named_graphs as u64,
286                "named_graph_quads": unique.saturating_sub(default),
287                "terms": s.terms as u64,
288            }),
289        );
290    }
291    serde_json::to_vec(&Value::Object(o)).unwrap_or_default()
292}
293
294/// The file every front end writes from an in-RAM `0x05` build image: the same
295/// sections, laid out as `0x06`, with `record` (the build record) in the card
296/// region beside the card. An image that is already `0x06` gets the record
297/// only. With `RETE_WRITE_FORMAT=5` the image stays `0x05` and the record is
298/// attached the `0x05` way.
299pub fn finish_in_ram(image: Vec<u8>, record: Option<&[u8]>) -> Result<Vec<u8>, FileError> {
300    let record = record.filter(|r| !r.is_empty());
301    if writes_legacy() {
302        return match record {
303            Some(r) => crate::file::attach_build_info(&image, r),
304            None => Ok(image),
305        };
306    }
307    if crate::v06::open::is_v06(&image) {
308        return match record {
309            Some(r) => crate::file::attach_build_info(&image, r),
310            None => Ok(image),
311        };
312    }
313    let src = crate::reader::SliceReader::new(&image);
314    let mut parts = match crate::v06::parts::from_v05(&src) {
315        Ok(p) => p,
316        // A per-graph (kind 5) image: the named graphs are re-encoded as the
317        // quad index, the only layout `0x06` has, and the result finished.
318        Err(FileError::Container(m)) if m.contains("kind 5") => {
319            let requad = reencode_per_graph(&image)?;
320            return finish_in_ram(requad, record);
321        }
322        Err(e) => return Err(e),
323    };
324    if let Some(r) = record {
325        parts.build = Some(r.to_vec());
326    }
327    let mut out = std::io::Cursor::new(Vec::with_capacity(image.len()));
328    crate::v06::write::assemble(
329        &parts,
330        &src,
331        &mut out,
332        crate::v06::write::WriteOpts::default(),
333    )?;
334    Ok(out.into_inner())
335}
336
337/// A `0x05` image whose named graphs are stored per graph (kind 5), rebuilt as
338/// a `0x05` image with the quad index: every statement is read back out and
339/// assembled again with what the file carries (pyramid, text index and its
340/// scope, label index, permutations, card). What `rete convert` does for such a
341/// file, and what the public per-graph writers fall back to.
342pub fn reencode_per_graph(image: &[u8]) -> Result<Vec<u8>, FileError> {
343    let rete = crate::Rete::open(image)?;
344    let card = rete.metadata().map(<[u8]>::to_vec);
345    let mut quads: Vec<crate::format::RawQuad> = rete
346        .dump(None)
347        .into_iter()
348        .map(|(s, p, o)| (s, p, o, None))
349        .collect();
350    let named: Vec<String> = rete.graph_names().iter().map(|g| g.to_string()).collect();
351    for g in &named {
352        for (s, p, o) in rete.dump(Some(g)) {
353            quads.push((s, p, o, Some(g.clone())));
354        }
355    }
356    let scope = rete.text_index_scope();
357    let mut opts = crate::ingest::AssembleOptions::sectioned();
358    opts.graph_layout = crate::GraphLayout::Quad;
359    opts.with_pyramid = rete.header().pyramid_meta_len > 0;
360    opts.with_text_index = rete.has_text_index();
361    opts.text_index_scope = scope.as_ref();
362    opts.label_index = rete.has_label_index();
363    opts.perms = rete.header().perms;
364    let (out, _) =
365        crate::ingest::assemble_dataset_opts(quads, &opts, |_, _| card.clone().unwrap_or_default());
366    Ok(out)
367}
368
369#[cfg(test)]
370mod tests {
371    use super::*;
372
373    #[test]
374    fn rfc3339_known_dates() {
375        assert_eq!(rfc3339(0), "1970-01-01T00:00:00Z");
376        assert_eq!(rfc3339(1_700_000_000), "2023-11-14T22:13:20Z");
377        assert_eq!(rfc3339(1_785_801_600), "2026-08-04T00:00:00Z");
378    }
379
380    #[test]
381    fn the_record_names_its_front_end_and_the_engine() {
382        let r = Record {
383            front_end: front_end::PYTHON.into(),
384            front_end_version: "9.9.9".into(),
385            built_at: Some("2026-01-02T03:04:05Z".into()),
386            options: Options {
387                command: "build".into(),
388                format: Some("nt".into()),
389                ..Default::default()
390            },
391            inputs: vec![Input::from_bytes("string", "nt", b"<a> <b> <c> .\n")],
392            statements_read: None,
393        };
394        let v: Value = serde_json::from_slice(&to_json(&r, None)).unwrap();
395        assert_eq!(v["front_end"], "python");
396        assert_eq!(v["builder"], "rete-python 9.9.9");
397        assert_eq!(v["engine"]["name"], "rete-core");
398        assert_eq!(v["engine"]["version"], env!("CARGO_PKG_VERSION"));
399        assert_eq!(v["params"]["command"], "build");
400        assert_eq!(v["inputs"][0]["bytes"], 14);
401        assert_eq!(v["schema"], 1);
402    }
403    #[test]
404    fn a_binding_style_record_says_the_file_has_a_label_index() {
405        // The callers of the shared assembler (Python, R, Java, the browser) set
406        // no `label_index` option, yet the file has the index; the record reads
407        // it from the build's stats.
408        let quads: Vec<crate::format::RawQuad> = vec![(
409            "<http://ex/a>".into(),
410            "<http://www.w3.org/2000/01/rdf-schema#label>".into(),
411            "\"A\"".into(),
412            None,
413        )];
414        let (image, stats) = crate::ingest::assemble_dataset_quad(quads, &[]);
415        assert!(crate::Rete::open(&image).unwrap().has_label_index());
416        let r = Record {
417            front_end: front_end::PYTHON.into(),
418            front_end_version: "9.9.9".into(),
419            built_at: None,
420            options: Options {
421                command: "build".into(),
422                ..Default::default()
423            },
424            inputs: vec![],
425            statements_read: Some(1),
426        };
427        let v: Value = serde_json::from_slice(&to_json(&r, Some(&stats))).unwrap();
428        assert_eq!(v["params"]["label_index"], true);
429        // Without stats (nothing known) the field stays absent.
430        let v: Value = serde_json::from_slice(&to_json(&r, None)).unwrap();
431        assert!(v["params"].get("label_index").is_none());
432    }
433}
rete_core/build_record.rs

rete_core/
build_record.rs
