1use serde_json::{json, Map, Value};
21
22use crate::file::FileError;
23use crate::ingest::BuildStats;
24
25pub mod front_end {
27 pub const CLI: &str = "cli";
29 pub const PYTHON: &str = "python";
31 pub const R: &str = "r";
33 pub const JAVA: &str = "java";
35 pub const WASM: &str = "wasm";
37}
38
39#[derive(Debug, Clone, Default)]
41pub struct Options {
42 pub command: String,
44 pub format: Option<String>,
46 pub no_pyramid: bool,
48 pub text_index: bool,
50 pub label_index: bool,
52 pub pyramid_algo: Option<String>,
54 pub permutations: Option<u8>,
56}
57
58#[derive(Debug, Clone, Default)]
60pub struct Input {
61 pub name: String,
63 pub format: String,
65 pub bytes: Option<u64>,
67 pub blake3: Option<String>,
69}
70
71impl Input {
72 pub fn from_bytes(name: &str, format: &str, data: &[u8]) -> Self {
74 Input {
75 name: name.to_string(),
76 format: format.to_string(),
77 bytes: Some(data.len() as u64),
78 blake3: Some(blake3::hash(data).to_hex().to_string()),
79 }
80 }
81}
82
83#[derive(Debug, Clone, Default)]
85pub struct Record {
86 pub front_end: String,
88 pub front_end_version: String,
90 pub built_at: Option<String>,
93 pub options: Options,
95 pub inputs: Vec<Input>,
97 pub statements_read: Option<u64>,
99}
100
101pub fn reproducible() -> bool {
103 #[cfg(not(target_arch = "wasm32"))]
104 {
105 std::env::var("RETE_BUILD_RECORD").is_ok_and(|v| v.trim() == "reproducible")
106 }
107 #[cfg(target_arch = "wasm32")]
108 {
109 false
110 }
111}
112
113pub fn writes_legacy() -> bool {
116 #[cfg(not(target_arch = "wasm32"))]
117 {
118 std::env::var("RETE_WRITE_FORMAT").is_ok_and(|v| v.trim() == "5")
119 }
120 #[cfg(target_arch = "wasm32")]
121 {
122 false
123 }
124}
125
126pub fn timestamp() -> Option<String> {
131 #[cfg(not(target_arch = "wasm32"))]
132 {
133 let epoch = std::env::var("SOURCE_DATE_EPOCH")
134 .ok()
135 .and_then(|v| v.parse::<i64>().ok());
136 if reproducible() && epoch.is_none() {
137 return None;
138 }
139 let secs = epoch.unwrap_or_else(|| {
140 std::time::SystemTime::now()
141 .duration_since(std::time::UNIX_EPOCH)
142 .map(|d| d.as_secs() as i64)
143 .unwrap_or(0)
144 });
145 Some(rfc3339(secs))
146 }
147 #[cfg(target_arch = "wasm32")]
148 {
149 None
150 }
151}
152
153pub fn rfc3339(secs: i64) -> String {
155 let days = secs.div_euclid(86_400);
156 let rem = secs.rem_euclid(86_400);
157 let z = days + 719_468;
158 let era = z.div_euclid(146_097);
159 let doe = z.rem_euclid(146_097);
160 let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
161 let y = yoe + era * 400;
162 let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
163 let mp = (5 * doy + 2) / 153;
164 let d = doy - (153 * mp + 2) / 5 + 1;
165 let m = if mp < 10 { mp + 3 } else { mp - 9 };
166 let y = if m <= 2 { y + 1 } else { y };
167 format!(
168 "{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}Z",
169 rem / 3600,
170 rem % 3600 / 60,
171 rem % 60
172 )
173}
174
175pub fn engine() -> Value {
177 json!({
178 "name": "rete-core",
179 "version": env!("CARGO_PKG_VERSION"),
180 "commit": env!("RETE_GIT_COMMIT"),
181 "target": env!("RETE_TARGET"),
182 "profile": env!("RETE_PROFILE"),
183 "features": crate::build_trace::enabled_features(),
184 })
185}
186
187pub fn to_json(r: &Record, stats: Option<&BuildStats>) -> Vec<u8> {
190 let mut o = Map::new();
191 o.insert("schema".into(), json!(1));
192 if let Some(t) = &r.built_at {
193 o.insert("built_at".into(), json!(t));
194 }
195 o.insert(
196 "builder".into(),
197 json!(format!("rete-{} {}", r.front_end, r.front_end_version)),
198 );
199 o.insert("front_end".into(), json!(r.front_end));
200 o.insert("engine".into(), engine());
201 #[cfg(not(target_arch = "wasm32"))]
202 let host = json!({"os": std::env::consts::OS, "arch": std::env::consts::ARCH});
203 #[cfg(target_arch = "wasm32")]
204 let host = json!({"os": "browser", "arch": "wasm32"});
205 o.insert("host".into(), host);
206
207 let mut p = Map::new();
208 p.insert("command".into(), json!(r.options.command));
209 if let Some(f) = &r.options.format {
210 p.insert("format".into(), json!(f));
211 }
212 for (k, v) in [
213 ("no_pyramid", r.options.no_pyramid),
214 ("text_index", r.options.text_index),
215 (
216 "label_index",
217 r.options.label_index || stats.is_some_and(|s| s.label_index),
218 ),
219 ] {
220 if v {
221 p.insert(k.into(), json!(true));
222 }
223 }
224 if let Some(a) = &r.options.pyramid_algo {
225 p.insert("pyramid_algo".into(), json!(a));
226 }
227 p.insert(
228 "codec".into(),
229 json!(if crate::file::writer_codec() == crate::CODEC_ZSTD {
230 "zstd"
231 } else {
232 "none"
233 }),
234 );
235 p.insert(
236 "permutations".into(),
237 json!(r.options.permutations.unwrap_or(6)),
238 );
239 p.insert("named_graphs".into(), json!("quad"));
240 let w = crate::write_opts::write_options();
241 p.insert(
242 "write".into(),
243 json!({
244 "tile_kib": w.tile_budget / 1024,
245 "dict_chunk_kib": w.dict_chunk_budget / 1024,
246 "zstd_level": w.zstd_level,
247 "zstd_frame": match w.frame {
248 crate::write_opts::FrameShape::Sized => "sized",
249 crate::write_opts::FrameShape::Stream => "stream",
250 },
251 }),
252 );
253 o.insert("params".into(), Value::Object(p));
254
255 if !r.inputs.is_empty() {
256 let inputs: Vec<Value> = r
257 .inputs
258 .iter()
259 .map(|i| {
260 let mut m = Map::new();
261 m.insert("name".into(), json!(i.name));
262 m.insert("format".into(), json!(i.format));
263 if let Some(b) = i.bytes {
264 m.insert("bytes".into(), json!(b));
265 }
266 if let Some(h) = &i.blake3 {
267 m.insert("blake3".into(), json!(h));
268 }
269 Value::Object(m)
270 })
271 .collect();
272 o.insert("inputs".into(), Value::Array(inputs));
273 }
274 if let Some(s) = stats {
275 let unique = s.statements as u64;
276 let default = s.default_triples as u64;
277 let read = r.statements_read.unwrap_or(unique);
278 o.insert(
279 "counts".into(),
280 json!({
281 "statements_read": read,
282 "statements_unique": unique,
283 "duplicates_dropped": read.saturating_sub(unique),
284 "default_graph_triples": default,
285 "named_graphs": s.named_graphs as u64,
286 "named_graph_quads": unique.saturating_sub(default),
287 "terms": s.terms as u64,
288 }),
289 );
290 }
291 serde_json::to_vec(&Value::Object(o)).unwrap_or_default()
292}
293
294pub fn finish_in_ram(image: Vec<u8>, record: Option<&[u8]>) -> Result<Vec<u8>, FileError> {
300 finish_in_ram_with(image, record, crate::v06::write::WriteOpts::default())
301}
302
303pub fn finish_in_ram_with(
305 image: Vec<u8>,
306 record: Option<&[u8]>,
307 opts: crate::v06::write::WriteOpts,
308) -> Result<Vec<u8>, FileError> {
309 let record = record.filter(|r| !r.is_empty());
310 if writes_legacy() {
311 return match record {
312 Some(r) => crate::file::attach_build_info(&image, r),
313 None => Ok(image),
314 };
315 }
316 if crate::v06::open::is_v06(&image) {
317 return match record {
318 Some(r) => crate::file::attach_build_info(&image, r),
319 None => Ok(image),
320 };
321 }
322 let src = crate::reader::SliceReader::new(&image);
323 let mut parts = match crate::v06::parts::from_v05(&src) {
324 Ok(p) => p,
325 Err(FileError::Container(m)) if m.contains("kind 5") => {
328 let requad = reencode_per_graph(&image)?;
329 return finish_in_ram_with(requad, record, opts);
330 }
331 Err(e) => return Err(e),
332 };
333 if let Some(r) = record {
334 parts.build = Some(r.to_vec());
335 }
336 let mut out = std::io::Cursor::new(Vec::with_capacity(image.len()));
337 crate::v06::write::assemble(&parts, &src, &mut out, opts)?;
338 Ok(out.into_inner())
339}
340
341pub fn reencode_per_graph(image: &[u8]) -> Result<Vec<u8>, FileError> {
347 let rete = crate::Rete::open(image)?;
348 let card = rete.metadata().map(<[u8]>::to_vec);
349 let mut quads: Vec<crate::format::RawQuad> = rete
350 .dump(None)
351 .into_iter()
352 .map(|(s, p, o)| (s, p, o, None))
353 .collect();
354 let named: Vec<String> = rete.graph_names().iter().map(|g| g.to_string()).collect();
355 for g in &named {
356 for (s, p, o) in rete.dump(Some(g)) {
357 quads.push((s, p, o, Some(g.clone())));
358 }
359 }
360 let scope = rete.text_index_scope();
361 let mut opts = crate::ingest::AssembleOptions::sectioned();
362 opts.graph_layout = crate::GraphLayout::Quad;
363 opts.with_pyramid = rete.header().pyramid_meta_len > 0;
364 opts.with_text_index = rete.has_text_index();
365 opts.text_index_scope = scope.as_ref();
366 opts.label_index = rete.has_label_index();
367 opts.perms = rete.header().perms;
368 let (out, _) =
369 crate::ingest::assemble_dataset_opts(quads, &opts, |_, _| card.clone().unwrap_or_default());
370 Ok(out)
371}
372
373#[cfg(test)]
374mod tests {
375 use super::*;
376
377 #[test]
378 fn rfc3339_known_dates() {
379 assert_eq!(rfc3339(0), "1970-01-01T00:00:00Z");
380 assert_eq!(rfc3339(1_700_000_000), "2023-11-14T22:13:20Z");
381 assert_eq!(rfc3339(1_785_801_600), "2026-08-04T00:00:00Z");
382 }
383
384 #[test]
385 fn the_record_names_its_front_end_and_the_engine() {
386 let r = Record {
387 front_end: front_end::PYTHON.into(),
388 front_end_version: "9.9.9".into(),
389 built_at: Some("2026-01-02T03:04:05Z".into()),
390 options: Options {
391 command: "build".into(),
392 format: Some("nt".into()),
393 ..Default::default()
394 },
395 inputs: vec![Input::from_bytes("string", "nt", b"<a> <b> <c> .\n")],
396 statements_read: None,
397 };
398 let v: Value = serde_json::from_slice(&to_json(&r, None)).unwrap();
399 assert_eq!(v["front_end"], "python");
400 assert_eq!(v["builder"], "rete-python 9.9.9");
401 assert_eq!(v["engine"]["name"], "rete-core");
402 assert_eq!(v["engine"]["version"], env!("CARGO_PKG_VERSION"));
403 assert_eq!(v["params"]["command"], "build");
404 assert_eq!(v["inputs"][0]["bytes"], 14);
405 assert_eq!(v["schema"], 1);
406 }
407 #[test]
408 fn a_binding_style_record_says_the_file_has_a_label_index() {
409 let quads: Vec<crate::format::RawQuad> = vec![(
413 "<http://ex/a>".into(),
414 "<http://www.w3.org/2000/01/rdf-schema#label>".into(),
415 "\"A\"".into(),
416 None,
417 )];
418 let (image, stats) = crate::ingest::assemble_dataset_quad(quads, &[]);
419 assert!(crate::Rete::open(&image).unwrap().has_label_index());
420 let r = Record {
421 front_end: front_end::PYTHON.into(),
422 front_end_version: "9.9.9".into(),
423 built_at: None,
424 options: Options {
425 command: "build".into(),
426 ..Default::default()
427 },
428 inputs: vec![],
429 statements_read: Some(1),
430 };
431 let v: Value = serde_json::from_slice(&to_json(&r, Some(&stats))).unwrap();
432 assert_eq!(v["params"]["label_index"], true);
433 let v: Value = serde_json::from_slice(&to_json(&r, None)).unwrap();
435 assert!(v["params"].get("label_index").is_none());
436 }
437}
438
439pub const DERIVE_REPORT_KEY: &str = "derive_iris";
442
443fn report_span(text: &str) -> Option<(usize, usize)> {
446 let needle = format!("\"{DERIVE_REPORT_KEY}\":");
447 let at = text.find(&needle)?;
448 let doc: serde_json::Value = serde_json::from_str(text).ok()?;
450 doc.get(DERIVE_REPORT_KEY)?;
451 let v0 = at + needle.len();
452 let mut it =
453 serde_json::Deserializer::from_str(&text[v0..]).into_iter::<serde::de::IgnoredAny>();
454 it.next()?.ok()?;
455 Some((at, v0 + it.byte_offset()))
456}
457
458pub fn with_derive_report(rec: &[u8], report: &serde_json::Value) -> Option<Vec<u8>> {
465 let text = std::str::from_utf8(rec).ok()?;
466 let body = text.trim_end_matches(|c: char| c.is_ascii_whitespace() || c == '\0');
467 let pad = &text[body.len()..];
468 let mut body = body.trim_start().to_string();
469 if !body.starts_with('{') || !body.ends_with('}') {
470 return None;
471 }
472 if let Some((a, b)) = report_span(&body) {
473 let (mut a, mut b) = (a, b);
475 if body[..a].trim_end().ends_with(',') {
476 a = body[..a].trim_end().len() - 1;
477 } else if body[b..].starts_with(',') {
478 b += 1;
479 }
480 body.replace_range(a..b, "");
481 }
482 let entry = format!("\"{DERIVE_REPORT_KEY}\":{report}");
483 let at = body.rfind(",\"conversions\":[").unwrap_or(body.len() - 1);
484 let head = body[..at].trim_end();
485 let comma = if head.ends_with('{') { "" } else { "," };
486 let out = format!("{head}{comma}{entry}{}{pad}", &body[at..]);
487 Some(out.into_bytes())
488}
489
490pub fn derive_report_text(rec: &[u8]) -> Option<String> {
492 let text = std::str::from_utf8(rec).ok()?;
493 let body = text.trim_matches(|c: char| c.is_ascii_whitespace() || c == '\0');
494 let (a, b) = report_span(body)?;
495 Some(body[a + DERIVE_REPORT_KEY.len() + 3..b].to_string())
496}
497
498#[cfg(test)]
499mod derive_report_tests {
500 use super::*;
501
502 #[test]
503 fn a_report_is_spliced_once_before_conversions() {
504 let r = serde_json::json!({"taken": [], "n": 3});
505 let rec = br#"{"schema":1,"builder":"x","conversions":[{"command":"c"}]} "#;
506 let out = with_derive_report(rec, &r).unwrap();
507 let text = String::from_utf8(out.clone()).unwrap();
508 assert!(text.contains(
509 r#""builder":"x","derive_iris":{"n":3,"taken":[]},"conversions":[{"command":"c"}]}"#
510 ));
511 assert!(text.ends_with("} "));
512 assert!(serde_json::from_str::<serde_json::Value>(&text).is_ok());
513 let r2 = serde_json::json!({"x": 1});
515 let again = with_derive_report(&out, &r2).unwrap();
516 let t2 = String::from_utf8(again.clone()).unwrap();
517 assert_eq!(t2.matches("derive_iris").count(), 1);
518 assert_eq!(derive_report_text(&again).as_deref(), Some(r#"{"x":1}"#));
519 let o = with_derive_report(br#"{"schema":1}"#, &r2).unwrap();
521 assert_eq!(o, br#"{"schema":1,"derive_iris":{"x":1}}"#);
522 let o = with_derive_report(b"{}", &r2).unwrap();
523 assert_eq!(o, br#"{"derive_iris":{"x":1}}"#);
524 assert!(with_derive_report(b"not json", &r2).is_none());
525 }
526}