pub fn compact(run: &EvidenceRun, include_catalog: bool) -> Result<Value>Expand description
Encode run in the compact form.
With include_catalog false the constraint catalog is omitted; decoding
then needs it supplied out of band (expand_with_catalog). Callers that
already hold the schema the run was produced against should omit it.
Examples found in repository?
examples/bench_evidence.rs (line 397)
132fn main() {
133 let args = parse_args();
134
135 eprintln!("loading shapes {}…", args.shapes.display());
136 let shapes_bytes = fs::read(&args.shapes).expect("cannot read --shapes");
137 let shapes = shifty_parse::load_turtle(&shapes_bytes, None).expect("cannot parse --shapes");
138 let compiled = CompiledShapes::compile(shapes).expect("cannot compile shapes");
139 drop(shapes_bytes);
140
141 if args.header {
142 println!("{COLUMNS}");
143 }
144
145 for model in models_in(&args.models) {
146 let name = model
147 .file_name()
148 .and_then(|name| name.to_str())
149 .unwrap_or("?")
150 .to_string();
151 eprint!("{name}… ");
152
153 // ---- untimed setup: everything common to all workflows --------------
154 let data_bytes = fs::read(&model).expect("cannot read model");
155 let data = shifty_parse::load_turtle(&data_bytes, None).expect("cannot parse model");
156 drop(data_bytes);
157 let data_graph = if args.no_infer {
158 data.graph.clone()
159 } else {
160 compiled
161 .session(
162 SessionData::Separate(data.graph.clone()),
163 SessionOptions {
164 inference: true,
165 ..SessionOptions::default()
166 },
167 )
168 .unwrap_or_else(|error| panic!("{name}: inference failed: {error}"))
169 .data()
170 .clone()
171 };
172 let data_triples = data_graph.len();
173
174 let prepared = PreparedEvidenceValidator::with_graphs(
175 &data_graph,
176 &compiled.source().graph,
177 compiled.authored_schema(),
178 ValidationGraphMode::Union,
179 )
180 .unwrap_or_else(|error| panic!("{name}: schema is non-stratifiable: {error}"));
181 let options = ValidationOptions::default();
182 let scan_options = ConformanceOptions::default();
183
184 // ---- timed: validation only -----------------------------------------
185 // One warm-up round, discarded: it faults in the caches all arms share.
186 // Its duration is only consulted when `--budget-ms` is in play.
187 let probe = Instant::now();
188 let _ = prepared.validate_conformance(&scan_options);
189 let _ = prepared.validate_canonical(&options);
190 let (_, warm_failures) = prepared.find_failures(&scan_options);
191 for pair in &warm_failures {
192 std::hint::black_box(prepared.explain_canonical(pair));
193 }
194 let probe_ms = probe.elapsed().as_secs_f64() * 1e3;
195 // Every model gets the same iteration count. `--budget-ms` only caps the
196 // count for a corpus slow enough that a fixed count is impractical; it
197 // is off by default, and the emitted `iters` column records what was used.
198 let iters = if args.budget_ms > 0.0 {
199 ((args.budget_ms / probe_ms.max(f64::MIN_POSITIVE)) as usize)
200 .clamp(args.min_iters.min(args.iters), args.iters)
201 } else {
202 args.iters
203 };
204
205 let mut conformance_samples = Vec::with_capacity(iters);
206 let mut evidence_samples = Vec::with_capacity(iters);
207 let mut discovery_samples = Vec::with_capacity(iters);
208 let mut explanation_samples = Vec::with_capacity(iters);
209 let mut on_demand_samples = Vec::with_capacity(iters);
210 let mut last_run = None;
211 let mut last_conformance = None;
212 let mut last_on_demand_conformance = None;
213 let mut last_failures = Vec::new();
214 let mut last_explanations = Vec::new();
215
216 // Rotate A/B/C so no arm systematically inherits the cache, allocator,
217 // scheduler, or thermal state left by another arm.
218 for iteration in 0..iters {
219 for arm in ARM_ORDERS[iteration % ARM_ORDERS.len()] {
220 match arm {
221 Arm::Conformance => {
222 let start = Instant::now();
223 let conformance_run = prepared.validate_conformance(&scan_options);
224 conformance_samples.push(start.elapsed());
225 std::hint::black_box(conformance_run.selected_pairs);
226 last_conformance = Some(conformance_run);
227 }
228 Arm::FullEvidence => {
229 let start = Instant::now();
230 let run = prepared.validate_canonical(&options);
231 evidence_samples.push(start.elapsed());
232 std::hint::black_box(run.statements.len());
233 last_run = Some(run);
234 }
235 Arm::OnDemand => {
236 let start = Instant::now();
237 let (on_demand_conformance, failures) =
238 prepared.find_failures(&scan_options);
239 let discovery = start.elapsed();
240
241 let start = Instant::now();
242 let explanations: Vec<_> = failures
243 .iter()
244 .flat_map(|pair| prepared.explain_canonical(pair))
245 .collect();
246 let explanation = start.elapsed();
247
248 discovery_samples.push(discovery);
249 explanation_samples.push(explanation);
250 on_demand_samples.push(discovery + explanation);
251 std::hint::black_box(explanations.len());
252 last_on_demand_conformance = Some(on_demand_conformance);
253 last_failures = failures;
254 last_explanations = explanations;
255 }
256 }
257 }
258 }
259
260 // ---- untimed: shape of the evidence produced -------------------------
261 // Reuse the final timed results rather than paying for another run.
262 let conformance = last_conformance.expect("at least one iteration runs");
263 let on_demand_conformance =
264 last_on_demand_conformance.expect("at least one iteration runs");
265 let run = last_run.expect("at least one iteration runs");
266 let authored_pairs: usize = run
267 .statements
268 .iter()
269 .map(|statement| statement.selected_foci.len())
270 .sum();
271 let authored_fail_pairs: usize = run
272 .statements
273 .iter()
274 .flat_map(|statement| &statement.selected_foci)
275 .filter(|focus| focus.status() == shifty_engine::EvaluationStatus::Fail)
276 .count();
277
278 // The three interfaces must describe the same normalized selection and
279 // polarity partition. Compare exact failed identities, not just the
280 // report-wide conformance bit.
281 assert_eq!(
282 conformance, on_demand_conformance,
283 "{name}: on-demand counts"
284 );
285 assert_eq!(
286 conformance.conforms, run.conforms,
287 "{name}: conformance bit"
288 );
289 let full_selected: HashSet<_> = run
290 .statements
291 .iter()
292 .flat_map(|statement| {
293 statement.selected_foci.iter().filter_map(|focus| {
294 statement
295 .normalized_statement_id
296 .map(|id| (id, focus.focus.clone()))
297 })
298 })
299 .collect();
300 let full_failures: HashSet<_> = run
301 .statements
302 .iter()
303 .flat_map(|statement| {
304 statement
305 .selected_foci
306 .iter()
307 .filter(|focus| focus.status() == shifty_engine::EvaluationStatus::Fail)
308 .filter_map(|focus| {
309 statement
310 .normalized_statement_id
311 .map(|id| (id, focus.focus.clone()))
312 })
313 })
314 .collect();
315 let discovered_failures: HashSet<_> = last_failures
316 .iter()
317 .map(|pair| (pair.normalized_statement(), pair.focus().clone()))
318 .collect();
319 assert_eq!(
320 full_selected.len(),
321 conformance.selected_pairs,
322 "{name}: selected pairs"
323 );
324 assert_eq!(
325 full_failures.len(),
326 conformance.failed,
327 "{name}: failed pairs"
328 );
329 assert_eq!(
330 full_selected.len() - full_failures.len(),
331 conformance.passed,
332 "{name}: passed pairs"
333 );
334 assert_eq!(
335 full_failures, discovered_failures,
336 "{name}: failed identities"
337 );
338
339 let full_by_authored: HashMap<_, _> = run
340 .statements
341 .iter()
342 .flat_map(|statement| {
343 statement.selected_foci.iter().map(|focus| {
344 (
345 (statement.source_statement_id, focus.focus.clone()),
346 &focus.evidence,
347 )
348 })
349 })
350 .collect();
351 let mut explained_foci = 0;
352 for statement in &last_explanations {
353 for focus in &statement.selected_foci {
354 explained_foci += 1;
355 assert_eq!(
356 focus.status(),
357 shifty_engine::EvaluationStatus::Fail,
358 "{name}: on-demand explanation polarity"
359 );
360 let key = (statement.source_statement_id, focus.focus.clone());
361 assert_eq!(
362 &focus.evidence,
363 *full_by_authored
364 .get(&key)
365 .unwrap_or_else(|| panic!("{name}: unexplained full-run pair {key:?}")),
366 "{name}: full and on-demand evidence differ for {key:?}"
367 );
368 }
369 }
370 assert_eq!(
371 explained_foci, authored_fail_pairs,
372 "{name}: authored explanations"
373 );
374
375 let evidence_nodes = run.walk().len();
376 let evidence_bytes: usize = run
377 .statements
378 .iter()
379 .flat_map(|statement| &statement.selected_foci)
380 .map(|focus| focus.evidence.to_json().expect("evidence serializes").len())
381 .sum();
382 let full_json = run.to_json().expect("evidence run serializes");
383 let full_run_bytes_with_catalog = full_json.len();
384 let mut full_value_no_catalog =
385 serde_json::to_value(&run).expect("evidence run converts to JSON");
386 full_value_no_catalog
387 .as_object_mut()
388 .expect("evidence run is a JSON object")
389 .remove("constraints");
390 let full_run_bytes_no_catalog = serde_json::to_string(&full_value_no_catalog)
391 .expect("catalog-free evidence run serializes")
392 .len();
393
394 // Compare complete runs with the same catalog policy on each side, and
395 // verify both encodings reconstruct the exact typed result.
396 let encoded_with_catalog =
397 shifty_engine::compact(&run, true).expect("compact encoding succeeds");
398 let compact_run_bytes_with_catalog = serde_json::to_string(&encoded_with_catalog)
399 .expect("compact encoding serializes")
400 .len();
401 assert_eq!(
402 shifty_engine::expand(&encoded_with_catalog).expect("compact run expands"),
403 run,
404 "{name}: compact run with catalog is not lossless"
405 );
406 let encoded_no_catalog =
407 shifty_engine::compact(&run, false).expect("compact encoding succeeds");
408 let compact_run_bytes_no_catalog = serde_json::to_string(&encoded_no_catalog)
409 .expect("catalog-free compact encoding serializes")
410 .len();
411 let catalog = serde_json::to_value(&run.constraints).expect("catalog serializes");
412 assert_eq!(
413 shifty_engine::expand_with_catalog(&encoded_no_catalog, catalog)
414 .expect("catalog-free compact run expands"),
415 run,
416 "{name}: compact run without catalog is not lossless"
417 );
418 // Sharing is reported over the evidence alone. The node and term tables
419 // of `encoded` also hold catalog entries, which would understate the
420 // redundancy by adding distinct entries with no evidence occurrences.
421 let sharing = shifty_engine::sharing(&run).expect("sharing measures");
422 let results = shifty_engine::result_sharing(&run);
423
424 // The two passes reach the same judgment nodes by routes that share no
425 // code: `sharing` interns serialized JSON, `result_sharing` walks the
426 // typed evidence. Agreement on both counts is what makes either number
427 // reportable; a mismatch means the node families are misclassified or a
428 // traversal is missing a variant.
429 assert_eq!(
430 sharing.result_occurrences + sharing.support_occurrences,
431 sharing.node_occurrences,
432 "{name}: node families do not partition occurrences"
433 );
434 assert_eq!(
435 sharing.distinct_results + sharing.distinct_support,
436 sharing.distinct_nodes,
437 "{name}: node families do not partition the table"
438 );
439 assert_eq!(
440 sharing.result_occurrences, evidence_nodes,
441 "{name}: interned judgments disagree with the typed walk"
442 );
443 assert_eq!(
444 results.occurrences, evidence_nodes,
445 "{name}: typed sharing pass disagrees with the typed walk"
446 );
447 assert_eq!(
448 results.distinct_payloads, sharing.distinct_results,
449 "{name}: distinct judgments disagree between the two passes"
450 );
451 // Structural equality implies key equality, never the reverse.
452 assert!(
453 results.distinct_keys <= results.distinct_payloads,
454 "{name}: more keys than payloads"
455 );
456 // Requests are the normalized selection the conformance pass counts;
457 // authored records are the source-preserving view over them.
458 assert_eq!(
459 results.normalized_requests, conformance.selected_pairs,
460 "{name}: requests disagree with the conformance selection"
461 );
462 assert_eq!(
463 results.authored_records, authored_pairs,
464 "{name}: authored records disagree with the run"
465 );
466
467 // Serialization is measured separately so its allocation traffic does
468 // not perturb the next validation arm.
469 let mut serialize_samples = Vec::with_capacity(iters);
470 for _ in 0..iters {
471 let start = Instant::now();
472 let json = run.to_json().expect("evidence run serializes");
473 serialize_samples.push(start.elapsed());
474 std::hint::black_box(json.len());
475 }
476
477 // Each ratio compares the corresponding raw samples from one rotated
478 // A/B/C round. The CSV retains those samples for independent checking.
479 let full_ratios: Vec<f64> = conformance_samples
480 .iter()
481 .zip(evidence_samples.iter())
482 .map(|(conformance, evidence)| {
483 evidence.as_secs_f64() / conformance.as_secs_f64().max(f64::MIN_POSITIVE)
484 })
485 .collect();
486 let on_demand_ratios: Vec<f64> = conformance_samples
487 .iter()
488 .zip(on_demand_samples.iter())
489 .map(|(conformance, on_demand)| {
490 on_demand.as_secs_f64() / conformance.as_secs_f64().max(f64::MIN_POSITIVE)
491 })
492 .collect();
493
494 let conformance_ms = summarize_durations(&conformance_samples);
495 let evidence_ms = summarize_durations(&evidence_samples);
496 let discovery_ms = summarize_durations(&discovery_samples);
497 let explanation_ms = summarize_durations(&explanation_samples);
498 let on_demand_ms = summarize_durations(&on_demand_samples);
499 let serialize_ms = summarize_durations(&serialize_samples);
500 let full_ratio = summarize_values(&full_ratios);
501 let on_demand_ratio = summarize_values(&on_demand_ratios);
502 let evaluated_pairs = conformance.selected_pairs.max(1) as f64;
503 let authored_pairs_denominator = authored_pairs.max(1) as f64;
504
505 println!(
506 "{suite},{name},{data_triples},{iters},\
507{c_med:.4},{c_mad:.4},{e_med:.4},{e_mad:.4},{full_ratio:.4},{full_ratio_mad:.4},\
508{find_med:.4},{find_mad:.4},{explain_med:.4},{explain_mad:.4},\
509{ondemand_med:.4},{ondemand_mad:.4},{ondemand_ratio:.4},{ondemand_ratio_mad:.4},\
510{serialize_med:.4},{serialize_mad:.4},\
511{conforms_c},{conforms_e},{conforms_o},\
512{pairs_eval},{pass_pairs},{fail_pairs},{fail_fraction:.6},\
513{authored_pairs},{authored_pass},{authored_fail},\
514{statements},\
515{evidence_nodes},{nodes_per_authored_pair:.3},\
516{evidence_bytes},{bytes_per_authored_pair:.1},\
517{full_with_catalog},{full_no_catalog},{compact_with_catalog},{compact_no_catalog},\
518{node_occurrences},{distinct_nodes},{node_redundancy:.3},\
519{result_occurrences},{distinct_results},{result_redundancy:.3},\
520{support_occurrences},{distinct_support},{support_redundancy:.3},{support_share:.4},\
521{term_occurrences},{distinct_terms},{term_redundancy:.3},\
522{normalized_requests},{duplicate_records},{divergent_duplicates},\
523{canonical_occurrences},{shared_payloads},{shared_payload_fraction:.4},\
524{shared_canonical_occurrences},{shared_occurrence_fraction:.4},\
525{request_reaches},{requests_per_payload:.3},{max_payload_requests},\
526{distinct_keys},{key_redundancy:.3},{multi_occurrence_keys},\
527{divergent_keys},{divergence_fraction:.4},{divergent_occurrences},\
528{distinct_payloads_per_key},{keys_over_payload_cap},{both_polarity_addresses},\
529{conformance_raw},{evidence_raw},{discovery_raw},{explanation_raw},{ondemand_raw}",
530 suite = args.suite,
531 node_occurrences = sharing.node_occurrences,
532 distinct_nodes = sharing.distinct_nodes,
533 node_redundancy = sharing.node_redundancy(),
534 result_occurrences = sharing.result_occurrences,
535 distinct_results = sharing.distinct_results,
536 result_redundancy = sharing.result_redundancy(),
537 support_occurrences = sharing.support_occurrences,
538 distinct_support = sharing.distinct_support,
539 support_redundancy = sharing.support_redundancy(),
540 support_share = sharing.support_share(),
541 term_occurrences = sharing.term_occurrences,
542 distinct_terms = sharing.distinct_terms,
543 term_redundancy = sharing.term_redundancy(),
544 normalized_requests = results.normalized_requests,
545 duplicate_records = results.duplicate_records,
546 divergent_duplicates = results.divergent_duplicates,
547 canonical_occurrences = results.canonical_occurrences,
548 shared_payloads = results.shared_payloads,
549 shared_payload_fraction = results.shared_payload_fraction(),
550 shared_canonical_occurrences = results.shared_canonical_occurrences,
551 shared_occurrence_fraction = results.shared_occurrence_fraction(),
552 request_reaches = results.request_reaches,
553 requests_per_payload = results.requests_per_payload(),
554 max_payload_requests = results.max_payload_requests,
555 distinct_keys = results.distinct_keys,
556 key_redundancy = results.key_redundancy(),
557 multi_occurrence_keys = results.multi_occurrence_keys,
558 divergent_keys = results.divergent_keys,
559 divergence_fraction = results.divergence_fraction(),
560 divergent_occurrences = results.divergent_occurrences,
561 distinct_payloads_per_key = results.distinct_payloads_per_key,
562 keys_over_payload_cap = results.keys_over_payload_cap,
563 both_polarity_addresses = results.both_polarity_addresses,
564 c_med = conformance_ms.median,
565 c_mad = conformance_ms.mad,
566 e_med = evidence_ms.median,
567 e_mad = evidence_ms.mad,
568 full_ratio = full_ratio.median,
569 full_ratio_mad = full_ratio.mad,
570 find_med = discovery_ms.median,
571 find_mad = discovery_ms.mad,
572 explain_med = explanation_ms.median,
573 explain_mad = explanation_ms.mad,
574 ondemand_med = on_demand_ms.median,
575 ondemand_mad = on_demand_ms.mad,
576 ondemand_ratio = on_demand_ratio.median,
577 ondemand_ratio_mad = on_demand_ratio.mad,
578 serialize_med = serialize_ms.median,
579 serialize_mad = serialize_ms.mad,
580 conforms_c = conformance.conforms,
581 conforms_e = run.conforms,
582 conforms_o = on_demand_conformance.conforms,
583 pairs_eval = conformance.selected_pairs,
584 pass_pairs = conformance.passed,
585 fail_pairs = conformance.failed,
586 fail_fraction = conformance.failed as f64 / evaluated_pairs,
587 authored_pass = authored_pairs - authored_fail_pairs,
588 authored_fail = authored_fail_pairs,
589 statements = run.statements.len(),
590 nodes_per_authored_pair = evidence_nodes as f64 / authored_pairs_denominator,
591 bytes_per_authored_pair = evidence_bytes as f64 / authored_pairs_denominator,
592 full_with_catalog = full_run_bytes_with_catalog,
593 full_no_catalog = full_run_bytes_no_catalog,
594 compact_with_catalog = compact_run_bytes_with_catalog,
595 compact_no_catalog = compact_run_bytes_no_catalog,
596 conformance_raw = samples_string(&conformance_samples),
597 evidence_raw = samples_string(&evidence_samples),
598 discovery_raw = samples_string(&discovery_samples),
599 explanation_raw = samples_string(&explanation_samples),
600 ondemand_raw = samples_string(&on_demand_samples),
601 );
602
603 eprintln!(
604 "median: {:.1}ms conformance, {:.1}ms full ({:.2}x), {:.1}ms on-demand ({:.2}x); {} failures / {} pairs; {iters} rounds",
605 conformance_ms.median,
606 evidence_ms.median,
607 full_ratio.median,
608 on_demand_ms.median,
609 on_demand_ratio.median,
610 conformance.failed,
611 conformance.selected_pairs,
612 );
613 }
614}