1use std::collections::{BTreeMap, BTreeSet};
33use std::path::{Path, PathBuf};
34
35use serde::Serialize;
36use sha2::{Digest, Sha256};
37
38use provable_contracts::lint::collect_yaml_files;
39use provable_contracts::schema::parse_contract;
40pub use provable_contracts::schema::{parse_external_corpora_str, ExternalCorpus};
46
47use crate::contract_walk::{ParseErrors, ZeroContracts};
48
49pub const SCHEMA: &str = "ont.paiml.dev/census/v1alpha1";
51
52pub const TIMING_RUNS: usize = 5;
58
59const QUARANTINE_DIR: &str = "quarantine";
61
62#[derive(Debug, Clone, Copy, PartialEq, Eq)]
64pub enum Anchoring {
65 Unanchored,
67 Class,
69 Instance,
71}
72
73#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize)]
75pub struct AnchoringCounts {
76 pub unanchored: usize,
77 pub class: usize,
78 pub instance: usize,
79}
80
81#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
83pub struct Timing {
84 pub census_cpu_ms_p50: Option<u64>,
85 pub lint_cpu_ms_p50: Option<u64>,
86 pub host_class: Option<String>,
87 pub n_runs: usize,
88}
89
90impl Default for Timing {
91 fn default() -> Self {
92 Self {
93 census_cpu_ms_p50: None,
94 lint_cpu_ms_p50: None,
95 host_class: None,
96 n_runs: TIMING_RUNS,
97 }
98 }
99}
100
101#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
103pub struct Census {
104 pub schema: String,
105 pub git_sha: Option<String>,
112 pub n_files: usize,
113 pub n_parsed: usize,
114 pub n_parse_errors: usize,
119 pub parse_errors: Vec<String>,
120 pub quarantined_n: usize,
121 pub by_kind: BTreeMap<String, usize>,
122 pub by_entity_type: BTreeMap<String, usize>,
123 pub by_anchoring: AnchoringCounts,
124 pub id_set_sha256: String,
128 pub declared_external: Vec<ExternalCorpus>,
129 pub timing: Timing,
130}
131
132#[must_use]
135pub fn classify(yaml: &str) -> (Anchoring, Option<String>) {
136 let Some(block) = entity_block(yaml) else {
137 return (Anchoring::Unanchored, None);
138 };
139 let ty = scalar_field(&block, "type");
140 let has_ref = scalar_field(&block, "ref").is_some();
141 match (ty, has_ref) {
142 (Some(t), true) => (Anchoring::Instance, Some(t)),
143 (Some(t), false) => (Anchoring::Class, Some(t)),
144 (None, _) => (Anchoring::Unanchored, None),
147 }
148}
149
150fn entity_block(yaml: &str) -> Option<String> {
152 let mut lines = yaml.lines();
153 while let Some(line) = lines.next() {
154 let Some(rest) = line.strip_prefix("entity:") else {
155 continue;
156 };
157 let rest = rest.trim();
158 if rest.is_empty() {
159 return Some(indented_block(&mut lines));
161 }
162 return Some(rest.to_string());
164 }
165 None
166}
167
168fn indented_block<'a>(lines: &mut impl Iterator<Item = &'a str>) -> String {
170 let mut block = String::new();
171 for next in lines {
172 if next.trim().is_empty() {
173 continue;
174 }
175 if !next.starts_with([' ', '\t']) {
176 break;
177 }
178 block.push_str(next.trim());
179 block.push('\n');
180 }
181 block
182}
183
184fn scalar_field(block: &str, key: &str) -> Option<String> {
186 let needle = format!("{key}:");
187 let idx = block.find(&needle)?;
188 let after = &block[idx + needle.len()..];
189 let val: String = after
190 .trim_start()
191 .chars()
192 .take_while(|c| !matches!(c, ',' | '}' | '\n' | '#'))
193 .collect();
194 let val = val.trim().trim_matches(['"', '\'']).to_string();
195 if val.is_empty() {
196 None
197 } else {
198 Some(val)
199 }
200}
201
202fn stem_of(path: &Path) -> String {
203 path.file_stem()
204 .and_then(|s| s.to_str())
205 .unwrap_or("unknown")
206 .to_string()
207}
208
209fn id_set_sha256(ids: &BTreeSet<String>) -> String {
211 let mut hasher = Sha256::new();
212 for id in ids {
213 hasher.update(id.as_bytes());
214 hasher.update(b"\n");
215 }
216 hasher
217 .finalize()
218 .iter()
219 .fold(String::with_capacity(64), |mut s, b| {
220 use std::fmt::Write;
221 let _ = write!(s, "{b:02x}");
222 s
223 })
224}
225
226fn declared_external(root: &Path) -> Result<Vec<ExternalCorpus>, Box<dyn std::error::Error>> {
230 let path = root.join("external-corpora.yaml");
231 if !path.is_file() {
232 return Ok(Vec::new());
233 }
234 let text = std::fs::read_to_string(&path)?;
235 let parsed = parse_external_corpora_str(&text)
236 .map_err(|e| format!("{} does not parse: {e}", path.display()))?;
237 let mut corpora = parsed.corpora;
238 corpora.sort_by(|a, b| a.name.cmp(&b.name));
239 Ok(corpora)
240}
241
242pub fn census_of(dir: &Path) -> Result<Census, Box<dyn std::error::Error>> {
245 let mut all = Vec::new();
246 if dir.is_dir() {
247 collect_yaml_files(dir, &mut all);
248 }
249 let mut files = all;
250 if files.is_empty() {
251 return Err(ZeroContracts {
252 path: dir.to_path_buf(),
253 filter: None,
254 }
255 .into());
256 }
257 files.sort();
258 let mut census = empty_census(files.len(), quarantined_n(dir));
259 let mut errors = Vec::new();
260 let mut ids = BTreeSet::new();
261 for path in &files {
262 tally(path, &mut census, &mut ids, &mut errors);
263 }
264 if !errors.is_empty() {
265 return Err(ParseErrors {
266 path: dir.to_path_buf(),
267 files: files.len(),
268 errors,
269 }
270 .into());
271 }
272 census.id_set_sha256 = id_set_sha256(&ids);
273 census.declared_external = declared_external(dir)?;
274 Ok(census)
275}
276
277fn quarantined_n(root: &Path) -> usize {
281 let mut out = Vec::new();
282 let dir = root.join(QUARANTINE_DIR);
283 if dir.is_dir() {
284 collect_quarantined(&dir, &mut out);
285 }
286 out.len()
287}
288
289fn collect_quarantined(dir: &Path, out: &mut Vec<PathBuf>) {
290 let Ok(entries) = std::fs::read_dir(dir) else {
291 return;
292 };
293 for entry in entries.flatten() {
294 let path = entry.path();
295 if path.is_dir() {
296 collect_quarantined(&path, out);
297 } else if path.extension().and_then(|e| e.to_str()) == Some("yaml") {
298 out.push(path);
299 }
300 }
301}
302
303fn empty_census(n_files: usize, quarantined_n: usize) -> Census {
304 Census {
305 schema: SCHEMA.to_string(),
306 git_sha: None,
307 n_files,
308 n_parsed: 0,
309 n_parse_errors: 0,
310 parse_errors: Vec::new(),
311 quarantined_n,
312 by_kind: BTreeMap::new(),
313 by_entity_type: BTreeMap::new(),
314 by_anchoring: AnchoringCounts::default(),
315 id_set_sha256: String::new(),
316 declared_external: Vec::new(),
317 timing: Timing::default(),
318 }
319}
320
321fn tally(
324 path: &Path,
325 census: &mut Census,
326 ids: &mut BTreeSet<String>,
327 errors: &mut Vec<(PathBuf, String)>,
328) {
329 let contract = match parse_contract(path) {
330 Ok(c) => c,
331 Err(e) => {
332 errors.push((path.to_path_buf(), e.to_string()));
333 census.n_parse_errors += 1;
334 census.parse_errors.push(path.display().to_string());
335 return;
336 }
337 };
338 census.n_parsed += 1;
339 ids.insert(stem_of(path));
340 *census
341 .by_kind
342 .entry(contract.kind().to_string())
343 .or_insert(0) += 1;
344 let Ok(text) = std::fs::read_to_string(path) else {
345 census.by_anchoring.unanchored += 1;
346 return;
347 };
348 let (anchoring, ty) = classify(&text);
349 match anchoring {
350 Anchoring::Unanchored => census.by_anchoring.unanchored += 1,
351 Anchoring::Class => census.by_anchoring.class += 1,
352 Anchoring::Instance => census.by_anchoring.instance += 1,
353 }
354 if let Some(t) = ty {
355 *census.by_entity_type.entry(t).or_insert(0) += 1;
356 }
357}
358
359pub fn render_json(census: &Census) -> Result<String, Box<dyn std::error::Error>> {
363 let mut json = serde_json::to_string_pretty(census)?;
364 json.push('\n');
365 Ok(json)
366}
367
368fn render_table(census: &Census) -> String {
369 use std::fmt::Write;
370 let mut out = String::new();
371 let _ = writeln!(out, "== pv census (ONT-001 ONT-1) ==");
372 let _ = writeln!(
373 out,
374 "contracts: {} parsed ({} file(s), {} parse error(s), {} quarantined)",
375 census.n_parsed, census.n_files, census.n_parse_errors, census.quarantined_n
376 );
377 let _ = writeln!(out, "id_set_sha256: {}", census.id_set_sha256);
378 let _ = writeln!(out, "\nby_anchoring");
379 let _ = writeln!(
380 out,
381 " unanchored {:>6} (no entity: — a law, pattern or policy; optional by R-5)",
382 census.by_anchoring.unanchored
383 );
384 let _ = writeln!(
385 out,
386 " class {:>6} (entity: {{type}} — a shape over every entity of that type)",
387 census.by_anchoring.class
388 );
389 let _ = writeln!(
390 out,
391 " instance {:>6} (entity: {{type, ref}} — one named thing)",
392 census.by_anchoring.instance
393 );
394 let _ = writeln!(out, "\nby_entity_type");
395 if census.by_entity_type.is_empty() {
396 let _ = writeln!(
397 out,
398 " (none — no contract in this corpus names an entity type)"
399 );
400 }
401 for (k, v) in &census.by_entity_type {
402 let _ = writeln!(out, " {k:<28} {v:>6}");
403 }
404 let _ = writeln!(out, "\nby_kind");
405 for (k, v) in &census.by_kind {
406 let _ = writeln!(out, " {k:<28} {v:>6}");
407 }
408 for ext in &census.declared_external {
409 let _ = writeln!(
410 out,
411 "\ndeclared external: {} — {} file(s), NOT in the count above ({})",
412 ext.name,
413 ext.n_files,
414 ext.mark.as_deref().unwrap_or("[U]")
415 );
416 }
417 out
418}
419
420pub fn run(contract_dir: &Path, json: bool) -> Result<(), Box<dyn std::error::Error>> {
422 let census = census_of(contract_dir)?;
423 if json {
424 print!("{}", render_json(&census)?);
425 } else {
426 print!("{}", render_table(&census));
427 }
428 Ok(())
429}
430
431#[cfg(test)]
432mod tests {
433 use super::*;
434 use crate::contract_walk::{exit_code_for, verdict_for, ZERO_CONTRACTS_EXIT};
435
436 fn write_valid(dir: &Path, name: &str) {
437 let path = dir.join(name);
438 if let Some(parent) = path.parent() {
439 std::fs::create_dir_all(parent).expect("fixture dir is creatable");
440 }
441 std::fs::write(
442 &path,
443 "metadata:\n version: 1.0.0\n description: ONT-1 fixture\n",
444 )
445 .expect("fixture contract is writable");
446 }
447
448 fn corpus(names: &[&str]) -> tempfile::TempDir {
449 let tmp = tempfile::tempdir().expect("temp dir is creatable");
450 for name in names {
451 write_valid(tmp.path(), name);
452 }
453 tmp
454 }
455
456 #[test]
461 fn an_empty_corpus_declines_at_exit_2() {
462 let tmp = tempfile::tempdir().expect("temp dir is creatable");
463 let err = census_of(tmp.path()).expect_err("an empty corpus is refused");
464 assert_eq!(
465 exit_code_for(err.as_ref()),
466 ZERO_CONTRACTS_EXIT,
467 "an empty corpus must decline (exit 2), not fail: {err}"
468 );
469 assert_eq!(verdict_for(err.as_ref()), "decline");
470 assert_eq!(
471 err.to_string(),
472 format!("0 contracts under {}", tmp.path().display())
473 );
474 }
475
476 #[test]
479 fn a_parse_error_rejects_at_exit_1_and_is_never_counted() {
480 let tmp = corpus(&["a.yaml", "b.yaml", "c.yaml"]);
481 std::fs::write(tmp.path().join("garbage.yaml"), "{{{ not yaml at all: [\n")
482 .expect("fixture file is writable");
483 let err = census_of(tmp.path())
484 .expect_err("a corpus with an unparseable file is rejected, never censused");
485 assert_eq!(
486 exit_code_for(err.as_ref()),
487 1,
488 "a parse error rejects: {err}"
489 );
490 assert_eq!(verdict_for(err.as_ref()), "reject");
491 assert_eq!(
492 err.to_string().lines().next().unwrap_or_default(),
493 format!("1 parse error under {}", tmp.path().display())
494 );
495 }
496
497 #[test]
500 fn n_files_equals_n_parsed_plus_n_parse_errors() {
501 let tmp = corpus(&["a.yaml", "b.yaml", "nested/c.yaml"]);
502 let c = census_of(tmp.path()).expect("a valid corpus censuses");
503 assert_eq!(c.n_files, 3);
504 assert_eq!(c.n_parsed + c.n_parse_errors, c.n_files);
505 assert_eq!(c.n_parse_errors, 0);
506 assert!(c.parse_errors.is_empty());
507 }
508
509 #[test]
511 fn the_shared_walker_decides_what_a_contract_file_is() {
512 let tmp = corpus(&["a.yaml", "binding.yaml", "kaizen/k.yaml"]);
513 let c = census_of(tmp.path()).expect("a valid corpus censuses");
514 assert_eq!(
515 c.n_files, 1,
516 "binding.yaml and kaizen/ are excluded by provable_contracts::lint's walker"
517 );
518 }
519
520 #[test]
521 fn git_sha_is_null_and_the_schema_is_named() {
522 let tmp = corpus(&["a.yaml"]);
523 let c = census_of(tmp.path()).expect("a valid corpus censuses");
524 assert_eq!(
525 c.git_sha, None,
526 "a commit sha is unknowable for its own commit"
527 );
528 assert_eq!(c.schema, SCHEMA);
529 }
530
531 #[test]
532 fn the_timing_baseline_declares_five_runs_and_measures_nothing_here() {
533 let tmp = corpus(&["a.yaml"]);
534 let c = census_of(tmp.path()).expect("a valid corpus censuses");
535 assert_eq!(c.timing.n_runs, 5);
536 assert_eq!(c.timing.census_cpu_ms_p50, None);
537 assert_eq!(c.timing.lint_cpu_ms_p50, None);
538 assert_eq!(c.timing.host_class, None);
539 }
540
541 #[test]
542 fn id_set_sha256_is_64_hex_stable_and_moves_with_the_id_set() {
543 let tmp = corpus(&["a.yaml", "b.yaml"]);
544 let first = census_of(tmp.path()).expect("censuses");
545 let again = census_of(tmp.path()).expect("censuses");
546 assert_eq!(first.id_set_sha256, again.id_set_sha256);
547 assert_eq!(first.id_set_sha256.len(), 64);
548 assert!(first.id_set_sha256.chars().all(|c| c.is_ascii_hexdigit()));
549 write_valid(tmp.path(), "c.yaml");
550 let after = census_of(tmp.path()).expect("censuses");
551 assert_ne!(first.id_set_sha256, after.id_set_sha256);
552 }
553
554 #[test]
555 fn by_kind_uses_the_contract_kind_vocabulary() {
556 let tmp = corpus(&["a.yaml"]);
557 std::fs::write(
558 tmp.path().join("p.yaml"),
559 "metadata:\n version: 1.0.0\n kind: pattern\n description: fixture\n",
560 )
561 .expect("fixture is writable");
562 let c = census_of(tmp.path()).expect("censuses");
563 assert_eq!(c.by_kind.get("pattern"), Some(&1));
564 assert_eq!(c.by_kind.get("kernel"), Some(&1), "kind defaults to kernel");
565 }
566
567 #[test]
568 fn quarantined_contracts_are_counted_and_not_censused() {
569 let tmp = corpus(&["a.yaml", "quarantine/broken.yaml"]);
570 let c = census_of(tmp.path()).expect("censuses");
571 assert_eq!(c.n_files, 1);
572 assert_eq!(c.quarantined_n, 1);
573 }
574
575 #[test]
576 fn declared_external_is_read_from_the_declaration_and_empty_without_one() {
577 let tmp = corpus(&["a.yaml"]);
578 assert!(census_of(tmp.path())
579 .expect("censuses")
580 .declared_external
581 .is_empty());
582 std::fs::write(
583 tmp.path().join("external-corpora.yaml"),
584 "schema: ont.paiml.dev/external-corpora/v1alpha1\ncorpora:\n - name: archived\n n_files: 397\n counted_by: gh api ...\n",
585 )
586 .expect("declaration is writable");
587 let c = census_of(tmp.path()).expect("censuses");
588 assert_eq!(c.declared_external.len(), 1);
589 assert_eq!(c.declared_external[0].n_files, 397);
590 assert_eq!(
591 c.n_files, 1,
592 "a declared external corpus is never added to the cardinality"
593 );
594 }
595
596 #[test]
598 fn render_json_is_deterministic_and_ends_in_one_newline() {
599 let tmp = corpus(&["a.yaml", "b.yaml"]);
600 let first = render_json(&census_of(tmp.path()).expect("censuses")).expect("renders");
601 let again = render_json(&census_of(tmp.path()).expect("censuses")).expect("renders");
602 assert_eq!(first, again);
603 assert!(first.ends_with("}\n"));
604 let parsed: serde_json::Value = serde_json::from_str(&first).expect("valid JSON");
605 for key in [
606 "schema",
607 "git_sha",
608 "n_files",
609 "n_parsed",
610 "n_parse_errors",
611 "parse_errors",
612 "quarantined_n",
613 "by_kind",
614 "by_entity_type",
615 "by_anchoring",
616 "id_set_sha256",
617 "declared_external",
618 "timing",
619 ] {
620 assert!(parsed.get(key).is_some(), "census.json must carry {key}");
621 }
622 assert!(parsed["git_sha"].is_null());
623 }
624
625 #[test]
628 fn absent_entity_is_unanchored() {
629 assert_eq!(classify("id: X\nkind: Kernel\n").0, Anchoring::Unanchored);
630 }
631
632 #[test]
633 fn inline_type_only_is_class_level() {
634 let (a, t) = classify("entity: { type: readme }\n");
635 assert_eq!(a, Anchoring::Class);
636 assert_eq!(t.as_deref(), Some("readme"));
637 }
638
639 #[test]
640 fn inline_type_and_ref_is_instance_level() {
641 let (a, t) = classify("entity: { type: readme, ref: README.md }\n");
642 assert_eq!(a, Anchoring::Instance);
643 assert_eq!(t.as_deref(), Some("readme"));
644 }
645
646 #[test]
647 fn nested_block_is_read_too() {
648 let (a, t) = classify("id: X\nentity:\n type: gguf\n ref: m.gguf\nshape: {}\n");
649 assert_eq!(a, Anchoring::Instance);
650 assert_eq!(t.as_deref(), Some("gguf"));
651 }
652
653 #[test]
654 fn nested_type_only_is_class_level() {
655 let (a, t) = classify("entity:\n type: csv\nshape: {}\n");
656 assert_eq!(a, Anchoring::Class);
657 assert_eq!(t.as_deref(), Some("csv"));
658 }
659
660 #[test]
663 fn entity_without_a_type_is_not_an_anchor() {
664 assert_eq!(
665 classify("entity: { ref: README.md }\n").0,
666 Anchoring::Unanchored
667 );
668 }
669
670 #[test]
672 fn indented_entity_is_not_the_top_level_block() {
673 assert_eq!(
674 classify("metadata:\n entity: { type: code }\n").0,
675 Anchoring::Unanchored
676 );
677 }
678
679 #[test]
680 fn quoted_values_are_unquoted() {
681 let (_, t) = classify("entity: { type: \"apr-model\", ref: 'm.apr' }\n");
682 assert_eq!(t.as_deref(), Some("apr-model"));
683 }
684
685 #[test]
686 fn a_trailing_comment_is_not_part_of_the_type() {
687 let (_, t) = classify("entity:\n type: sqlite # the db\n");
688 assert_eq!(t.as_deref(), Some("sqlite"));
689 }
690
691 #[test]
693 fn a_missing_directory_is_a_decline_not_a_zero_census() {
694 let err = census_of(Path::new("/nonexistent/contracts")).expect_err("refused");
695 assert_eq!(exit_code_for(err.as_ref()), ZERO_CONTRACTS_EXIT);
696 }
697}