1use std::collections::{BTreeMap, BTreeSet};
33use std::path::Path;
34
35use super::code::{manifest_names, workspace_membership};
36use super::gguf::ExtractError;
37use crate::ontology::rdf::{iri_path, ont, Graph, Term, PROV_ENTITY, RDF_TYPE};
38
39pub const SNAPSHOT: &str = "evidence/binary/snapshot.jsonl";
41pub const SCHEMA: &str = "binary-snapshot/v1";
43pub const LEDGER: &str = "docs/audits/surface_audit.csv";
45pub const LEDGER_HEADER: &str =
47 "binary,feature,quality_1_10,verified_hardware,top_competitor,in_dogfood_skill,\
48cluster_id,cluster_label,evidence_path,confidence";
49pub const EMPTY_SHA256: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
51
52const HTTP_VERBS: &[&str] = &["GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"];
53
54#[must_use]
56pub fn bin(name: &str) -> String {
57 ont(&format!("bin/{name}"))
58}
59
60#[must_use]
62pub fn node(package: &str, target: &str) -> String {
63 iri_path("binary", &[package, target])
64}
65
66#[derive(Debug, Clone, PartialEq, Eq, Default)]
68pub struct Target {
69 pub package: String,
70 pub target: String,
71 pub name: String,
72 pub src_path: String,
73 pub git_sha: String,
74 pub version: String,
75 pub version_sha256: String,
76 pub binary_sha256: String,
77 pub help_sha256: String,
78 pub help_failed: bool,
79 pub commands: BTreeSet<String>,
80 pub routes: BTreeSet<String>,
81 pub tools: BTreeSet<String>,
82}
83
84impl Target {
85 #[must_use]
87 pub fn version_names_sha(&self) -> bool {
88 self.git_sha
89 .get(..9)
90 .is_some_and(|sha9| self.version.contains(sha9))
91 }
92}
93
94#[derive(Debug, Clone, Default, PartialEq, Eq)]
96pub struct LedgerEntry {
97 pub rows: BTreeSet<String>,
99 pub commands: BTreeSet<String>,
100 pub routes: BTreeSet<String>,
101 pub tools: BTreeSet<String>,
102}
103
104#[derive(Debug, Clone, Default, PartialEq, Eq)]
106pub struct BinaryStats {
107 pub at_workspace_root: bool,
109 pub targets: usize,
111 pub names: usize,
113 pub census: Option<usize>,
115 pub ledger_rows: usize,
117 pub version_lacks_sha: usize,
118 pub name_pair_mismatch: usize,
120 pub help_failed: usize,
121 pub unledgered: usize,
122 pub orphans: usize,
123 pub no_audit_row: usize,
125 pub errors: Vec<ExtractError>,
126}
127
128fn refuse(errors: &mut Vec<ExtractError>, file: &str, what: String) {
129 errors.push(ExtractError {
130 file: file.to_string(),
131 what,
132 });
133}
134
135fn str_key(v: &serde_json::Value, k: &str) -> Result<String, String> {
136 v.get(k)
137 .and_then(serde_json::Value::as_str)
138 .map(str::to_string)
139 .ok_or_else(|| format!("key `{k}` is missing or not a string"))
140}
141
142fn set_key(v: &serde_json::Value, k: &str) -> Result<BTreeSet<String>, String> {
143 let arr = v
144 .get(k)
145 .and_then(serde_json::Value::as_array)
146 .ok_or_else(|| {
147 format!("key `{k}` is missing or not an array (an empty surface is `[]`)")
148 })?;
149 arr.iter()
150 .map(|x| {
151 x.as_str()
152 .map(str::to_string)
153 .ok_or_else(|| format!("`{k}` holds a non-string"))
154 })
155 .collect()
156}
157
158fn hex_of(s: &str, len: usize) -> bool {
159 s.len() == len
160 && s.bytes()
161 .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
162}
163
164fn parse_line(line: &str) -> Result<Target, String> {
166 let v: serde_json::Value = serde_json::from_str(line).map_err(|e| format!("not JSON: {e}"))?;
167 let schema = str_key(&v, "schema")?;
168 if schema != SCHEMA {
169 return Err(format!("schema `{schema}` is not `{SCHEMA}`"));
170 }
171 let t = Target {
172 package: str_key(&v, "package")?,
173 target: str_key(&v, "target")?,
174 name: str_key(&v, "name")?,
175 src_path: str_key(&v, "src_path")?,
176 git_sha: str_key(&v, "git_sha")?,
177 version: str_key(&v, "version")?,
178 version_sha256: str_key(&v, "version_sha256")?,
179 binary_sha256: str_key(&v, "binary_sha256")?,
180 help_sha256: str_key(&v, "help_sha256")?,
181 help_failed: v
182 .get("help_failed")
183 .and_then(serde_json::Value::as_bool)
184 .unwrap_or(false),
185 commands: set_key(&v, "commands")?,
186 routes: set_key(&v, "routes")?,
187 tools: set_key(&v, "tools")?,
188 };
189 if t.package.is_empty() || t.target.is_empty() || t.name.is_empty() {
190 return Err("package, target and name must be non-empty".into());
191 }
192 if !hex_of(&t.git_sha, 40) {
193 return Err(format!("git_sha `{}` is not 40 lowercase hex", t.git_sha));
194 }
195 Ok(t)
196}
197
198#[must_use]
200pub fn parse_snapshot(text: &str) -> (Vec<Target>, Vec<ExtractError>) {
201 let mut errors = Vec::new();
202 let mut by_key: BTreeMap<(String, String), Target> = BTreeMap::new();
203 for (i, line) in text.lines().enumerate() {
204 if line.trim().is_empty() {
205 continue;
206 }
207 let file = format!("{SNAPSHOT}:{}", i + 1);
208 match parse_line(line) {
209 Ok(t) => match by_key.entry((t.package.clone(), t.target.clone())) {
210 std::collections::btree_map::Entry::Occupied(e) => refuse(
211 &mut errors,
212 &file,
213 format!("duplicate bin target {}/{}", e.key().0, e.key().1),
214 ),
215 std::collections::btree_map::Entry::Vacant(e) => {
216 e.insert(t);
217 }
218 },
219 Err(what) => refuse(&mut errors, &file, what),
220 }
221 }
222 (by_key.into_values().collect(), errors)
223}
224
225fn csv_records(text: &str) -> Vec<Vec<String>> {
227 let mut out = Vec::new();
228 let mut rec = Vec::new();
229 let mut field = String::new();
230 let mut quoted = false;
231 let mut chars = text.chars().peekable();
232 while let Some(c) = chars.next() {
233 match (quoted, c) {
234 (true, '"') if chars.peek() == Some(&'"') => {
235 field.push('"');
236 chars.next();
237 }
238 (true, '"') => quoted = false,
239 (true, c) => field.push(c),
240 (false, '"') => quoted = true,
241 (false, ',') => rec.push(std::mem::take(&mut field)),
242 (false, '\r') => {}
243 (false, '\n') => {
244 rec.push(std::mem::take(&mut field));
245 out.push(std::mem::take(&mut rec));
246 }
247 (false, c) => field.push(c),
248 }
249 }
250 if !field.is_empty() || !rec.is_empty() {
251 rec.push(field);
252 out.push(rec);
253 }
254 out
255}
256
257#[must_use]
260pub fn command_path(name: &str, feature: &str) -> Option<String> {
261 let mut tokens = feature.split_whitespace();
262 if tokens.next() != Some(name) {
263 return None;
264 }
265 let path: Vec<&str> = tokens
266 .take_while(|t| !t.starts_with(['-', '<', '[', '(']))
267 .collect();
268 Some(path.join(" "))
269}
270
271#[must_use]
273pub fn route_key(feature: &str) -> Option<String> {
274 let mut tokens = feature.split_whitespace();
275 let verb = tokens.next()?;
276 let path = tokens.next()?;
277 (HTTP_VERBS.contains(&verb) && path.starts_with('/')).then(|| format!("{verb} {path}"))
278}
279
280#[must_use]
282pub fn tool_key(feature: &str) -> Option<String> {
283 let first = feature.split_whitespace().next()?;
284 first
285 .strip_prefix("mcp:")
286 .filter(|t| !t.is_empty())
287 .map(str::to_string)
288}
289
290pub fn parse_ledger(text: &str) -> Result<BTreeMap<String, LedgerEntry>, String> {
292 let mut records = csv_records(text).into_iter();
293 let header = records.next().ok_or("the ledger is empty")?.join(",");
294 if header != LEDGER_HEADER {
295 return Err(format!("header `{header}` is not the surface-audit header"));
296 }
297 let mut out: BTreeMap<String, LedgerEntry> = BTreeMap::new();
298 for rec in records {
299 let (Some(binary), Some(feature)) = (rec.first(), rec.get(1)) else {
300 continue;
301 };
302 if binary.is_empty() {
303 continue;
304 }
305 let e = out.entry(binary.clone()).or_default();
306 e.rows.insert(feature.clone());
307 if let Some(r) = route_key(feature) {
308 e.routes.insert(r);
309 } else if let Some(t) = tool_key(feature) {
310 e.tools.insert(t);
311 } else if let Some(p) = command_path(binary, feature).filter(|p| !p.is_empty()) {
312 e.commands.insert(p);
313 }
314 }
315 Ok(out)
316}
317
318#[must_use]
323pub fn bins_of(dir: &Path, manifest: &str) -> Vec<(String, String)> {
324 let Some(package) = manifest_names(manifest).package else {
325 return Vec::new();
326 };
327 let (mut names, paths, autobins) = declared_bins(manifest);
328 if autobins {
329 if dir.join("src/main.rs").is_file() && !paths.contains("src/main.rs") {
331 names.insert(package.clone());
332 }
333 if let Ok(rd) = std::fs::read_dir(dir.join("src/bin")) {
334 names.extend(
335 rd.flatten()
336 .filter_map(|e| discovered_bin(&e.path(), &paths)),
337 );
338 }
339 }
340 names.into_iter().map(|n| (package.clone(), n)).collect()
341}
342
343fn declared_bins(manifest: &str) -> (BTreeSet<String>, BTreeSet<String>, bool) {
345 let mut names = BTreeSet::new();
346 let mut paths = BTreeSet::new();
347 let mut autobins = true;
348 let mut section = String::new();
349 for line in manifest.lines() {
350 let t = line.trim();
351 if t.starts_with('[') && t.ends_with(']') {
352 section = t.trim_matches(['[', ']']).trim().to_string();
353 continue;
354 }
355 let Some((k, v)) = t.split_once('=') else {
356 continue;
357 };
358 let v = v.trim().trim_matches('"');
359 match (section.as_str(), k.trim()) {
360 ("bin", "name") => {
361 names.insert(v.to_string());
362 }
363 ("bin", "path") => {
364 paths.insert(v.trim_start_matches("./").to_string());
365 }
366 ("package", "autobins") => autobins = v != "false",
367 _ => {}
368 }
369 }
370 (names, paths, autobins)
371}
372
373fn discovered_bin(p: &Path, declared_paths: &BTreeSet<String>) -> Option<String> {
375 let stem = p.file_stem().and_then(|s| s.to_str()).unwrap_or_default();
376 let (file, is_target) = if p.is_dir() {
377 (
378 format!("src/bin/{stem}/main.rs"),
379 p.join("main.rs").is_file(),
380 )
381 } else {
382 (
383 format!("src/bin/{stem}.rs"),
384 p.extension().is_some_and(|e| e == "rs"),
385 )
386 };
387 (is_target && !stem.is_empty() && !declared_paths.contains(&file)).then(|| stem.to_string())
388}
389
390#[must_use]
392pub fn census(root: &Path) -> Option<BTreeSet<(String, String)>> {
393 let membership = std::fs::read_to_string(root.join("Cargo.toml"))
394 .ok()
395 .and_then(|t| workspace_membership(&t))?;
396 let mut out = BTreeSet::new();
397 for manifest in super::example::manifests(root) {
398 let dir = manifest.parent().unwrap_or(root);
399 if !membership.admits(root, dir) {
400 continue;
401 }
402 if let Ok(text) = std::fs::read_to_string(&manifest) {
403 out.extend(bins_of(dir, &text));
404 }
405 }
406 Some(out)
407}
408
409pub fn emit(g: &mut Graph, t: &Target, ledger: Option<&LedgerEntry>, pair_mismatch: &[&Target]) {
411 let n = node(&t.package, &t.target);
412 g.insert(n.clone(), RDF_TYPE, Term::iri(ont("Binary")));
413 g.insert(n.clone(), RDF_TYPE, Term::iri(PROV_ENTITY));
414 g.insert(
415 n.clone(),
416 RDF_TYPE,
417 Term::iri(bin(&format!("target/{}/{}", t.package, t.target))),
418 );
419 for (p, v) in [
420 ("package", &t.package),
421 ("target", &t.target),
422 ("name", &t.name),
423 ("srcPath", &t.src_path),
424 ("gitSha", &t.git_sha),
425 ("versionString", &t.version),
426 ("versionSha256", &t.version_sha256),
427 ("sha256", &t.binary_sha256),
428 ("helpSha256", &t.help_sha256),
429 ] {
430 g.insert(n.clone(), bin(p), Term::string(v.as_str()));
431 }
432 for c in &t.commands {
433 g.insert(n.clone(), bin("command"), Term::string(c.as_str()));
434 }
435 for r in &t.routes {
436 g.insert(n.clone(), bin("route"), Term::string(r.as_str()));
437 }
438 for x in &t.tools {
439 g.insert(n.clone(), bin("mcpTool"), Term::string(x.as_str()));
440 }
441 if !t.version_names_sha() {
442 let sha9 = t.git_sha.get(..9).unwrap_or(&t.git_sha);
443 g.insert(n.clone(), bin("versionLacksSha"), Term::string(sha9));
444 }
445 if t.help_failed || t.help_sha256 == EMPTY_SHA256 {
446 g.insert(n.clone(), bin("helpFailed"), Term::boolean(true));
447 }
448 for other in pair_mismatch {
449 g.insert(
450 n.clone(),
451 bin("namePairMismatch"),
452 Term::iri(node(&other.package, &other.target)),
453 );
454 }
455 let empty = LedgerEntry::default();
456 let l = ledger.unwrap_or(&empty);
457 for row in &l.rows {
458 g.insert(n.clone(), bin("auditRow"), Term::string(row.as_str()));
459 }
460 for (pred, snap, led) in [
461 ("unledgeredCommand", &t.commands, &l.commands),
462 ("unledgeredRoute", &t.routes, &l.routes),
463 ("unledgeredTool", &t.tools, &l.tools),
464 ("ledgerOrphan", &l.commands, &t.commands),
465 ("ledgerOrphanRoute", &l.routes, &t.routes),
466 ] {
467 for x in snap.difference(led) {
468 g.insert(n.clone(), bin(pred), Term::string(x.as_str()));
469 }
470 }
471}
472
473#[must_use]
475pub fn pair_mismatches(targets: &[Target]) -> Vec<Vec<&Target>> {
476 targets
477 .iter()
478 .map(|t| {
479 targets
480 .iter()
481 .filter(|o| {
482 o.name == t.name
483 && !(o.package == t.package && o.target == t.target)
484 && (o.help_sha256 != t.help_sha256 || o.version_sha256 != t.version_sha256)
485 })
486 .collect()
487 })
488 .collect()
489}
490
491pub fn emit_all(
493 g: &mut Graph,
494 targets: &[Target],
495 ledger: &BTreeMap<String, LedgerEntry>,
496 stats: &mut BinaryStats,
497) {
498 let pairs = pair_mismatches(targets);
499 for (t, mismatch) in targets.iter().zip(&pairs) {
500 let l = ledger.get(&t.name);
501 emit(g, t, l, mismatch);
502 stats.version_lacks_sha += usize::from(!t.version_names_sha());
503 stats.name_pair_mismatch += usize::from(!mismatch.is_empty());
504 stats.help_failed += usize::from(t.help_failed || t.help_sha256 == EMPTY_SHA256);
505 stats.no_audit_row += usize::from(l.is_none_or(|l| l.rows.is_empty()));
506 let empty = LedgerEntry::default();
507 let l = l.unwrap_or(&empty);
508 stats.unledgered += t.commands.difference(&l.commands).count()
509 + t.routes.difference(&l.routes).count()
510 + t.tools.difference(&l.tools).count();
511 stats.orphans +=
512 l.commands.difference(&t.commands).count() + l.routes.difference(&t.routes).count();
513 }
514 stats.targets = targets.len();
515 stats.names = targets
516 .iter()
517 .map(|t| t.name.as_str())
518 .collect::<BTreeSet<_>>()
519 .len();
520}
521
522pub fn extract_at(root: &Path, g: &mut Graph) -> BinaryStats {
524 let census = census(root);
525 let mut stats = BinaryStats {
526 at_workspace_root: census.is_some(),
527 census: census.as_ref().map(BTreeSet::len),
528 ..BinaryStats::default()
529 };
530 let snapshot = std::fs::read_to_string(root.join(SNAPSHOT));
531 let (targets, mut errors) = match &snapshot {
532 Ok(text) => parse_snapshot(text),
533 Err(_) => (Vec::new(), Vec::new()),
534 };
535 stats.errors.append(&mut errors);
536 let ledger = match std::fs::read_to_string(root.join(LEDGER)) {
537 Ok(text) => parse_ledger(&text).unwrap_or_else(|what| {
538 refuse(&mut stats.errors, LEDGER, what);
539 BTreeMap::new()
540 }),
541 Err(_) => BTreeMap::new(),
542 };
543 stats.ledger_rows = ledger.values().map(|e| e.rows.len()).sum();
544 emit_all(g, &targets, &ledger, &mut stats);
545 if let Some(census) = &census {
546 if targets.is_empty() && !census.is_empty() {
550 refuse(
551 &mut stats.errors,
552 SNAPSHOT,
553 format!(
554 "a [workspace] root with {} bin target(s) read zero snapshot lines ({}) — the extractor \
555 measured nothing",
556 census.len(),
557 if snapshot.is_ok() { "empty" } else { "absent" }
558 ),
559 );
560 } else {
561 let read: BTreeSet<(String, String)> = targets
562 .iter()
563 .map(|t| (t.package.clone(), t.target.clone()))
564 .collect();
565 for (p, t) in census.difference(&read) {
566 refuse(
567 &mut stats.errors,
568 SNAPSHOT,
569 format!(
570 "bin target {p}/{t} is declared by its manifest and has no snapshot line"
571 ),
572 );
573 }
574 for (p, t) in read.difference(census) {
575 refuse(
576 &mut stats.errors,
577 SNAPSHOT,
578 format!("snapshot line {p}/{t} names no bin target of a workspace member"),
579 );
580 }
581 }
582 }
583 stats
584}
585
586pub fn extract(contract_dir: &Path, g: &mut Graph) -> BinaryStats {
588 extract_at(&super::repo_root(contract_dir), g)
589}
590
591#[must_use]
594pub fn control_sample() -> Vec<Target> {
595 let sha = "0123456789abcdef0123456789abcdef01234567";
596 let base = Target {
597 name: "apr".into(),
598 git_sha: sha.into(),
599 version: "apr 0.70.0 (012345678)".into(),
600 help_sha256: "a".repeat(64),
601 version_sha256: "b".repeat(64),
602 commands: ["run".to_string()].into_iter().collect(),
603 ..Target::default()
604 };
605 let facade = Target {
606 package: "aprender".into(),
607 target: "apr".into(),
608 ..base.clone()
609 };
610 let cli = Target {
611 package: "apr-cli".into(),
612 target: "apr".into(),
613 version: "apr 0.70.0".into(),
614 help_sha256: "c".repeat(64),
615 ..base
616 };
617 vec![cli, facade]
618}
619
620#[must_use]
625pub fn positive_control(targets: &[Target]) -> bool {
626 let ledger = parse_ledger(&format!(
627 "{LEDGER_HEADER}\napr,apr run,,,,,,,,\napr,apr gone,,,,,,,,\n"
628 ))
629 .unwrap_or_default();
630 let mut g = Graph::default();
631 let mut stats = BinaryStats::default();
632 emit_all(&mut g, targets, &ledger, &mut stats);
633 let a = node("apr-cli", "apr");
634 let b = node("aprender", "apr");
635 g.instances_of(&ont("Binary")).len() == 2
636 && !g.objects(&a, &bin("namePairMismatch")).is_empty()
637 && !g.objects(&b, &bin("namePairMismatch")).is_empty()
638 && !g.objects(&a, &bin("versionLacksSha")).is_empty()
639 && g.objects(&b, &bin("versionLacksSha")).is_empty()
640 && g.objects(&b, &bin("ledgerOrphan")).len() == 1
641}
642
643#[cfg(test)]
644#[path = "binary_tests.rs"]
645mod tests;