use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use crate::query::json::J;
mod assertions;
mod fp_sites;
use assertions::evaluate_assert;
use fp_sites::FpSite;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
enum Tier {
Precise,
Ext,
Guess,
Heuristic,
}
impl Tier {
const ORDER: [Tier; 4] = [Tier::Precise, Tier::Ext, Tier::Guess, Tier::Heuristic];
fn key(self) -> &'static str {
match self {
Tier::Precise => "precise",
Tier::Ext => "ext",
Tier::Guess => "guess",
Tier::Heuristic => "heuristic",
}
}
}
struct EdgeRow {
from_file: String,
from_line: usize,
to: String,
to_file: String,
tier: Tier,
member: Option<String>,
}
#[derive(Debug, Clone, serde::Deserialize)]
struct DefRow {
id: String,
file: String,
#[allow(dead_code)]
kind: String,
#[serde(default)]
test: bool,
}
#[derive(Debug, Clone, serde::Deserialize)]
struct OracleRef {
file: String,
#[serde(rename = "startLine")]
start_line: usize,
shape: String,
#[serde(rename = "receiverKind")]
receiver_kind: String,
member: String,
target: Option<String>,
#[serde(rename = "targetKind")]
target_kind: Option<String>,
#[serde(rename = "targetFile")]
target_file: Option<String>,
external: bool,
ambiguous: bool,
}
#[derive(Debug, Clone, serde::Deserialize)]
struct Unit {
name: String,
status: String,
#[serde(default)]
refs: Vec<String>,
#[serde(default)]
files: Vec<String>,
}
struct Inputs {
root: PathBuf,
graph_defs: Vec<DefRow>,
oracle_defs: Vec<DefRow>,
edges: Vec<EdgeRow>,
records: Vec<OracleRef>,
units: Vec<Unit>,
universe: HashSet<String>,
collect_fp_sites: bool,
}
struct AuditOptions<'a> {
semantic: &'a Path,
units: Option<&'a Path>,
defs: Option<&'a Path>,
}
fn load(root: &Path, opts: &AuditOptions) -> Result<Inputs, String> {
let graph_path = crate::graph::graph_json_path(root);
let graph_text = std::fs::read_to_string(&graph_path).map_err(|_| {
"no graph.json for this repo — run `devscout map` on a C# scope first".to_string()
})?;
let graph_value: serde_json::Value = serde_json::from_str(&graph_text).map_err(|e| {
format!(
"graph.json at {} is not valid JSON: {e}",
graph_path.display()
)
})?;
let (graph_defs, edges) = parse_graph(&graph_value)?;
let records = read_jsonl::<OracleRef>(opts.semantic)?;
let units = match opts.units {
Some(p) => read_jsonl::<Unit>(p)?,
None => Vec::new(),
};
let oracle_defs = match opts.defs {
Some(p) => read_jsonl::<DefRow>(p)?,
None => Vec::new(),
};
let universe = build_universe(root, &graph_value, &units);
Ok(Inputs {
root: root.to_path_buf(),
graph_defs,
oracle_defs,
edges,
records,
units,
universe,
collect_fp_sites: false,
})
}
fn read_jsonl<T: serde::de::DeserializeOwned>(path: &Path) -> Result<Vec<T>, String> {
let text = std::fs::read_to_string(path)
.map_err(|e| format!("failed to read {}: {e}", path.display()))?;
let mut out = Vec::new();
for (i, line) in text.lines().enumerate() {
if line.trim().is_empty() {
continue;
}
let rec: T =
serde_json::from_str(line).map_err(|e| format!("{}:{}: {e}", path.display(), i + 1))?;
out.push(rec);
}
Ok(out)
}
fn parse_graph(v: &serde_json::Value) -> Result<(Vec<DefRow>, Vec<EdgeRow>), String> {
let defs_arr = v
.get("defs")
.and_then(|d| d.as_array())
.ok_or("graph.json has no 'defs' array")?;
let mut defs = Vec::with_capacity(defs_arr.len());
for d in defs_arr {
defs.push(DefRow {
id: str_field(d, "id"),
file: str_field(d, "file"),
kind: str_field(d, "kind"),
test: d
.get("testMethods")
.and_then(|x| x.as_array())
.is_some_and(|a| !a.is_empty()),
});
}
let edges_arr = v
.get("edges")
.and_then(|e| e.as_array())
.ok_or("graph.json has no 'edges' array")?;
let mut edges = Vec::new();
for e in edges_arr {
if e.get("kind").and_then(|k| k.as_str()) != Some("uses-member") {
continue;
}
edges.push(EdgeRow {
from_file: str_field(e, "from_file"),
from_line: e.get("from_line").and_then(|x| x.as_u64()).unwrap_or(0) as usize,
to: str_field(e, "to"),
to_file: str_field(e, "to_file"),
tier: tier_of(e),
member: e
.get("member")
.and_then(|x| x.as_str())
.map(|s| s.to_string()),
});
}
Ok((defs, edges))
}
fn str_field(v: &serde_json::Value, key: &str) -> String {
v.get(key)
.and_then(|x| x.as_str())
.unwrap_or("")
.to_string()
}
fn tier_of(e: &serde_json::Value) -> Tier {
match e.get("tier").and_then(|x| x.as_str()) {
Some("ext") => return Tier::Ext,
Some("guess") => return Tier::Guess,
_ => {}
}
if e.get("heuristic")
.and_then(|x| x.as_bool())
.unwrap_or(false)
{
Tier::Heuristic
} else {
Tier::Precise
}
}
fn build_universe(root: &Path, graph_value: &serde_json::Value, units: &[Unit]) -> HashSet<String> {
let mut universe: HashSet<String> = HashSet::new();
if let Ok(Some(m)) = crate::manifest::read_manifest(root) {
if let Some(entries) = m.get("entries").and_then(crate::manifest::Value::as_object) {
universe.extend(entries.iter().map(|(k, _)| k.clone()));
}
}
if universe.is_empty() {
for key in ["defs", "edges"] {
if let Some(arr) = graph_value.get(key).and_then(|x| x.as_array()) {
for row in arr {
for file_key in ["file", "from_file", "to_file"] {
if let Some(f) = row.get(file_key).and_then(|x| x.as_str()) {
universe.insert(f.to_string());
}
}
}
}
}
}
if units.is_empty() {
return universe;
}
let ok_unit_files: HashSet<&str> = units
.iter()
.filter(|u| u.status == "ok")
.flat_map(|u| u.files.iter().map(String::as_str))
.collect();
universe.retain(|f| ok_unit_files.contains(f.as_str()));
universe
}
fn target_matches(r: &OracleRef, edge_to: &str) -> bool {
let Some(t) = &r.target else { return false };
if t == edge_to {
return true;
}
if r.target_kind.as_deref() == Some("enum-member") {
if let Some((prefix, _)) = t.rsplit_once('.') {
if prefix == edge_to {
return true;
}
}
}
false
}
fn member_matches(edge_member: &Option<String>, record_member: &str) -> bool {
match edge_member {
Some(m) => m == record_member,
None => true,
}
}
fn target_known(defs_by_id: &HashMap<&str, &DefRow>, r: &OracleRef) -> bool {
let Some(t) = &r.target else { return false };
if defs_by_id.contains_key(t.as_str()) {
return true;
}
if r.target_kind.as_deref() == Some("enum-member") {
if let Some((prefix, _)) = t.rsplit_once('.') {
if defs_by_id.contains_key(prefix) {
return true;
}
}
}
false
}
fn short_name(id: &str) -> &str {
id.rsplit(['.', '+']).next().unwrap_or(id)
}
fn file_to_unit(units: &[Unit]) -> HashMap<String, String> {
let mut m = HashMap::new();
for u in units {
for f in &u.files {
m.entry(f.clone()).or_insert_with(|| u.name.clone());
}
}
m
}
fn reach(units: &[Unit]) -> HashMap<String, HashSet<String>> {
let by_name: HashMap<&str, &Unit> = units.iter().map(|u| (u.name.as_str(), u)).collect();
let mut result = HashMap::new();
for u in units {
let mut seen: HashSet<String> = HashSet::new();
let mut stack = vec![u.name.clone()];
while let Some(n) = stack.pop() {
if seen.contains(&n) {
continue;
}
seen.insert(n.clone());
if let Some(unit) = by_name.get(n.as_str()) {
for r in &unit.refs {
if !seen.contains(r) {
stack.push(r.clone());
}
}
}
}
result.insert(u.name.clone(), seen);
}
result
}
fn is_structural(
e: &EdgeRow,
units_method: bool,
file_unit: &HashMap<String, String>,
reach_map: &HashMap<String, HashSet<String>>,
test_by_id: &HashMap<String, bool>,
test_files: &HashSet<String>,
) -> Option<bool> {
if units_method {
let u1 = file_unit.get(&e.from_file)?;
let u2 = file_unit.get(&e.to_file)?;
let reachable = reach_map.get(u1).is_some_and(|s| s.contains(u2));
Some(!reachable)
} else {
let target_test = *test_by_id.get(&e.to)?;
if !target_test {
return Some(false);
}
Some(!test_files.contains(&e.from_file))
}
}
#[derive(Debug, Clone, Copy, Default)]
struct TierStats {
edges: usize,
tp: usize,
fp: usize,
fp_no_site: usize,
fp_external_site: usize,
fp_wrong_target: usize,
structural: usize,
}
impl TierStats {
fn precision(&self) -> f64 {
if self.edges == 0 {
0.0
} else {
self.tp as f64 / self.edges as f64
}
}
}
struct AuditReport {
root: String,
oracle_records: usize,
oracle_sites: usize,
oracle_external_sites: usize,
oracle_ambiguous: usize,
oracle_dropped: usize,
units_ok: usize,
units_failed: usize,
structural_method: &'static str,
tiers: Vec<(Tier, TierStats)>,
recall_denominator: usize,
recall_precise: usize,
recall_precise_ext: usize,
recall_all: usize,
by_receiver: Vec<(&'static str, Option<f64>)>,
recall_conditional: usize,
recall_bare: usize,
silent_correct: usize,
silent_leak: usize,
structural_impossible: usize,
structural_checked: usize,
fanout: [usize; 4],
unknown_targets: Vec<(String, usize)>,
top_fp: Vec<(String, usize)>,
top_missed: Vec<(String, usize)>,
partial_file_mismatch: usize,
edges_outside_universe: usize,
fp_sites: Vec<FpSite>,
}
fn push_fp_site(
fp_rows: &mut Vec<FpSite>,
collect: bool,
e: &EdgeRow,
class: &'static str,
evidence: &[&OracleRef],
structural: bool,
) {
if collect {
fp_rows.push(FpSite::new(e, class, evidence, structural));
}
}
#[allow(
clippy::too_many_lines,
reason = "one ordered scoring pass building every report tier from the same inputs; the tiers must stay in the order the report promises"
)]
fn score(inputs: Inputs) -> AuditReport {
let root = inputs.root.display().to_string();
let collect_fp_sites = inputs.collect_fp_sites;
let units_method = !inputs.units.is_empty();
let mut oracle_dropped = 0usize;
let records: Vec<&OracleRef> = inputs
.records
.iter()
.filter(|r| {
let keep = inputs.universe.contains(&r.file);
if !keep {
oracle_dropped += 1;
}
keep
})
.collect();
let mut edges_outside_universe = 0usize;
let edges: Vec<&EdgeRow> = inputs
.edges
.iter()
.filter(|e| {
let keep = !units_method || inputs.universe.contains(&e.from_file);
if !keep {
edges_outside_universe += 1;
}
keep
})
.collect();
let graph_defs_by_id: HashMap<&str, &DefRow> = inputs
.graph_defs
.iter()
.map(|d| (d.id.as_str(), d))
.collect();
let mut by_site: HashMap<(String, usize), Vec<&OracleRef>> = HashMap::new();
for r in &records {
by_site
.entry((r.file.clone(), r.start_line))
.or_default()
.push(r);
}
let mut edges_by_site: HashMap<(String, usize), Vec<&EdgeRow>> = HashMap::new();
for e in edges.iter().copied() {
edges_by_site
.entry((e.from_file.clone(), e.from_line))
.or_default()
.push(e);
}
let oracle_sites = by_site.len();
let oracle_ambiguous = records.iter().filter(|r| r.ambiguous).count();
let mut access_by_site: HashMap<(String, usize), Vec<&OracleRef>> = HashMap::new();
for r in records.iter().filter(|r| r.shape == "access") {
access_by_site
.entry((r.file.clone(), r.start_line))
.or_default()
.push(r);
}
let mut silent_correct = 0usize;
let mut silent_leak = 0usize;
for (site, recs) in &access_by_site {
if recs.iter().all(|r| r.external) {
if edges_by_site.contains_key(site) {
silent_leak += 1;
} else {
silent_correct += 1;
}
}
}
let oracle_external_sites = silent_correct + silent_leak;
let structural_method: &'static str = if units_method { "units" } else { "test-defs" };
let file_unit = file_to_unit(&inputs.units);
let reach_map = reach(&inputs.units);
let mut test_by_id: HashMap<String, bool> = HashMap::new();
let mut file_by_id: HashMap<String, String> = HashMap::new();
for d in inputs.graph_defs.iter().chain(inputs.oracle_defs.iter()) {
test_by_id.insert(d.id.clone(), d.test);
file_by_id.insert(d.id.clone(), d.file.clone());
}
let test_files: HashSet<String> = test_by_id
.iter()
.filter(|&(_, &t)| t)
.filter_map(|(id, _)| file_by_id.get(id).cloned())
.collect();
let mut structural_impossible = 0usize;
let mut structural_checked = 0usize;
let edge_structural: Vec<bool> = edges
.iter()
.copied()
.map(|e| {
match is_structural(
e,
units_method,
&file_unit,
&reach_map,
&test_by_id,
&test_files,
) {
Some(flag) => {
structural_checked += 1;
if flag {
structural_impossible += 1;
}
flag
}
None => false,
}
})
.collect();
let mut tiers: HashMap<Tier, TierStats> = HashMap::new();
let mut fp_targets: HashMap<String, usize> = HashMap::new();
let mut fanout_sites: HashMap<(String, usize), usize> = HashMap::new();
for e in edges.iter().copied() {
*fanout_sites
.entry((e.from_file.clone(), e.from_line))
.or_insert(0) += 1;
}
let mut partial_file_mismatch = 0usize;
let mut fp_rows: Vec<FpSite> = Vec::new();
for (i, e) in edges.iter().copied().enumerate() {
let stats = tiers.entry(e.tier).or_default();
stats.edges += 1;
let site_key = (e.from_file.clone(), e.from_line);
let is_tp = match by_site.get(&site_key) {
None => {
stats.fp += 1;
stats.fp_no_site += 1;
*fp_targets.entry(short_name(&e.to).to_string()).or_insert(0) += 1;
push_fp_site(
&mut fp_rows,
collect_fp_sites,
e,
"no-site",
&[],
edge_structural[i],
);
false
}
Some(recs) => {
let tp_record = recs.iter().find(|r| {
!r.external && target_matches(r, &e.to) && member_matches(&e.member, &r.member)
});
if let Some(r) = tp_record {
stats.tp += 1;
if let Some(tf) = &r.target_file {
if tf != &e.to_file {
partial_file_mismatch += 1;
}
}
true
} else {
let scoped: Vec<&&OracleRef> = recs
.iter()
.filter(|r| member_matches(&e.member, &r.member))
.collect();
stats.fp += 1;
let evidence: Vec<&OracleRef> = if scoped.is_empty() {
recs.clone()
} else {
scoped.iter().map(|r| **r).collect()
};
let external_site = evidence.iter().all(|r| r.external);
let class = if external_site {
stats.fp_external_site += 1;
"external"
} else {
stats.fp_wrong_target += 1;
"wrong-target"
};
*fp_targets.entry(short_name(&e.to).to_string()).or_insert(0) += 1;
push_fp_site(
&mut fp_rows,
collect_fp_sites,
e,
class,
&evidence,
edge_structural[i],
);
false
}
}
};
if !is_tp && edge_structural[i] {
stats.structural += 1;
}
}
let tiers_ordered: Vec<(Tier, TierStats)> = Tier::ORDER
.into_iter()
.filter_map(|t| tiers.get(&t).filter(|ts| ts.edges > 0).map(|ts| (t, *ts)))
.collect();
let site_hit = |r: &OracleRef, tier_ok: &dyn Fn(Tier) -> bool| -> bool {
edges_by_site
.get(&(r.file.clone(), r.start_line))
.is_some_and(|es| {
es.iter().any(|e| {
tier_ok(e.tier)
&& target_matches(r, &e.to)
&& member_matches(&e.member, &r.member)
})
})
};
let d_records: Vec<&&OracleRef> = records
.iter()
.filter(|r| r.shape == "access" && !r.external && target_known(&graph_defs_by_id, r))
.collect();
let recall_denominator = d_records.len();
let recall_precise = d_records
.iter()
.filter(|r| site_hit(r, &|t| t == Tier::Precise))
.count();
let recall_precise_ext = d_records
.iter()
.filter(|r| site_hit(r, &|t| t == Tier::Precise || t == Tier::Ext))
.count();
let recall_all = d_records.iter().filter(|r| site_hit(r, &|_| true)).count();
let mut top_missed: HashMap<String, usize> = HashMap::new();
for r in &d_records {
if !site_hit(r, &|_| true) {
*top_missed.entry(r.target.clone().unwrap()).or_insert(0) += 1;
}
}
let by_receiver: Vec<(&'static str, Option<f64>)> =
["ident", "qualified", "this", "base", "call"]
.into_iter()
.map(|k| {
let subset: Vec<_> = d_records.iter().filter(|r| r.receiver_kind == k).collect();
if subset.is_empty() {
(k, None)
} else {
let hits = subset.iter().filter(|r| site_hit(r, &|_| true)).count();
(k, Some(hits as f64 / subset.len() as f64))
}
})
.collect();
let recall_conditional = records
.iter()
.filter(|r| r.shape == "conditional" && !r.external && target_known(&graph_defs_by_id, r))
.count();
let recall_bare = records
.iter()
.filter(|r| r.shape == "bare" && !r.external && target_known(&graph_defs_by_id, r))
.count();
let mut unknown_by_kind: HashMap<String, HashSet<String>> = HashMap::new();
for r in &records {
if r.external {
continue;
}
let Some(t) = &r.target else { continue };
if target_known(&graph_defs_by_id, r) {
continue;
}
let kind = r
.target_kind
.clone()
.unwrap_or_else(|| "unknown".to_string());
unknown_by_kind.entry(kind).or_default().insert(t.clone());
}
let mut unknown_targets: Vec<(String, usize)> = unknown_by_kind
.into_iter()
.map(|(k, set)| (k, set.len()))
.collect();
unknown_targets.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
let mut fanout = [0usize; 4];
for n in fanout_sites.values() {
match n {
1 => fanout[0] += 1,
2 => fanout[1] += 1,
3 => fanout[2] += 1,
_ => fanout[3] += 1,
}
}
let mut top_fp: Vec<(String, usize)> = fp_targets.into_iter().collect();
top_fp.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
top_fp.truncate(20);
let mut top_missed: Vec<(String, usize)> = top_missed.into_iter().collect();
top_missed.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
top_missed.truncate(20);
let units_ok = inputs.units.iter().filter(|u| u.status == "ok").count();
let units_failed = inputs.units.len() - units_ok;
AuditReport {
root,
oracle_records: records.len(),
oracle_sites,
oracle_external_sites,
oracle_ambiguous,
oracle_dropped,
units_ok,
units_failed,
structural_method,
tiers: tiers_ordered,
recall_denominator,
recall_precise,
recall_precise_ext,
recall_all,
by_receiver,
recall_conditional,
recall_bare,
silent_correct,
silent_leak,
structural_impossible,
structural_checked,
fanout,
unknown_targets,
top_fp,
top_missed,
partial_file_mismatch,
edges_outside_universe,
fp_sites: fp_rows,
}
}
fn ratio_text(hits: usize, denom: usize) -> String {
if denom == 0 {
"-".to_string()
} else {
format!("{:.3}", hits as f64 / denom as f64)
}
}
fn ratio_j(hits: usize, denom: usize) -> J {
if denom == 0 {
J::RawNum("null".to_string())
} else {
J::RawNum(format!("{:.3}", hits as f64 / denom as f64))
}
}
#[allow(
clippy::too_many_lines,
reason = "one ordered pass rendering every report section in the fixed order the text output promises"
)]
fn render_text(r: &AuditReport) -> String {
let mut lines = Vec::new();
lines.push(format!(
"devscout audit --semantic root {} oracle {} records / {} sites units ok {} failed {} method {}",
r.root, r.oracle_records, r.oracle_sites, r.units_ok, r.units_failed, r.structural_method
));
if !r.tiers.is_empty() {
lines.push(format!(
"{:<10}{:>7}{:>7}{:>7}{:>12}{:>13}{:>13}{:>10}{:>12}",
"tier",
"edges",
"tp",
"fp",
"precision",
"fp:no-site",
"fp:external",
"fp:wrong",
"structural"
));
for (tier, ts) in &r.tiers {
lines.push(format!(
"{:<10}{:>7}{:>7}{:>7}{:>12.3}{:>13}{:>13}{:>10}{:>12}",
tier.key(),
ts.edges,
ts.tp,
ts.fp,
ts.precision(),
ts.fp_no_site,
ts.fp_external_site,
ts.fp_wrong_target,
ts.structural,
));
}
}
lines.push(format!(
"recall ({} in-graph member sites) precise {} precise+ext {} all {}",
r.recall_denominator,
ratio_text(r.recall_precise, r.recall_denominator),
ratio_text(r.recall_precise_ext, r.recall_denominator),
ratio_text(r.recall_all, r.recall_denominator),
));
let by_receiver = r
.by_receiver
.iter()
.map(|(k, v)| match v {
Some(x) => format!("{k} {x:.3}"),
None => format!("{k} -"),
})
.collect::<Vec<_>>()
.join(" ");
lines.push(format!(" by receiver {by_receiver}"));
lines.push(format!(
"external sites {} silent-correct {} leaked {}",
r.oracle_external_sites, r.silent_correct, r.silent_leak
));
lines.push(format!(
"structural impossible {} checked {}",
r.structural_impossible, r.structural_checked
));
lines.push(format!(
"fan-out 1: {} 2: {} 3: {} 4+: {}",
r.fanout[0], r.fanout[1], r.fanout[2], r.fanout[3]
));
if !r.top_fp.is_empty() {
let s = r
.top_fp
.iter()
.map(|(n, c)| format!("{n} {c}"))
.collect::<Vec<_>>()
.join(" ");
lines.push(format!("top fp targets {s}"));
}
if !r.top_missed.is_empty() {
let s = r
.top_missed
.iter()
.map(|(id, c)| format!("{id} {c}"))
.collect::<Vec<_>>()
.join(" ");
lines.push(format!("top missed {s}"));
}
if !r.unknown_targets.is_empty() {
let s = r
.unknown_targets
.iter()
.map(|(k, c)| format!("{k} {c}"))
.collect::<Vec<_>>()
.join(" ");
lines.push(format!("unknown targets {s}"));
}
if r.partial_file_mismatch > 0 {
lines.push(format!("partial file mismatch {}", r.partial_file_mismatch));
}
if r.oracle_ambiguous > 0 {
lines.push(format!("ambiguous {}", r.oracle_ambiguous));
}
if r.oracle_dropped > 0 {
lines.push(format!("dropped (outside universe) {}", r.oracle_dropped));
}
if r.edges_outside_universe > 0 {
lines.push(format!(
"edges outside universe (not judged) {}",
r.edges_outside_universe
));
}
lines.join("\n")
}
#[allow(
clippy::too_many_lines,
reason = "one ordered pass assembling every report field into the JSON shape render_text mirrors"
)]
fn render_json(r: &AuditReport) -> String {
let mut tiers_fields: Vec<(&'static str, J)> = Vec::new();
for tier in Tier::ORDER {
if let Some((_, ts)) = r.tiers.iter().find(|(t, _)| *t == tier) {
tiers_fields.push((
tier.key(),
J::Obj(vec![
("edges", J::UInt(ts.edges as u64)),
("tp", J::UInt(ts.tp as u64)),
("fp", J::UInt(ts.fp as u64)),
("precision", J::RawNum(format!("{:.3}", ts.precision()))),
("fp_no_site", J::UInt(ts.fp_no_site as u64)),
("fp_external_site", J::UInt(ts.fp_external_site as u64)),
("fp_wrong_target", J::UInt(ts.fp_wrong_target as u64)),
("structural", J::UInt(ts.structural as u64)),
]),
));
}
}
let by_receiver_j: Vec<(&'static str, J)> = r
.by_receiver
.iter()
.map(|(k, v)| {
(
*k,
match v {
Some(x) => J::RawNum(format!("{x:.3}")),
None => J::RawNum("null".to_string()),
},
)
})
.collect();
J::Obj(vec![
("status", J::Str("ok".to_string())),
("root", J::Str(r.root.clone())),
(
"oracle",
J::Obj(vec![
("records", J::UInt(r.oracle_records as u64)),
("sites", J::UInt(r.oracle_sites as u64)),
("external_sites", J::UInt(r.oracle_external_sites as u64)),
("ambiguous", J::UInt(r.oracle_ambiguous as u64)),
("dropped", J::UInt(r.oracle_dropped as u64)),
]),
),
(
"units",
J::Obj(vec![
("ok", J::UInt(r.units_ok as u64)),
("failed", J::UInt(r.units_failed as u64)),
("method", J::Str(r.structural_method.to_string())),
]),
),
("tiers", J::Obj(tiers_fields)),
(
"recall",
J::Obj(vec![
("denominator", J::UInt(r.recall_denominator as u64)),
("precise", ratio_j(r.recall_precise, r.recall_denominator)),
(
"precise_ext",
ratio_j(r.recall_precise_ext, r.recall_denominator),
),
("all", ratio_j(r.recall_all, r.recall_denominator)),
("by_receiver", J::Obj(by_receiver_j)),
("conditional", J::UInt(r.recall_conditional as u64)),
("bare", J::UInt(r.recall_bare as u64)),
]),
),
(
"silent",
J::Obj(vec![
("correct", J::UInt(r.silent_correct as u64)),
("leak", J::UInt(r.silent_leak as u64)),
]),
),
(
"structural",
J::Obj(vec![
("impossible", J::UInt(r.structural_impossible as u64)),
("checked", J::UInt(r.structural_checked as u64)),
("method", J::Str(r.structural_method.to_string())),
]),
),
(
"fanout",
J::Obj(vec![
("1", J::UInt(r.fanout[0] as u64)),
("2", J::UInt(r.fanout[1] as u64)),
("3", J::UInt(r.fanout[2] as u64)),
("4+", J::UInt(r.fanout[3] as u64)),
]),
),
(
"unknown_targets",
J::Arr(
r.unknown_targets
.iter()
.map(|(k, c)| {
J::Obj(vec![
("kind", J::Str(k.clone())),
("count", J::UInt(*c as u64)),
])
})
.collect(),
),
),
(
"top_fp",
J::Arr(
r.top_fp
.iter()
.map(|(n, c)| {
J::Obj(vec![
("name", J::Str(n.clone())),
("count", J::UInt(*c as u64)),
])
})
.collect(),
),
),
(
"top_missed",
J::Arr(
r.top_missed
.iter()
.map(|(id, c)| {
J::Obj(vec![
("id", J::Str(id.clone())),
("count", J::UInt(*c as u64)),
])
})
.collect(),
),
),
(
"partial_file_mismatch",
J::UInt(r.partial_file_mismatch as u64),
),
(
"edges_outside_universe",
J::UInt(r.edges_outside_universe as u64),
),
])
.to_json_string()
}
pub(crate) fn cmd_audit(cwd: &Path, args: &[String]) -> (i32, String) {
const USAGE: &str = "usage: devscout audit --semantic <refs.jsonl> [--units F] [--defs F] [--json] [--assert F] [--fp-sites F]";
let mut semantic: Option<String> = None;
let mut units: Option<String> = None;
let mut defs: Option<String> = None;
let mut assert_path: Option<String> = None;
let mut fp_sites_path: Option<String> = None;
let mut json = false;
let mut i = 0;
while i < args.len() {
let flag = args[i].as_str();
if matches!(
flag,
"--semantic" | "--units" | "--defs" | "--assert" | "--fp-sites"
) {
let Some(val) = args.get(i + 1) else {
return (1, format!("error: missing value for '{flag}'\n{USAGE}"));
};
match flag {
"--semantic" => semantic = Some(val.clone()),
"--units" => units = Some(val.clone()),
"--defs" => defs = Some(val.clone()),
"--assert" => assert_path = Some(val.clone()),
"--fp-sites" => fp_sites_path = Some(val.clone()),
_ => unreachable!(),
}
i += 2;
} else if flag == "--json" {
json = true;
i += 1;
} else {
return (1, format!("error: unrecognized argument '{flag}'\n{USAGE}"));
}
}
let Some(semantic) = semantic else {
return (
1,
format!("error: --semantic <refs.jsonl> is required\n{USAGE}"),
);
};
let root = match crate::cli::require_repo(cwd) {
Ok(r) => r,
Err(e) => return (1, format!("error: {e}")),
};
let semantic_path = crate::repo::resolve_from(cwd, Path::new(&semantic));
let units_path = units.map(|u| crate::repo::resolve_from(cwd, Path::new(&u)));
let defs_path = defs.map(|d| crate::repo::resolve_from(cwd, Path::new(&d)));
let opts = AuditOptions {
semantic: &semantic_path,
units: units_path.as_deref(),
defs: defs_path.as_deref(),
};
let mut inputs = match load(&root, &opts) {
Ok(i) => i,
Err(e) => return (1, format!("error: {e}")),
};
inputs.collect_fp_sites = fp_sites_path.is_some();
let report = score(inputs);
if let Some(fp_path) = fp_sites_path {
let abs = crate::repo::resolve_from(cwd, Path::new(&fp_path));
if let Err(e) = std::fs::write(&abs, fp_sites::render(&report.fp_sites)) {
return (1, format!("error: failed to write {}: {e}", abs.display()));
}
}
let json_string = render_json(&report);
let mut out = if json {
json_string.clone()
} else {
render_text(&report)
};
if let Some(assert_path) = assert_path {
let abs = crate::repo::resolve_from(cwd, Path::new(&assert_path));
let text = match std::fs::read_to_string(&abs) {
Ok(t) => t,
Err(e) => return (1, format!("error: failed to read {}: {e}", abs.display())),
};
let violations = match evaluate_assert(&json_string, &text) {
Ok(v) => v,
Err(e) => return (1, format!("error: {e}")),
};
if !violations.is_empty() {
if json {
eprintln!("{}", violations.join("\n"));
} else {
out.push('\n');
out.push_str(&violations.join("\n"));
}
return (1, out);
}
}
(0, out)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{extract, graph, resolve};
fn fragments_for(files: &[(&str, &str)]) -> Vec<(String, graph::Fragment)> {
files
.iter()
.map(|(rel, src)| {
(
(*rel).to_string(),
graph::fragment_from_extraction(&extract::extract(src)),
)
})
.collect()
}
fn no_git_root() -> PathBuf {
std::env::temp_dir().join("scout-audit-test-not-a-repo")
}
fn graph_value_for(files: &[(&str, &str)]) -> serde_json::Value {
let frags = fragments_for(files);
let g = resolve::resolve_graph(&no_git_root(), &frags);
let text = serde_json::to_string(&g).expect("graph serializes");
serde_json::from_str(&text).expect("graph.json round-trips through Value")
}
#[allow(clippy::too_many_arguments)]
fn oracle_ref(
file: &str,
start_line: usize,
shape: &str,
receiver_kind: &str,
member: &str,
target: Option<&str>,
target_kind: Option<&str>,
external: bool,
) -> OracleRef {
OracleRef {
file: file.to_string(),
start_line,
shape: shape.to_string(),
receiver_kind: receiver_kind.to_string(),
member: member.to_string(),
target: target.map(str::to_string),
target_kind: target_kind.map(str::to_string),
target_file: None,
external,
ambiguous: false,
}
}
#[test]
fn precise_edge_at_a_matching_in_solution_site_scores_as_a_true_positive() {
let files = [
(
"Other/MessageUrn.cs",
"namespace App.Consumers { public static class MessageUrn { public static string Prefix { get; } } }",
),
(
"Consumers/UsesProperty.cs",
"\nnamespace App.Consumers;\n\npublic class UsesProperty\n{\n public object Get() => MessageUrn.Prefix;\n}\n",
),
];
let value = graph_value_for(&files);
let (graph_defs, edges) = parse_graph(&value).expect("graph.json parses");
assert_eq!(edges.len(), 1, "exactly one uses-member edge expected");
assert_eq!(edges[0].tier, Tier::Precise);
let record = oracle_ref(
&edges[0].from_file,
edges[0].from_line,
"access",
"ident",
edges[0].member.as_deref().unwrap_or(""),
Some(&edges[0].to),
Some("class"),
false,
);
let universe: HashSet<String> = [edges[0].from_file.clone()].into_iter().collect();
let report = score(Inputs {
root: PathBuf::from("/repo"),
graph_defs,
oracle_defs: Vec::new(),
edges,
records: vec![record],
units: Vec::new(),
universe,
collect_fp_sites: false,
});
assert_eq!(report.tiers.len(), 1);
let (tier, ts) = &report.tiers[0];
assert_eq!(*tier, Tier::Precise);
assert_eq!(ts.tp, 1);
assert_eq!(ts.fp, 0);
assert_eq!(report.oracle_dropped, 0);
}
#[test]
fn guessed_edge_at_an_all_external_site_scores_as_a_leaked_false_positive() {
let files = [
(
"Other/Widget.cs",
"namespace App.Other { public class Widget { public void Frob() { } } }",
),
(
"Consumers/Guess.cs",
"\nnamespace App.Consumers;\n\npublic class Guess\n{\n public void Unknown()\n {\n var w = Compute();\n w.Frob();\n }\n private object Compute() => null;\n}\n",
),
];
let value = graph_value_for(&files);
let (graph_defs, edges) = parse_graph(&value).expect("graph.json parses");
let heuristic_count = edges.iter().filter(|e| e.tier == Tier::Guess).count();
assert_eq!(heuristic_count, 1, "expected exactly one guessed edge");
let g_edge = edges.iter().find(|e| e.tier == Tier::Guess).unwrap();
let record = oracle_ref(
&g_edge.from_file,
g_edge.from_line,
"access",
"ident",
g_edge.member.as_deref().unwrap_or("Frob"),
Some("Some.External.Type"),
Some("class"),
true,
);
let universe: HashSet<String> = [g_edge.from_file.clone()].into_iter().collect();
let report = score(Inputs {
root: PathBuf::from("/repo"),
graph_defs,
oracle_defs: Vec::new(),
edges,
records: vec![record],
units: Vec::new(),
universe,
collect_fp_sites: false,
});
assert_eq!(report.tiers.len(), 1);
let (tier, ts) = &report.tiers[0];
assert_eq!(*tier, Tier::Guess);
assert_eq!(ts.tp, 0);
assert_eq!(ts.fp, 1);
assert_eq!(ts.fp_external_site, 1);
assert_eq!(report.silent_leak, 1);
assert_eq!(report.silent_correct, 0);
assert_eq!(report.oracle_external_sites, 1);
}
#[test]
fn a_matching_record_among_several_at_one_site_still_earns_a_true_positive() {
let edge = EdgeRow {
from_file: "F.cs".into(),
from_line: 5,
to: "Ns.Right".into(),
to_file: "F.cs".into(),
tier: Tier::Precise,
member: None,
};
let wrong = oracle_ref(
"F.cs",
5,
"access",
"ident",
"M",
Some("Ns.Wrong"),
Some("class"),
false,
);
let right = oracle_ref(
"F.cs",
5,
"access",
"ident",
"M",
Some("Ns.Right"),
Some("class"),
false,
);
let report = score(Inputs {
root: PathBuf::from("/repo"),
graph_defs: vec![DefRow {
id: "Ns.Right".into(),
file: "F.cs".into(),
kind: "class".into(),
test: false,
}],
oracle_defs: Vec::new(),
edges: vec![edge],
records: vec![wrong, right],
units: Vec::new(),
universe: ["F.cs".to_string()].into_iter().collect(),
collect_fp_sites: false,
});
let (_, ts) = &report.tiers[0];
assert_eq!(ts.tp, 1);
assert_eq!(ts.fp, 0);
}
#[test]
fn member_equality_picks_the_right_record_at_a_shared_site_chain_tail_vs_qualifier() {
let edge = EdgeRow {
from_file: "F.cs".into(),
from_line: 5,
to: "Ns.Order".into(),
to_file: "F.cs".into(),
tier: Tier::Precise,
member: Some("Load".to_string()),
};
let load_record = oracle_ref(
"F.cs",
5,
"access",
"ident",
"Load",
Some("Ns.Order"),
Some("class"),
false,
);
let validate_record = oracle_ref(
"F.cs",
5,
"access",
"call",
"Validate",
Some("Ns.Order"),
Some("class"),
false,
);
let report = score(Inputs {
root: PathBuf::from("/repo"),
graph_defs: vec![DefRow {
id: "Ns.Order".into(),
file: "F.cs".into(),
kind: "class".into(),
test: false,
}],
oracle_defs: Vec::new(),
edges: vec![edge],
records: vec![load_record, validate_record],
units: Vec::new(),
universe: ["F.cs".to_string()].into_iter().collect(),
collect_fp_sites: false,
});
let (_, ts) = &report.tiers[0];
assert_eq!(ts.tp, 1);
assert_eq!(ts.fp, 0);
assert_eq!(report.recall_denominator, 2);
assert_eq!(report.recall_all, 1);
assert_eq!(report.top_missed, vec![("Ns.Order".to_string(), 1)]);
}
#[test]
fn a_different_member_non_external_record_does_not_block_an_external_leak_classification() {
let edge = EdgeRow {
from_file: "F.cs".into(),
from_line: 24,
to: "Ns.FilterConfig".into(),
to_file: "F.cs".into(),
tier: Tier::Guess,
member: Some("Property".to_string()),
};
let has_max_length = oracle_ref(
"F.cs",
24,
"access",
"call",
"HasMaxLength",
Some("Ext.PropertyBuilder"),
Some("class"),
true,
);
let e_name = oracle_ref(
"F.cs",
24,
"access",
"ident",
"Name",
Some("Ns.Order"),
Some("class"),
false,
);
let entity_property = oracle_ref(
"F.cs",
24,
"access",
"ident",
"Property",
Some("Ext.EntityTypeBuilder"),
Some("class"),
true,
);
let report = score(Inputs {
root: PathBuf::from("/repo"),
graph_defs: vec![DefRow {
id: "Ns.Order".into(),
file: "F.cs".into(),
kind: "class".into(),
test: false,
}],
oracle_defs: Vec::new(),
edges: vec![edge],
records: vec![has_max_length, e_name, entity_property],
units: Vec::new(),
universe: ["F.cs".to_string()].into_iter().collect(),
collect_fp_sites: false,
});
let (_, ts) = &report.tiers[0];
assert_eq!(ts.fp, 1);
assert_eq!(
ts.fp_external_site, 1,
"the member-scoped view sees only `entity_property`, which is external"
);
assert_eq!(
ts.fp_wrong_target, 0,
"`e_name` is non-external but names a different member, so it must not \
count as an in-tree answer this edge got wrong"
);
}
#[test]
fn an_edge_naming_a_member_no_record_at_the_site_names_falls_back_to_the_whole_site() {
let edge = EdgeRow {
from_file: "F.cs".into(),
from_line: 24,
to: "Ns.FilterConfig".into(),
to_file: "F.cs".into(),
tier: Tier::Guess,
member: Some("Nowhere".to_string()),
};
let in_tree = oracle_ref(
"F.cs",
24,
"access",
"ident",
"Name",
Some("Ns.Order"),
Some("class"),
false,
);
let report = score(Inputs {
root: PathBuf::from("/repo"),
graph_defs: vec![DefRow {
id: "Ns.Order".into(),
file: "F.cs".into(),
kind: "class".into(),
test: false,
}],
oracle_defs: Vec::new(),
edges: vec![edge],
records: vec![in_tree],
units: Vec::new(),
universe: ["F.cs".to_string()].into_iter().collect(),
collect_fp_sites: false,
});
let (_, ts) = &report.tiers[0];
assert_eq!(ts.fp, 1);
assert_eq!(
ts.fp_wrong_target, 1,
"the site's only record is in-tree, so the edge is a wrong target, not a leak"
);
assert_eq!(
ts.fp_external_site, 0,
"an empty member scope must not be read as `every record here is external`"
);
}
#[test]
fn an_edge_naming_an_unwritten_member_at_an_all_external_site_is_still_a_leak() {
let edge = EdgeRow {
from_file: "F.cs".into(),
from_line: 24,
to: "Ns.FilterConfig".into(),
to_file: "F.cs".into(),
tier: Tier::Guess,
member: Some("Nowhere".to_string()),
};
let external_only = oracle_ref(
"F.cs",
24,
"access",
"call",
"HasMaxLength",
Some("Ext.PropertyBuilder"),
Some("class"),
true,
);
let report = score(Inputs {
root: PathBuf::from("/repo"),
graph_defs: vec![DefRow {
id: "Ns.FilterConfig".into(),
file: "F.cs".into(),
kind: "class".into(),
test: false,
}],
oracle_defs: Vec::new(),
edges: vec![edge],
records: vec![external_only],
units: Vec::new(),
universe: ["F.cs".to_string()].into_iter().collect(),
collect_fp_sites: false,
});
let (_, ts) = &report.tiers[0];
assert_eq!(ts.fp, 1);
assert_eq!(ts.fp_external_site, 1);
assert_eq!(ts.fp_wrong_target, 0);
}
#[test]
fn enum_member_edge_matches_both_the_full_and_the_bare_spelling() {
let edge_full = EdgeRow {
from_file: "F.cs".into(),
from_line: 10,
to: "Ns.OrderStatus.Open".into(),
to_file: "Ns/OrderStatus.cs".into(),
tier: Tier::Precise,
member: Some("Open".to_string()),
};
let edge_bare = EdgeRow {
from_file: "F.cs".into(),
from_line: 20,
to: "Ns.OrderStatus".into(),
to_file: "Ns/OrderStatus.cs".into(),
tier: Tier::Precise,
member: Some("Open".to_string()),
};
let rec_at_full = oracle_ref(
"F.cs",
10,
"access",
"ident",
"Open",
Some("Ns.OrderStatus.Open"),
Some("enum-member"),
false,
);
let rec_at_bare = oracle_ref(
"F.cs",
20,
"access",
"ident",
"Open",
Some("Ns.OrderStatus.Open"),
Some("enum-member"),
false,
);
let report = score(Inputs {
root: PathBuf::from("/repo"),
graph_defs: vec![DefRow {
id: "Ns.OrderStatus".into(),
file: "Ns/OrderStatus.cs".into(),
kind: "enum".into(),
test: false,
}],
oracle_defs: Vec::new(),
edges: vec![edge_full, edge_bare],
records: vec![rec_at_full, rec_at_bare],
units: Vec::new(),
universe: ["F.cs".to_string()].into_iter().collect(),
collect_fp_sites: false,
});
let (_, ts) = &report.tiers[0];
assert_eq!(ts.tp, 2);
assert_eq!(ts.fp, 0);
}
#[test]
fn tier_is_read_from_the_tier_string_when_present_else_from_legacy_heuristic_bool() {
let text = r#"{"defs":[],"edges":[
{"kind":"uses-member","from_file":"F.cs","from_line":1,"to":"Ns.A","to_file":"A.cs","heuristic":true},
{"kind":"uses-member","from_file":"F.cs","from_line":2,"to":"Ns.B","to_file":"B.cs","tier":"ext"},
{"kind":"uses-member","from_file":"F.cs","from_line":3,"to":"Ns.C","to_file":"C.cs","tier":"guess"},
{"kind":"uses-member","from_file":"F.cs","from_line":4,"to":"Ns.D","to_file":"D.cs"},
{"kind":"inherits","from_file":"F.cs","from_line":5,"to":"Ns.E","to_file":"E.cs"}
]}"#;
let value: serde_json::Value = serde_json::from_str(text).unwrap();
let (_, edges) = parse_graph(&value).unwrap();
assert_eq!(
edges.len(),
4,
"the 'inherits' edge is not uses-member and must be filtered out"
);
let tiers: Vec<Tier> = edges.iter().map(|e| e.tier).collect();
assert_eq!(
tiers,
vec![Tier::Heuristic, Tier::Ext, Tier::Guess, Tier::Precise]
);
}
#[test]
fn an_edge_from_a_file_outside_units_files_is_dropped_not_a_false_positive() {
let edge = EdgeRow {
from_file: "Other/B.cs".into(),
from_line: 1,
to: "Ns.Something".into(),
to_file: "Other/B.cs".into(),
tier: Tier::Heuristic,
member: None,
};
let units = vec![Unit {
name: "App".into(),
status: "ok".into(),
refs: vec![],
files: vec!["App/A.cs".into()],
}];
let report = score(Inputs {
root: PathBuf::from("/repo"),
graph_defs: Vec::new(),
oracle_defs: Vec::new(),
edges: vec![edge],
records: Vec::new(),
units,
universe: ["App/A.cs".to_string()].into_iter().collect(),
collect_fp_sites: false,
});
assert_eq!(report.edges_outside_universe, 1);
assert!(
report.tiers.is_empty(),
"the dropped edge must not appear in any tier's stats, fp_no_site included: {:?}",
report.tiers
);
assert_eq!(report.structural_checked, 0);
}
#[test]
fn structural_check_via_units_flags_an_edge_the_caller_project_cannot_reach() {
let edge = EdgeRow {
from_file: "App/A.cs".into(),
from_line: 9,
to: "Tests.Foo".into(),
to_file: "Tests/Foo.cs".into(),
tier: Tier::Heuristic,
member: None,
};
let units = vec![
Unit {
name: "App".into(),
status: "ok".into(),
refs: vec!["Domain".into()],
files: vec!["App/A.cs".into()],
},
Unit {
name: "Domain".into(),
status: "ok".into(),
refs: vec![],
files: vec![],
},
Unit {
name: "Tests".into(),
status: "ok".into(),
refs: vec!["App".into()],
files: vec!["Tests/Foo.cs".into()],
},
];
let record = oracle_ref(
"App/A.cs",
9,
"access",
"ident",
"Foo",
Some("Tests.Foo"),
Some("class"),
false,
);
let report = score(Inputs {
root: PathBuf::from("/repo"),
graph_defs: vec![DefRow {
id: "Tests.Foo".into(),
file: "Tests/Foo.cs".into(),
kind: "class".into(),
test: false,
}],
oracle_defs: Vec::new(),
edges: vec![edge],
records: vec![record],
units,
universe: ["App/A.cs".to_string()].into_iter().collect(),
collect_fp_sites: false,
});
assert_eq!(report.structural_method, "units");
assert_eq!(report.structural_checked, 1);
assert_eq!(report.structural_impossible, 1);
let (_, ts) = &report.tiers[0];
assert_eq!(ts.tp, 1, "the structurally-impossible edge is still a TP");
assert_eq!(
ts.structural, 0,
"a TP is never counted structural, however structurally impossible its edge -- \
the tier column reports resolver false positives, not every structural oddity"
);
}
#[test]
fn structural_fallback_flags_a_non_test_caller_reaching_a_test_attributed_def() {
let edge_bad = EdgeRow {
from_file: "App/A.cs".into(),
from_line: 3,
to: "Tests.Helper".into(),
to_file: "Tests/Helper.cs".into(),
tier: Tier::Heuristic,
member: None,
};
let edge_ok = EdgeRow {
from_file: "Tests/Caller.cs".into(),
from_line: 4,
to: "Tests.Helper".into(),
to_file: "Tests/Helper.cs".into(),
tier: Tier::Heuristic,
member: None,
};
let report = score(Inputs {
root: PathBuf::from("/repo"),
graph_defs: vec![
DefRow {
id: "Tests.Helper".into(),
file: "Tests/Helper.cs".into(),
kind: "class".into(),
test: true,
},
DefRow {
id: "Tests.Caller".into(),
file: "Tests/Caller.cs".into(),
kind: "class".into(),
test: true,
},
],
oracle_defs: Vec::new(),
edges: vec![edge_bad, edge_ok],
records: Vec::new(),
units: Vec::new(), universe: HashSet::new(),
collect_fp_sites: false,
});
assert_eq!(report.structural_method, "test-defs");
assert_eq!(report.structural_checked, 2);
assert_eq!(report.structural_impossible, 1);
}
#[test]
fn an_oracle_def_row_saying_test_false_un_marks_the_file_for_the_structural_fallback() {
let inputs = |oracle_defs: Vec<DefRow>| Inputs {
root: PathBuf::from("/repo"),
graph_defs: vec![
DefRow {
id: "Tests.Helper".into(),
file: "Tests/Helper.cs".into(),
kind: "class".into(),
test: true,
},
DefRow {
id: "Tests.Caller".into(),
file: "Tests/Caller.cs".into(),
kind: "class".into(),
test: true,
},
],
oracle_defs,
edges: vec![EdgeRow {
from_file: "Tests/Caller.cs".into(),
from_line: 4,
to: "Tests.Helper".into(),
to_file: "Tests/Helper.cs".into(),
tier: Tier::Heuristic,
member: None,
}],
records: Vec::new(),
units: Vec::new(), universe: HashSet::new(),
collect_fp_sites: false,
};
let without_oracle = score(inputs(Vec::new()));
assert_eq!(
without_oracle.structural_impossible, 0,
"with only the graph's view the caller declares a test def, so the edge is fine"
);
let with_oracle = score(inputs(vec![DefRow {
id: "Tests.Caller".into(),
file: "Tests/Caller.cs".into(),
kind: "class".into(),
test: false,
}]));
assert_eq!(with_oracle.structural_checked, 1);
assert_eq!(
with_oracle.structural_impossible, 1,
"the merged view has no test def in Tests/Caller.cs any more, so the file is not a test file and the edge into a test-only def is impossible"
);
}
#[test]
fn assert_thresholds_report_pass_fail_and_missing_path_distinctly() {
let report_json = r#"{"tiers":{"guess":{"fp":13}},"recall":{"all":0.867}}"#;
let pass = evaluate_assert(report_json, r#"{"recall.all":{"min":0.8}}"#).unwrap();
assert!(pass.is_empty());
let fail = evaluate_assert(report_json, r#"{"tiers.guess.fp":{"max":0}}"#).unwrap();
assert_eq!(fail, vec!["assert: tiers.guess.fp = 13 > max 0"]);
let missing =
evaluate_assert(report_json, r#"{"tiers.precise.precision":{"min":1.0}}"#).unwrap();
assert_eq!(missing, vec!["assert: tiers.precise.precision missing"]);
}
#[test]
fn json_output_key_order_is_pinned() {
let report = AuditReport {
root: "/repo".to_string(),
oracle_records: 34,
oracle_sites: 25,
oracle_external_sites: 8,
oracle_ambiguous: 1,
oracle_dropped: 2,
units_ok: 6,
units_failed: 0,
structural_method: "units",
tiers: vec![
(
Tier::Precise,
TierStats {
edges: 6,
tp: 6,
fp: 0,
fp_no_site: 0,
fp_external_site: 0,
fp_wrong_target: 0,
structural: 0,
},
),
(
Tier::Heuristic,
TierStats {
edges: 21,
tp: 8,
fp: 13,
fp_no_site: 0,
fp_external_site: 13,
fp_wrong_target: 0,
structural: 2,
},
),
],
recall_denominator: 15,
recall_precise: 6,
recall_precise_ext: 7,
recall_all: 13,
by_receiver: vec![
("ident", Some(0.917)),
("qualified", None),
("this", Some(0.0)),
("base", None),
("call", Some(0.0)),
],
recall_conditional: 1,
recall_bare: 0,
silent_correct: 4,
silent_leak: 4,
structural_impossible: 2,
structural_checked: 21,
fanout: [20, 4, 0, 0],
unknown_targets: vec![("class".to_string(), 2)],
top_fp: vec![("FilterConfig".to_string(), 2), ("Mailer".to_string(), 1)],
top_missed: vec![("Fixture.Domain.Order".to_string(), 2)],
partial_file_mismatch: 0,
edges_outside_universe: 5,
fp_sites: Vec::new(),
};
let json = render_json(&report);
assert_eq!(
json,
concat!(
"{\"status\":\"ok\",\"root\":\"/repo\",",
"\"oracle\":{\"records\":34,\"sites\":25,\"external_sites\":8,\"ambiguous\":1,\"dropped\":2},",
"\"units\":{\"ok\":6,\"failed\":0,\"method\":\"units\"},",
"\"tiers\":{",
"\"precise\":{\"edges\":6,\"tp\":6,\"fp\":0,\"precision\":1.000,\"fp_no_site\":0,\"fp_external_site\":0,\"fp_wrong_target\":0,\"structural\":0},",
"\"heuristic\":{\"edges\":21,\"tp\":8,\"fp\":13,\"precision\":0.381,\"fp_no_site\":0,\"fp_external_site\":13,\"fp_wrong_target\":0,\"structural\":2}",
"},",
"\"recall\":{\"denominator\":15,\"precise\":0.400,\"precise_ext\":0.467,\"all\":0.867,",
"\"by_receiver\":{\"ident\":0.917,\"qualified\":null,\"this\":0.000,\"base\":null,\"call\":0.000},",
"\"conditional\":1,\"bare\":0},",
"\"silent\":{\"correct\":4,\"leak\":4},",
"\"structural\":{\"impossible\":2,\"checked\":21,\"method\":\"units\"},",
"\"fanout\":{\"1\":20,\"2\":4,\"3\":0,\"4+\":0},",
"\"unknown_targets\":[{\"kind\":\"class\",\"count\":2}],",
"\"top_fp\":[{\"name\":\"FilterConfig\",\"count\":2},{\"name\":\"Mailer\",\"count\":1}],",
"\"top_missed\":[{\"id\":\"Fixture.Domain.Order\",\"count\":2}],",
"\"partial_file_mismatch\":0,",
"\"edges_outside_universe\":5}",
)
);
}
#[test]
fn text_output_layout_is_pinned() {
let report = AuditReport {
root: "/repo".to_string(),
oracle_records: 34,
oracle_sites: 25,
oracle_external_sites: 8,
oracle_ambiguous: 1,
oracle_dropped: 0,
units_ok: 6,
units_failed: 0,
structural_method: "units",
tiers: vec![
(
Tier::Precise,
TierStats {
edges: 6,
tp: 6,
fp: 0,
fp_no_site: 0,
fp_external_site: 0,
fp_wrong_target: 0,
structural: 0,
},
),
(
Tier::Guess,
TierStats {
edges: 21,
tp: 8,
fp: 13,
fp_no_site: 0,
fp_external_site: 13,
fp_wrong_target: 0,
structural: 2,
},
),
],
recall_denominator: 15,
recall_precise: 6,
recall_precise_ext: 7,
recall_all: 13,
by_receiver: vec![
("ident", Some(0.917)),
("qualified", None),
("this", Some(0.0)),
],
recall_conditional: 1,
recall_bare: 0,
silent_correct: 4,
silent_leak: 4,
structural_impossible: 3,
structural_checked: 21,
fanout: [20, 4, 0, 0],
unknown_targets: vec![("class".to_string(), 2)],
top_fp: vec![("FilterConfig".to_string(), 2), ("Mailer".to_string(), 1)],
top_missed: vec![("Fixture.Domain.Order".to_string(), 2)],
partial_file_mismatch: 0,
edges_outside_universe: 5,
fp_sites: Vec::new(),
};
let text = render_text(&report);
assert_eq!(
text,
concat!(
"devscout audit --semantic root /repo oracle 34 records / 25 sites units ok 6 failed 0 method units\n",
"tier edges tp fp precision fp:no-site fp:external fp:wrong structural\n",
"precise 6 6 0 1.000 0 0 0 0\n",
"guess 21 8 13 0.381 0 13 0 2\n",
"recall (15 in-graph member sites) precise 0.400 precise+ext 0.467 all 0.867\n",
" by receiver ident 0.917 qualified - this 0.000\n",
"external sites 8 silent-correct 4 leaked 4\n",
"structural impossible 3 checked 21\n",
"fan-out 1: 20 2: 4 3: 0 4+: 0\n",
"top fp targets FilterConfig 2 Mailer 1\n",
"top missed Fixture.Domain.Order 2\n",
"unknown targets class 2\n",
"ambiguous 1\n",
"edges outside universe (not judged) 5",
)
);
}
}