pounce_cli/diagnose/
report.rs

1//! The findings schema `pounce diagnose` reports in (`pounce.diagnose/v1`).
2//!
3//! A finding is one observation about the model, with what it rests on kept
4//! separate from how bad it is:
5//!
6//! * [`Severity`] — how much the user should care: `error` (the model as
7//!   written cannot be solved, or a solve from this point will abort),
8//!   `warning` (likely trouble), `info` (worth knowing).
9//! * [`Basis`] — what kind of evidence backs it: a `proof` from the
10//!   represented algebra and bounds, a `structural` fact about the sparsity
11//!   pattern, a `numerical` observation at one named point and tolerance, or
12//!   a `heuristic` pattern match.
13//! * [`Confidence`] — how sure the check is that the observation is real.
14//!
15//! The two are deliberately independent: a numerical rank loss at one point
16//! can be an `error` for the solve from that point and still only a
17//! `numerical` observation about the model.
18//!
19//! A check that could not run is not silently absent. It is recorded in
20//! [`Report::checks`] with status `unavailable` and a reason, so an empty
21//! findings list can be told apart from "nothing was checked".
22
23use pounce_common::types::Number;
24use serde_json::{Value, json};
25use std::collections::BTreeMap;
26
27/// How much the user should care about a finding.
28#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
29pub enum Severity {
30    Info,
31    Warning,
32    Error,
33}
34
35impl Severity {
36    pub fn as_str(self) -> &'static str {
37        match self {
38            Severity::Info => "info",
39            Severity::Warning => "warning",
40            Severity::Error => "error",
41        }
42    }
43}
44
45/// What kind of evidence a finding rests on.
46#[derive(Debug, Clone, Copy, PartialEq, Eq)]
47pub enum Basis {
48    /// Follows from the represented algebra and bounds alone; holds at
49    /// every point.
50    Proof,
51    /// Follows from the sparsity pattern alone. A structural rank loss is a
52    /// bound on the numerical rank at every point.
53    Structural,
54    /// Observed at one named point under a stated tolerance. Says nothing
55    /// by itself about any other point.
56    Numerical,
57    /// A pattern that often signals trouble but proves nothing.
58    Heuristic,
59}
60
61impl Basis {
62    pub fn as_str(self) -> &'static str {
63        match self {
64            Basis::Proof => "proof",
65            Basis::Structural => "structural",
66            Basis::Numerical => "numerical",
67            Basis::Heuristic => "heuristic",
68        }
69    }
70}
71
72/// How sure the check is that the observation is real.
73#[derive(Debug, Clone, Copy, PartialEq, Eq)]
74pub enum Confidence {
75    Low,
76    Medium,
77    High,
78    Certain,
79}
80
81impl Confidence {
82    pub fn as_str(self) -> &'static str {
83        match self {
84            Confidence::Low => "low",
85            Confidence::Medium => "medium",
86            Confidence::High => "high",
87            Confidence::Certain => "certain",
88        }
89    }
90}
91
92/// One finding. Build with [`Finding::new`] and the chained setters.
93#[derive(Debug, Clone)]
94pub struct Finding {
95    /// Stable identifier, `snake_case`. Codes are part of the schema:
96    /// renaming one is a breaking change for anyone filtering on it.
97    pub code: &'static str,
98    /// The check that produced it (one of [`Report::checks`]).
99    pub check: &'static str,
100    pub severity: Severity,
101    pub basis: Basis,
102    pub confidence: Confidence,
103    /// What was seen, in one or two sentences, naming the entities.
104    pub observation: String,
105    /// Why the solver or the user should care.
106    pub why_it_matters: String,
107    /// The cheapest next thing to look at. An investigation step, never an
108    /// instruction to change the model.
109    pub suggested_next_check: String,
110    /// Variable names involved.
111    pub variables: Vec<String>,
112    /// Constraint names involved (`objective` for the objective).
113    pub constraints: Vec<String>,
114    /// Supporting numbers: intervals, values, tolerances, the point label.
115    pub evidence: Value,
116}
117
118impl Finding {
119    pub fn new(
120        check: &'static str,
121        code: &'static str,
122        severity: Severity,
123        basis: Basis,
124        confidence: Confidence,
125        observation: impl Into<String>,
126    ) -> Self {
127        Finding {
128            code,
129            check,
130            severity,
131            basis,
132            confidence,
133            observation: observation.into(),
134            why_it_matters: String::new(),
135            suggested_next_check: String::new(),
136            variables: Vec::new(),
137            constraints: Vec::new(),
138            evidence: Value::Null,
139        }
140    }
141
142    pub fn why(mut self, s: impl Into<String>) -> Self {
143        self.why_it_matters = s.into();
144        self
145    }
146
147    pub fn next(mut self, s: impl Into<String>) -> Self {
148        self.suggested_next_check = s.into();
149        self
150    }
151
152    pub fn vars(mut self, v: Vec<String>) -> Self {
153        self.variables = v;
154        self
155    }
156
157    pub fn cons(mut self, c: Vec<String>) -> Self {
158        self.constraints = c;
159        self
160    }
161
162    pub fn evidence(mut self, e: Value) -> Self {
163        self.evidence = e;
164        self
165    }
166
167    fn to_json(&self) -> Value {
168        json!({
169            "code": self.code,
170            "check": self.check,
171            "severity": self.severity.as_str(),
172            "basis": self.basis.as_str(),
173            "confidence": self.confidence.as_str(),
174            "observation": self.observation,
175            "why_it_matters": self.why_it_matters,
176            "suggested_next_check": self.suggested_next_check,
177            "affected": {
178                "variables": self.variables,
179                "constraints": self.constraints,
180            },
181            "evidence": self.evidence,
182        })
183    }
184}
185
186/// Whether a check ran.
187#[derive(Debug, Clone)]
188pub enum CheckStatus {
189    Ran,
190    /// The check did not run; the string says why.
191    Unavailable(String),
192}
193
194/// One entry of [`Report::checks`].
195#[derive(Debug, Clone)]
196pub struct CheckRecord {
197    pub name: &'static str,
198    pub status: CheckStatus,
199}
200
201/// Collects findings for one check, capping how many of each code are kept
202/// so a model with ten thousand `log(x)` rows does not produce a
203/// ten-thousand-entry report. The cap is per code; the number dropped is
204/// reported in [`Report::truncated`].
205pub struct Sink<'a> {
206    pub check: &'static str,
207    report: &'a mut Report,
208}
209
210impl Sink<'_> {
211    pub fn push(&mut self, f: Finding) {
212        debug_assert_eq!(f.check, self.check, "finding filed under the wrong check");
213        let kept = self.report.kept_per_code.entry(f.code).or_insert(0);
214        if *kept < self.report.max_per_code {
215            *kept += 1;
216            self.report.findings.push(f);
217        } else {
218            *self.report.truncated.entry(f.code).or_insert(0) += 1;
219            *self.report.dropped.entry(f.severity).or_insert(0) += 1;
220        }
221    }
222}
223
224/// The whole report.
225#[derive(Debug, Clone)]
226pub struct Report {
227    pub source: String,
228    pub sha256: Option<String>,
229    pub n_vars: usize,
230    pub n_cons: usize,
231    /// Label of the point the numerical checks evaluated at.
232    pub point: String,
233    pub checks: Vec<CheckRecord>,
234    pub findings: Vec<Finding>,
235    /// Per-code count of findings dropped by the cap.
236    pub truncated: BTreeMap<&'static str, usize>,
237    /// Findings dropped by the cap, per severity. A code can mix severities
238    /// (a non-finite value the clamp clears is a warning, one it does not is
239    /// an error), so the per-code count alone cannot say which were dropped.
240    dropped: BTreeMap<Severity, usize>,
241    kept_per_code: BTreeMap<&'static str, usize>,
242    pub max_per_code: usize,
243    /// Tolerances used, by name, so a reader can reproduce the run.
244    pub tolerances: BTreeMap<&'static str, Number>,
245    /// Extra structured sections (the embedded check-x0 report, summary
246    /// statistics), keyed by section name.
247    pub sections: BTreeMap<&'static str, Value>,
248}
249
250impl Report {
251    pub fn new(source: String, n_vars: usize, n_cons: usize, max_per_code: usize) -> Self {
252        Report {
253            source,
254            sha256: None,
255            n_vars,
256            n_cons,
257            point: String::new(),
258            checks: Vec::new(),
259            findings: Vec::new(),
260            truncated: BTreeMap::new(),
261            dropped: BTreeMap::new(),
262            kept_per_code: BTreeMap::new(),
263            max_per_code: max_per_code.max(1),
264            tolerances: BTreeMap::new(),
265            sections: BTreeMap::new(),
266        }
267    }
268
269    /// Record that `check` ran and return a sink for its findings.
270    pub fn ran(&mut self, check: &'static str) -> Sink<'_> {
271        self.checks.push(CheckRecord {
272            name: check,
273            status: CheckStatus::Ran,
274        });
275        Sink {
276            check,
277            report: self,
278        }
279    }
280
281    /// Record that `check` could not run, and why.
282    pub fn unavailable(&mut self, check: &'static str, reason: impl Into<String>) {
283        self.checks.push(CheckRecord {
284            name: check,
285            status: CheckStatus::Unavailable(reason.into()),
286        });
287    }
288
289    /// Findings of severity `sev`, counting those the cap dropped.
290    pub fn count(&self, sev: Severity) -> usize {
291        self.findings.iter().filter(|f| f.severity == sev).count()
292            + self.dropped.get(&sev).copied().unwrap_or(0)
293    }
294
295    /// `true` when any finding is an error, including ones the cap dropped.
296    pub fn has_errors(&self) -> bool {
297        self.count(Severity::Error) > 0
298    }
299
300    /// `ERRORS` / `WARNINGS` / `CLEAN`.
301    pub fn verdict(&self) -> &'static str {
302        if self.has_errors() {
303            "ERRORS"
304        } else if self.count(Severity::Warning) > 0 {
305            "WARNINGS"
306        } else {
307            "CLEAN"
308        }
309    }
310
311    /// Findings sorted most severe first, stable within a severity (checks
312    /// run in a fixed order, so the order is deterministic).
313    pub fn sorted(&self) -> Vec<&Finding> {
314        let mut v: Vec<&Finding> = self.findings.iter().collect();
315        v.sort_by_key(|f| std::cmp::Reverse(f.severity));
316        v
317    }
318
319    pub fn to_json(&self) -> Value {
320        let checks: Vec<Value> = self
321            .checks
322            .iter()
323            .map(|c| match &c.status {
324                CheckStatus::Ran => json!({"check": c.name, "status": "ran"}),
325                CheckStatus::Unavailable(r) => {
326                    json!({"check": c.name, "status": "unavailable", "reason": r})
327                }
328            })
329            .collect();
330        let findings: Vec<Value> = self.sorted().iter().map(|f| f.to_json()).collect();
331        let mut sections = serde_json::Map::new();
332        for (k, v) in &self.sections {
333            sections.insert((*k).to_string(), v.clone());
334        }
335        json!({
336            "schema": "pounce.diagnose/v1",
337            "solver": format!("pounce {}", env!("CARGO_PKG_VERSION")),
338            "problem": {
339                "source": self.source,
340                "sha256": self.sha256,
341                "n_vars": self.n_vars,
342                "n_cons": self.n_cons,
343            },
344            "point": self.point,
345            "tolerances": self.tolerances,
346            "verdict": self.verdict(),
347            "summary": {
348                "errors": self.count(Severity::Error),
349                "warnings": self.count(Severity::Warning),
350                "info": self.count(Severity::Info),
351            },
352            "checks": checks,
353            "findings": findings,
354            "truncated": self.truncated,
355            "sections": Value::Object(sections),
356        })
357    }
358
359    /// The human-readable report. Errors and warnings are listed in full;
360    /// info findings are summarized per code after the first few, so the
361    /// signal is not buried. The JSON carries everything.
362    pub fn to_text(&self) -> String {
363        use std::fmt::Write;
364        const INFO_SHOWN_PER_CODE: usize = 3;
365        let mut out = String::new();
366        let _ = writeln!(out, "pounce diagnose: {}", self.source);
367        let _ = writeln!(
368            out,
369            "  {} variables, {} constraints; numerical checks at {}",
370            self.n_vars, self.n_cons, self.point
371        );
372        let _ = writeln!(
373            out,
374            "  verdict: {}  ({} error(s), {} warning(s), {} info)",
375            self.verdict(),
376            self.count(Severity::Error),
377            self.count(Severity::Warning),
378            self.count(Severity::Info)
379        );
380        let mut info_shown: BTreeMap<&str, usize> = BTreeMap::new();
381        let mut info_hidden: BTreeMap<&str, usize> = BTreeMap::new();
382        for f in self.sorted() {
383            if f.severity == Severity::Info {
384                let shown = info_shown.entry(f.code).or_insert(0);
385                if *shown >= INFO_SHOWN_PER_CODE {
386                    *info_hidden.entry(f.code).or_insert(0) += 1;
387                    continue;
388                }
389                *shown += 1;
390            }
391            let _ = writeln!(out);
392            let _ = writeln!(
393                out,
394                "  [{}] {}  ({}, {} confidence)",
395                f.severity.as_str().to_uppercase(),
396                f.code,
397                f.basis.as_str(),
398                f.confidence.as_str()
399            );
400            let _ = writeln!(out, "    {}", f.observation);
401            if !f.why_it_matters.is_empty() {
402                let _ = writeln!(out, "    why: {}", f.why_it_matters);
403            }
404            if !f.suggested_next_check.is_empty() {
405                let _ = writeln!(out, "    next: {}", f.suggested_next_check);
406            }
407        }
408        if !info_hidden.is_empty() || !self.truncated.is_empty() {
409            let _ = writeln!(out);
410            for (code, n) in &info_hidden {
411                let _ = writeln!(out, "  ... {n} more info `{code}` finding(s) (see --json)");
412            }
413            for (code, n) in &self.truncated {
414                let _ = writeln!(
415                    out,
416                    "  ... {n} more `{code}` finding(s) dropped by --max-per-code {}",
417                    self.max_per_code
418                );
419            }
420        }
421        let unavailable: Vec<&CheckRecord> = self
422            .checks
423            .iter()
424            .filter(|c| matches!(c.status, CheckStatus::Unavailable(_)))
425            .collect();
426        if !unavailable.is_empty() {
427            let _ = writeln!(out);
428            let _ = writeln!(out, "  checks that did not run:");
429            for c in unavailable {
430                if let CheckStatus::Unavailable(r) = &c.status {
431                    let _ = writeln!(out, "    {}: {r}", c.name);
432                }
433            }
434        }
435        out
436    }
437}
438
439/// A number for JSON evidence: finite values as numbers, non-finite as the
440/// strings `"inf"`, `"-inf"`, `"nan"` (JSON has no spelling for them).
441pub fn num(v: Number) -> Value {
442    if v.is_finite() {
443        json!(v)
444    } else {
445        json!(v.to_string())
446    }
447}
448
449/// A number for prose: plain decimal between 1e-4 and 1e6, scientific
450/// otherwise, and no 17-digit tails. `inf`/`-inf`/`NaN` as Rust prints them.
451pub fn fmt_num(v: Number) -> String {
452    if !v.is_finite() {
453        return v.to_string();
454    }
455    if v == 0.0 {
456        return "0".to_string();
457    }
458    let a = v.abs();
459    if (1e-4..1e6).contains(&a) {
460        let s = format!("{v:.6}");
461        let s = s.trim_end_matches('0').trim_end_matches('.');
462        s.to_string()
463    } else {
464        format!("{v:.3e}")
465    }
466}
467
468/// An interval for JSON evidence.
469pub fn interval(lo: Number, hi: Number) -> Value {
470    json!([num(lo), num(hi)])
471}
472
473#[cfg(test)]
474mod tests {
475    use super::*;
476
477    #[test]
478    fn an_error_dropped_by_the_cap_still_sets_the_verdict() {
479        let mut r = Report::new("t".into(), 0, 0, 1);
480        let mut sink = r.ran("c");
481        let f = |sev| Finding::new("c", "mixed", sev, Basis::Numerical, Confidence::High, "x");
482        sink.push(f(Severity::Warning));
483        sink.push(f(Severity::Error)); // dropped: the cap is 1 per code
484        assert!(r.findings.iter().all(|f| f.severity == Severity::Warning));
485        assert!(r.has_errors());
486        assert_eq!(r.verdict(), "ERRORS");
487        assert_eq!(r.count(Severity::Error), 1);
488        assert_eq!(r.count(Severity::Warning), 1);
489    }
490}
pounce_cli/diagnose/report.rs

pounce_cli/diagnose/
report.rs
