Skip to main content

deq_runtime/decoder/
test_harness.rs

1//! Standard decoder test harness
2//!
3//! Runs the curated set of [`StandardTestProblem`]s against any
4//! [`DynDecoder`], exercising both the one-shot
5//! `decode` path and the `load_hypergraph` + `decode_loaded` path for every
6//! case. Returns a [`SuiteReport`] describing the actual outcome of each call;
7//! callers decide what is acceptable.
8
9use crate::decoder::DynDecoder;
10use crate::decoder::blackbox_decoder::{self, DecodingHypergraph, ParityFactor};
11use crate::decoder::blackbox_util::is_parity_factor;
12use crate::decoder::test_problems::{StandardTestProblem, TestCase, case_id, standard_test_problems};
13use crate::util::BitVector;
14
15/// Which API path produced a [`CaseResult`].
16#[derive(Copy, Clone, PartialEq, Eq, Debug)]
17pub enum Path {
18    /// One-shot `decode(hypergraph, syndrome)`.
19    Decode,
20    /// `load_hypergraph(hypergraph)` followed by `decode_loaded(hid, syndrome)`.
21    DecodeLoaded,
22}
23
24impl Path {
25    pub fn as_str(self) -> &'static str {
26        match self {
27            Path::Decode => "decode",
28            Path::DecodeLoaded => "decode_loaded",
29        }
30    }
31}
32
33/// Outcome of running one case on one API path.
34#[derive(Clone, Debug, PartialEq)]
35pub enum Outcome {
36    /// The decoder returned a subgraph that explains the syndrome.
37    Pass,
38    /// The decoder returned a subgraph that does not explain the syndrome.
39    InvalidSubgraph { returned: Vec<u64> },
40    /// The RPC failed (panic, error status, etc.).
41    RpcError(String),
42}
43
44impl Outcome {
45    pub fn is_pass(&self) -> bool {
46        matches!(self, Outcome::Pass)
47    }
48}
49
50/// Outcome of a single (problem, case, path) entry.
51#[derive(Clone, Debug)]
52pub struct CaseResult {
53    pub problem: &'static str,
54    pub case: &'static str,
55    pub path: Path,
56    pub outcome: Outcome,
57}
58
59/// Aggregated outcomes for one suite run.
60#[derive(Clone, Debug, Default)]
61pub struct SuiteReport {
62    pub results: Vec<CaseResult>,
63}
64
65impl SuiteReport {
66    /// Look up the outcome for a specific (problem, case, path) entry.
67    pub fn get(&self, problem: &str, case: &str, path: Path) -> Option<&Outcome> {
68        self.results
69            .iter()
70            .find(|r| r.problem == problem && r.case == case && r.path == path)
71            .map(|r| &r.outcome)
72    }
73
74    /// Number of entries that passed.
75    pub fn pass_count(&self) -> usize {
76        self.results.iter().filter(|r| r.outcome.is_pass()).count()
77    }
78
79    /// Total number of entries.
80    pub fn total(&self) -> usize {
81        self.results.len()
82    }
83
84    /// One human-readable line per entry, suitable for CLI output.
85    pub fn summary_lines(&self) -> Vec<String> {
86        self.results
87            .iter()
88            .map(|r| {
89                let id = case_id(r.problem, r.case);
90                match &r.outcome {
91                    Outcome::Pass => format!("[PASS] {}/{}", id, r.path.as_str()),
92                    Outcome::InvalidSubgraph { returned } => {
93                        format!("[FAIL] {}/{}  invalid subgraph: returned {:?}", id, r.path.as_str(), returned)
94                    }
95                    Outcome::RpcError(msg) => {
96                        format!("[FAIL] {}/{}  rpc error: {}", id, r.path.as_str(), msg)
97                    }
98                }
99            })
100            .collect()
101    }
102}
103
104fn classify(hypergraph: &DecodingHypergraph, syndrome: &BitVector, response: &ParityFactor) -> Outcome {
105    if is_parity_factor(hypergraph, response, syndrome) {
106        Outcome::Pass
107    } else {
108        Outcome::InvalidSubgraph {
109            returned: response.subgraph.clone(),
110        }
111    }
112}
113
114async fn run_one_problem(decoder: &DynDecoder, problem: &StandardTestProblem, out: &mut Vec<CaseResult>) {
115    for case in &problem.cases {
116        out.push(run_decode_path(decoder, problem, case).await);
117    }
118
119    let load_outcome = decoder.load_hypergraph(problem.hypergraph.clone()).await;
120    let hid = match load_outcome {
121        Ok(response) => Some(response.hid),
122        Err(status) => {
123            let msg = format!("load_hypergraph failed: {status}");
124            for case in &problem.cases {
125                out.push(CaseResult {
126                    problem: problem.name,
127                    case: case.name,
128                    path: Path::DecodeLoaded,
129                    outcome: Outcome::RpcError(msg.clone()),
130                });
131            }
132            None
133        }
134    };
135
136    if let Some(hid) = hid {
137        for case in &problem.cases {
138            out.push(run_decode_loaded_path(decoder, problem, case, hid).await);
139        }
140    }
141
142    // Best-effort reset between problems. Failures are recorded as a synthetic
143    // entry so callers see the issue but do not crash the rest of the suite.
144    if let Err(status) = decoder
145        .reset(blackbox_decoder::ResetRequest {
146            reset_hypergraphs: true,
147            ..Default::default()
148        })
149        .await
150    {
151        out.push(CaseResult {
152            problem: problem.name,
153            case: "reset",
154            path: Path::DecodeLoaded,
155            outcome: Outcome::RpcError(format!("reset failed: {status}")),
156        });
157    }
158}
159
160async fn run_decode_path(decoder: &DynDecoder, problem: &StandardTestProblem, case: &TestCase) -> CaseResult {
161    let problem_payload = blackbox_decoder::DecodingProblem {
162        hypergraph: Some(problem.hypergraph.clone()),
163        syndrome: Some(case.syndrome.clone()),
164        loss: None,
165        decoder_seed: None,
166    };
167    let outcome = match decoder.decode(problem_payload).await {
168        Ok(response) => classify(&problem.hypergraph, &case.syndrome, &response),
169        Err(status) => Outcome::RpcError(status.to_string()),
170    };
171    CaseResult {
172        problem: problem.name,
173        case: case.name,
174        path: Path::Decode,
175        outcome,
176    }
177}
178
179async fn run_decode_loaded_path(
180    decoder: &DynDecoder,
181    problem: &StandardTestProblem,
182    case: &TestCase,
183    hid: u64,
184) -> CaseResult {
185    let problem_payload = blackbox_decoder::LoadedDecodingProblem {
186        hid,
187        syndrome: Some(case.syndrome.clone()),
188        ..Default::default()
189    };
190    let outcome = match decoder.decode_loaded(problem_payload).await {
191        Ok(response) => classify(&problem.hypergraph, &case.syndrome, &response),
192        Err(status) => Outcome::RpcError(status.to_string()),
193    };
194    CaseResult {
195        problem: problem.name,
196        case: case.name,
197        path: Path::DecodeLoaded,
198        outcome,
199    }
200}
201
202/// Run the full standard suite against `client` and collect outcomes.
203pub async fn run_standard_suite(decoder: &DynDecoder) -> SuiteReport {
204    let mut results: Vec<CaseResult> = Vec::new();
205    for problem in standard_test_problems() {
206        run_one_problem(decoder, &problem, &mut results).await;
207    }
208    SuiteReport { results }
209}