Skip to main content

persistant_relational/
conformance.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2//! Contract tests for the relational kind: what [`RelationalBackend`] promises, decided against a real
3//! database. Each leaf produces a matrix row, **on** only when the controls held.
4//!
5//! - `rel.values`: integers, text, bytes and floats written as parameters come back as written, and a
6//!   `NULL` comes back as `Value::Null`.
7//! - `rel.transaction`: a committed write is visible, a rolled-back one is not, and a transaction
8//!   dropped without `commit` rolls back.
9//! - `rel.error.unique`: a duplicate key is reported as [`RelError::UniqueViolation`], not as an
10//!   opaque backend error.
11//! - `rel.foreign_key`: a reference to a missing row is rejected as [`RelError::ForeignKeyViolation`].
12//! - `rel.check_constraint`: a row violating a `CHECK` is rejected as [`RelError::CheckViolation`].
13//! - `rel.isolation.no_dirty_read`: a write inside an open transaction is invisible to another
14//!   connection until it commits, and visible after (read committed or stronger).
15//!
16//! **Controls.** There is no reference relational engine to use as a positive control, so the positive
17//! control is weaker than in the kv and blob suites: it is that the leaf's schema could be created on
18//! the target at all (a failure to create it is "missing", not "wrong"). The **known-bad** controls are
19//! real: a [`Defect`] wrapper that makes the same target misbehave in one way (no transactions, lossy
20//! bytes, unclassified errors), and, for the two constraint leaves, the same test run against a schema
21//! created *without* the constraint, which must be caught accepting the bad row.
22//!
23//! Statements are written per dialect (this is a probe of the dialect itself, not of translation).
24//! Booleans are left out of `rel.values`: how a driver returns one (`BOOL`, `TINYINT(1)`, `0/1`) is a
25//! documented difference, not a contract.
26
27use crate::{Dialect, RelError, RelationalBackend, RelationalTx, Row, Value};
28use persistant_core::{CapabilityId, Config, Controls, Reason, Row as MatrixRow, Verdict};
29use std::collections::BTreeMap;
30
31/// A leaf this module can decide.
32#[derive(Debug, Clone, Copy, PartialEq, Eq)]
33#[non_exhaustive]
34pub enum Leaf {
35    /// `rel.values`
36    Values,
37    /// `rel.transaction`
38    Transaction,
39    /// `rel.error.unique`
40    UniqueClassified,
41    /// `rel.foreign_key`
42    ForeignKey,
43    /// `rel.check_constraint`
44    CheckConstraint,
45    /// `rel.isolation.no_dirty_read`
46    IsolationNoDirtyRead,
47}
48
49impl Leaf {
50    /// Every leaf with a test here.
51    pub const ALL: [Leaf; 6] = [
52        Leaf::Values,
53        Leaf::Transaction,
54        Leaf::UniqueClassified,
55        Leaf::ForeignKey,
56        Leaf::CheckConstraint,
57        Leaf::IsolationNoDirtyRead,
58    ];
59
60    /// The capability id.
61    pub fn id(self) -> &'static str {
62        match self {
63            Leaf::Values => "rel.values",
64            Leaf::Transaction => "rel.transaction",
65            Leaf::UniqueClassified => "rel.error.unique",
66            Leaf::ForeignKey => "rel.foreign_key",
67            Leaf::CheckConstraint => "rel.check_constraint",
68            Leaf::IsolationNoDirtyRead => "rel.isolation.no_dirty_read",
69        }
70    }
71
72    fn defects(self) -> &'static [Defect] {
73        match self {
74            Leaf::Values => &[Defect::LossyBytes],
75            Leaf::Transaction | Leaf::IsolationNoDirtyRead => &[Defect::NoTransactions],
76            Leaf::UniqueClassified => &[Defect::UnclassifiedErrors],
77            Leaf::ForeignKey | Leaf::CheckConstraint => &[],
78        }
79    }
80
81    /// Whether the leaf has a "schema without the constraint" known-bad run.
82    fn has_unconstrained_control(self) -> bool {
83        matches!(self, Leaf::ForeignKey | Leaf::CheckConstraint)
84    }
85}
86
87/// One way for a wrapped backend to be wrong, for the known-bad controls.
88#[derive(Debug, Clone, Copy, PartialEq, Eq)]
89#[non_exhaustive]
90pub enum Defect {
91    /// `begin` returns a transaction whose statements run at once and whose `rollback` and drop do
92    /// nothing.
93    NoTransactions,
94    /// A byte parameter is cut to its first byte.
95    LossyBytes,
96    /// Every error comes back as `RelError::Backend`.
97    UnclassifiedErrors,
98}
99
100/// A backend with one [`Defect`] injected. The wrapped backend does the work.
101#[derive(Debug)]
102pub struct Defective<'a, R> {
103    inner: &'a R,
104    defect: Defect,
105}
106
107impl<'a, R: RelationalBackend> Defective<'a, R> {
108    /// Wrap `inner` so it is wrong in exactly the way `defect` says.
109    pub fn new(inner: &'a R, defect: Defect) -> Self {
110        Defective { inner, defect }
111    }
112}
113
114/// The transaction of a [`Defective`] backend.
115pub enum DefectiveTx<'a, R: RelationalBackend + 'a> {
116    /// The real transaction (the defect is elsewhere).
117    Real(R::Tx<'a>, Defect),
118    /// No transaction at all: statements run on the backend directly.
119    Direct(&'a R),
120}
121
122impl<R: RelationalBackend> RelationalBackend for Defective<'_, R> {
123    type Tx<'a>
124        = DefectiveTx<'a, R>
125    where
126        Self: 'a;
127
128    fn backend(&self) -> &str {
129        self.inner.backend()
130    }
131
132    fn version(&self) -> &str {
133        self.inner.version()
134    }
135
136    fn dialect(&self) -> Dialect {
137        self.inner.dialect()
138    }
139
140    async fn execute(&self, sql: &str, params: &[Value]) -> Result<u64, RelError> {
141        self.inner
142            .execute(sql, &lossy(self.defect, params))
143            .await
144            .map_err(|e| unclassify(self.defect, e))
145    }
146
147    async fn query(&self, sql: &str, params: &[Value]) -> Result<Vec<Row>, RelError> {
148        self.inner
149            .query(sql, &lossy(self.defect, params))
150            .await
151            .map_err(|e| unclassify(self.defect, e))
152    }
153
154    async fn begin(&self) -> Result<DefectiveTx<'_, R>, RelError> {
155        if self.defect == Defect::NoTransactions {
156            return Ok(DefectiveTx::Direct(self.inner));
157        }
158        Ok(DefectiveTx::Real(
159            self.inner
160                .begin()
161                .await
162                .map_err(|e| unclassify(self.defect, e))?,
163            self.defect,
164        ))
165    }
166}
167
168fn lossy(defect: Defect, params: &[Value]) -> Vec<Value> {
169    params
170        .iter()
171        .map(|p| match (defect, p) {
172            (Defect::LossyBytes, Value::Bytes(b)) => {
173                Value::Bytes(b.iter().take(1).copied().collect())
174            }
175            _ => p.clone(),
176        })
177        .collect()
178}
179
180fn unclassify(defect: Defect, e: RelError) -> RelError {
181    if defect == Defect::UnclassifiedErrors {
182        RelError::Backend(e.to_string())
183    } else {
184        e
185    }
186}
187
188impl<R: RelationalBackend> RelationalTx for DefectiveTx<'_, R> {
189    async fn execute(&mut self, sql: &str, params: &[Value]) -> Result<u64, RelError> {
190        match self {
191            DefectiveTx::Real(tx, d) => tx
192                .execute(sql, &lossy(*d, params))
193                .await
194                .map_err(|e| unclassify(*d, e)),
195            DefectiveTx::Direct(db) => db.execute(sql, params).await,
196        }
197    }
198
199    async fn query(&mut self, sql: &str, params: &[Value]) -> Result<Vec<Row>, RelError> {
200        match self {
201            DefectiveTx::Real(tx, d) => tx
202                .query(sql, &lossy(*d, params))
203                .await
204                .map_err(|e| unclassify(*d, e)),
205            DefectiveTx::Direct(db) => db.query(sql, params).await,
206        }
207    }
208
209    async fn commit(self) -> Result<(), RelError> {
210        match self {
211            DefectiveTx::Real(tx, d) => tx.commit().await.map_err(|e| unclassify(d, e)),
212            DefectiveTx::Direct(_) => Ok(()),
213        }
214    }
215
216    async fn rollback(self) -> Result<(), RelError> {
217        match self {
218            DefectiveTx::Real(tx, d) => tx.rollback().await.map_err(|e| unclassify(d, e)),
219            DefectiveTx::Direct(_) => Ok(()),
220        }
221    }
222}
223
224/// Why a leaf's test did not pass.
225#[derive(Debug, Clone, PartialEq, Eq)]
226pub enum Failure {
227    /// The backend returned something other than the contract says.
228    Wrong(String),
229    /// The backend could not do something the test needs (including creating the leaf's schema).
230    Backend(RelError),
231}
232
233impl From<RelError> for Failure {
234    fn from(e: RelError) -> Self {
235        Failure::Backend(e)
236    }
237}
238
239fn wrong<T>(msg: impl Into<String>) -> Result<T, Failure> {
240    Err(Failure::Wrong(msg.into()))
241}
242
243// ---- per-dialect statement text -------------------------------------------------------------
244
245#[derive(Clone, Copy)]
246enum Ty {
247    Int,
248    Text,
249    Bytes,
250    Float,
251}
252
253struct Sql(Dialect);
254
255impl Sql {
256    /// The placeholder for parameter `n` (1-based).
257    fn p(&self, n: usize) -> String {
258        match self.0 {
259            Dialect::Postgres => format!("${n}"),
260            Dialect::Sqlite => format!("?{n}"),
261            _ => "?".to_string(),
262        }
263    }
264
265    fn ty(&self, t: Ty) -> &'static str {
266        match (&self.0, t) {
267            (_, Ty::Int) => "BIGINT",
268            (Dialect::Postgres, Ty::Text) => "TEXT",
269            (Dialect::Sqlite, Ty::Text) => "TEXT",
270            (_, Ty::Text) => "VARCHAR(255)",
271            (Dialect::Postgres, Ty::Bytes) => "BYTEA",
272            (Dialect::Sqlite, Ty::Bytes) => "BLOB",
273            (_, Ty::Bytes) => "LONGBLOB",
274            (Dialect::Postgres, Ty::Float) => "DOUBLE PRECISION",
275            (_, Ty::Float) => "DOUBLE",
276        }
277    }
278
279    /// Table options (InnoDB is explicit on MySQL so constraints and transactions exist).
280    fn tail(&self) -> &'static str {
281        match self.0 {
282            Dialect::MySql | Dialect::MariaDb => " ENGINE=InnoDB",
283            _ => "",
284        }
285    }
286
287    /// An integer primary key column definition.
288    fn pk(&self, name: &str) -> String {
289        format!("{name} BIGINT NOT NULL PRIMARY KEY")
290    }
291}
292
293fn supported(d: &Dialect) -> Result<(), Failure> {
294    match d {
295        Dialect::Postgres | Dialect::Sqlite | Dialect::MySql | Dialect::MariaDb => Ok(()),
296        other => Err(Failure::Backend(RelError::Unsupported(format!(
297            "the relational contract tests have no statements for {other}"
298        )))),
299    }
300}
301
302async fn drop_tables<R: RelationalBackend>(b: &R, names: &[&str]) -> Result<(), RelError> {
303    for n in names {
304        b.execute(&format!("DROP TABLE IF EXISTS {n}"), &[]).await?;
305    }
306    Ok(())
307}
308
309/// Run one leaf's test against `b`. `constrained = false` creates the constraint leaves' schema without
310/// the constraint (the known-bad control); it makes no difference to the other leaves. Writes only to
311/// tables named `persistant_rel_*`, which it drops first and last.
312pub async fn run_leaf<R: RelationalBackend>(
313    leaf: Leaf,
314    b: &R,
315    constrained: bool,
316) -> Result<(), Failure> {
317    let d = b.dialect();
318    supported(&d)?;
319    let s = Sql(d);
320    match leaf {
321        Leaf::Values => values(b, &s).await,
322        Leaf::Transaction => transaction(b, &s).await,
323        Leaf::UniqueClassified => unique(b, &s).await,
324        Leaf::ForeignKey => foreign_key(b, &s, constrained).await,
325        Leaf::CheckConstraint => check(b, &s, constrained).await,
326        Leaf::IsolationNoDirtyRead => no_dirty_read(b, &s).await,
327    }
328}
329
330async fn values<R: RelationalBackend>(b: &R, s: &Sql) -> Result<(), Failure> {
331    const T: &str = "persistant_rel_values";
332    drop_tables(b, &[T]).await?;
333    b.execute(
334        &format!(
335            "CREATE TABLE {T} ({}, i {}, t {}, b {}, f {}){}",
336            s.pk("id"),
337            s.ty(Ty::Int),
338            s.ty(Ty::Text),
339            s.ty(Ty::Bytes),
340            s.ty(Ty::Float),
341            s.tail()
342        ),
343        &[],
344    )
345    .await?;
346    let cases: Vec<(i64, Value, Value, Value, Value)> = vec![
347        (
348            1,
349            Value::Int(i64::MIN),
350            Value::Text("ünï ✓ \u{1F980}".to_string()),
351            Value::Bytes(vec![0x00, 0xFF, 0x80, 0x00, 0x7F]),
352            Value::Float(1.5),
353        ),
354        (
355            2,
356            Value::Int(i64::MAX),
357            Value::Text(String::new()),
358            Value::Bytes(vec![]),
359            Value::Float(-0.25),
360        ),
361        (3, Value::Null, Value::Null, Value::Null, Value::Null),
362    ];
363    for (id, i, t, bytes, f) in &cases {
364        b.execute(
365            &format!(
366                "INSERT INTO {T} (id, i, t, b, f) VALUES ({}, {}, {}, {}, {})",
367                s.p(1),
368                s.p(2),
369                s.p(3),
370                s.p(4),
371                s.p(5)
372            ),
373            &[
374                Value::Int(*id),
375                i.clone(),
376                t.clone(),
377                bytes.clone(),
378                f.clone(),
379            ],
380        )
381        .await?;
382    }
383    let rows = b
384        .query(&format!("SELECT id, i, t, b, f FROM {T} ORDER BY id"), &[])
385        .await?;
386    drop_tables(b, &[T]).await?;
387    if rows.len() != cases.len() {
388        return wrong(format!("wrote {} rows, read {}", cases.len(), rows.len()));
389    }
390    for (row, (id, i, t, bytes, f)) in rows.iter().zip(&cases) {
391        let want = [
392            Value::Int(*id),
393            i.clone(),
394            t.clone(),
395            bytes.clone(),
396            f.clone(),
397        ];
398        // An empty byte string and an empty text may come back as NULL on no engine here; they must not.
399        if row.values != want {
400            return wrong(format!("row {id}: wrote {want:?}, read {:?}", row.values));
401        }
402    }
403    Ok(())
404}
405
406async fn count<R: RelationalBackend>(b: &R, table: &str) -> Result<i64, Failure> {
407    let r = b
408        .query(&format!("SELECT COUNT(*) FROM {table}"), &[])
409        .await?;
410    match r.first().and_then(|r| r.int(0)) {
411        Some(n) => Ok(n),
412        None => wrong("COUNT(*) did not come back as an integer"),
413    }
414}
415
416async fn transaction<R: RelationalBackend>(b: &R, s: &Sql) -> Result<(), Failure> {
417    const T: &str = "persistant_rel_tx";
418    drop_tables(b, &[T]).await?;
419    b.execute(
420        &format!(
421            "CREATE TABLE {T} ({}, n BIGINT NOT NULL){}",
422            s.pk("id"),
423            s.tail()
424        ),
425        &[],
426    )
427    .await?;
428    let ins = format!("INSERT INTO {T} (id, n) VALUES ({}, {})", s.p(1), s.p(2));
429    let result = async {
430        // Commit: visible.
431        let mut tx = b.begin().await?;
432        tx.execute(&ins, &[Value::Int(1), Value::Int(10)]).await?;
433        let own = tx.query(&format!("SELECT COUNT(*) FROM {T}"), &[]).await?;
434        if own.first().and_then(|r| r.int(0)) != Some(1) {
435            return wrong("a transaction cannot see its own write");
436        }
437        tx.commit().await?;
438        if count(b, T).await? != 1 {
439            return wrong("a committed write is not visible");
440        }
441        // Rollback: not visible.
442        let mut tx = b.begin().await?;
443        tx.execute(&ins, &[Value::Int(2), Value::Int(20)]).await?;
444        tx.rollback().await?;
445        if count(b, T).await? != 1 {
446            return wrong("a rolled-back write is visible");
447        }
448        // Dropped without commit: rolled back.
449        {
450            let mut tx = b.begin().await?;
451            tx.execute(&ins, &[Value::Int(3), Value::Int(30)]).await?;
452        }
453        if count(b, T).await? != 1 {
454            return wrong("a transaction dropped without commit was not rolled back");
455        }
456        Ok(())
457    }
458    .await;
459    drop_tables(b, &[T]).await?;
460    result
461}
462
463async fn no_dirty_read<R: RelationalBackend>(b: &R, s: &Sql) -> Result<(), Failure> {
464    const T: &str = "persistant_rel_isolation";
465    drop_tables(b, &[T]).await?;
466    b.execute(
467        &format!(
468            "CREATE TABLE {T} ({}, n BIGINT NOT NULL){}",
469            s.pk("id"),
470            s.tail()
471        ),
472        &[],
473    )
474    .await?;
475    let ins = format!("INSERT INTO {T} (id, n) VALUES ({}, {})", s.p(1), s.p(2));
476    let result = async {
477        let mut tx = b.begin().await?;
478        tx.execute(&ins, &[Value::Int(1), Value::Int(1)]).await?;
479        // `b.query` runs on another connection, outside `tx`.
480        let outside = count(b, T).await?;
481        if outside != 0 {
482            let _ = tx.rollback().await;
483            return wrong(format!(
484                "another connection saw {outside} uncommitted row(s) (dirty read)"
485            ));
486        }
487        tx.commit().await?;
488        if count(b, T).await? != 1 {
489            return wrong("a committed write is not visible to another connection");
490        }
491        Ok(())
492    }
493    .await;
494    drop_tables(b, &[T]).await?;
495    result
496}
497
498async fn unique<R: RelationalBackend>(b: &R, s: &Sql) -> Result<(), Failure> {
499    const T: &str = "persistant_rel_unique";
500    drop_tables(b, &[T]).await?;
501    b.execute(
502        &format!(
503            "CREATE TABLE {T} ({}, n BIGINT NOT NULL){}",
504            s.pk("id"),
505            s.tail()
506        ),
507        &[],
508    )
509    .await?;
510    let ins = format!("INSERT INTO {T} (id, n) VALUES ({}, {})", s.p(1), s.p(2));
511    let result = async {
512        b.execute(&ins, &[Value::Int(1), Value::Int(1)]).await?;
513        match b.execute(&ins, &[Value::Int(1), Value::Int(2)]).await {
514            Err(RelError::UniqueViolation { .. }) => {}
515            other => return wrong(format!("a duplicate key came back as {other:?}")),
516        }
517        // Inside a transaction too.
518        let mut tx = b.begin().await?;
519        let dup = tx.execute(&ins, &[Value::Int(1), Value::Int(3)]).await;
520        let _ = tx.rollback().await;
521        match dup {
522            Err(RelError::UniqueViolation { .. }) => Ok(()),
523            other => wrong(format!(
524                "a duplicate key in a transaction came back as {other:?}"
525            )),
526        }
527    }
528    .await;
529    drop_tables(b, &[T]).await?;
530    result
531}
532
533async fn foreign_key<R: RelationalBackend>(
534    b: &R,
535    s: &Sql,
536    constrained: bool,
537) -> Result<(), Failure> {
538    const P: &str = "persistant_rel_fk_parent";
539    const C: &str = "persistant_rel_fk_child";
540    drop_tables(b, &[C, P]).await?;
541    b.execute(
542        &format!("CREATE TABLE {P} ({}){}", s.pk("id"), s.tail()),
543        &[],
544    )
545    .await?;
546    let refs = if constrained {
547        format!(", FOREIGN KEY (pid) REFERENCES {P}(id)")
548    } else {
549        String::new()
550    };
551    b.execute(
552        &format!(
553            "CREATE TABLE {C} ({}, pid BIGINT NOT NULL{refs}){}",
554            s.pk("id"),
555            s.tail()
556        ),
557        &[],
558    )
559    .await?;
560    let result = async {
561        b.execute(
562            &format!("INSERT INTO {P} (id) VALUES ({})", s.p(1)),
563            &[Value::Int(1)],
564        )
565        .await?;
566        let ins = format!("INSERT INTO {C} (id, pid) VALUES ({}, {})", s.p(1), s.p(2));
567        b.execute(&ins, &[Value::Int(1), Value::Int(1)]).await?;
568        match b.execute(&ins, &[Value::Int(2), Value::Int(99)]).await {
569            Err(RelError::ForeignKeyViolation { .. }) => {}
570            Ok(_) => return wrong("a reference to a missing row was accepted"),
571            other => {
572                return wrong(format!(
573                    "a reference to a missing row came back as {other:?}"
574                ));
575            }
576        }
577        match b
578            .execute(
579                &format!("DELETE FROM {P} WHERE id = {}", s.p(1)),
580                &[Value::Int(1)],
581            )
582            .await
583        {
584            Err(RelError::ForeignKeyViolation { .. }) => Ok(()),
585            Ok(_) => wrong("a referenced row was deleted"),
586            other => wrong(format!("deleting a referenced row came back as {other:?}")),
587        }
588    }
589    .await;
590    drop_tables(b, &[C, P]).await?;
591    result
592}
593
594async fn check<R: RelationalBackend>(b: &R, s: &Sql, constrained: bool) -> Result<(), Failure> {
595    const T: &str = "persistant_rel_check";
596    drop_tables(b, &[T]).await?;
597    let ck = if constrained { " CHECK (n >= 0)" } else { "" };
598    b.execute(
599        &format!(
600            "CREATE TABLE {T} ({}, n BIGINT NOT NULL{ck}){}",
601            s.pk("id"),
602            s.tail()
603        ),
604        &[],
605    )
606    .await?;
607    let ins = format!("INSERT INTO {T} (id, n) VALUES ({}, {})", s.p(1), s.p(2));
608    let result = async {
609        b.execute(&ins, &[Value::Int(1), Value::Int(0)]).await?;
610        match b.execute(&ins, &[Value::Int(2), Value::Int(-1)]).await {
611            Err(RelError::CheckViolation { .. }) => Ok(()),
612            Ok(_) => wrong("a row violating a CHECK was accepted"),
613            other => wrong(format!("a row violating a CHECK came back as {other:?}")),
614        }
615    }
616    .await;
617    drop_tables(b, &[T]).await?;
618    result
619}
620
621// ---- certification --------------------------------------------------------------------------
622
623/// The outcome of certifying one leaf.
624#[derive(Debug, Clone, PartialEq, Eq)]
625pub struct Certified {
626    /// The leaf.
627    pub leaf: Leaf,
628    /// The row to record.
629    pub row: MatrixRow,
630}
631
632/// Run every leaf here against `target` and return one row per leaf.
633///
634/// A leaf is **on** only if the target passed *and* every known-bad control failed (see the module
635/// docs for the positive control). `config` is the facts a gate can read for this target; `metadata`
636/// is copied into every row with `test=rel::<leaf>` added.
637pub async fn certify<R: RelationalBackend>(
638    target: &R,
639    config: &Config,
640    metadata: &Config,
641) -> Vec<Certified> {
642    let mut out = Vec::new();
643    for leaf in Leaf::ALL {
644        let result = run_leaf(leaf, target, true).await;
645        let positive_passed = !matches!(
646            &result,
647            Err(Failure::Backend(
648                RelError::Unsupported(_) | RelError::Unavailable(_)
649            ))
650        );
651        let mut known_bad_failed = true;
652        for d in leaf.defects() {
653            let bad = Defective::new(target, *d);
654            if run_leaf(leaf, &bad, true).await.is_ok() {
655                known_bad_failed = false;
656            }
657        }
658        if leaf.has_unconstrained_control() && run_leaf(leaf, target, false).await.is_ok() {
659            known_bad_failed = false;
660        }
661        let controls = Controls {
662            positive_passed,
663            known_bad_failed,
664        };
665        let id = CapabilityId::new(leaf.id()).expect("leaf id is valid");
666        let md = metadata.clone().with("test", format!("rel::{}", leaf.id()));
667        let off = |reason: Reason, note: String| MatrixRow {
668            subject_kind: "capability".to_string(),
669            subject: id.clone(),
670            backend: target.backend().to_string(),
671            version: target.version().to_string(),
672            config: config.clone(),
673            enabled: false,
674            reason: Some(reason),
675            metadata: md.clone(),
676            note: note.replace(['\t', '\n'], " "),
677            extra: BTreeMap::new(),
678        };
679        let row = match (
680            &result,
681            controls.positive_passed && controls.known_bad_failed,
682        ) {
683            (Ok(()), true) => Verdict::Exact
684                .into_row(
685                    id.clone(),
686                    target.backend(),
687                    target.version(),
688                    config.clone(),
689                    md.clone(),
690                    controls,
691                )
692                .expect("both controls hold"),
693            (Ok(()), false) => off(
694                Reason::Untested,
695                format!("controls did not hold: {controls:?}"),
696            ),
697            (Err(Failure::Wrong(m)), _) => off(Reason::Wrong, m.clone()),
698            (Err(Failure::Backend(e)), _) => off(Reason::Missing, e.to_string()),
699        };
700        out.push(Certified { leaf, row });
701    }
702    out
703}