rusty_erasure 0.4.0

Intel ISA-L's erasure coding, remade with Rust: matrix-flexible GF(2^8) Reed-Solomon encode, incremental update and first-class recovery, plus RAID-6 P+Q. Byte-identical to ISA-L, faster than its AVX2-GFNI assembly, validated parameters instead of undefined behaviour. No C, no NASM, no GPL. MIT OR Apache-2.0.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120

//! Full-grid conformance replay (env-gated; part of the benchmark campaign,
//! not the per-commit suite): every Cauchy config k=1..32 × p=1..8 (three
//! lengths incl. tails) plus every Vandermonde-safe config, generated by real
//! ISA-L on the rig (`tools/oracle/gen_fullgrid.c`), replayed through the
//! SHIPPING dispatched path (`coder()` — GFNI on this box), with recovery
//! spot-checks per case.
//!
//! Run: RUSTY_ERASURE_FULLGRID=<path> cargo test -p rusty_erasure --release \
//!        --test fullgrid -- --ignored

use rusty_erasure::{Matrix, coder};

struct Cursor {
    buf: Vec<u8>,
    pos: usize,
}

impl Cursor {
    fn take(&mut self, n: usize) -> &[u8] {
        let s = &self.buf[self.pos..self.pos + n];
        self.pos += n;
        s
    }
    fn u8(&mut self) -> u8 {
        self.take(1)[0]
    }
    fn u16(&mut self) -> u16 {
        u16::from_le_bytes(self.take(2).try_into().expect("2 bytes"))
    }
    fn u32(&mut self) -> u32 {
        u32::from_le_bytes(self.take(4).try_into().expect("4 bytes"))
    }
}

#[test]
#[ignore = "env-gated: set RUSTY_ERASURE_FULLGRID to the generated vector file"]
fn full_grid_matches_isal_through_the_shipping_path() {
    let path = std::env::var("RUSTY_ERASURE_FULLGRID").expect("RUSTY_ERASURE_FULLGRID not set");
    let buf = std::fs::read(&path).expect("vector file readable");
    let mut c = Cursor { buf, pos: 0 };
    assert_eq!(c.take(4), b"REV1", "magic");
    let count = c.u32();
    assert!(count > 0);
    let mut kernel_name = "";

    for case in 0..count {
        let kind = c.u8();
        let k = c.u16() as usize;
        let p = c.u16() as usize;
        let len = c.u32() as usize;
        let data_bytes = c.take(k * len).to_vec();
        let gftbls_expect = c.take(p * k * 32).to_vec();
        let parity_expect = c.take(p * len).to_vec();
        let tag = format!("case {case}: kind={kind} k={k} p={p} len={len}");

        let matrix = match kind {
            0 => Matrix::reed_solomon(k, p),
            _ => Matrix::cauchy(k, p),
        }
        .unwrap_or_else(|e| panic!("{tag}: matrix: {e}"));
        let cdr = coder(matrix).unwrap_or_else(|e| panic!("{tag}: coder: {e}"));
        kernel_name = cdr.kernels().name;

        // Table layout is only comparable for nibble-format sets; the GFNI
        // set stores affine tables, so gate tables only when formats match.
        if cdr.kernels().table_bytes == 32 {
            assert_eq!(cdr.gftbls(), gftbls_expect, "{tag}: gftbls");
        }

        let data: Vec<&[u8]> = (0..k)
            .map(|j| &data_bytes[j * len..(j + 1) * len])
            .collect();
        let mut parity = vec![vec![0u8; len]; p];
        {
            let mut refs: Vec<&mut [u8]> = parity.iter_mut().map(|b| b.as_mut_slice()).collect();
            cdr.encode(&data, &mut refs)
                .unwrap_or_else(|e| panic!("{tag}: encode: {e}"));
        }
        let flat: Vec<u8> = parity.iter().flat_map(|b| b.iter().copied()).collect();
        assert_eq!(flat, parity_expect, "{tag}: encode output");

        // Recovery spot-checks: worst case (drop min(p,k) data shards) and a
        // spread pattern.
        let n = k + p;
        for missing in [
            (0..p.min(k)).collect::<Vec<usize>>(),
            (0..p).map(|i| (i * n) / p).collect::<Vec<usize>>(),
        ] {
            let mut missing = missing;
            missing.sort_unstable();
            missing.dedup();
            if missing.is_empty() {
                continue;
            }
            let shards: Vec<Option<&[u8]>> = (0..n)
                .map(|i| {
                    if missing.contains(&i) {
                        None
                    } else if i < k {
                        Some(data[i])
                    } else {
                        Some(parity[i - k].as_slice())
                    }
                })
                .collect();
            let mut out = vec![vec![0u8; len]; missing.len()];
            {
                let mut orefs: Vec<&mut [u8]> = out.iter_mut().map(|b| b.as_mut_slice()).collect();
                cdr.recover(&shards, &missing, &mut orefs)
                    .unwrap_or_else(|e| panic!("{tag}: recover {missing:?}: {e}"));
            }
            for (&x, got) in missing.iter().zip(&out) {
                let expect: &[u8] = if x < k { data[x] } else { &parity[x - k] };
                assert_eq!(got.as_slice(), expect, "{tag}: recovered shard {x}");
            }
        }
    }
    assert_eq!(c.pos, c.buf.len(), "trailing bytes");
    eprintln!("full grid: {count} cases conformant through kernel set {kernel_name}");
}