saorsa_fec/
lib.rs

1// Copyright 2024 Saorsa Labs
2// SPDX-License-Identifier: AGPL-3.0-or-later
3
4//! # Saorsa FEC - High-Performance Erasure Coding
5//!
6//! This crate provides exceptional performance systematic Reed-Solomon erasure coding
7//! with integrated encryption, storage, and content addressing.
8//!
9//! ## Performance (v0.2.1)
10//! - **1MB files**: 1,193 MB/s (2.4x specification target)
11//! - **10MB files**: 7,545 MB/s (15x specification target)  
12//! - **50MB files**: 5,366 MB/s (10.7x specification target)
13//!
14//! ## Features
15//! - **High-Performance**: reed-solomon-simd with SIMD acceleration (AVX2, AVX, SSE4.1, NEON)
16//! - **Systematic Encoding**: Original data preserved in first k shares
17//! - **Authenticated Encryption**: AES-256-GCM with deterministic nonces
18//! - **Content Addressing**: Blake3-based deduplication
19//! - **Storage Pipeline**: High-level API with pluggable backends
20//! - **Cross-Platform**: Pure Rust with no C dependencies
21
22use std::fmt;
23use thiserror::Error;
24
25pub mod backends;
26pub mod chunk_registry;
27pub mod config;
28pub mod crypto;
29pub mod fec;
30pub mod gc;
31pub mod gf256;
32pub mod ida;
33pub mod metadata;
34pub mod pipeline;
35pub mod quantum_crypto;
36pub mod storage;
37pub mod traits;
38pub mod types;
39pub mod version;
40
41pub use ida::{IDAConfig, IDADescriptor, ShareMetadata};
42pub use traits::{Fec, FecBackend};
43
44// v0.3 API exports
45pub use config::{Config, EncryptionMode};
46pub use pipeline::{Meta, PipelineStats, StoragePipeline};
47pub use quantum_crypto::{QuantumCryptoEngine, QuantumEncryptionMetadata};
48pub use storage::{
49    ChunkMeta, Cid, FileMetadata, GcReport, LocalStorage, MemoryStorage, MultiStorage,
50    MultiStorageStrategy, NetworkStorage, NodeEndpoint, Shard, ShardHeader, StorageBackend,
51    StorageStats,
52};
53
54/// Errors that can occur during FEC operations
55#[derive(Debug, Error)]
56pub enum FecError {
57    #[error("Invalid parameters: k={k}, n={n}")]
58    InvalidParameters { k: usize, n: usize },
59
60    #[error("Insufficient shares for reconstruction: have {have}, need {need}")]
61    InsufficientShares { have: usize, need: usize },
62
63    #[error("Share index out of bounds: {index} >= {max}")]
64    InvalidShareIndex { index: usize, max: usize },
65
66    #[error("Data size mismatch: expected {expected}, got {actual}")]
67    SizeMismatch { expected: usize, actual: usize },
68
69    #[error("Matrix is not invertible")]
70    SingularMatrix,
71
72    #[error("Backend error: {0}")]
73    Backend(String),
74
75    #[error("IO error: {0}")]
76    Io(#[from] std::io::Error),
77}
78
79pub type Result<T> = std::result::Result<T, FecError>;
80
81/// FEC parameters for encoding/decoding
82#[derive(Debug, Clone, Copy, PartialEq, Eq)]
83pub struct FecParams {
84    /// Number of data shares (k)
85    pub data_shares: u16,
86    /// Number of parity shares (n - k)
87    pub parity_shares: u16,
88    /// Size of each symbol in bytes
89    pub symbol_size: u32,
90}
91
92impl FecParams {
93    /// Create new FEC parameters
94    pub fn new(data_shares: u16, parity_shares: u16) -> Result<Self> {
95        if data_shares == 0 || parity_shares == 0 {
96            return Err(FecError::InvalidParameters {
97                k: data_shares as usize,
98                n: (data_shares + parity_shares) as usize,
99            });
100        }
101
102        // GF(256) limits us to 255 total shares
103        if data_shares as u32 + parity_shares as u32 > 255 {
104            return Err(FecError::InvalidParameters {
105                k: data_shares as usize,
106                n: (data_shares + parity_shares) as usize,
107            });
108        }
109
110        Ok(Self {
111            data_shares,
112            parity_shares,
113            symbol_size: 64 * 1024, // 64KB default
114        })
115    }
116
117    /// Get total number of shares (n)
118    pub fn total_shares(&self) -> u16 {
119        self.data_shares + self.parity_shares
120    }
121
122    /// Calculate parameters based on content size
123    pub fn from_content_size(size: usize) -> Self {
124        match size {
125            // These parameters are hardcoded and guaranteed to be valid
126            // k=8, n=10 (8+2=10), total=10 < 255 ✓
127            // k=16, n=20 (16+4=20), total=20 < 255 ✓
128            // k=20, n=25 (20+5=25), total=25 < 255 ✓
129            0..=1_000_000 => Self {
130                data_shares: 8,
131                parity_shares: 2,
132                symbol_size: 64 * 1024, // 64KB default
133            },
134            1_000_001..=10_000_000 => Self {
135                data_shares: 16,
136                parity_shares: 4,
137                symbol_size: 64 * 1024, // 64KB default
138            },
139            _ => Self {
140                data_shares: 20,
141                parity_shares: 5,
142                symbol_size: 64 * 1024, // 64KB default
143            },
144        }
145    }
146}
147
148impl fmt::Display for FecParams {
149    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
150        write!(
151            f,
152            "FEC({}/{}, {}KB symbols)",
153            self.data_shares,
154            self.total_shares(),
155            self.symbol_size / 1024
156        )
157    }
158}
159
160/// Main FEC encoder/decoder
161#[derive(Debug)]
162pub struct FecCodec {
163    params: FecParams,
164    #[allow(dead_code)]
165    backend: Box<dyn FecBackend>,
166}
167
168impl FecCodec {
169    /// Create a new FEC codec with the given parameters
170    pub fn new(params: FecParams) -> Result<Self> {
171        let backend = backends::create_backend()?;
172        Ok(Self { params, backend })
173    }
174
175    /// Create with specific backend
176    pub fn with_backend(params: FecParams, backend: Box<dyn FecBackend>) -> Self {
177        Self { params, backend }
178    }
179
180    /// Encode data into shares
181    pub fn encode(&self, data: &[u8]) -> Result<Vec<Vec<u8>>> {
182        let k = self.params.data_shares as usize;
183        let m = self.params.parity_shares as usize;
184
185        // Split data into k blocks
186        let block_size = data.len().div_ceil(k);
187        let mut data_blocks = vec![vec![0u8; block_size]; k];
188
189        for (i, chunk) in data.chunks(block_size).enumerate() {
190            if i < k {
191                data_blocks[i][..chunk.len()].copy_from_slice(chunk);
192            }
193        }
194
195        let data_refs: Vec<&[u8]> = data_blocks.iter().map(|v| v.as_slice()).collect();
196
197        // Generate parity blocks
198        let mut parity_blocks = vec![vec![]; m];
199        self.backend
200            .encode_blocks(&data_refs, &mut parity_blocks, self.params)?;
201
202        // Combine data and parity blocks
203        let mut shares = data_blocks;
204        shares.extend(parity_blocks);
205
206        Ok(shares)
207    }
208
209    /// Decode from available shares
210    pub fn decode(&self, shares: &[Option<Vec<u8>>]) -> Result<Vec<u8>> {
211        let k = self.params.data_shares as usize;
212
213        // Clone shares for decoding
214        let mut work_shares = shares.to_vec();
215
216        // Decode
217        self.backend.decode_blocks(&mut work_shares, self.params)?;
218
219        // Reconstruct original data from first k shares
220        let mut data = Vec::new();
221        for maybe_block in work_shares.iter().take(k) {
222            if let Some(block) = maybe_block {
223                data.extend_from_slice(block);
224            } else {
225                return Err(FecError::InsufficientShares { have: 0, need: k });
226            }
227        }
228
229        Ok(data)
230    }
231}
232
233#[cfg(test)]
234mod tests {
235    use super::*;
236
237    #[test]
238    fn test_fec_params_validation() {
239        assert!(FecParams::new(0, 10).is_err());
240        assert!(FecParams::new(10, 0).is_err());
241        assert!(FecParams::new(200, 100).is_err()); // > 255 total
242        assert!(FecParams::new(10, 5).is_ok());
243    }
244
245    #[test]
246    fn test_content_size_params() {
247        let small = FecParams::from_content_size(500_000);
248        assert_eq!(small.data_shares, 8);
249        assert_eq!(small.parity_shares, 2);
250
251        let medium = FecParams::from_content_size(5_000_000);
252        assert_eq!(medium.data_shares, 16);
253        assert_eq!(medium.parity_shares, 4);
254
255        let large = FecParams::from_content_size(50_000_000);
256        assert_eq!(large.data_shares, 20);
257        assert_eq!(large.parity_shares, 5);
258    }
259}
saorsa_fec/lib.rs

saorsa_fec/
lib.rs
