1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
//! This crate provides thread pool([`ErasureCoderPool`]) for managing executions of erasure coding.
//!
//! `ecpool` also provides [`ErasureCode`] trait defines erasure coding interface
//! and of which implemtations can be executed via [`ErasureCoderPool`].
//!
//! There are some built-in implementations of the trait:
//! - [`LibErasureCoder`]:
//!   - This implementation uses [`liberasurecode`] crate that is a wrapper for [openstack/liberasurecode] library.
//!   - It is highly optimized and stable but only available in Unix environments.
//! - [`ReplicaCoder`]:
//!   - This implementation simply replicates the input data.
//!   - It is provided for example and testing purposes only and not intended to use in production.
//!
//!
//! # Build Prerequisites
//!
//! It is required to install [openstack/liberasurecode] and its dependencies by executing
//! the following commands before building this crate:
//!
//! ```console
//! $ git clone https://github.com/frugalos/liberasurecode
//! $ cd liberasurecode && sudo ./install_deps.sh
//! ```
//!
//! # Examples
//!
//! Basic usage:
//! ```
//! # extern crate ecpool;
//! # extern crate fibers_global;
//! # extern crate trackable;
//! use ecpool::replica::ReplicaCoder;
//! use ecpool::{ErrorKind, ErasureCoderPool};
//! use std::num::NonZeroUsize;
//! use std::result::Result;
//! use trackable::error::{Failure, Failed};
//!
//! # fn main() -> Result<(), trackable::error::MainError> {
//! // Creates a pool
//! let data_fragments = NonZeroUsize::new(4).ok_or_else(|| Failure::from(Failed))?;
//! let parity_fragments = NonZeroUsize::new(2).ok_or_else(|| Failure::from(Failed))?;
//! let coder = ErasureCoderPool::new(ReplicaCoder::new(data_fragments, parity_fragments));
//!
//! // Encodes
//! let data = vec![0, 1, 2, 3];
//! let encoded = fibers_global::execute(coder.encode(data.clone()))?;
//!
//! // Decodes
//! assert_eq!(
//!     Some(&data),
//!     fibers_global::execute(coder.decode(encoded[0..].to_vec()))
//!         .as_ref()
//!         .ok()
//! );
//! assert_eq!(
//!     Some(&data),
//!     fibers_global::execute(coder.decode(encoded[1..].to_vec()))
//!         .as_ref()
//!         .ok()
//! );
//! assert_eq!(
//!     Some(&data),
//!     fibers_global::execute(coder.decode(encoded[2..].to_vec()))
//!         .as_ref()
//!         .ok()
//! );
//! assert_eq!(
//!     Err(ErrorKind::InvalidInput),
//!     fibers_global::execute(coder.decode(encoded[3..].to_vec())).map_err(|e| *e.kind())
//! );
//! # Ok(())
//! # }
//! ```
//!
//! [`ErasureCoderPool`]: ./struct.ErasureCoderPool.html
//! [`ErasureCode`]: ./trait.ErasureCode.html
//! [`liberasurecode`]: https://github.com/frugalos/liberasurecode
//! [openstack/liberasurecode]: https://github.com/openstack/liberasurecode
//! [`LibErasureCoder`]: ./liberasurecode/struct.LibErasureCoder.html
//! [`ReplicaCoder`]: ./replica/struct.ReplicaCoder.html
#![warn(missing_docs)]
extern crate fibers;
#[cfg(test)]
extern crate fibers_global;
extern crate fibers_tasque;
extern crate futures;
#[macro_use]
extern crate trackable;

#[cfg(unix)]
extern crate liberasurecode as libec;

use std::num::NonZeroUsize;

pub use error::{Error, ErrorKind};
pub use pool::ErasureCoderPool;

#[cfg(unix)]
pub mod liberasurecode;
pub mod replica;

mod error;
mod pool;

/// This crate specific [`Result`] type.
///
/// [`Result`]: https://doc.rust-lang.org/std/result/enum.Result.html
pub type Result<T> = std::result::Result<T, Error>;

/// A fragment.
pub type Fragment = [u8];

/// An owned fragment.
pub type FragmentBuf = Vec<u8>;

/// This trait allows for encoding and decoding data by using some erasure coding algorithm.
pub trait ErasureCode {
    /// Returns the number of data fragments that the instance uses when encoding/decoding data.
    fn data_fragments(&self) -> NonZeroUsize;

    /// Returns the number of parity fragments that the instance uses when encoding/decoding data.
    fn parity_fragments(&self) -> NonZeroUsize;

    /// The total number of data fragments and parity fragments of the instance.
    fn fragments(&self) -> NonZeroUsize {
        unsafe {
            NonZeroUsize::new_unchecked(self.data_fragments().get() + self.parity_fragments().get())
        }
    }

    /// Encodes the given data to fragments.
    ///
    /// The result vector contains `N` data fragments and `M` parity fragments
    /// (where `N = self.data_fragments()` and `M = self.parity_fragments()`).
    fn encode(&mut self, data: &[u8]) -> Result<Vec<FragmentBuf>>;

    /// Decodes the original data from the given fragments.
    ///
    /// Note whether the correctness of the result data has been validated depends on the implementations.
    fn decode(&mut self, fragments: &[&Fragment]) -> Result<Vec<u8>>;

    /// Reconstructs the fragment specified by the given index from other fragments.
    fn reconstruct(&mut self, index: usize, fragments: &[&Fragment]) -> Result<Vec<u8>> {
        track_assert!(
            index < self.fragments().get(),
            ErrorKind::Other,
            "Too large index: index={}, fragments={}",
            index,
            self.fragments()
        );
        let decoded = self.decode(fragments)?;
        let mut encoded = self.encode(&decoded)?;
        Ok(encoded.swap_remove(index))
    }
}

/// This trait allows for building instances of an implementaion of [`ErasureCode`] trait.
///
/// [`ErasureCode`]: ./trait.ErasureCode.html
pub trait BuildCoder: Clone + Send + 'static {
    /// The type of `ErasureCode` implementaion to be built.
    type Coder: ErasureCode;

    /// Builds an instance of the `ErasureCode` implementaion.
    fn build_coder(&self) -> Result<Self::Coder>;

    /// Returns the identifier that distinguishes the kind of instances to be built.
    ///
    /// If two coder instances use different parameters for encoding/decoding,
    /// the identifiers that associated to those must be different.
    fn coder_id(&self) -> String;
}