use crate::config::EncodingConfig;
use crate::decoding::{DecodingConfig, DecodingError, DecodingPipeline};
use crate::encoding::{EncodingError, EncodingPipeline};
use crate::security::AuthenticatedSymbol;
use crate::trace::raptorq_journal::{
BlockSymbols, EpochManifest, JournalFrame, ObjectParamsRecord, serialize_epoch,
};
#[cfg(not(target_arch = "wasm32"))]
use crate::trace::raptorq_journal::{
JOURNAL_FLAG_CHECKPOINT_BOUNDARY, latest_complete_epoch, scan_frames,
};
use crate::types::resource::{PoolConfig, SymbolPool};
use crate::types::{ObjectId, ObjectParams, Symbol, SymbolId, SymbolKind};
use std::collections::BTreeMap;
pub fn encode_checkpoint_blocks(
epoch: u64,
data: &[u8],
config: EncodingConfig,
repair_count: usize,
) -> Result<Vec<BlockSymbols>, EncodingError> {
let symbol_size = u32::from(config.symbol_size);
let mut pipeline = EncodingPipeline::new(config, SymbolPool::new(PoolConfig::default()));
let object_id = ObjectId::new(epoch, 0);
let mut by_block: BTreeMap<u8, (Vec<(u32, Vec<u8>)>, u32)> = BTreeMap::new();
for result in pipeline.encode_with_repair(object_id, data, repair_count) {
let symbol = result?;
let id = symbol.id();
let is_source = symbol.kind().is_source();
let entry = by_block.entry(id.sbn()).or_default();
entry.0.push((id.esi(), symbol.symbol().data().to_vec()));
if is_source {
entry.1 += 1;
}
}
Ok(by_block
.into_iter()
.map(|(sbn, (symbols, source_symbol_count))| BlockSymbols {
source_block_number: u32::from(sbn),
source_symbol_count,
symbol_size,
symbols,
})
.collect())
}
pub fn encode_and_serialize_epoch(
epoch: u64,
data: &[u8],
config: EncodingConfig,
repair_count: usize,
stripe_count: usize,
flags: u16,
) -> Result<Option<(Vec<Vec<u8>>, EpochManifest)>, EncodingError> {
let blocks = encode_checkpoint_blocks(epoch, data, config, repair_count)?;
Ok(serialize_epoch(epoch, stripe_count, flags, &blocks))
}
#[must_use]
pub fn stripe_file_name(epoch: u64, index: usize) -> String {
format!("epoch-{epoch}-stripe-{index}.rqj")
}
#[cfg(not(target_arch = "wasm32"))]
pub async fn write_epoch_stripes(
dir: &std::path::Path,
epoch: u64,
stripes: &[Vec<u8>],
) -> std::io::Result<Vec<std::path::PathBuf>> {
crate::fs::create_dir_all(dir).await?;
let mut paths = Vec::with_capacity(stripes.len());
for (index, bytes) in stripes.iter().enumerate() {
let path = dir.join(stripe_file_name(epoch, index));
crate::fs::write_atomic(&path, bytes).await?;
paths.push(path);
}
Ok(paths)
}
#[cfg(not(target_arch = "wasm32"))]
pub async fn read_epoch_stripes(
dir: &std::path::Path,
epoch: u64,
stripe_count: usize,
) -> std::io::Result<Vec<u8>> {
let mut out = Vec::new();
for index in 0..stripe_count {
let path = dir.join(stripe_file_name(epoch, index));
match crate::fs::read(&path).await {
Ok(bytes) => out.extend_from_slice(&bytes),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => return Err(error),
}
}
Ok(out)
}
#[must_use]
pub fn manifest_file_name(epoch: u64) -> String {
format!("epoch-{epoch}-manifest.rqm")
}
#[cfg(not(target_arch = "wasm32"))]
pub async fn write_epoch_manifest(
dir: &std::path::Path,
manifest: EpochManifest,
) -> std::io::Result<std::path::PathBuf> {
crate::fs::create_dir_all(dir).await?;
let path = dir.join(manifest_file_name(manifest.epoch));
crate::fs::write_atomic(&path, &manifest.encode()).await?;
Ok(path)
}
#[cfg(not(target_arch = "wasm32"))]
pub async fn read_epoch_manifest(
dir: &std::path::Path,
epoch: u64,
) -> std::io::Result<Option<EpochManifest>> {
let path = dir.join(manifest_file_name(epoch));
match crate::fs::read(&path).await {
Ok(bytes) => EpochManifest::decode(&bytes)
.map(Some)
.map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error)),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(error) => Err(error),
}
}
#[must_use]
pub fn params_file_name(epoch: u64) -> String {
format!("epoch-{epoch}-params.rqp")
}
#[cfg(not(target_arch = "wasm32"))]
pub async fn write_epoch_params(
dir: &std::path::Path,
record: ObjectParamsRecord,
) -> std::io::Result<std::path::PathBuf> {
crate::fs::create_dir_all(dir).await?;
let path = dir.join(params_file_name(record.epoch));
crate::fs::write_atomic(&path, &record.encode()).await?;
Ok(path)
}
#[cfg(not(target_arch = "wasm32"))]
pub async fn read_epoch_params(
dir: &std::path::Path,
epoch: u64,
) -> std::io::Result<Option<ObjectParamsRecord>> {
let path = dir.join(params_file_name(epoch));
match crate::fs::read(&path).await {
Ok(bytes) => ObjectParamsRecord::decode(&bytes)
.map(Some)
.map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error)),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(error) => Err(error),
}
}
pub fn decode_epoch_frames(
record: ObjectParamsRecord,
frames: &[JournalFrame],
) -> Result<Vec<u8>, DurableJournalError> {
if record.object_size == 0 {
return Ok(Vec::new());
}
let object_id = ObjectId::new(record.epoch, 0);
let (source_blocks, symbols_per_block) = record.block_layout();
let mut config = DecodingConfig::without_auth();
config.symbol_size = record.symbol_size;
config.max_block_size = record.max_block_size as usize;
let mut decoder = DecodingPipeline::new(config);
decoder.set_object_params(ObjectParams::new(
object_id,
record.object_size,
record.symbol_size,
source_blocks,
symbols_per_block,
))?;
for frame in frames {
if frame.header.epoch != record.epoch {
continue;
}
let sbn = u8::try_from(frame.header.source_block_number).map_err(|_| {
DurableJournalError::Decoding(DecodingError::InconsistentMetadata {
sbn: 0,
details: format!(
"source block number {} exceeds RaptorQ SBN range",
frame.header.source_block_number
),
})
})?;
let kind = if frame.header.encoding_symbol_id < frame.header.source_symbol_count {
SymbolKind::Source
} else {
SymbolKind::Repair
};
let symbol = Symbol::new(
SymbolId::new(object_id, sbn, frame.header.encoding_symbol_id),
frame.payload.clone(),
kind,
);
decoder.feed(AuthenticatedSymbol::new_unauthenticated(symbol))?;
}
if !decoder.is_complete() {
return Err(DurableJournalError::Incomplete);
}
decoder.into_data().map_err(DurableJournalError::Decoding)
}
#[derive(Debug)]
pub enum DurableJournalError {
Encoding(EncodingError),
Decoding(DecodingError),
Io(std::io::Error),
NoStripes,
MissingParams,
Incomplete,
}
impl core::fmt::Display for DurableJournalError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
match self {
Self::Encoding(error) => write!(f, "durable journal encode error: {error}"),
Self::Decoding(error) => write!(f, "durable journal decode error: {error}"),
Self::Io(error) => write!(f, "durable journal I/O error: {error}"),
Self::NoStripes => f.write_str("durable journal configured with zero stripes"),
Self::MissingParams => {
f.write_str("durable journal object-params record missing; cannot decode epoch")
}
Self::Incomplete => f.write_str(
"durable journal epoch incomplete; fewer than K' symbols survived to decode",
),
}
}
}
impl std::error::Error for DurableJournalError {}
impl From<EncodingError> for DurableJournalError {
fn from(error: EncodingError) -> Self {
Self::Encoding(error)
}
}
impl From<DecodingError> for DurableJournalError {
fn from(error: DecodingError) -> Self {
Self::Decoding(error)
}
}
impl From<std::io::Error> for DurableJournalError {
fn from(error: std::io::Error) -> Self {
Self::Io(error)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RecoveryProof {
pub epoch: u64,
pub source_block_count: u16,
pub recovered_len: usize,
pub surviving_frames: usize,
}
#[cfg(not(target_arch = "wasm32"))]
#[derive(Debug, Clone)]
pub struct DurableTraceJournalConfig {
pub directory: std::path::PathBuf,
pub encoding: EncodingConfig,
pub repair_count: usize,
pub stripe_count: usize,
}
#[cfg(not(target_arch = "wasm32"))]
#[derive(Debug, Clone)]
pub struct DurableTraceJournal {
config: DurableTraceJournalConfig,
}
#[cfg(not(target_arch = "wasm32"))]
impl DurableTraceJournal {
#[must_use]
pub fn new(config: DurableTraceJournalConfig) -> Self {
Self { config }
}
#[must_use]
pub fn config(&self) -> &DurableTraceJournalConfig {
&self.config
}
pub async fn record_epoch(
&self,
epoch: u64,
data: &[u8],
) -> Result<EpochManifest, DurableJournalError> {
let (stripes, manifest) = encode_and_serialize_epoch(
epoch,
data,
self.config.encoding.clone(),
self.config.repair_count,
self.config.stripe_count,
JOURNAL_FLAG_CHECKPOINT_BOUNDARY,
)?
.ok_or(DurableJournalError::NoStripes)?;
write_epoch_stripes(&self.config.directory, epoch, &stripes).await?;
write_epoch_manifest(&self.config.directory, manifest).await?;
let record = ObjectParamsRecord {
epoch,
object_size: data.len() as u64,
symbol_size: self.config.encoding.symbol_size,
max_block_size: u32::try_from(self.config.encoding.max_block_size).unwrap_or(u32::MAX),
};
write_epoch_params(&self.config.directory, record).await?;
Ok(manifest)
}
pub async fn recover_epoch(&self, epoch: u64) -> Result<Vec<u8>, DurableJournalError> {
let record = read_epoch_params(&self.config.directory, epoch)
.await?
.ok_or(DurableJournalError::MissingParams)?;
let survivors =
read_epoch_stripes(&self.config.directory, epoch, self.config.stripe_count).await?;
let (frames, _) = scan_frames(&survivors);
decode_epoch_frames(record, &frames)
}
pub async fn recover_epoch_with_proof(
&self,
epoch: u64,
) -> Result<(Vec<u8>, RecoveryProof), DurableJournalError> {
let record = read_epoch_params(&self.config.directory, epoch)
.await?
.ok_or(DurableJournalError::MissingParams)?;
let (source_block_count, _) = record.block_layout();
let survivors =
read_epoch_stripes(&self.config.directory, epoch, self.config.stripe_count).await?;
let (frames, _) = scan_frames(&survivors);
let surviving_frames = frames.iter().filter(|f| f.header.epoch == epoch).count();
let data = decode_epoch_frames(record, &frames)?;
let proof = RecoveryProof {
epoch,
source_block_count,
recovered_len: data.len(),
surviving_frames,
};
Ok((data, proof))
}
pub async fn recover_latest(&self) -> Result<Option<(u64, Vec<u8>)>, DurableJournalError> {
let Some(epoch) = self.latest_recoverable_epoch().await? else {
return Ok(None);
};
let bytes = self.recover_epoch(epoch).await?;
Ok(Some((epoch, bytes)))
}
pub async fn epoch_recoverable(&self, epoch: u64) -> std::io::Result<bool> {
let Some(manifest) = read_epoch_manifest(&self.config.directory, epoch).await? else {
return Ok(false);
};
let survivors =
read_epoch_stripes(&self.config.directory, epoch, self.config.stripe_count).await?;
let (frames, _) = scan_frames(&survivors);
Ok(latest_complete_epoch(&frames, &[manifest]) == Some(epoch))
}
pub async fn recorded_epochs(&self) -> std::io::Result<Vec<u64>> {
discover_epochs(&self.config.directory).await
}
pub async fn latest_recoverable_epoch(&self) -> std::io::Result<Option<u64>> {
let mut epochs = discover_epochs(&self.config.directory).await?;
epochs.sort_unstable_by(|left, right| right.cmp(left)); for epoch in epochs {
if self.epoch_recoverable(epoch).await? {
return Ok(Some(epoch));
}
}
Ok(None)
}
}
#[cfg(not(target_arch = "wasm32"))]
fn parse_manifest_epoch(file_name: &str) -> Option<u64> {
file_name
.strip_prefix("epoch-")?
.strip_suffix("-manifest.rqm")?
.parse::<u64>()
.ok()
}
#[cfg(not(target_arch = "wasm32"))]
pub async fn discover_epochs(dir: &std::path::Path) -> std::io::Result<Vec<u64>> {
let mut entries = match crate::fs::read_dir(dir).await {
Ok(entries) => entries,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
Err(error) => return Err(error),
};
let mut epochs = Vec::new();
while let Some(entry) = entries.next_entry().await? {
let name = entry.file_name();
if let Some(epoch) = parse_manifest_epoch(&name.to_string_lossy()) {
epochs.push(epoch);
}
}
epochs.sort_unstable();
Ok(epochs)
}