parmesan-par2 0.6.0

Fast, standalone PAR2 (Reed-Solomon) creation, verification and repair tool
Documentation
use super::{check_cancellation, CreateCancellation, CreateEvent};
use crate::encoder::{FileHashes, RecoveryEncoder, RecoverySlice};
use crate::ops::{
    ingest_files_ex, ingest_files_with_progress, IngestHashes, InputFile, MemoryLayout, SliceWindow,
};
use crate::packet::SliceChecksum;
use crate::{EncoderLayout, SimdPath};
use anyhow::Result;
use std::sync::Arc;

pub(super) struct PassRequest<'a> {
    pub input_files: &'a [InputFile],
    pub slice_size: usize,
    pub total_slices: usize,
    pub first_exponent: u32,
    pub recovery_blocks: usize,
    pub pass: usize,
    pub input_bytes: u64,
    pub memory_plan: MemoryLayout,
    pub memory_limit: usize,
    pub layout: EncoderLayout,
    pub simd: SimdPath,
    pub thread_pool: Arc<rayon::ThreadPool>,
}

pub(super) struct PassResult {
    pub recovery_slices: Vec<RecoverySlice>,
    pub slice_checksums: Vec<SliceChecksum>,
    pub hashes: Vec<FileHashes>,
}

pub(super) async fn encode_pass<F>(
    request: PassRequest<'_>,
    cancellation: Option<&CreateCancellation>,
    on_event: &mut F,
) -> Result<PassResult>
where
    F: FnMut(CreateEvent),
{
    if request.memory_plan.slice_chunk < request.slice_size {
        encode_chunked(request, cancellation, on_event).await
    } else {
        encode_full_slices(request, cancellation, on_event).await
    }
}

async fn encode_chunked<F>(
    request: PassRequest<'_>,
    cancellation: Option<&CreateCancellation>,
    on_event: &mut F,
) -> Result<PassResult>
where
    F: FnMut(CreateEvent),
{
    let mut ingest_hashes = IngestHashes::default();
    let mut accumulated: Vec<Vec<u8>> = vec![Vec::new(); request.recovery_blocks];
    let mut offset = 0usize;
    let mut bytes_read = 0u64;
    while offset < request.slice_size {
        check_cancellation(cancellation, on_event)?;
        let len = request
            .memory_plan
            .slice_chunk
            .min(request.slice_size - offset);
        let encoder = configured_encoder(&request, len);
        let worker = crate::worker::Par2Worker::spawn_with_thread_pool(
            encoder,
            false,
            crate::worker::DEFAULT_CHANNEL_DEPTH,
            Arc::clone(&request.thread_pool),
        );
        let hash_slot = (request.pass == 0 && offset == 0).then_some(&mut ingest_hashes);
        ingest_files_ex(
            request.input_files,
            &worker,
            request.slice_size,
            cancellation.map(CreateCancellation::as_atomic),
            |_| Ok(()),
            Some(SliceWindow { offset, len }),
            hash_slot,
        )
        .await?;
        bytes_read += request.input_bytes;
        on_event(CreateEvent::BytesRead {
            pass: request.pass,
            bytes_read,
        });
        let (part, _, _) = tokio::task::block_in_place(|| worker.finish());
        check_cancellation(cancellation, on_event)?;
        for (index, slice) in part.into_iter().enumerate() {
            accumulated[index].extend_from_slice(&slice.data);
        }
        offset += len;
    }

    let recovery_slices = accumulated
        .into_iter()
        .enumerate()
        .map(|(index, data)| RecoverySlice {
            exponent: request.first_exponent + index as u32,
            data,
        })
        .collect();
    Ok(PassResult {
        recovery_slices,
        slice_checksums: ingest_hashes.checksums,
        hashes: ingest_hashes.hashes,
    })
}

async fn encode_full_slices<F>(
    request: PassRequest<'_>,
    cancellation: Option<&CreateCancellation>,
    on_event: &mut F,
) -> Result<PassResult>
where
    F: FnMut(CreateEvent),
{
    let mut encoder = configured_encoder(&request, request.slice_size);
    if request.pass == 0 {
        encoder = encoder.with_checksums();
    }
    let worker = crate::worker::Par2Worker::spawn_with_thread_pool(
        encoder,
        request.pass == 0,
        crate::worker::DEFAULT_CHANNEL_DEPTH,
        Arc::clone(&request.thread_pool),
    );
    let mut bytes_read = 0u64;
    ingest_files_with_progress(
        request.input_files,
        &worker,
        request.slice_size,
        cancellation.map(CreateCancellation::as_atomic),
        |_| Ok(()),
        |read| {
            bytes_read += read as u64;
            on_event(CreateEvent::BytesRead {
                pass: request.pass,
                bytes_read,
            });
            Ok(())
        },
    )
    .await?;
    let (recovery_slices, slice_checksums, hashes) =
        tokio::task::block_in_place(|| worker.finish());
    check_cancellation(cancellation, on_event)?;
    Ok(PassResult {
        recovery_slices,
        slice_checksums,
        hashes,
    })
}

fn configured_encoder(request: &PassRequest<'_>, slice_size: usize) -> RecoveryEncoder {
    let encoder = match request.layout {
        EncoderLayout::Smart => RecoveryEncoder::new_smart(
            slice_size,
            request.total_slices,
            request.first_exponent,
            request.recovery_blocks,
        ),
        EncoderLayout::Normal => RecoveryEncoder::new(
            slice_size,
            request.total_slices,
            request.first_exponent,
            request.recovery_blocks,
        ),
        EncoderLayout::Affine => RecoveryEncoder::new_affine(
            slice_size,
            request.total_slices,
            request.first_exponent,
            request.recovery_blocks,
        ),
        EncoderLayout::Affine512 => RecoveryEncoder::new_affine512(
            slice_size,
            request.total_slices,
            request.first_exponent,
            request.recovery_blocks,
        ),
        EncoderLayout::Shuffle2x => RecoveryEncoder::new_shuffle2x(
            slice_size,
            request.total_slices,
            request.first_exponent,
            request.recovery_blocks,
        ),
    };
    encoder
        .with_simd_path(request.simd)
        .with_flush_limit((request.memory_limit / 4).clamp(256 * 1024 * 1024, 1024 * 1024 * 1024))
}