structured-zstd 0.0.54

use super::super::blocks::block::BlockHeader;
use super::super::blocks::block::BlockType;
use super::super::blocks::literals_section::LiteralsSection;
use super::super::blocks::literals_section::LiteralsSectionType;
use super::super::blocks::sequence_section::SequencesHeader;
use super::literals_section_decoder::{LiteralsView, decode_literals_zerocopy};
use super::sequence_section_decoder::decode_and_execute_sequences;
use crate::common::MAX_BLOCK_SIZE;
use crate::cpu_kernel::CpuKernelTag;
#[cfg(any(test, feature = "bench-internals"))]
use crate::cpu_kernel::detect_cpu_kernel;
use crate::decoding::errors::DecodeSequenceError;
use crate::decoding::errors::{
    BlockHeaderReadError, BlockSizeError, BlockTypeError, DecodeBlockContentError,
    DecompressBlockError,
};
use crate::decoding::scratch::Workspace;
use crate::io::Read;

pub struct BlockDecoder {
    header_buffer: [u8; 3],
    internal_state: DecoderState,
    /// Best kernel available on this CPU, resolved ONCE here rather than
    /// per block. Feature detection belongs before the first decode, not
    /// inside it: the two section decoders take this tag and `match` it
    /// straight to their per-tier monomorph, so the only thing left on the
    /// hot path is the dispatch branch itself.
    kernel: CpuKernelTag,
}

enum DecoderState {
    ReadyToDecodeNextHeader,
    ReadyToDecodeNextBody,
    #[allow(dead_code)]
    Failed, //TODO put "self.internal_state = DecoderState::Failed;" everywhere an unresolvable error occurs
}

/// The most a block of a frame with `window_size` may produce: the smaller of
/// the window and 128 KiB (RFC 8878 3.1.1.2.4), which upstream keeps per frame
/// as `blockSizeMax`. A single-segment frame's window is its content size, so
/// such a frame's blocks are bounded by the content as well.
#[inline]
pub(crate) fn block_maximum(window_size: usize) -> usize {
    window_size.min(MAX_BLOCK_SIZE as usize)
}

/// A Raw or RLE block states its output in its header, so it is held to the
/// frame's block maximum before anything is written, as upstream checks `rSize`
/// against `blockSizeMax` in `ZSTD_decompressContinue`. A compressed block's
/// output is only known as it decodes, and is checked there.
#[inline]
fn block_fits_the_maximum(
    header: &BlockHeader,
    window_size: usize,
) -> Result<(), DecodeBlockContentError> {
    let size = header.decompressed_size as usize;
    let maximum = block_maximum(window_size);
    if size > maximum {
        return Err(DecodeBlockContentError::DecompressBlockError(
            DecompressBlockError::ExpandsPastBlockMaximum { size, maximum },
        ));
    }
    Ok(())
}

/// A block with no sequences: its literals ARE its output. Out of line, so the
/// per-block body keeps the shape the sequence executor is laid out around.
///
/// A growable backend allocates rather than refusing, so its write cannot fail
/// and takes the infallible path; the compile-time const folds the other arm
/// away. A fixed-capacity backend reports a short target, where the infallible
/// write would assert.
///
/// No per-block ceiling is armed for this write, and none is needed. The
/// ceiling bounds SEQUENCE writes, which is why it is armed beside the
/// sequence reserve; the write here goes through `try_extend`, whose bound is
/// the caller's slice. Arming the previous block's ceiling over it would
/// REJECT valid frames: a small block leaves the ceiling near its own output,
/// and a following literal-only block of a whole block maximum would exceed it
/// while the slice still had room. The literals were already held to the block
/// maximum where their section was parsed, so this write cannot exceed it
/// either.
#[inline(never)]
fn write_literals_only<B: super::buffer_backend::BufferBackend>(
    buffer: &mut crate::decoding::decode_buffer::DecodeBuffer<B>,
    literals: &[u8],
) -> Result<(), DecompressBlockError> {
    if !B::FIXED_CAPACITY {
        buffer.push(literals);
        return Ok(());
    }
    buffer
        .try_push(literals)
        .map_err(|overflow| DecompressBlockError::LiteralsOutputOverflow {
            tail: overflow.tail,
            requested: overflow.requested,
            capacity: overflow.capacity,
        })
}

/// Create a new [BlockDecoder], detecting the CPU kernel. Detection belongs at
/// the decoder's entry, so the decode paths take [`with_kernel`] instead; this
/// is for callers that decode a block in isolation.
#[cfg(any(test, feature = "bench-internals"))]
pub fn new() -> BlockDecoder {
    with_kernel(detect_cpu_kernel())
}

/// Create a new [BlockDecoder] over a kernel the caller already resolved. A
/// decoder that builds one per call (a chunked decode does) detects once and
/// passes the tag here, rather than reading the detection cache every time.
pub(crate) fn with_kernel(kernel: CpuKernelTag) -> BlockDecoder {
    BlockDecoder {
        internal_state: DecoderState::ReadyToDecodeNextHeader,
        header_buffer: [0u8; 3],
        kernel,
    }
}

impl BlockDecoder {
    /// Decode the body of a single block described by `header` from `source` into `workspace`.
    ///
    /// Returns the number of bytes consumed from `source`.
    /// The decode buffer inside `workspace` may be reserved or grown during
    /// decoding. For some block types the decompressed size is known up front,
    /// but this is not guaranteed before any data is written.
    /// Slice-source fast path for `decode_block_content`. Consumes
    /// the right number of bytes from `*source` (advancing the slice)
    /// without going through the persistent `block_content_buffer`.
    /// Used by `FrameDecoder::decode_all` where `source` is
    /// already a `&[u8]` view into the user's input.
    ///
    /// Returns the number of bytes consumed from `*source`.
    pub fn decode_block_content_from_slice<W: Workspace>(
        &mut self,
        header: &BlockHeader,
        workspace: &mut W,
        dict: Option<&crate::decoding::dictionary::Dictionary>,
        source: &mut &[u8],
    ) -> Result<u64, DecodeBlockContentError> {
        use DecoderState as State;
        match self.internal_state {
            State::ReadyToDecodeNextBody => { /* ok */ }
            State::Failed => return Err(DecodeBlockContentError::DecoderStateIsFailed),
            State::ReadyToDecodeNextHeader => {
                return Err(DecodeBlockContentError::ExpectedHeaderOfPreviousBlock);
            }
        }

        let block_type = header.block_type;
        match block_type {
            BlockType::RLE => {
                // 1 byte from source via slice; no Read overhead.
                if source.is_empty() {
                    // ErrorKind::UnexpectedEof matches what the streaming path
                    // gets from Read::read_exact on truncated input — callers
                    // and tests can detect truncation by kind alone, identical
                    // across slice-source and Read-source decode entry points.
                    return Err(DecodeBlockContentError::ReadError {
                        step: block_type,
                        source: crate::io::Error::from(crate::io::ErrorKind::UnexpectedEof),
                    });
                }
                // Peek the fill byte without advancing `*source` yet —
                // `try_extend_and_fill` is fallible on fixed-capacity
                // backends, and on `Err` the caller exits early. If we
                // advanced first, the input cursor would diverge from
                // the bytes_read accounting by one byte on the error
                // path. Advance ONLY after the write succeeds, matching
                // the Raw arm's split_at-then-try_push-then-advance shape.
                let fill = source[0];
                let parts = workspace.split();
                block_fits_the_maximum(header, parts.buffer.window_size)?;
                parts
                    .buffer
                    .try_extend_and_fill(fill, header.decompressed_size as usize)
                    .map_err(|_| DecodeBlockContentError::BackendOverflow { step: block_type })?;
                *source = &source[1..];
                self.internal_state = State::ReadyToDecodeNextHeader;
                Ok(1)
            }
            BlockType::Raw => {
                // Raw payload IS the source bytes; push them
                // directly to the buffer. For UserSliceBackend this
                // is a single memcpy from input -> output, no
                // intermediate Vec.
                let n = header.decompressed_size as usize;
                if source.len() < n {
                    return Err(DecodeBlockContentError::ReadError {
                        step: block_type,
                        source: crate::io::Error::from(crate::io::ErrorKind::UnexpectedEof),
                    });
                }
                let (payload, tail) = source.split_at(n);
                // `try_push` returns `Err(BackendOverflow)` on
                // `UserSliceBackend` when the Raw payload would push
                // past the caller's output slice. Growable backends
                // grow on demand and always succeed.
                let parts = workspace.split();
                block_fits_the_maximum(header, parts.buffer.window_size)?;
                parts
                    .buffer
                    .try_push(payload)
                    .map_err(|_| DecodeBlockContentError::BackendOverflow { step: block_type })?;
                *source = tail;
                self.internal_state = State::ReadyToDecodeNextHeader;
                Ok(u64::from(header.decompressed_size))
            }
            BlockType::Reserved => {
                panic!("Reserved-type block should be rejected during header parsing");
            }
            BlockType::Compressed => {
                let n = header.content_size as usize;
                if source.len() < n {
                    return Err(DecodeBlockContentError::ReadError {
                        step: block_type,
                        source: crate::io::Error::from(crate::io::ErrorKind::UnexpectedEof),
                    });
                }
                let (payload, tail) = source.split_at(n);
                self.decompress_block_inplace(header, workspace, dict, payload)?;
                *source = tail;
                self.internal_state = State::ReadyToDecodeNextHeader;
                Ok(u64::from(header.content_size))
            }
        }
    }

    pub fn decode_block_content<W: Workspace>(
        &mut self,
        header: &BlockHeader,
        workspace: &mut W,
        dict: Option<&crate::decoding::dictionary::Dictionary>,
        mut source: impl Read,
    ) -> Result<u64, DecodeBlockContentError> {
        match self.internal_state {
            DecoderState::ReadyToDecodeNextBody => { /* Happy :) */ }
            DecoderState::Failed => return Err(DecodeBlockContentError::DecoderStateIsFailed),
            DecoderState::ReadyToDecodeNextHeader => {
                return Err(DecodeBlockContentError::ExpectedHeaderOfPreviousBlock);
            }
        }

        let block_type = header.block_type;
        match block_type {
            BlockType::RLE => {
                let mut buf = [0u8; 1];
                source.read_exact(&mut buf[..]).map_err(|err| {
                    DecodeBlockContentError::ReadError {
                        step: block_type,
                        source: err,
                    }
                })?;
                let parts = workspace.split();
                block_fits_the_maximum(header, parts.buffer.window_size)?;
                parts
                    .buffer
                    .extend_and_fill(buf[0], header.decompressed_size as usize);

                self.internal_state = DecoderState::ReadyToDecodeNextHeader;

                Ok(1)
            }
            BlockType::Raw => {
                // Pass `source` by value rather than `&mut source` — it isn't
                // used after this match arm anyway, so moving avoids the
                // borrow-by-reference indirection. (Both io shims provide a
                // blanket `Read for &mut T`, so `&mut source` would also
                // compile; the by-value form is just cleaner here.)
                let parts = workspace.split();
                block_fits_the_maximum(header, parts.buffer.window_size)?;
                parts
                    .buffer
                    .extend_from_reader(source, header.decompressed_size as usize)
                    .map_err(|err| DecodeBlockContentError::ReadError {
                        step: block_type,
                        source: err,
                    })?;

                self.internal_state = DecoderState::ReadyToDecodeNextHeader;
                Ok(u64::from(header.decompressed_size))
            }

            BlockType::Reserved => {
                panic!(
                    "How did you even get this. The decoder should error out if it detects a reserved-type block"
                );
            }

            BlockType::Compressed => {
                self.decompress_block(header, workspace, dict, source)?;

                self.internal_state = DecoderState::ReadyToDecodeNextHeader;
                Ok(u64::from(header.content_size))
            }
        }
    }

    fn decompress_block<W: Workspace>(
        &mut self,
        header: &BlockHeader,
        workspace: &mut W,
        dict: Option<&crate::decoding::dictionary::Dictionary>,
        mut source: impl Read,
    ) -> Result<(), DecompressBlockError> {
        // Streaming-path entry: copy `content_size` bytes from the
        // `Read` source into `block_content_buffer`, then dispatch
        // to the in-place body via per-field borrows. The direct-
        // decode path (`decode_all`) skips this copy by passing
        // a borrowed slice of the input straight to
        // `decompress_block_inplace_with_parts`.
        let parts = workspace.split();
        parts
            .block_content_buffer
            .resize(header.content_size as usize, 0);
        source.read_exact(parts.block_content_buffer.as_mut_slice())?;
        // Disjoint-fields borrow: `as_slice()` reborrows
        // block_content_buffer as `&[u8]`; the other WorkspaceRef
        // fields stay independently movable into the helper since
        // each is a distinct struct field. No `unsafe` needed —
        // Rust's borrow checker tracks per-field disjointness.
        let raw = parts.block_content_buffer.as_slice();
        self.decompress_block_inplace_with_parts(
            header,
            parts.huf,
            parts.fse,
            parts.buffer,
            parts.offset_hist,
            parts.literals_buffer,
            raw,
            dict,
        )
    }

    /// Compressed-block fast path that takes the block content as a
    /// borrowed slice (no per-block memcpy into
    /// `block_content_buffer`).
    ///
    /// Called by:
    /// - `decompress_block` (streaming path) after it has copied the
    ///   compressed bytes from the source `Read` into the persistent
    ///   `block_content_buffer`.
    /// - `decode_block_content_from_slice` (direct-decode path) where
    ///   the source is a `&[u8]` already, so the slice IS the input —
    ///   saving the per-block memcpy + anonymous-page first-touch on
    ///   `block_content_buffer.resize(...)`. At L-1 fast on
    ///   `decodecorpus-z000033` this `block_content_buffer` traffic
    ///   was the largest single contributor to the
    ///   `__memmove_avx_unaligned_erms` + `exc_page_fault` flame
    ///   on the post-#244 baseline (~30% of decode time combined).
    pub(crate) fn decompress_block_inplace<W: Workspace>(
        &mut self,
        header: &BlockHeader,
        workspace: &mut W,
        dict: Option<&crate::decoding::dictionary::Dictionary>,
        raw: &[u8],
    ) -> Result<(), DecompressBlockError> {
        let parts = workspace.split();
        // block_content_buffer is intentionally NOT passed — the
        // in-place body does not need it (raw already IS the block
        // content). Dropping it here keeps the helper signature
        // free of an unused-field reference.
        self.decompress_block_inplace_with_parts(
            header,
            parts.huf,
            parts.fse,
            parts.buffer,
            parts.offset_hist,
            parts.literals_buffer,
            raw,
            dict,
        )
    }

    /// Inner body of [`Self::decompress_block_inplace`] that takes
    /// the scratch fields as disjoint `&mut` references. The split
    /// happens at the call site (so the streaming path can keep its
    /// `block_content_buffer` borrow alive while passing `raw =
    /// block_content_buffer.as_slice()` here without aliasing the
    /// other scratch fields). All `unsafe` from the previous
    /// lifetime-widening trick is gone — disjoint-field borrows
    /// type-check naturally.
    #[allow(clippy::too_many_arguments)]
    fn decompress_block_inplace_with_parts<'d, B: super::buffer_backend::BufferBackend>(
        &mut self,
        header: &BlockHeader,
        huf: &mut crate::decoding::scratch::HuffmanScratch,
        fse: &'d mut crate::decoding::scratch::FSEScratch,
        buffer: &mut crate::decoding::decode_buffer::DecodeBuffer<B>,
        offset_hist: &mut [u32; 3],
        literals_buffer: &mut alloc::vec::Vec<u8>,
        raw: &[u8],
        dict: Option<&'d crate::decoding::dictionary::Dictionary>,
    ) -> Result<(), DecompressBlockError> {
        // A block produces at most its frame's block maximum, its literals and
        // its matches together: the smaller of the window and 128 KiB (RFC 8878
        // 3.1.1.2.4), as upstream derives it once per frame
        // (`zstd_decompress.c`: `blockSizeMax = MIN(windowSize,
        // ZSTD_BLOCKSIZE_MAX)`). Upstream bounds both halves by the same
        // `oend`: the literals up front (`litSize > blockSizeMax` is corruption
        // in `ZSTD_decodeLiteralsBlock`) and every write after. Sequence writes
        // stop at the per-block ceiling; the literals are checked here and the
        // whole block after it decodes, which catches literals left over after
        // the last sequence.
        let block_maximum = block_maximum(buffer.window_size);
        let len_before = buffer.len();
        let mut section = LiteralsSection::new();
        let bytes_in_literals_header = section.parse_from_header(raw)?;
        if section.regenerated_size as usize > block_maximum {
            return Err(DecompressBlockError::ExpandsPastBlockMaximum {
                size: section.regenerated_size as usize,
                maximum: block_maximum,
            });
        }
        let raw = &raw[bytes_in_literals_header as usize..];
        vprintln!(
            "Found {} literalssection with regenerated size: {}, and compressed size: {:?}",
            section.ls_type,
            section.regenerated_size,
            section.compressed_size
        );

        let upper_limit_for_literals = match section.compressed_size {
            Some(x) => x as usize,
            None => match section.ls_type {
                LiteralsSectionType::RLE => 1,
                LiteralsSectionType::Raw => section.regenerated_size as usize,
                _ => panic!("Bug in this library"),
            },
        };

        if raw.len() < upper_limit_for_literals {
            return Err(DecompressBlockError::MalformedSectionHeader {
                expected_len: upper_limit_for_literals,
                remaining_bytes: raw.len(),
            });
        }

        let raw_literals = &raw[..upper_limit_for_literals];
        vprintln!("Slice for literals: {}", raw_literals.len());

        literals_buffer.clear(); //all literals of the previous block must have been used in the sequence execution anyways. just be defensive here
        // Zero-copy literals view — for Raw sections this borrows
        // straight into `raw_literals` (no memcpy into the Vec).
        // For RLE / HUF it materialises into `literals_buffer`
        // and `data` is a slice over that buffer. Eliminates a major
        // memcpy contributor (Vec::extend_from_slice → spec_extend)
        // flagged at ~20% of decode time on the L-1 fast c_stream
        // flamegraph.
        let LiteralsView {
            data: literals_view,
            bytes_used: bytes_used_in_literals_section,
        } = decode_literals_zerocopy(
            &section,
            huf,
            dict,
            raw_literals,
            literals_buffer,
            self.kernel,
        )?;
        assert!(
            section.regenerated_size as usize == literals_view.len(),
            "Wrong number of literals: {}, Should have been: {}",
            literals_view.len(),
            section.regenerated_size
        );
        assert!(bytes_used_in_literals_section == upper_limit_for_literals as u32);

        let raw = &raw[upper_limit_for_literals..];
        vprintln!("Slice for sequences with headers: {}", raw.len());

        let mut seq_section = SequencesHeader::new();
        let bytes_in_sequence_header = seq_section.parse_from_header(raw)?;
        let raw = &raw[bytes_in_sequence_header as usize..];
        vprintln!(
            "Found sequencessection with sequences: {} and size: {}",
            seq_section.num_sequences,
            raw.len()
        );

        assert!(
            u32::from(bytes_in_literals_header)
                + bytes_used_in_literals_section
                + u32::from(bytes_in_sequence_header)
                + raw.len() as u32
                == header.content_size
        );
        vprintln!("Slice for sequences: {}", raw.len());

        if seq_section.num_sequences != 0 {
            // Fused decode + execute: avoids the Vec<Sequence> round-trip
            // and inlines the per-iter execute_one_sequence work next to
            // the FSE state advance. RLE-mode axes are handled in the same
            // fused loop via a degenerate single-state FSE table built by
            // `maybe_update_fse_tables`, so there is no separate two-pass
            // path. Pass field-level borrows from the WorkspaceRef so `raw`
            // (immutable view into block_content_buffer) can coexist
            // with the mutable borrows on the FSE / decode-buffer /
            // offset-hist fields.
            // Room for this block's output, and the ceiling that bounds it.
            // Exact growth: the reservation is a no-op while the frame-entry
            // window reservation covers it, and on the frame's last block (a
            // tail worth a fraction of a block) the amortized policy would
            // DOUBLE a window-sized buffer. The ceiling is what stops a
            // malformed block's sequences from growing the buffer past
            // `len + block_maximum` (a decompression-bomb OOM on the growable
            // RingBuffer); `DecodeBuffer::repeat` rejects the crossing match.
            // Both belong here, where the block maximum is already in hand: the
            // arithmetic then stays out of the per-kernel sequence monomorphs.
            // The reservation stops at what the frame has left to produce; the
            // ceiling stays the block maximum, since it decides whether a block
            // is malformed and a frame that outruns its declared size is caught
            // by the size check instead, which says so.
            buffer.reserve_for_block(block_maximum);
            buffer.set_block_output_ceiling(block_maximum);
            decode_and_execute_sequences(
                &seq_section,
                raw,
                fse,
                buffer,
                offset_hist,
                literals_view,
                dict,
                self.kernel,
            )?;
        } else {
            if !raw.is_empty() {
                return Err(DecompressBlockError::DecodeSequenceError(
                    DecodeSequenceError::ExtraBits {
                        bits_remaining: raw.len() as isize * 8,
                    },
                ));
            }
            // The literals ARE this block's output, and their length was held
            // to the block maximum above, so the post-block check below has
            // nothing left to say: hand the write over and return its result.
            // A tail call rather than `?` on purpose. Carrying the fallible
            // write's error path through this body cost 9.9% of cycles on a
            // 1 MiB level-19 stream while issuing 0.6% FEWER instructions: the
            // sequence executor it calls is laid out around this body.
            return write_literals_only(buffer, literals_view);
        }

        // Nothing drains the buffer inside a block, so the growth of its live
        // length is this block's output.
        let produced = buffer.len() - len_before;
        if produced > block_maximum {
            return Err(DecompressBlockError::ExpandsPastBlockMaximum {
                size: produced,
                maximum: block_maximum,
            });
        }
        Ok(())
    }

    /// Reads 3 bytes from the provided reader and returns
    /// the deserialized header and the number of bytes read.
    pub fn read_block_header(
        &mut self,
        mut r: impl Read,
    ) -> Result<(BlockHeader, u8), BlockHeaderReadError> {
        //match self.internal_state {
        //    DecoderState::ReadyToDecodeNextHeader => {/* Happy :) */},
        //    DecoderState::Failed => return Err(format!("Cant decode next block if failed along the way. Results will be nonsense")),
        //    DecoderState::ReadyToDecodeNextBody => return Err(format!("Cant decode next block header, while expecting to decode the body of the previous block. Results will be nonsense")),
        //}

        r.read_exact(&mut self.header_buffer[0..3])?;

        let btype = self.block_type()?;
        if let BlockType::Reserved = btype {
            return Err(BlockHeaderReadError::FoundReservedBlock);
        }

        let block_size = self.block_content_size()?;
        let decompressed_size = match btype {
            BlockType::Raw => block_size,
            BlockType::RLE => block_size,
            BlockType::Reserved => 0, //should be caught above, this is an error state
            BlockType::Compressed => 0, //unknown but will be smaller than 128kb (or window_size if that is smaller than 128kb)
        };
        let content_size = match btype {
            BlockType::Raw => block_size,
            BlockType::Compressed => block_size,
            BlockType::RLE => 1,
            BlockType::Reserved => 0, //should be caught above, this is an error state
        };

        let last_block = self.is_last();

        self.reset_buffer();
        self.internal_state = DecoderState::ReadyToDecodeNextBody;

        //just return 3. Blockheaders always take 3 bytes
        Ok((
            BlockHeader {
                last_block,
                block_type: btype,
                decompressed_size,
                content_size,
            },
            3,
        ))
    }

    fn reset_buffer(&mut self) {
        self.header_buffer[0] = 0;
        self.header_buffer[1] = 0;
        self.header_buffer[2] = 0;
    }

    fn is_last(&self) -> bool {
        self.header_buffer[0] & 0x1 == 1
    }

    fn block_type(&self) -> Result<BlockType, BlockTypeError> {
        let t = (self.header_buffer[0] >> 1) & 0x3;
        match t {
            0 => Ok(BlockType::Raw),
            1 => Ok(BlockType::RLE),
            2 => Ok(BlockType::Compressed),
            3 => Ok(BlockType::Reserved),
            other => Err(BlockTypeError::InvalidBlocktypeNumber { num: other }),
        }
    }

    fn block_content_size(&self) -> Result<u32, BlockSizeError> {
        let val = self.block_content_size_unchecked();
        if val > MAX_BLOCK_SIZE {
            Err(BlockSizeError::BlockSizeTooLarge { size: val })
        } else {
            Ok(val)
        }
    }

    fn block_content_size_unchecked(&self) -> u32 {
        u32::from(self.header_buffer[0] >> 3) //push out type and last_block flags. Retain 5 bit
            | (u32::from(self.header_buffer[1]) << 5)
            | (u32::from(self.header_buffer[2]) << 13)
    }
}

#[cfg(test)]
mod tests;