structured-zstd 0.2.5

//! Optimal-parse machinery for the binary-tree strategies (btopt / btultra /
//! btultra2): the `build_optimal_plan` DP + its per-CPU-tier kernels, the
//! candidate-collection + price-set body macros, and the `btlazy2` cost
//! helpers. Split out of `hc/generator.rs` (no behaviour change) as a second
//! `impl HcMatchGenerator` block over the same matcher, so the HC chain matcher
//! and the optimal parser live in separate files.

// The DP body macros reference many opt-parser types / cost-model constants
// UNqualified inside their `macro_rules!` bodies; rustc's `unused_imports` lint
// does not count macro-body references, so it false-positives on every such
// import (gating or removing any one breaks the lib build — the expansions
// need them). This module is macro-dense enough that per-import suppression is
// noise, so the lint is disabled module-wide here.
#![allow(unused_imports)]

use super::generator::HcMatchGenerator;
use crate::encoding::Sequence;
use crate::encoding::blocks::encode_offset_with_history;
use crate::encoding::hc::HC_OPT_CANDIDATE_CAPACITY;
use crate::encoding::hc::priceset::{
    priceset_avx2, priceset_neon, priceset_scalar, priceset_simd128, priceset_sse2, priceset_sse41,
};
use alloc::vec::Vec;

// The DP body macros reference these opt-parser types and cost-model constants
// UNqualified inside their `macro_rules!` bodies. rustc's `unused_imports` lint
// does not count macro-body references, so it reports every one of these as
// "unused" even though each macro expansion requires them (gating or removing
// any one breaks the lib build). Suppress the false positive on the group.
#[cfg(test)]
use crate::encoding::opt::types::HcCandidateQuery;
#[allow(unused_imports)]
use crate::encoding::{
    bt::BtMatcher,
    cost_model::{
        HC_BITCOST_MULTIPLIER, HC_OPT_NODE_LEN, HC_OPT_NUM, HC_OPT_PRICE_ARENA_LEN,
        HC_OPT_PRICE_STRIDE, HC_PREDEF_THRESHOLD, HcOptState, HcOptimalCostProfile,
    },
    hc::HC_MIN_MATCH_LEN,
    levels::config::HcConfig,
    match_generator::{HC_OPT_MIN_MATCH_LEN, HC_SEARCH_DEPTH, HC_TARGET_LEN, HcBackend},
    match_table::storage::HC3_HASH_LOG,
    opt::ldm::{HcOptLdmState, HcRawSeqStore},
    opt::types::{
        HcBlockPass, HcOptimalNode, HcOptimalPlanBuffers, HcOptimalPlanState, HcOptimalSequence,
        MatchCandidate,
    },
};

/// One position's search, built for the block's strategy, key width and kernel
/// (`collect_optimal_candidates_initialized_<kernel>`), as the block's pass
/// calls it.
type OptimalCollectFn = unsafe fn(
    &mut HcMatchGenerator,
    usize,
    usize,
    usize,
    &[u32; 3],
    bool,
    &mut Vec<MatchCandidate>,
);

macro_rules! build_optimal_plan_impl_body {
    (
        $self:expr,
        $strategy_ty:ty,
        $current:ident,
        $current_abs_start:ident,
        $current_len:ident,
        $initial_state:ident,
        $stats:ident,
        $out:ident,
        $buffers:expr,
        $collect:ident,
        // The tier's price-set macro (`priceset::priceset_<tier>`).
        $priceset:ident,
        // The whole block and its statistics: the traceback records each
        // sequence it settles into them (upstream `ZSTD_updateStats`).
        $block:ident,
        $opt_state:ident,
        // False for the btultra2 seed pass, which keeps only the statistics.
        $keep_plan:expr,
        // The block the caller wraps this body in: every exit is a `break` out
        // of it with the segment's result, so the body runs inside the caller's
        // segment loop rather than as a function of its own.
        $seg:lifetime $(,)?
    ) => {{
        let current_abs_end = $current_abs_start + $current_len;
        let min_match_len = HC_OPT_MIN_MATCH_LEN;
        // `HC_OPT_NUM > 0` by const definition, so `HC_OPT_NUM - 1` is safe.
        let frontier_limit = $current_len.min(HC_OPT_NUM - 1);
        let initial_reps = $initial_state.reps;
        let initial_litlen = $initial_state.litlen;
        let ldm_block_offset = $initial_state.block_offset;
        // `sufficient_match_len` arrives already clamped for the pass: it is a
        // block constant (the profile's value against `target_len`), and this
        // body runs once per LITERAL on input the search finds nothing in, so
        // deriving it here was an out-of-line call per literal.
        let profile = $initial_state.profile;
        debug_assert_eq!(
            profile.sufficient_match_len,
            $self.hc.sufficient_match_len_for_pass(profile),
            "the caller must clamp sufficient_match_len for the pass",
        );
        // Const-fold from the strategy's associated `OPT_LEVEL`
        // (upstream zstd `optLevel`): BtOpt = 0, BtUltra / BtUltra2 = 2.
        // The two flags below are the only places the inner DP loop
        // used to consult `parse_mode`; lifting them into const
        // expressions drops one indirect read + one branch on every
        // candidate insertion and every traceback step.
        // `let` (not `const`) — nested `const` items inside a
        // generic fn cannot project through the outer fn's type
        // parameter, but a `let` binding from a const expression
        // does get folded by the optimiser per monomorphisation,
        // which is what we actually want here.
        debug_assert!(
            <$strategy_ty as crate::encoding::strategy::Strategy>::USE_BT,
            "build_optimal_plan_impl_body called on non-BT strategy"
        );
        let abort_on_worse_match: bool =
            <$strategy_ty as crate::encoding::strategy::Strategy>::OPT_LEVEL == 0;
        let opt_level: bool = <$strategy_ty as crate::encoding::strategy::Strategy>::OPT_LEVEL >= 2;
        // `frontier_limit + 2 <= HC_OPT_NODE_LEN` — bounded by const.
        let frontier_buffer_size = frontier_limit + 2;
        // The five DP scratch buffers are owned one level up
        // (`start_matching_optimal` / `run_btultra2_seed_pass` take them once per
        // block via `take_optimal_plan_buffers`, size them, and reuse them across
        // every per-segment call, then restore them once). Borrow the fields here
        // and reset the per-call ones. Hoisting the take/restore out of this
        // per-segment body removes five `mem::take`s plus five restores per call,
        // which on near-random input runs once per literal.
        let HcOptimalPlanBuffers {
            nodes,
            node_prices,
            candidates,
            store,
            price_arena,
            candidates_searched_at: searched_at,
            pass,
        } = &mut *$buffers;
        // The node arenas are indexed through base pointers resolved once, the
        // way upstream zstd indexes `opt[]`: nothing in this body resizes them,
        // and a raw base leaves no slice header to reload per access. Within
        // this call every price up to `last_pos + 1` is written before it is
        // read (seed, reset of the cells the frontier extends over, or a match
        // / literal update). A node is read only once its price is finite, and
        // each transition that makes a price finite writes the whole node, so
        // cells the frontier reaches but no transition does stay unwritten.
        debug_assert!(
            nodes.len() >= frontier_buffer_size && node_prices.len() >= frontier_buffer_size
        );
        let nodes: *mut HcOptimalNode = nodes.as_mut_ptr().cast();
        let node_prices: *mut u32 = node_prices.as_mut_ptr().cast();
        // The run this call re-enters on already searched this position and left
        // its answer in `candidates`; searching again would insert the position
        // into the binary tree a second time, so keep the buffer as it stands.
        let carried_candidates = *searched_at == Some(($current_abs_start, initial_litlen));
        *searched_at = None;
        if !carried_candidates {
            candidates.clear();
        }
        store.clear();
        // Price-cache slices + monotonic stamps feed ONLY the priced paths (the
        // matched-seed block and the forward DP loop), both of which run only
        // when the seed found a candidate. On a no-match seed (the per-literal
        // case that dominates near-random input, where the forward loop is
        // skipped) they are never touched, so their setup (the arena slice
        // creation and the three stamp bumps) is deferred into the
        // `!candidates.is_empty()` block below. Declared here so they stay in
        // scope for the forward DP loop; the empty slices / zero stamps are
        // never read on the no-match path.
        let mut ll_cache: &mut [[u32; 2]] = &mut [];
        let mut ml_cache: &mut [[u32; 2]] = &mut [];
        let mut ll_price_stamp = 0u32;
        let mut lit_price_stamp = 0u32;
        let mut ml_price_stamp = 0u32;
        let sufficient_len = profile.sufficient_match_len;
        // `node0_price` / `ll0_price` / `ll1_price` (and the `nodes[0]` base node)
        // feed ONLY the forward-DP and seed paths below, which run only when the
        // seed found a candidate. On a no-match seed — the per-literal case that
        // dominates incompressible / near-random input — they are never read, so
        // defer the three cached price lookups + base-node init until a candidate
        // is confirmed (mirrors upstream zstd doing the literal `ip++` without
        // touching the DP price state). Initialised to 0; assigned in the
        // `!candidates.is_empty()` block before any reader.
        let mut ll0_price = 0u32;
        let mut ll1_price = 0u32;
        // A position the search finds nothing at is one literal (upstream zstd
        // `ZSTD_compressBlock_opt_generic`: `if (!nbMatches) { ip++; continue; }`
        // — inside its own loop). We return it to the caller instead, and on
        // input the search finds nothing in that is EVERY position, so the
        // caller re-entered this body per literal and paid its stack frame each
        // time. Walk the run here and hand back the whole run.
        //
        // The number of searches is unchanged: the run stops at the first
        // position with candidates and those candidates are the ones the seed
        // below then uses. The bound is the caller's own re-entry condition
        // (it enters while more than `HASH_READ_SIZE` bytes remain), so the
        // cursor lands exactly where the per-literal returns would have left it.
        //
        // Skipped when the LDM producer is active: its state machine is rebuilt
        // per call from the segment's block offset, so advancing inside one call
        // is not the same thing. `HAS_LDM` is a const generic, so this whole
        // block folds away there.
        //
        // What the walk is worth, per frame on the i9 (10 KiB frames at level 19
        // with a 16 KiB dictionary, two prebuilt binaries and libzstd alternated
        // in one session, `perf stat -r 3`, three rounds, ranges
        // non-overlapping). Near-random input, the shape this exists for:
        // 2,380,845 -> 1,679,325 cycles (-29.5%) and 6,132,013 -> 4,731,553
        // retired instructions (-22.8%), which is 1.94x -> 1.37x of libzstd on
        // cycles and 2.04x -> 1.57x on instructions. Compressible input, where
        // the search finds matches and the run is short: 4,616,838 -> 4,390,463
        // cycles (-4.9%) and 10,291,065 -> 9,993,577 instructions (-2.9%). The
        // control arm is level 1, whose Fast backend never enters this parser:
        // its instruction count is bit-identical between the two binaries
        // (71,276), and its cycles differ by 3.4%, which bounds what code layout
        // alone can account for. Output is byte-identical on both fixtures.
        let mut skipped_literals = 0usize;
        let mut seed_candidates_ready = carried_candidates;
        if !HAS_LDM && !seed_candidates_ready {
            while $current_len - skipped_literals > 8 {
                candidates.clear();
                // SAFETY: as the seed below — the wrapper shares the `$collect`
                // kernel's target_feature umbrella, entered under the runtime
                // detector.
                unsafe {
                    $collect(
                        &mut *$self,
                        $current_abs_start + skipped_literals,
                        current_abs_end,
                        profile.sufficient_match_len,
                        &initial_reps,
                        initial_litlen + skipped_literals == 0,
                        &mut *candidates,
                    )
                };
                if !candidates.is_empty() {
                    if skipped_literals > 0 {
                        // Hand the answer to the call that re-enters here.
                        *searched_at = Some((
                            $current_abs_start + skipped_literals,
                            initial_litlen + skipped_literals,
                        ));
                    } else {
                        seed_candidates_ready = true;
                    }
                    break;
                }
                skipped_literals += 1;
            }
            if skipped_literals > 0 {
                // Literals only: no sequence was emitted and the repcodes are
                // untouched, exactly as the per-literal returns left them. The
                // price is discarded by the caller.
                break $seg (
                    0u32,
                    initial_reps,
                    initial_litlen + skipped_literals,
                    skipped_literals,
                );
            }
        }
        let mut pos = 1usize;
        let mut last_pos = 0usize;
        let mut forced_end: Option<usize> = None;
        let mut forced_end_state: Option<HcOptimalNode> = None;
        // Price companion of `forced_end_state` (price no longer lives in the
        // node struct; tracked alongside the forced-seed node).
        let mut forced_end_price: Option<u32> = None;
        let mut seed_forced_shortest_path = false;
        // LDM presence is a frame constant (decided before entering the DP), so
        // it is a const-generic monomorph here, not a hot-loop runtime check:
        // with HAS_LDM == false every LDM branch below const-folds away and the
        // opt_ldm workspace is never built.
        let mut opt_ldm = if HAS_LDM {
            HcOptLdmState {
                seq_store: HcRawSeqStore {
                    pos: 0,
                    pos_in_sequence: 0,
                    size: $self.backend.bt_mut().ldm_sequences.len(),
                },
                ..HcOptLdmState::default()
            }
        } else {
            HcOptLdmState::default()
        };
        if HAS_LDM {
            // `ldm_sequences` are emitted in BLOCK-relative coordinates,
            // but this optimal-parser pass runs over a SEGMENT of the
            // block starting at block-offset `$block_offset` and uses
            // segment-relative positions throughout. Fast-forward the raw
            // seq-store cursor past the bytes covered by earlier segments
            // so the (segment-relative) LDM windows below land at the
            // correct positions. Idempotent: `ldm_skip_raw_seq_store_bytes`
            // recomputes from `pos = 0`, so re-running it per segment is
            // safe. Without this, every segment after the first re-applied
            // the block's leading LDM windows at the wrong offset, emitting
            // matches that copy the wrong bytes (undecodable frame).
            if ldm_block_offset > 0 {
                $self
                    .backend
                    .bt_mut()
                    .ldm_skip_raw_seq_store_bytes(&mut opt_ldm.seq_store, ldm_block_offset);
            }
            $self
                .backend
                .bt_mut()
                .ldm_get_next_match_and_update_seq_store(&mut opt_ldm, 0, $current_len);
        }

        // Upstream zstd-like seed at rPos=0: initialize frontier with matches starting
        // at current position before entering the generic forward DP loop.
        if $current_len >= min_match_len {
            let seed_ldm = if HAS_LDM {
                $self.backend.bt_mut().ldm_process_match_candidate(
                    &mut opt_ldm,
                    0,
                    $current_len,
                    min_match_len,
                )
            } else {
                None
            };
            // The no-match run above already searched this exact position with
            // this exact query and left its candidates in the buffer, so the
            // seed reads them rather than repeating the search.
            if !seed_candidates_ready {
                candidates.clear();
                // SAFETY: wrapper is in the same target_feature umbrella as the
                // `$collect` kernel variant; the runtime kernel detector already
                // gated entry into the wrapper.
                unsafe {
                    $collect(
                        &mut *$self,
                        $current_abs_start,
                        current_abs_end,
                        profile.sufficient_match_len,
                        &initial_reps,
                        initial_litlen == 0,
                        &mut *candidates,
                    )
                };
                // The long-distance candidate joins after the search, as
                // upstream's `ZSTD_optLdm_processMatchCandidate` does.
                if let Some(ldm) = seed_ldm {
                    BtMatcher::push_ldm_candidate(&mut *candidates, ldm, min_match_len);
                }
            }
            if !candidates.is_empty() {
                // Deferred price-cache setup: the arena slices are two disjoint
                // STRIDE-wide regions of `price_arena` (LL, ML); the fixed STRIDE
                // keeps each code's cell at a constant offset so the monotonic
                // stamps stay valid across calls. price_arena is exactly
                // HC_OPT_PRICE_ARENA_LEN = 2 * HC_OPT_PRICE_STRIDE pairs (sized by
                // take_optimal_plan_buffers).
                // SAFETY: the two STRIDE regions are in bounds and disjoint; the
                // raw slices are confined to this priced path and never outlive
                // `price_arena` (owned by the caller for the whole call).
                let arena_base = price_arena.as_mut_ptr();
                ll_cache =
                    unsafe { core::slice::from_raw_parts_mut(arena_base, HC_OPT_PRICE_STRIDE) };
                ml_cache = unsafe {
                    core::slice::from_raw_parts_mut(
                        arena_base.add(HC_OPT_PRICE_STRIDE),
                        HC_OPT_PRICE_STRIDE,
                    )
                };
                $self.backend.bt_mut().opt_ll_price_stamp =
                    $self.backend.bt_mut().opt_ll_price_stamp.wrapping_add(1).max(1);
                ll_price_stamp = $self.backend.bt_mut().opt_ll_price_stamp;
                $self.backend.bt_mut().opt_lit_price_stamp =
                    $self.backend.bt_mut().opt_lit_price_stamp.wrapping_add(1).max(1);
                lit_price_stamp = $self.backend.bt_mut().opt_lit_price_stamp;
                $self.backend.bt_mut().opt_ml_price_stamp =
                    $self.backend.bt_mut().opt_ml_price_stamp.wrapping_add(1).max(1);
                ml_price_stamp = $self.backend.bt_mut().opt_ml_price_stamp;
                // Deferred base/seed prices: only reached on a matched seed (see
                // the declarations above). Assign before the forward DP / seed
                // paths below read them.
                let node0_price = BtMatcher::cached_lit_length_price::<ACCURATE_PRICE>(
                    profile,
                    $stats,
                    initial_litlen,
                    &mut ll_cache,
                    ll_price_stamp,
                );
                // SAFETY: cell 0 is in both arenas (`frontier_buffer_size >= 2`).
                unsafe {
                    node_prices.write(node0_price);
                    nodes.write(HcOptimalNode {
                        litlen: initial_litlen as u32,
                        reps: initial_reps,
                        ..HcOptimalNode::default()
                    });
                }
                ll0_price = BtMatcher::cached_lit_length_price::<ACCURATE_PRICE>(
                    profile,
                    $stats,
                    0,
                    &mut ll_cache,
                    ll_price_stamp,
                );
                ll1_price = BtMatcher::cached_lit_length_price::<ACCURATE_PRICE>(
                    profile,
                    $stats,
                    1,
                    &mut ll_cache,
                    ll_price_stamp,
                );
                // `min_match_len >= HC_FORMAT_MINMATCH (3)` by invariant.
                last_pos = (min_match_len - 1).min(frontier_limit);
                // Cells `1..min_match_len` start unreached; the forward loop
                // reaches them through the literal transition, which writes the
                // node it makes reachable.
                let seed_end = min_match_len.min(frontier_buffer_size);
                if seed_end > 1 {
                    // SAFETY: `seed_end - 1 < frontier_buffer_size`.
                    unsafe { BtMatcher::reset_opt_node_prices(node_prices, 1, seed_end - 1) };
                }
            }

            if let Some(candidate) = candidates.last() {
                let longest_len = candidate.match_len.min($current_len);
                if longest_len > sufficient_len {
                    let off_base = BtMatcher::encode_offset_base_with_reps(
                        candidate.offset as u32,
                        initial_litlen,
                        initial_reps,
                    );
                    let off_price = profile
                        .offset_price_for::<ACCURATE_PRICE, FAVOR_SMALL_OFFSETS>($stats, off_base);
                    let ml_price = BtMatcher::cached_match_length_price::<ACCURATE_PRICE>(
                        profile,
                        $stats,
                        longest_len,
                        &mut ml_cache,
                        ml_price_stamp,
                    );
                    let seq_cost = BtMatcher::add_prices(
                        ll0_price,
                        profile.match_price_from_parts(off_price, ml_price, $stats),
                    );
                    // SAFETY: cell 0 was written by the seed above (a candidate exists).
                    let forced_price = BtMatcher::add_prices(unsafe { *node_prices }, seq_cost);
                    let forced_state = HcOptimalNode {
                        off: candidate.offset as u32,
                        mlen: longest_len as u32,
                        litlen: 0,
                        reps: initial_reps,
                    };
                    // Not stored in `nodes[longest_len]`: that cell lies past the
                    // frontier, and the traceback takes this stretch from
                    // `forced_end_state`, never from the cell.
                    forced_end = Some(longest_len);
                    forced_end_state = Some(forced_state);
                    forced_end_price = Some(forced_price);
                    seed_forced_shortest_path = true;
                }
            }
            if !seed_forced_shortest_path {
                let mut prev_max_len = min_match_len - 1;
                for candidate in candidates.iter() {
                    let max_match_len = candidate.match_len.min(frontier_limit);
                    if max_match_len < min_match_len {
                        continue;
                    }
                    let start_len = (prev_max_len + 1).max(min_match_len);
                    if start_len > max_match_len {
                        prev_max_len = prev_max_len.max(max_match_len);
                        continue;
                    }
                    debug_assert!(max_match_len < frontier_buffer_size);
                    if max_match_len > last_pos {
                        // SAFETY: `max_match_len < frontier_buffer_size`.
                        unsafe {
                            BtMatcher::reset_opt_node_prices(
                                node_prices,
                                last_pos + 1,
                                max_match_len,
                            )
                        };
                    }
                    let off_base = BtMatcher::encode_offset_base_with_reps(
                        candidate.offset as u32,
                        initial_litlen,
                        initial_reps,
                    );
                    let off_price = profile
                        .offset_price_for::<ACCURATE_PRICE, FAVOR_SMALL_OFFSETS>($stats, off_base);
                    // SAFETY: cell 0 was written by the seed above.
                    let nodes0_price = unsafe { *node_prices };
                    for match_len in (start_len..=max_match_len).rev() {
                        let ml_price = BtMatcher::cached_match_length_price::<ACCURATE_PRICE>(
                            profile,
                            $stats,
                            match_len,
                            &mut ml_cache,
                            ml_price_stamp,
                        );
                        let seq_cost = BtMatcher::add_prices(
                            ll0_price,
                            profile.match_price_from_parts(off_price, ml_price, $stats),
                        );
                        let next_cost = BtMatcher::add_prices(nodes0_price, seq_cost);
                        // SAFETY: `match_len <= max_match_len`, a cell the seed or
                        // the reset above wrote.
                        let node_price = unsafe { *node_prices.add(match_len) };
                        if match_len > last_pos || next_cost < node_price {
                            unsafe {
                                HcOptimalNode::write_match_end(
                                    nodes.add(match_len),
                                    candidate.offset as u32,
                                    match_len as u32,
                                );
                                *node_prices.add(match_len) = next_cost;
                            }
                            if match_len > last_pos {
                                last_pos = match_len;
                            }
                        } else if abort_on_worse_match {
                            break;
                        }
                    }
                    prev_max_len = prev_max_len.max(max_match_len);
                }
                if last_pos + 1 < frontier_buffer_size {
                    // SAFETY: bounded by the arena length just checked.
                    unsafe { *node_prices.add(last_pos + 1) = u32::MAX };
                }
            }
        }
        while !seed_forced_shortest_path && pos <= last_pos && pos <= frontier_limit {
            debug_assert!(pos + 1 < frontier_buffer_size);
            // SAFETY (every arena access in this loop): prices `0..=last_pos + 1`
            // were written in this call, and a match cell past `last_pos` is
            // read only after `reset_opt_node_prices` wrote its price. A node is
            // read only where its price is finite: the transition that made it
            // finite wrote the node.
            let prev_node_price = unsafe { *node_prices.add(pos - 1) };
            if prev_node_price != u32::MAX {
                let prev_node = unsafe { *nodes.add(pos - 1) };
                let lit_len = prev_node.litlen as usize + 1;
                let lit_price = {
                    let bt = $self.backend.bt_mut();
                    BtMatcher::cached_literal_price::<ACCURATE_PRICE>(
                        profile,
                        $stats,
                        $current[pos - 1],
                        &mut bt.opt_lit_price_scratch,
                        &mut bt.opt_lit_price_generation,
                        lit_price_stamp,
                    )
                };
                let ll_delta = BtMatcher::cached_lit_length_delta_price::<ACCURATE_PRICE>(
                    profile,
                    $stats,
                    lit_len,
                    &mut ll_cache,
                    ll_price_stamp,
                );
                let lit_cost = BtMatcher::add_price_delta(prev_node_price, lit_price, ll_delta);
                // `node_pos_price` is the OLD price at `pos` (before the write
                // below) — also the price of `prev_match`, the pre-overwrite copy.
                let node_pos_price = unsafe { *node_prices.add(pos) };
                if lit_cost <= node_pos_price {
                    // The cell being replaced, read field by field: a match
                    // written it without a repeat history (the pass derives that
                    // on arrival), so the node as a whole may not be initialised.
                    // An unreached cell has no node to read; a zero length fails
                    // the end-of-match test below exactly as a reset node would.
                    let (prev_match_off, prev_match_mlen, prev_match_litlen) =
                        if node_pos_price != u32::MAX {
                            unsafe {
                                let cell = nodes.add(pos);
                                ((*cell).off, (*cell).mlen, (*cell).litlen)
                            }
                        } else {
                            (0, 0, u32::MAX)
                        };
                    unsafe {
                        *nodes.add(pos) = HcOptimalNode {
                            litlen: lit_len as u32,
                            ..prev_node
                        };
                        *node_prices.add(pos) = lit_cost;
                    }
                    #[allow(clippy::collapsible_if)]
                    if opt_level
                        && prev_match_mlen > 0
                        && prev_match_litlen == 0
                        && pos < $current_len
                    {
                        if ll1_price < ll0_price {
                            let next_lit_price = {
                                let bt = $self.backend.bt_mut();
                                BtMatcher::cached_literal_price::<ACCURATE_PRICE>(
                                    profile,
                                    $stats,
                                    $current[pos],
                                    &mut bt.opt_lit_price_scratch,
                                    &mut bt.opt_lit_price_generation,
                                    lit_price_stamp,
                                )
                            };
                            let with1literal = BtMatcher::add_price_delta(
                                node_pos_price,
                                next_lit_price,
                                ll1_price as i32 - ll0_price as i32,
                            );
                            let ll_delta_next =
                                BtMatcher::cached_lit_length_delta_price::<ACCURATE_PRICE>(
                                profile,
                                $stats,
                                lit_len + 1,
                                &mut ll_cache,
                                ll_price_stamp,
                            );
                            let with_more_literals =
                                BtMatcher::add_price_delta(lit_cost, next_lit_price, ll_delta_next);
                            let next = pos + 1;
                            let next_price = unsafe { *node_prices.add(next) };
                            if with1literal < with_more_literals && with1literal < next_price {
                                // Upstream zstd parity (zstd_opt.c:1232): `cur >= prevMatch.mlen`.
                                debug_assert!(pos >= prev_match_mlen as usize);
                                let prev_pos = pos - prev_match_mlen as usize;
                                {
                                    debug_assert!(unsafe { *node_prices.add(prev_pos) } != u32::MAX);
                                    let prev_state = unsafe { *nodes.add(prev_pos) };
                                    let (_, reps_after_match) = BtMatcher::encode_offset_with_reps(
                                        prev_match_off,
                                        prev_state.litlen as usize,
                                        prev_state.reps,
                                    );
                                    // Writes the whole node, so `next` (at most
                                    // `last_pos + 1`) is initialised when it
                                    // joins the frontier below.
                                    unsafe {
                                        *nodes.add(next) = HcOptimalNode {
                                            off: prev_match_off,
                                            mlen: prev_match_mlen,
                                            litlen: 1,
                                            reps: reps_after_match,
                                        };
                                        *node_prices.add(next) = with1literal;
                                    }
                                    if next > last_pos {
                                        last_pos = next;
                                        // The next iteration reads the price
                                        // past the frontier before this one's
                                        // closing sentinel runs (its `continue`
                                        // paths skip it), so set it now.
                                        if next + 1 < frontier_buffer_size {
                                            unsafe { *node_prices.add(next + 1) = u32::MAX };
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            }

            // Memory-resident DP (upstream zstd parity): read opt[cur] fields on
            // demand instead of holding a 28-byte node copy live across the
            // per-position `$collect` call below. The held copy forced LLVM
            // to spill reps[3] + litlen around the (non-inlinable) call;
            // reading the fields fresh on each side keeps them out of the
            // cross-call live set. `nodes[pos]` is stable across `$collect`
            // (it only fills `candidates`), so post-call reads are identical.
            let base_cost = unsafe { *node_prices.add(pos) };
            if base_cost == u32::MAX {
                pos += 1;
                continue;
            }
            {
                // Field by field: a match-written cell has no repeat history
                // yet, which this block is about to derive.
                let (base_off, base_mlen, base_litlen) = unsafe {
                    let cell = nodes.add(pos);
                    ((*cell).off, (*cell).mlen, (*cell).litlen)
                };
                if base_mlen > 0 && base_litlen == 0 {
                    // Upstream zstd parity (zstd_opt.c:1255): `cur >= opt[cur].mlen`.
                    debug_assert!(pos >= base_mlen as usize);
                    let prev_pos = pos - base_mlen as usize;
                    debug_assert!(unsafe { *node_prices.add(prev_pos) } != u32::MAX);
                    let prev_state = unsafe { *nodes.add(prev_pos) };
                    let (_, reps_after_match) = BtMatcher::encode_offset_with_reps(
                        base_off,
                        prev_state.litlen as usize,
                        prev_state.reps,
                    );
                    unsafe { (*nodes.add(pos)).reps = reps_after_match };
                }
            }

            if pos + 8 > $current_len {
                pos += 1;
                continue;
            }

            if pos == last_pos {
                break;
            }

            let next_price = unsafe { *node_prices.add(pos + 1) };
            // `saturating_add` is REQUIRED here, not a masked bug: `base_cost`
            // is a node price that can be the `u32::MAX` "unreachable" sentinel,
            // and saturating keeps `base_cost + margin` pinned at MAX so the
            // comparison stays correct. Plain `+` would wrap the sentinel and
            // flip the abort decision (a ratio bug / debug overflow panic).
            if abort_on_worse_match
                && next_price <= base_cost.saturating_add(HC_BITCOST_MULTIPLIER / 2)
            {
                pos += 1;
                continue;
            }

            let abs_pos = $current_abs_start + pos;
            candidates.clear();
            // SAFETY: same umbrella as `$collect`. The search reads the repeat
            // history in place through `nodes[pos]`, as upstream passes
            // `opt[cur].rep`, so nothing is copied into the call and nothing
            // stays live across it; the post-call reads below are a fresh load
            // of the same stable `nodes[pos]`.
            unsafe {
                $collect(
                    &mut *$self,
                    abs_pos,
                    current_abs_end,
                    profile.sufficient_match_len,
                    &(*nodes.add(pos)).reps,
                    (*nodes.add(pos)).litlen == 0,
                    &mut *candidates,
                )
            };
            if HAS_LDM {
                // Upstream `ZSTD_optLdm_processMatchCandidate`: the producer
                // advances past this position and its candidate joins after
                // the search.
                if let Some(ldm) = $self.backend.bt_mut().ldm_process_match_candidate(
                    &mut opt_ldm,
                    pos,
                    $current_len - pos,
                    min_match_len,
                ) {
                    BtMatcher::push_ldm_candidate(&mut *candidates, ldm, min_match_len);
                }
            }
            // Post-call reads of opt[cur]: fresh, born after `$collect`, so
            // never part of the cross-call live set (see memory-resident note
            // above). `nodes[pos]` is untouched by `$collect`.
            let base_reps = unsafe { (*nodes.add(pos)).reps };
            let base_litlen = unsafe { (*nodes.add(pos)).litlen as usize };
            if let Some(candidate) = candidates.last() {
                let longest_len = candidate.match_len.min($current_len - pos);
                if longest_len > sufficient_len
                    || pos + longest_len >= HC_OPT_NUM
                    || pos + longest_len >= $current_len
                {
                    let lit_len = base_litlen;
                    let off_base = BtMatcher::encode_offset_base_with_reps(
                        candidate.offset as u32,
                        lit_len,
                        base_reps,
                    );
                    let off_price = profile
                        .offset_price_for::<ACCURATE_PRICE, FAVOR_SMALL_OFFSETS>($stats, off_base);
                    let ml_price = BtMatcher::cached_match_length_price::<ACCURATE_PRICE>(
                        profile,
                        $stats,
                        longest_len,
                        &mut ml_cache,
                        ml_price_stamp,
                    );
                    let seq_cost = BtMatcher::add_prices(
                        ll0_price,
                        profile.match_price_from_parts(off_price, ml_price, $stats),
                    );
                    let forced_price = BtMatcher::add_prices(base_cost, seq_cost);
                    let end_pos = (pos + longest_len).min($current_len);
                    forced_end = Some(end_pos);
                    forced_end_state = Some(HcOptimalNode {
                        off: candidate.offset as u32,
                        mlen: longest_len as u32,
                        litlen: 0,
                        reps: base_reps,
                    });
                    forced_end_price = Some(forced_price);
                    break;
                }
            }
            let mut prev_max_len = min_match_len - 1;
            for candidate in candidates.iter() {
                // Outer loop guards `pos <= frontier_limit` (see the
                // `while ... pos <= frontier_limit` condition); the
                // subtraction below is therefore safe.
                debug_assert!(pos <= frontier_limit);
                let max_match_len = candidate
                    .match_len
                    .min($current_len - pos)
                    .min(frontier_limit - pos);
                let min_len = min_match_len;
                if max_match_len < min_len {
                    continue;
                }
                let start_len = (prev_max_len + 1).max(min_len);
                if start_len > max_match_len {
                    prev_max_len = prev_max_len.max(max_match_len);
                    continue;
                }
                let max_next = pos + max_match_len;
                debug_assert!(max_next < frontier_buffer_size);
                if max_next > last_pos {
                    // SAFETY: `max_next < frontier_buffer_size`.
                    unsafe { BtMatcher::reset_opt_node_prices(node_prices, last_pos + 1, max_next) };
                }
                let lit_len = base_litlen;
                let off_base = BtMatcher::encode_offset_base_with_reps(
                    candidate.offset as u32,
                    lit_len,
                    base_reps,
                );
                let off_price = profile
                    .offset_price_for::<ACCURATE_PRICE, FAVOR_SMALL_OFFSETS>($stats, off_base);
                debug_assert!(pos + max_match_len < frontier_buffer_size);
                if abort_on_worse_match {
                    // btopt (OPT_LEVEL == 0): reverse-iterate with early break —
                    // once a longer match stops improving, shorter ones are
                    // skipped. Order-dependent, stays scalar.
                    for match_len in (start_len..=max_match_len).rev() {
                        let next = pos + match_len;
                        let ml_price = BtMatcher::cached_match_length_price::<ACCURATE_PRICE>(
                            profile,
                            $stats,
                            match_len,
                            &mut ml_cache,
                            ml_price_stamp,
                        );
                        let seq_cost = BtMatcher::add_prices(
                            ll0_price,
                            profile.match_price_from_parts(off_price, ml_price, $stats),
                        );
                        let next_cost = BtMatcher::add_prices(base_cost, seq_cost);
                        let node_next_price = unsafe { *node_prices.add(next) };
                        if next > last_pos || next_cost < node_next_price {
                            unsafe {
                                HcOptimalNode::write_match_end(
                                    nodes.add(next),
                                    candidate.offset as u32,
                                    match_len as u32,
                                );
                                *node_prices.add(next) = next_cost;
                            }
                            if next > last_pos {
                                last_pos = next;
                            }
                        } else {
                            break;
                        }
                    }
                } else {
                    // btultra / btultra2 (OPT_LEVEL >= 2): no abort, each
                    // match_len writes a distinct node => order-independent.
                    // The tier's price-set (`$priceset` is its macro). The AVX2
                    // one expands its vector loop here, inside this wrapper's
                    // umbrella, the way upstream walks the matches inside opt2:
                    // left as an `#[inline]` target_feature function it stayed
                    // out of line, a call per priced candidate.
                    // The kernel reaches cells `pos + start_len ..= max_next`, so
                    // the arenas go in as slices ending at `max_next`. The prices
                    // there are initialised (`0..=last_pos` written in this call,
                    // the rest just reset); the nodes may not be, so they go in
                    // as `MaybeUninit` and the kernel only writes them.
                    let written = max_next + 1;
                    let (node_prices_s, nodes_s) = unsafe {
                        (
                            core::slice::from_raw_parts_mut(node_prices, written),
                            core::slice::from_raw_parts_mut(
                                nodes.cast::<core::mem::MaybeUninit<HcOptimalNode>>(),
                                written,
                            ),
                        )
                    };
                    #[allow(unused_unsafe)]
                    {
                        last_pos = last_pos.max(unsafe {
                            $priceset!(
                                ACCURATE_PRICE;
                                node_prices_s,
                                nodes_s,
                                ml_cache,
                                ml_price_stamp,
                                profile,
                                $stats,
                                pos,
                                start_len,
                                max_match_len,
                                ll0_price,
                                off_price,
                                base_cost,
                                candidate.offset as u32,
                                last_pos,
                            )
                        });
                    }
                }
                prev_max_len = prev_max_len.max(max_match_len);
            }

            if last_pos + 1 < frontier_buffer_size {
                unsafe { *node_prices.add(last_pos + 1) = u32::MAX };
            }
            pos += 1;
        }

        if last_pos == 0 {
            if $current_len == 0 {
                let price = 0u32; // deferred: node_prices[0] unset on no-match; caller discards price
                break $seg (price, initial_reps, initial_litlen, 0);
            }
            // No match at this position: it is a single literal (upstream zstd
            // `ZSTD_compressBlock_opt_generic` `if (!nbMatches) { ip++; }`). The
            // previous code recomputed a literal+length price here purely to fill
            // the returned price slot — which the per-segment caller discards
            // (`let (_, ..)`). Skip that recompute: on near-random input this
            // no-match return runs once per literal, so the two cached price
            // lookups were pure waste. Byte-identical — the tree, the carried
            // litlen (+1), and the reps are unchanged; only the discarded price
            // value differs (now the base-literal node price).
            let next_litlen = initial_litlen
                .checked_add(1)
                .expect("optimal parser next litlen out of usize range");
            // node_prices[0] is unset on a no-match seed (deferred); the caller
            // discards this price anyway.
            let price = 0u32;
            break $seg (price, initial_reps, next_litlen, 1);
        }

        let target_pos = forced_end.unwrap_or(last_pos.min(frontier_limit));
        // Price lives in `node_prices`, not the node struct, so carry the
        // final-stretch price alongside its node (forced-seed companion or the
        // frontier price at `target_pos`).
        let (last_stretch, last_stretch_price) = if let Some(forced_state) = forced_end_state {
            (forced_state, forced_end_price.expect("forced state has a price"))
        } else {
            // SAFETY (every arena read of the traceback): it walks back from
            // `target_pos <= last_pos` through cells the DP wrote in this call.
            debug_assert!(target_pos <= last_pos);
            unsafe { (*nodes.add(target_pos), *node_prices.add(target_pos)) }
        };
        if last_stretch_price == u32::MAX {
            break $seg (u32::MAX, initial_reps, initial_litlen, $current_len);
        }

        if last_stretch.mlen == 0 {
            break $seg (
                last_stretch_price,
                last_stretch.reps,
                last_stretch.litlen as usize,
                target_pos.min($current_len),
            );
        }

        let mut cur = target_pos.saturating_sub(last_stretch.mlen as usize);
        let end_reps = if last_stretch.litlen == 0 {
            debug_assert!(cur <= last_pos);
            debug_assert!(unsafe { *node_prices.add(cur) } != u32::MAX, "unreached node read");
            let prev_state = unsafe { *nodes.add(cur) };
            let (_, reps_after_match) = BtMatcher::encode_offset_with_reps(
                last_stretch.off,
                prev_state.litlen as usize,
                prev_state.reps,
            );
            reps_after_match
        } else {
            let tail_literals = last_stretch.litlen as usize;
            if cur < tail_literals {
                break $seg (
                    last_stretch_price,
                    last_stretch.reps,
                    tail_literals,
                    target_pos.min($current_len),
                );
            }
            cur -= tail_literals;
            last_stretch.reps
        };
        let store_end = cur + 2;
        if store.len() <= store_end {
            store.resize(store_end + 1, HcOptimalNode::default());
        }
        let mut store_start;
        let mut stretch_pos = cur;

        if last_stretch.litlen > 0 {
            store[store_end] = HcOptimalNode {
                litlen: last_stretch.litlen,
                mlen: 0,
                ..HcOptimalNode::default()
            };
            store_start = store_end.saturating_sub(1);
            store[store_start] = last_stretch;
        }
        store[store_end] = last_stretch;
        store_start = store_end;

        loop {
            debug_assert!(stretch_pos <= last_pos);
            debug_assert!(
                unsafe { *node_prices.add(stretch_pos) } != u32::MAX,
                "unreached node read"
            );
            let next_stretch = unsafe { *nodes.add(stretch_pos) };
            store[store_start].litlen = next_stretch.litlen;
            if next_stretch.mlen == 0 {
                break;
            }
            if store_start == 0 {
                break;
            }
            store_start -= 1;
            store[store_start] = next_stretch;
            // Parser invariant: every emitted stretch is bounded by the
            // current block, so `litlen + mlen <= current_len <=
            // HC_BLOCKSIZE_MAX (128 KiB)`. The `as usize` widening + raw
            // `+` is safe on 32-bit targets — two u32 values do NOT
            // automatically fit in `usize` on i686, the block bound is
            // what makes this addition safe.
            let litlen = next_stretch.litlen as usize;
            let mlen = next_stretch.mlen as usize;
            debug_assert!(litlen + mlen <= $current_len);
            let step = litlen + mlen;
            if step == 0 || stretch_pos < step {
                break;
            }
            stretch_pos -= step;
        }

        let mut tail_literals = initial_litlen;
        let mut store_pos = store_start;
        // Each settled sequence goes into the statistics here, as upstream's
        // traceback calls `ZSTD_updateStats` per stored sequence, instead of in
        // a second pass over the plan after the segment. The repeat history
        // runs from the segment's own.
        let mut stats_reps = initial_reps;
        let mut recorded = false;
        while store_pos <= store_end {
            let stretch = store[store_pos];
            let llen = stretch.litlen as usize;
            let mlen = stretch.mlen as usize;
            if mlen == 0 {
                tail_literals = llen;
                store_pos += 1;
                continue;
            }
            let sequence = HcOptimalSequence {
                offset: stretch.off,
                match_len: mlen as u32,
                lit_len: llen as u32,
            };
            if $keep_plan {
                $out.push(sequence);
            }
            BtMatcher::record_sequence_stats(
                $block,
                sequence,
                &mut pass.literals_cursor,
                &mut stats_reps,
                &mut *$opt_state,
            );
            recorded = true;
            tail_literals = 0;
            store_pos += 1;
        }
        if recorded {
            $opt_state
                .set_base_prices(<$strategy_ty as crate::encoding::strategy::Strategy>::ACCURATE_PRICE);
        }
        let result = (
            last_stretch_price,
            end_reps,
            if last_stretch.litlen > 0 {
                last_stretch.litlen as usize
            } else {
                tail_literals
            },
            target_pos.min($current_len),
        );
        result
    }};
}

/// The optimal parser's pass over one block, upstream zstd
/// `ZSTD_compressBlock_opt_generic`: the segment loop around each segment's
/// forward pass and traceback ([`build_optimal_plan_impl_body!`]), whose
/// traceback also records the segment's sequences into the statistics. The
/// frame, the buffer set-up and the arguments are paid once per block rather
/// than once per segment, which on input with few matches is once per literal
/// run. `$keep_plan` is false for the btultra2 seed pass, which keeps only the
/// statistics.
macro_rules! optimal_block_body {
    (
        $self:expr,
        $strategy_ty:ty,
        $current:ident,
        $current_abs_start:ident,
        $cursor:ident,
        $litlen:ident,
        $reps:ident,
        $profile:ident,
        $opt_state:ident,
        $plan:ident,
        $buffers:ident,
        $keep_plan:expr,
        $collect:ident,
        $priceset:ident $(,)?
    ) => {{
        // The block is armed, so the tree's coordinates hold for the whole pass.
        $self.table.capture_block_coords();
        // The tree's insert cursor starts inside the history, so no search has
        // to pull it up: within the pass it only moves forward.
        $self.table.skip_insert_until_abs = $self
            .table
            .skip_insert_until_abs
            .max($self.table.history_abs_start);
        // The search's key width is fixed for the block, so the search built
        // for it is chosen once here and every position calls it directly, as
        // upstream picks `ZSTD_btGetAllMatches_<dictMode>_<mls>` per block
        // (`ZSTD_selectBtGetAllMatches`).
        let collect: OptimalCollectFn = match $self.table.search_mls {
            3 => HcMatchGenerator::$collect::<$strategy_ty, 3>,
            5 => HcMatchGenerator::$collect::<$strategy_ty, 5>,
            6 => HcMatchGenerator::$collect::<$strategy_ty, 6>,
            mls => {
                debug_assert_eq!(mls, 4, "the binary tree keys on three to six bytes");
                HcMatchGenerator::$collect::<$strategy_ty, 4>
            }
        };
        // Everything the pass carries between segments lives in `$buffers.pass`,
        // read at the top of a segment and written at its end, so none of it is
        // live across the segment body (see `HcOptimalPlanBuffers::pass`).
        $buffers.pass = HcBlockPass {
            cursor: $cursor,
            litlen: $litlen,
            reps: $reps,
            literals_cursor: 0,
        };
        // The last 8 bytes are left as literals. Written as an addition on the
        // cursor, which never passes the block's end, so a block shorter than
        // the tail needs no floor: it simply has no segment.
        let block_len = $current.len();
        while $buffers.pass.cursor + 8 < block_len {
            let cursor = $buffers.pass.cursor;
            let segment = &$current[cursor..];
            let segment_abs_start = $current_abs_start + cursor;
            let segment_len = $current.len() - cursor;
            let segment_state = HcOptimalPlanState {
                block_offset: cursor,
                reps: $buffers.pass.reps,
                litlen: $buffers.pass.litlen,
                profile: $profile,
            };
            let segment_stats: &HcOptState = &*$opt_state;
            let (_, end_reps, end_litlen, consumed_len) = 'segment: {
                build_optimal_plan_impl_body!(
                    $self,
                    $strategy_ty,
                    segment,
                    segment_abs_start,
                    segment_len,
                    segment_state,
                    segment_stats,
                    $plan,
                    $buffers,
                    collect,
                    $priceset,
                    $current,
                    $opt_state,
                    $keep_plan,
                    'segment,
                )
            };
            let pass = &mut $buffers.pass;
            pass.reps = end_reps;
            pass.litlen = end_litlen;
            pass.cursor += consumed_len;
        }
    }};
}

/// `collect_optimal_candidates_initialized` body parameterized over the per-CPU
/// kernel: the `$cpl` path is the kernel's `common_prefix_len_ptr` (used in
/// the HC chain walk fallback), and the four method-name substitutions
/// (`$bt_update`, `$bt_insert`, `$for_each_rep`, `$hash3`) route to the
/// kernel-specific wrappers of the inner helpers. With every helper under
/// the same `target_feature` umbrella, the entire per-position pipeline
/// (BT-tree fill + rep probing + hash3 probing + BT match collection /
/// HC chain walk) inlines without ABI barriers on the level22 hot path.
macro_rules! collect_optimal_candidates_initialized_body {
    (
        $self:expr,
        $strategy_ty:ty,
        $abs_pos:ident,
        $current_abs_end:ident,
        $sufficient_match_len:ident,
        $reps:ident,
        $ll0:ident,
        $out:ident,
        $mls:expr,
        $bt_insert_range:ident,
        $cpl:path,
        $cmf:path $(,)?
    ) => {{
        debug_assert_eq!(
            $mls, $self.table.search_mls,
            "the block chose the search for its key width"
        );
        // Per-strategy compile-time const: only BtUltra2 drives the
        // hash3 short-match table. All other monomorphisations drop
        // the entire hash3 lookup block at codegen time. The relaxed
        // implication enforces only the direction we depend on:
        // if the strategy declares hash3, the table must be live.
        // The reverse (`hash3_log != 0` without `USE_HASH3`) is OK —
        // a future caller may pre-allocate hash3 storage without
        // wiring the BtUltra2 path through.
        let use_hash3: bool = <$strategy_ty as crate::encoding::strategy::Strategy>::USE_HASH3;
        debug_assert!(!$self.table.hash_table().is_empty());
        debug_assert!($self.table.hash3_log == 0 || !$self.table.hash3_table().is_empty());
        debug_assert!(
            !use_hash3 || $self.table.hash3_log != 0,
            "Strategy::USE_HASH3 = true but runtime hash3_log is 0 — call configure() first",
        );
        debug_assert!(!$self.table.chain_table().is_empty());
        let min_match_len = HC_OPT_MIN_MATCH_LEN;
        $out.clear();
        if $abs_pos < $self.table.skip_insert_until_abs {
            return;
        }
        {
            // BT tree catch-up: insert the positions the parser skipped into the
            // binary tree before this position's search, as one run (upstream
            // zstd ZSTD_updateTree). `$bt_insert_range` overshoots the target
            // with no clamp (a long match skips the positions it covers),
            // exactly as upstream. The pass starts the cursor inside the
            // history and only ever moves it forward.
            let update_abs = $self.table.skip_insert_until_abs;
            debug_assert!(update_abs >= $self.table.history_abs_start);
            // No rebase guard: `arm_block_positions` cleared every position in
            // this block before the parse began, and the catch-up only inserts
            // positions below `$abs_pos`, which is in it.
            debug_assert!($self.table.can_skip_rebase_check($abs_pos));
            if update_abs < $abs_pos {
                // SAFETY: caller is in the same target_feature umbrella as
                // `$bt_insert_range`; the runtime kernel detector gated entry.
                let _ = unsafe {
                    $self
                        .table
                        .$bt_insert_range(update_abs, $abs_pos, $current_abs_end, $abs_pos)
                };
            }
            $self.table.skip_insert_until_abs = $abs_pos;
        }
        let current_idx = $abs_pos - $self.table.history_abs_start;
        // Every probe below reads up to eight bytes at the position; the parser
        // searches only positions with that much of the block left, so this is
        // the one length check the search needs.
        if current_idx + 8 > $self.table.block_live.1 {
            return;
        }
        let mut best_len_for_skip = 0usize;
        {
            // Per-position match-finder: the rep-code probe, the hash3
            // short-match probe and the binary-tree walk folded into one body
            // (upstream ZSTD_insertBtAndGetAllMatches, FORCE_INLINE_TEMPLATE into
            // the opt loop). Seeds `best_len_for_skip` and handles the
            // sufficient-match early-out internally.
            let bt_search_depth = $self.table.search_depth;
            let best_len_ref = &mut best_len_for_skip;
            crate::encoding::hc::generator::bt_insert_and_collect_matches_body!(
                $self.table,
                bt_search_depth,
                $abs_pos,
                $current_abs_end,
                // Not a strategy const: the pass adjusts it (btultra2's seed
                // pass runs a different sufficient length from its main pass),
                // so it arrives as a scalar. The chain depth beside it IS the
                // strategy's own const and needs no argument at all.
                $sufficient_match_len,
                <$strategy_ty as crate::encoding::strategy::Strategy>::MAX_CHAIN_DEPTH,
                min_match_len,
                best_len_ref,
                $out,
                $reps,
                $ll0,
                use_hash3,
                $mls,
                $cpl,
                $cmf,
            );
        }
    }};
}
impl HcMatchGenerator {
    // The scalar arm of `run_main_loop!` wraps a safe `*_scalar` call in
    // `unsafe` (the SIMD arms need it); allow the redundant block here.
    #[allow(unused_unsafe)]
    pub(crate) fn start_matching_optimal<S: crate::encoding::strategy::Strategy>(
        &mut self,
        mut handle_sequence: impl FnMut(Sequence),
    ) {
        self.table.ensure_tables();
        // Borrowed-aware: owned → last committed chunk; borrowed → staged
        // in-place block range.
        let (current_abs_start, current_len) = self.table.current_block_range();
        if current_len == 0 {
            return;
        }
        let current_ptr = self.table.get_last_space().as_ptr();
        // `start_matching_optimal()` mutates tables/state but never mutates or
        // reallocates `self.table.history`, so this tail slice remains valid for
        // the duration of the routine and avoids cloning the full block.
        let current = unsafe { core::slice::from_raw_parts(current_ptr, current_len) };

        let current_abs_end = current_abs_start + current_len;
        // Settle index representability for the whole block here, as upstream
        // settles it before compressing one, so nothing below has to ask again
        // per position.
        self.table.arm_block_positions(current_abs_end);
        self.table
            .apply_limited_update_after_long_match(current_abs_start);
        let hash3_start_cursor = self
            .table
            .skip_insert_until_abs
            .max(self.table.history_abs_start);
        self.table
            .backfill_boundary_positions(current_abs_start, current_abs_end);
        self.table.next_to_update3 = hash3_start_cursor;

        // Take the five DP scratch buffers ONCE for both the seed pass and the
        // main pass (upstream shares one workspace across btultra2's two
        // passes). Threading the same `&mut` into the seed pass removes its
        // separate take + restore round-trip.
        let mut plan_buffers = self.take_optimal_plan_buffers();
        if self.should_run_btultra2_seed_pass::<S>(current_len) {
            self.run_btultra2_seed_pass(current, current_abs_start, current_len, &mut plan_buffers);
            // The seed pass pushes the stored-index offset past everything it
            // inserted, so the block has to be cleared again for the main pass
            // against the offset it leaves behind.
            self.table.arm_block_positions(current_abs_end);
        }

        // Const-generic profile selection: every field is folded from
        // S's associated consts (MAX_CHAIN_DEPTH /
        // SUFFICIENT_MATCH_LEN / ACCURATE_PRICE / FAVOR_SMALL_OFFSETS),
        // so the optimiser produces the literal at codegen time
        // without a runtime match.
        let mut profile = HcOptimalCostProfile::const_for_strategy::<S>();
        // Clamped here, once for the block, rather than inside the per-segment
        // DP body (which on input without matches is entered per literal).
        profile.sufficient_match_len = self.hc.sufficient_match_len_for_pass(profile);
        // The DP bodies read the strategy's `FAVOR_SMALL_OFFSETS` const directly;
        // verify the runtime profile (built from the same strategy) agrees.
        debug_assert_eq!(profile.favor_small_offsets, S::FAVOR_SMALL_OFFSETS);
        let mut opt_state =
            core::mem::replace(&mut self.backend.bt_mut().opt_state, HcOptState::new());
        opt_state.rescale_freqs(current, S::ACCURATE_PRICE);
        let mut best_plan = core::mem::take(&mut self.backend.bt_mut().opt_segment_plan_scratch);
        best_plan.clear();
        let plan_reps = self.table.offset_hist;
        let (cursor, plan_litlen) = self.table.opt_start_cursor_and_litlen(current_abs_start);
        // Frame-constant LDM presence, resolved once per block (not per segment
        // and not in the DP hot loop): drives the HAS_LDM const-generic dispatch.
        let has_ldm = !self.backend.bt_mut().ldm_sequences.is_empty();
        // Resolve the SIMD tier ONCE here: the whole block then runs under one
        // kernel-monomorphized pass (`run_optimal_block_<kernel>`) instead of
        // hitting select_kernel()'s OnceLock atomic + a CPU-tier match per
        // segment. Mirrors the Fast matcher dispatch shape.
        macro_rules! run_main_loop {
            ($block:ident) => {{
                match (S::ACCURATE_PRICE, S::FAVOR_SMALL_OFFSETS, has_ldm) {
                    (true, false, false) => unsafe {
                        self.$block::<S, true, false, false, true>(
                            current,
                            current_abs_start,
                            cursor,
                            plan_litlen,
                            plan_reps,
                            profile,
                            &mut opt_state,
                            &mut best_plan,
                            &mut plan_buffers,
                        )
                    },
                    (true, false, true) => unsafe {
                        self.$block::<S, true, false, true, true>(
                            current,
                            current_abs_start,
                            cursor,
                            plan_litlen,
                            plan_reps,
                            profile,
                            &mut opt_state,
                            &mut best_plan,
                            &mut plan_buffers,
                        )
                    },
                    (true, true, false) => unsafe {
                        self.$block::<S, true, true, false, true>(
                            current,
                            current_abs_start,
                            cursor,
                            plan_litlen,
                            plan_reps,
                            profile,
                            &mut opt_state,
                            &mut best_plan,
                            &mut plan_buffers,
                        )
                    },
                    (true, true, true) => unsafe {
                        self.$block::<S, true, true, true, true>(
                            current,
                            current_abs_start,
                            cursor,
                            plan_litlen,
                            plan_reps,
                            profile,
                            &mut opt_state,
                            &mut best_plan,
                            &mut plan_buffers,
                        )
                    },
                    (false, false, false) => unsafe {
                        self.$block::<S, false, false, false, true>(
                            current,
                            current_abs_start,
                            cursor,
                            plan_litlen,
                            plan_reps,
                            profile,
                            &mut opt_state,
                            &mut best_plan,
                            &mut plan_buffers,
                        )
                    },
                    (false, false, true) => unsafe {
                        self.$block::<S, false, false, true, true>(
                            current,
                            current_abs_start,
                            cursor,
                            plan_litlen,
                            plan_reps,
                            profile,
                            &mut opt_state,
                            &mut best_plan,
                            &mut plan_buffers,
                        )
                    },
                    (false, true, false) => unsafe {
                        self.$block::<S, false, true, false, true>(
                            current,
                            current_abs_start,
                            cursor,
                            plan_litlen,
                            plan_reps,
                            profile,
                            &mut opt_state,
                            &mut best_plan,
                            &mut plan_buffers,
                        )
                    },
                    (false, true, true) => unsafe {
                        self.$block::<S, false, true, true, true>(
                            current,
                            current_abs_start,
                            cursor,
                            plan_litlen,
                            plan_reps,
                            profile,
                            &mut opt_state,
                            &mut best_plan,
                            &mut plan_buffers,
                        )
                    },
                }
            }};
        }
        #[cfg(all(
            target_arch = "aarch64",
            target_endian = "little",
            feature = "kernel-neon"
        ))]
        unsafe {
            run_main_loop!(run_optimal_block_neon);
        }
        #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
        {
            use crate::encoding::fastpath::FastpathKernel;
            match self.table.kernel {
                #[cfg(feature = "kernel-avx2")]
                FastpathKernel::Avx2Bmi2 => run_main_loop!(run_optimal_block_avx2_bmi2),
                #[cfg(feature = "kernel-sse")]
                FastpathKernel::Sse2 => run_main_loop!(run_optimal_block_sse2),
                #[cfg(feature = "kernel-sse")]
                FastpathKernel::Sse42 => run_main_loop!(run_optimal_block_sse42),
                FastpathKernel::Scalar => run_main_loop!(run_optimal_block_scalar),
            }
        }
        #[cfg(all(
            target_arch = "wasm32",
            target_feature = "simd128",
            feature = "kernel-simd128"
        ))]
        unsafe {
            run_main_loop!(run_optimal_block_simd128);
        }
        #[cfg(not(any(
            all(
                target_arch = "aarch64",
                target_endian = "little",
                feature = "kernel-neon"
            ),
            target_arch = "x86",
            target_arch = "x86_64",
            all(
                target_arch = "wasm32",
                target_feature = "simd128",
                feature = "kernel-simd128"
            )
        )))]
        {
            run_main_loop!(run_optimal_block_scalar);
        }

        self.table
            .emit_optimal_plan(current_len, &best_plan, &mut handle_sequence);
        best_plan.clear();
        self.backend.bt_mut().opt_segment_plan_scratch = best_plan;
        let _ = self
            .backend
            .bt_mut()
            .finish_optimal_plan(plan_buffers, (0, [0; 3], 0, 0));
        self.backend.bt_mut().opt_state = opt_state;
    }

    // The scalar arm of `run_seed_loop!` wraps a safe call in `unsafe`.
    #[allow(unused_unsafe)]
    fn run_btultra2_seed_pass(
        &mut self,
        current: &[u8],
        current_abs_start: usize,
        current_len: usize,
        plan_buffers: &mut HcOptimalPlanBuffers,
    ) {
        // The seed pass is BtUltra2-exclusive by name (the only
        // caller is `should_run_btultra2_seed_pass`), so pin `S` to
        // `BtUltra2` for both the cost-profile lookup and the
        // kernel-dispatched segment loop below.
        type S = crate::encoding::strategy::BtUltra2;
        // `S` is a concrete type here (not a generic bound), so the `Strategy`
        // trait must be in scope to read its associated consts in
        // `run_seed_loop!`.
        use crate::encoding::strategy::Strategy;
        let mut seed_profile = HcOptimalCostProfile::const_for_strategy::<S>();
        // Same per-block clamp the main pass does; the DP body expects it done.
        seed_profile.sufficient_match_len = self.hc.sufficient_match_len_for_pass(seed_profile);
        debug_assert_eq!(seed_profile.favor_small_offsets, S::FAVOR_SMALL_OFFSETS);
        let mut opt_state =
            core::mem::replace(&mut self.backend.bt_mut().opt_state, HcOptState::new());
        opt_state.rescale_freqs(current, S::ACCURATE_PRICE);
        let seed_reps = self.table.offset_hist;
        let (cursor, seed_litlen) = self.table.opt_start_cursor_and_litlen(current_abs_start);
        let mut seed_plan = core::mem::take(&mut self.backend.bt_mut().opt_seed_plan_scratch);
        seed_plan.clear();
        let has_ldm = !self.backend.bt_mut().ldm_sequences.is_empty();
        // SIMD tier resolved ONCE (see start_matching_optimal): the seed pass
        // runs as one kernel-monomorphized block pass. It keeps the statistics
        // and drops the plan (`KEEP_PLAN = false`).
        macro_rules! run_seed_loop {
            ($block:ident) => {{
                match (S::ACCURATE_PRICE, S::FAVOR_SMALL_OFFSETS, has_ldm) {
                    (true, false, false) => unsafe {
                        self.$block::<S, true, false, false, false>(
                            current,
                            current_abs_start,
                            cursor,
                            seed_litlen,
                            seed_reps,
                            seed_profile,
                            &mut opt_state,
                            &mut seed_plan,
                            &mut *plan_buffers,
                        )
                    },
                    (true, false, true) => unsafe {
                        self.$block::<S, true, false, true, false>(
                            current,
                            current_abs_start,
                            cursor,
                            seed_litlen,
                            seed_reps,
                            seed_profile,
                            &mut opt_state,
                            &mut seed_plan,
                            &mut *plan_buffers,
                        )
                    },
                    (true, true, false) => unsafe {
                        self.$block::<S, true, true, false, false>(
                            current,
                            current_abs_start,
                            cursor,
                            seed_litlen,
                            seed_reps,
                            seed_profile,
                            &mut opt_state,
                            &mut seed_plan,
                            &mut *plan_buffers,
                        )
                    },
                    (true, true, true) => unsafe {
                        self.$block::<S, true, true, true, false>(
                            current,
                            current_abs_start,
                            cursor,
                            seed_litlen,
                            seed_reps,
                            seed_profile,
                            &mut opt_state,
                            &mut seed_plan,
                            &mut *plan_buffers,
                        )
                    },
                    (false, false, false) => unsafe {
                        self.$block::<S, false, false, false, false>(
                            current,
                            current_abs_start,
                            cursor,
                            seed_litlen,
                            seed_reps,
                            seed_profile,
                            &mut opt_state,
                            &mut seed_plan,
                            &mut *plan_buffers,
                        )
                    },
                    (false, false, true) => unsafe {
                        self.$block::<S, false, false, true, false>(
                            current,
                            current_abs_start,
                            cursor,
                            seed_litlen,
                            seed_reps,
                            seed_profile,
                            &mut opt_state,
                            &mut seed_plan,
                            &mut *plan_buffers,
                        )
                    },
                    (false, true, false) => unsafe {
                        self.$block::<S, false, true, false, false>(
                            current,
                            current_abs_start,
                            cursor,
                            seed_litlen,
                            seed_reps,
                            seed_profile,
                            &mut opt_state,
                            &mut seed_plan,
                            &mut *plan_buffers,
                        )
                    },
                    (false, true, true) => unsafe {
                        self.$block::<S, false, true, true, false>(
                            current,
                            current_abs_start,
                            cursor,
                            seed_litlen,
                            seed_reps,
                            seed_profile,
                            &mut opt_state,
                            &mut seed_plan,
                            &mut *plan_buffers,
                        )
                    },
                }
            }};
        }
        #[cfg(all(
            target_arch = "aarch64",
            target_endian = "little",
            feature = "kernel-neon"
        ))]
        unsafe {
            run_seed_loop!(run_optimal_block_neon);
        }
        #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
        {
            use crate::encoding::fastpath::FastpathKernel;
            match self.table.kernel {
                #[cfg(feature = "kernel-avx2")]
                FastpathKernel::Avx2Bmi2 => run_seed_loop!(run_optimal_block_avx2_bmi2),
                #[cfg(feature = "kernel-sse")]
                FastpathKernel::Sse2 => run_seed_loop!(run_optimal_block_sse2),
                #[cfg(feature = "kernel-sse")]
                FastpathKernel::Sse42 => run_seed_loop!(run_optimal_block_sse42),
                FastpathKernel::Scalar => run_seed_loop!(run_optimal_block_scalar),
            }
        }
        #[cfg(all(
            target_arch = "wasm32",
            target_feature = "simd128",
            feature = "kernel-simd128"
        ))]
        unsafe {
            run_seed_loop!(run_optimal_block_simd128);
        }
        #[cfg(not(any(
            all(
                target_arch = "aarch64",
                target_endian = "little",
                feature = "kernel-neon"
            ),
            target_arch = "x86",
            target_arch = "x86_64",
            all(
                target_arch = "wasm32",
                target_feature = "simd128",
                feature = "kernel-simd128"
            )
        )))]
        {
            run_seed_loop!(run_optimal_block_scalar);
        }
        seed_plan.clear();
        self.backend.bt_mut().opt_seed_plan_scratch = seed_plan;
        // The DP scratch buffers are owned by the caller (start_matching_optimal)
        // and shared across both passes, so the seed pass leaves them in place
        // (`plan_buffers` is a borrow) instead of restoring them here.
        self.backend.bt_mut().opt_state = opt_state;

        // Upstream zstd initStats_ultra keeps the collected entropy statistics but
        // invalidates the first-pass matchfinder history before the real pass.
        // The offset between stored and absolute positions grows past every
        // entry the first pass stored, counted from the offset the frame
        // started with: a reused compressor carries the previous frame's, and
        // from any other start some first-pass entries would decode back into
        // the window. From a fresh start this is `index_shift = current_len`.
        // Formed relative to the floor: it is the stored index of the block's
        // end, which the first pass kept representable, while the block's
        // absolute end plus the old offset need not fit a 32-bit word.
        debug_assert!(self.table.position_base <= current_abs_start);
        let index_shift =
            current_abs_start - self.table.position_base + current_len + self.table.index_shift;
        self.table.position_base = self.table.history_abs_start;
        self.table.index_shift = index_shift;
        self.table.next_to_update3 = current_abs_start;
        self.table.skip_insert_until_abs = current_abs_start;
    }

    /// Take the five DP scratch buffers out of the backend ONCE per block and
    /// size them for the optimal-parser frontier, so the per-segment
    /// `build_optimal_plan` calls borrow and reuse them instead of taking +
    /// restoring all five every call (which on near-random input runs once per
    /// literal). Restore with [`BtMatcher::finish_optimal_plan`] after the
    /// per-segment loop.
    fn take_optimal_plan_buffers(&mut self) -> HcOptimalPlanBuffers {
        let bt = self.backend.bt_mut();
        let mut nodes = core::mem::take(&mut bt.opt_nodes_scratch);
        let mut node_prices = core::mem::take(&mut bt.opt_node_prices_scratch);
        let mut candidates = core::mem::take(&mut bt.opt_candidates_scratch);
        let store = core::mem::take(&mut bt.opt_store_scratch);
        let mut price_arena = core::mem::take(&mut bt.opt_price_arena);
        // Allocated, never filled: the DP reads only cells it wrote in the same
        // call, so a frame whose frontier stays short touches only what it
        // reaches.
        if nodes.len() < HC_OPT_NODE_LEN {
            nodes = alloc::boxed::Box::new_uninit_slice(HC_OPT_NODE_LEN);
        }
        if node_prices.len() < HC_OPT_NODE_LEN {
            node_prices = alloc::boxed::Box::new_uninit_slice(HC_OPT_NODE_LEN);
        }
        candidates.clear();
        candidates.reserve_exact(HC_OPT_CANDIDATE_CAPACITY);
        if price_arena.len() < HC_OPT_PRICE_ARENA_LEN {
            price_arena = alloc::vec![[0u32; 2]; HC_OPT_PRICE_ARENA_LEN].into_boxed_slice();
        }
        HcOptimalPlanBuffers {
            nodes,
            node_prices,
            candidates,
            store,
            price_arena,
            // Nothing in the buffer answers a query yet: the block that filled
            // it is over, and the parser is about to start another.
            candidates_searched_at: None,
            pass: HcBlockPass::default(),
        }
    }

    /// NEON-umbrella pass over one block (see [`optimal_block_body!`]).
    /// `collect_optimal_candidates_initialized_neon` shares the umbrella, so
    /// the per-position pipeline needs no feature re-dispatch.
    #[cfg(all(
        target_arch = "aarch64",
        target_endian = "little",
        feature = "kernel-neon"
    ))]
    #[target_feature(enable = "neon")]
    #[allow(clippy::too_many_arguments)]
    unsafe fn run_optimal_block_neon<
        S: crate::encoding::strategy::Strategy,
        const ACCURATE_PRICE: bool,
        const FAVOR_SMALL_OFFSETS: bool,
        const HAS_LDM: bool,
        const KEEP_PLAN: bool,
    >(
        &mut self,
        current: &[u8],
        current_abs_start: usize,
        cursor: usize,
        litlen: usize,
        reps: [u32; 3],
        profile: HcOptimalCostProfile,
        opt_state: &mut HcOptState,
        plan: &mut Vec<HcOptimalSequence>,
        buffers: &mut HcOptimalPlanBuffers,
    ) {
        optimal_block_body!(
            self,
            S,
            current,
            current_abs_start,
            cursor,
            litlen,
            reps,
            profile,
            opt_state,
            plan,
            buffers,
            KEEP_PLAN,
            collect_optimal_candidates_initialized_neon,
            priceset_neon,
        )
    }

    // Stays on the SSE4.2 umbrella: the DP body calls
    // `priceset_range_nonabort_sse41`, whose `_mm_min_epu32` is SSE4.1, so
    // this tier needs more than the SSE2 vector ops the fastpath module uses.
    #[cfg(all(
        any(target_arch = "x86", target_arch = "x86_64"),
        feature = "kernel-sse"
    ))]
    #[target_feature(enable = "sse4.2")]
    #[allow(clippy::too_many_arguments)]
    unsafe fn run_optimal_block_sse42<
        S: crate::encoding::strategy::Strategy,
        const ACCURATE_PRICE: bool,
        const FAVOR_SMALL_OFFSETS: bool,
        const HAS_LDM: bool,
        const KEEP_PLAN: bool,
    >(
        &mut self,
        current: &[u8],
        current_abs_start: usize,
        cursor: usize,
        litlen: usize,
        reps: [u32; 3],
        profile: HcOptimalCostProfile,
        opt_state: &mut HcOptState,
        plan: &mut Vec<HcOptimalSequence>,
        buffers: &mut HcOptimalPlanBuffers,
    ) {
        optimal_block_body!(
            self,
            S,
            current,
            current_abs_start,
            cursor,
            litlen,
            reps,
            profile,
            opt_state,
            plan,
            buffers,
            KEEP_PLAN,
            collect_optimal_candidates_initialized_sse42,
            priceset_sse41,
        )
    }

    /// SSE2 twin of [`Self::run_optimal_block_sse42`] for x86 CPUs
    /// without SSE4.1/4.2: same 128-bit pipeline, with the price set using
    /// the SSE2 unsigned-compare emulation.
    #[cfg(all(
        any(target_arch = "x86", target_arch = "x86_64"),
        feature = "kernel-sse"
    ))]
    #[target_feature(enable = "sse2")]
    #[allow(clippy::too_many_arguments)]
    unsafe fn run_optimal_block_sse2<
        S: crate::encoding::strategy::Strategy,
        const ACCURATE_PRICE: bool,
        const FAVOR_SMALL_OFFSETS: bool,
        const HAS_LDM: bool,
        const KEEP_PLAN: bool,
    >(
        &mut self,
        current: &[u8],
        current_abs_start: usize,
        cursor: usize,
        litlen: usize,
        reps: [u32; 3],
        profile: HcOptimalCostProfile,
        opt_state: &mut HcOptState,
        plan: &mut Vec<HcOptimalSequence>,
        buffers: &mut HcOptimalPlanBuffers,
    ) {
        optimal_block_body!(
            self,
            S,
            current,
            current_abs_start,
            cursor,
            litlen,
            reps,
            profile,
            opt_state,
            plan,
            buffers,
            KEEP_PLAN,
            collect_optimal_candidates_initialized_sse2,
            priceset_sse2,
        )
    }

    #[cfg(all(
        any(target_arch = "x86", target_arch = "x86_64"),
        feature = "kernel-avx2"
    ))]
    #[target_feature(enable = "avx2,bmi2")]
    #[allow(clippy::too_many_arguments)]
    unsafe fn run_optimal_block_avx2_bmi2<
        S: crate::encoding::strategy::Strategy,
        const ACCURATE_PRICE: bool,
        const FAVOR_SMALL_OFFSETS: bool,
        const HAS_LDM: bool,
        const KEEP_PLAN: bool,
    >(
        &mut self,
        current: &[u8],
        current_abs_start: usize,
        cursor: usize,
        litlen: usize,
        reps: [u32; 3],
        profile: HcOptimalCostProfile,
        opt_state: &mut HcOptState,
        plan: &mut Vec<HcOptimalSequence>,
        buffers: &mut HcOptimalPlanBuffers,
    ) {
        optimal_block_body!(
            self,
            S,
            current,
            current_abs_start,
            cursor,
            litlen,
            reps,
            profile,
            opt_state,
            plan,
            buffers,
            KEEP_PLAN,
            collect_optimal_candidates_initialized_avx2_bmi2,
            priceset_avx2,
        )
    }

    #[cfg(not(all(
        target_arch = "aarch64",
        target_endian = "little",
        feature = "kernel-neon"
    )))]
    // Body macros wrap callees in `unsafe { }` for the NEON/AVX/SSE
    // variants where callees are `unsafe fn`. The scalar wrappers route
    // through safe fns, so those blocks are redundant on this path.
    #[allow(unused_unsafe)]
    // The dispatch reaches this only on non-SIMD x86 (Scalar tier) and the
    // portable fallback; on wasm+simd128 the simd128 wrapper is selected, so
    // this is cfg-dead there.
    #[cfg_attr(
        all(
            target_arch = "wasm32",
            target_feature = "simd128",
            feature = "kernel-simd128"
        ),
        allow(dead_code)
    )]
    #[allow(clippy::too_many_arguments)]
    fn run_optimal_block_scalar<
        S: crate::encoding::strategy::Strategy,
        const ACCURATE_PRICE: bool,
        const FAVOR_SMALL_OFFSETS: bool,
        const HAS_LDM: bool,
        const KEEP_PLAN: bool,
    >(
        &mut self,
        current: &[u8],
        current_abs_start: usize,
        cursor: usize,
        litlen: usize,
        reps: [u32; 3],
        profile: HcOptimalCostProfile,
        opt_state: &mut HcOptState,
        plan: &mut Vec<HcOptimalSequence>,
        buffers: &mut HcOptimalPlanBuffers,
    ) {
        optimal_block_body!(
            self,
            S,
            current,
            current_abs_start,
            cursor,
            litlen,
            reps,
            profile,
            opt_state,
            plan,
            buffers,
            KEEP_PLAN,
            collect_optimal_candidates_initialized_scalar,
            priceset_scalar,
        )
    }

    /// wasm `simd128`-umbrella DP body: scalar candidate collection (no wasm
    /// collect kernel) but the simd128 4-lane price-set.
    #[cfg(all(
        target_arch = "wasm32",
        target_feature = "simd128",
        feature = "kernel-simd128"
    ))]
    #[target_feature(enable = "simd128")]
    // With `+simd128` in the wasm baseline the shared body macro's `unsafe`
    // blocks (needed by the safe scalar wrapper) are redundant inside this
    // target_feature fn.
    #[allow(unused_unsafe)]
    #[allow(clippy::too_many_arguments)]
    unsafe fn run_optimal_block_simd128<
        S: crate::encoding::strategy::Strategy,
        const ACCURATE_PRICE: bool,
        const FAVOR_SMALL_OFFSETS: bool,
        const HAS_LDM: bool,
        const KEEP_PLAN: bool,
    >(
        &mut self,
        current: &[u8],
        current_abs_start: usize,
        cursor: usize,
        litlen: usize,
        reps: [u32; 3],
        profile: HcOptimalCostProfile,
        opt_state: &mut HcOptState,
        plan: &mut Vec<HcOptimalSequence>,
        buffers: &mut HcOptimalPlanBuffers,
    ) {
        optimal_block_body!(
            self,
            S,
            current,
            current_abs_start,
            cursor,
            litlen,
            reps,
            profile,
            opt_state,
            plan,
            buffers,
            KEEP_PLAN,
            collect_optimal_candidates_initialized_simd128,
            priceset_simd128,
        )
    }

    #[cfg(test)]
    pub(crate) fn collect_optimal_candidates(
        &mut self,
        abs_pos: usize,
        current_abs_end: usize,
        sufficient_match_len: usize,
        query: HcCandidateQuery,
        out: &mut Vec<MatchCandidate>,
    ) {
        use crate::encoding::strategy::{self, StrategyTag};
        self.table.ensure_tables();
        self.table.capture_block_coords();
        // As a block's pass starts it (see `optimal_block_body!`), and with
        // the room the search writes its candidates into.
        self.table.skip_insert_until_abs = self
            .table
            .skip_insert_until_abs
            .max(self.table.history_abs_start);
        out.clear();
        out.reserve_exact(HC_OPT_CANDIDATE_CAPACITY);
        let reps = &query.reps;
        let ll0 = query.lit_len == 0;
        // Dispatch purely from `self.strategy_tag` (set by
        // `configure()`). Tests must configure the matcher the same
        // way production does — wiring up `table.hash3_log` directly
        // without setting a matching `strategy_tag` is no longer
        // allowed.
        // The key width picks the search the way the block's pass picks it.
        macro_rules! search {
            ($strategy:ty) => {
                match self.table.search_mls {
                    3 => self.collect_optimal_candidates_initialized::<$strategy, 3>(
                        abs_pos,
                        current_abs_end,
                        sufficient_match_len,
                        reps,
                        ll0,
                        out,
                    ),
                    5 => self.collect_optimal_candidates_initialized::<$strategy, 5>(
                        abs_pos,
                        current_abs_end,
                        sufficient_match_len,
                        reps,
                        ll0,
                        out,
                    ),
                    6 => self.collect_optimal_candidates_initialized::<$strategy, 6>(
                        abs_pos,
                        current_abs_end,
                        sufficient_match_len,
                        reps,
                        ll0,
                        out,
                    ),
                    _ => self.collect_optimal_candidates_initialized::<$strategy, 4>(
                        abs_pos,
                        current_abs_end,
                        sufficient_match_len,
                        reps,
                        ll0,
                        out,
                    ),
                }
            };
        }
        match self.strategy_tag {
            StrategyTag::BtUltra2 => search!(strategy::BtUltra2),
            StrategyTag::BtUltra => search!(strategy::BtUltra),
            StrategyTag::Btlazy2 => search!(strategy::Btlazy2),
            StrategyTag::BtOpt => search!(strategy::BtOpt),
            StrategyTag::Fast | StrategyTag::Dfast | StrategyTag::Greedy | StrategyTag::Lazy => {
                // Optimal candidate collection is binary-tree only (btopt /
                // btultra / btultra2 / btlazy2). The Fast / Dfast / Greedy / Lazy
                // strategies never run the optimal parser — they drive their own
                // match finders — and their generators keep `chain_table` as an
                // HC chain, not BT pair slots. Routing them here would walk that
                // chain as a binary tree. Reaching this arm is a caller bug.
                unreachable!(
                    "collect_optimal_candidates is binary-tree only; \
                     non-BT strategies use their own match finder"
                )
            }
        }
        if let Some(ldm) = query.ldm_candidate {
            BtMatcher::push_ldm_candidate(out, ldm, HC_OPT_MIN_MATCH_LEN);
        }
    }

    /// Cross-platform entry. Picks the kernel-specific variant so the per-
    /// position pipeline (BT-tree fill, rep probing, hash3 probing, BT
    /// collect / HC chain walk) runs inside a single `target_feature`
    /// umbrella — all inner SIMD probes inline without ABI barriers.
    ///
    /// The on-encode hot path bypasses this dispatcher: `run_optimal_block_<kernel>`
    /// calls the matching `_<kernel>` variant directly. This entry is kept
    /// for the cfg(test)-only `collect_optimal_candidates` shim and any
    /// future caller that isn't already inside a kernel umbrella.
    ///
    /// Six arms, and no host compiles more than half of them: aarch64 sees the
    /// NEON one, x86 sees three behind a runtime match, and the wasm and
    /// portable ones are only reachable on their own targets. So a change to
    /// this function's argument list has to be applied to every arm by reading,
    /// not by compiling — a local `cargo check` on either development
    /// architecture will happily accept a call that is missing an argument in
    /// the arms it cannot see. `cargo clippy --target wasm32-unknown-unknown`
    /// and a `--no-default-features --features kernel-scalar` build are what
    /// cover the rest.
    #[allow(dead_code)]
    #[inline(always)]
    pub(crate) fn collect_optimal_candidates_initialized<
        S: crate::encoding::strategy::Strategy,
        const MLS: usize,
    >(
        &mut self,
        abs_pos: usize,
        current_abs_end: usize,
        sufficient_match_len: usize,
        reps: &[u32; 3],
        ll0: bool,
        out: &mut Vec<MatchCandidate>,
    ) {
        #[cfg(all(
            target_arch = "aarch64",
            target_endian = "little",
            feature = "kernel-neon"
        ))]
        unsafe {
            self.collect_optimal_candidates_initialized_neon::<S, MLS>(
                abs_pos,
                current_abs_end,
                sufficient_match_len,
                reps,
                ll0,
                out,
            )
        }
        #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
        {
            use crate::encoding::fastpath::FastpathKernel;
            match self.table.kernel {
                #[cfg(feature = "kernel-avx2")]
                FastpathKernel::Avx2Bmi2 => unsafe {
                    self.collect_optimal_candidates_initialized_avx2_bmi2::<S, MLS>(
                        abs_pos,
                        current_abs_end,
                        sufficient_match_len,
                        reps,
                        ll0,
                        out,
                    )
                },
                #[cfg(feature = "kernel-sse")]
                FastpathKernel::Sse2 => unsafe {
                    self.collect_optimal_candidates_initialized_sse2::<S, MLS>(
                        abs_pos,
                        current_abs_end,
                        sufficient_match_len,
                        reps,
                        ll0,
                        out,
                    )
                },
                #[cfg(feature = "kernel-sse")]
                FastpathKernel::Sse42 => unsafe {
                    self.collect_optimal_candidates_initialized_sse42::<S, MLS>(
                        abs_pos,
                        current_abs_end,
                        sufficient_match_len,
                        reps,
                        ll0,
                        out,
                    )
                },
                FastpathKernel::Scalar => self
                    .collect_optimal_candidates_initialized_scalar::<S, MLS>(
                        abs_pos,
                        current_abs_end,
                        sufficient_match_len,
                        reps,
                        ll0,
                        out,
                    ),
            }
        }
        // wasm resolves `simd128` at compile time (no runtime detection), so
        // the tier comes from `cfg`, not from `self.table.kernel`.
        #[cfg(all(
            target_arch = "wasm32",
            target_feature = "simd128",
            feature = "kernel-simd128"
        ))]
        // SAFETY: the `cfg` above establishes `simd128` at compile time, which
        // is exactly the umbrella the callee declares.
        unsafe {
            self.collect_optimal_candidates_initialized_simd128::<S, MLS>(
                abs_pos,
                current_abs_end,
                sufficient_match_len,
                reps,
                ll0,
                out,
            )
        }
        #[cfg(not(any(
            all(
                target_arch = "aarch64",
                target_endian = "little",
                feature = "kernel-neon"
            ),
            target_arch = "x86",
            target_arch = "x86_64",
            all(
                target_arch = "wasm32",
                target_feature = "simd128",
                feature = "kernel-simd128"
            )
        )))]
        {
            self.collect_optimal_candidates_initialized_scalar::<S, MLS>(
                abs_pos,
                current_abs_end,
                sufficient_match_len,
                reps,
                ll0,
                out,
            )
        }
    }

    /// NEON-umbrella variant: the repeat and hash3 probes, the tree walk and
    /// `fastpath::neon::common_prefix_len_ptr` share the NEON umbrella so the
    /// per-position search runs as one straight-line sequence, and the tree
    /// catch-up (`bt_insert_range_neon`) runs under the same tier.
    #[cfg(all(
        target_arch = "aarch64",
        target_endian = "little",
        feature = "kernel-neon"
    ))]
    #[target_feature(enable = "neon")]
    #[inline(never)]
    unsafe fn collect_optimal_candidates_initialized_neon<
        S: crate::encoding::strategy::Strategy,
        const MLS: usize,
    >(
        &mut self,
        abs_pos: usize,
        current_abs_end: usize,
        sufficient_match_len: usize,
        reps: &[u32; 3],
        ll0: bool,
        out: &mut Vec<MatchCandidate>,
    ) {
        collect_optimal_candidates_initialized_body!(
            self,
            S,
            abs_pos,
            current_abs_end,
            sufficient_match_len,
            reps,
            ll0,
            out,
            MLS,
            bt_insert_range_neon,
            crate::encoding::fastpath::neon::common_prefix_len_ptr,
            crate::encoding::fastpath::neon::count_match_from_indices,
        )
    }

    #[cfg(all(
        any(target_arch = "x86", target_arch = "x86_64"),
        feature = "kernel-sse"
    ))]
    #[target_feature(enable = "sse4.2")]
    #[inline(never)]
    unsafe fn collect_optimal_candidates_initialized_sse42<
        S: crate::encoding::strategy::Strategy,
        const MLS: usize,
    >(
        &mut self,
        abs_pos: usize,
        current_abs_end: usize,
        sufficient_match_len: usize,
        reps: &[u32; 3],
        ll0: bool,
        out: &mut Vec<MatchCandidate>,
    ) {
        collect_optimal_candidates_initialized_body!(
            self,
            S,
            abs_pos,
            current_abs_end,
            sufficient_match_len,
            reps,
            ll0,
            out,
            MLS,
            bt_insert_range_sse2,
            crate::encoding::fastpath::sse2::common_prefix_len_ptr,
            crate::encoding::fastpath::sse2::count_match_from_indices,
        )
    }

    /// SSE2 twin of [`Self::collect_optimal_candidates_initialized_sse42`].
    /// The per-position pipeline is identical; only the enclosing umbrella
    /// differs, so the SSE2 tier keeps the same straight-line inlining.
    ///
    /// # Safety
    /// x86/x86_64 with SSE2.
    #[cfg(all(
        any(target_arch = "x86", target_arch = "x86_64"),
        feature = "kernel-sse"
    ))]
    #[target_feature(enable = "sse2")]
    #[inline(never)]
    unsafe fn collect_optimal_candidates_initialized_sse2<
        S: crate::encoding::strategy::Strategy,
        const MLS: usize,
    >(
        &mut self,
        abs_pos: usize,
        current_abs_end: usize,
        sufficient_match_len: usize,
        reps: &[u32; 3],
        ll0: bool,
        out: &mut Vec<MatchCandidate>,
    ) {
        collect_optimal_candidates_initialized_body!(
            self,
            S,
            abs_pos,
            current_abs_end,
            sufficient_match_len,
            reps,
            ll0,
            out,
            MLS,
            bt_insert_range_sse2,
            crate::encoding::fastpath::sse2::common_prefix_len_ptr,
            crate::encoding::fastpath::sse2::count_match_from_indices,
        )
    }

    #[cfg(all(
        any(target_arch = "x86", target_arch = "x86_64"),
        feature = "kernel-avx2"
    ))]
    #[target_feature(enable = "avx2,bmi2")]
    #[inline(never)]
    unsafe fn collect_optimal_candidates_initialized_avx2_bmi2<
        S: crate::encoding::strategy::Strategy,
        const MLS: usize,
    >(
        &mut self,
        abs_pos: usize,
        current_abs_end: usize,
        sufficient_match_len: usize,
        reps: &[u32; 3],
        ll0: bool,
        out: &mut Vec<MatchCandidate>,
    ) {
        collect_optimal_candidates_initialized_body!(
            self,
            S,
            abs_pos,
            current_abs_end,
            sufficient_match_len,
            reps,
            ll0,
            out,
            MLS,
            bt_insert_range_avx2_bmi2,
            crate::encoding::fastpath::avx2_bmi2::common_prefix_len_ptr,
            crate::encoding::fastpath::avx2_bmi2::count_match_from_indices,
        )
    }

    /// WebAssembly `simd128` umbrella variant: the BT step, the match-length
    /// probe and the prefix compare all share the `simd128` umbrella so the
    /// per-position pipeline stays one inline sequence.
    ///
    /// # Safety
    /// wasm32 with `simd128` enabled at compile time.
    #[cfg(all(
        target_arch = "wasm32",
        target_feature = "simd128",
        feature = "kernel-simd128"
    ))]
    #[target_feature(enable = "simd128")]
    #[inline(never)]
    unsafe fn collect_optimal_candidates_initialized_simd128<
        S: crate::encoding::strategy::Strategy,
        const MLS: usize,
    >(
        &mut self,
        abs_pos: usize,
        current_abs_end: usize,
        sufficient_match_len: usize,
        reps: &[u32; 3],
        ll0: bool,
        out: &mut Vec<MatchCandidate>,
    ) {
        collect_optimal_candidates_initialized_body!(
            self,
            S,
            abs_pos,
            current_abs_end,
            sufficient_match_len,
            reps,
            ll0,
            out,
            MLS,
            bt_insert_range_simd128,
            crate::encoding::fastpath::simd128::common_prefix_len_ptr,
            crate::encoding::fastpath::simd128::count_match_from_indices,
        )
    }

    #[cfg(not(all(
        target_arch = "aarch64",
        target_endian = "little",
        feature = "kernel-neon"
    )))]
    // Macro emits `unsafe { }` wrappers for NEON/AVX/SSE variants; scalar
    // callees are safe so the blocks are redundant here only.
    #[allow(unused_unsafe)]
    #[inline(never)]
    pub(crate) fn collect_optimal_candidates_initialized_scalar<
        S: crate::encoding::strategy::Strategy,
        const MLS: usize,
    >(
        &mut self,
        abs_pos: usize,
        current_abs_end: usize,
        sufficient_match_len: usize,
        reps: &[u32; 3],
        ll0: bool,
        out: &mut Vec<MatchCandidate>,
    ) {
        collect_optimal_candidates_initialized_body!(
            self,
            S,
            abs_pos,
            current_abs_end,
            sufficient_match_len,
            reps,
            ll0,
            out,
            MLS,
            bt_insert_range_scalar,
            crate::encoding::fastpath::scalar::common_prefix_len_ptr,
            crate::encoding::fastpath::scalar::count_match_from_indices,
        )
    }
}