rusty_h264_encoder/

mb16.rs

//! I_16x16 macroblock encoding (DC prediction) — the compressing intra path.
//!
//! Index-based loops below drive pixel/block-position arithmetic and read
//! clearer than iterator adapters for this raster math.
#![allow(clippy::needless_range_loop)]
//!
//! Each macroblock is DC-predicted from already-reconstructed neighbors, the
//! residual is transformed/quantized (luma DC via the secondary Hadamard), the
//! coefficients are CAVLC-coded, and the macroblock is reconstructed so the next
//! one can predict from it. `nnz` grids feed the CAVLC `nC` context exactly as a
//! conforming decoder derives it.

#[allow(unused_imports)]
use alloc::vec;
#[allow(unused_imports)]
use alloc::vec::Vec;
#[allow(unused_imports)]
use rusty_h264_common::fmath::{F32Ext as _, F64Ext as _};
#[allow(unused_imports)]
use rusty_h264_common::once::OnceLock;

use crate::cabac::CabacEncoder;
use crate::config::EncoderConfig;
use crate::signals::{self, FrameSignals};
use rusty_h264_common::aligned::AlignedBytes;
use rusty_h264_common::cavlc::{
    encode_residual_block, scan_4x4_ac, scan_4x4_dcac, write_cbp_inter, write_cbp_intra,
};
use rusty_h264_common::inter::{
    inter_partitions, mc_chroma, mc_luma, predict_mv, predict_partition_mv, MvNeighbor,
};
#[cfg(not(accel))]
use rusty_h264_common::predict::add_residual_4x4;
use rusty_h264_common::predict::{
    add_residual_8x8, chroma8x8_pred, chroma_mode_available, chroma_qp, intra4x4_pred,
    intra8x8_pred, luma16x16_pred, reconstruct_4x4, reconstruct_4x4_into, I16Mode,
    CHROMA_4X4_SCAN_XY, LUMA_4X4_SCAN_XY,
};
#[cfg(not(accel))]
use rusty_h264_common::transform::inverse_dct_blocks;
use rusty_h264_common::transform::{
    dequantize, forward_core, forward_core_8x8, forward_dct_blocks, forward_quant_chroma_dc,
    forward_quant_luma_dc, inverse_quant_8x8, inverse_quant_chroma_dc, inverse_quant_luma_dc,
    quantize, quantize_8x8, satd_4x4_sum,
};
use rusty_h264_common::{BitWriter, YuvPlanes};

/// A/B switch for the batched full-pel rescue grid (`RFF_ME_BATCH=0` disables).
///
/// Read ONCE per process, not per call: this sits inside the motion-search rescue
/// path, and `std::env::var` allocates a `String` and takes the process-wide
/// environment lock every time. A runtime switch inside a hot loop is its own
/// measurable tax — cache it.

/// λ-normalised threshold for the partition-split search gate (U2).
///
/// The existing `split_gate` is a function of qstep ALONE, so it does not scale with
/// the rate/distortion trade the search is actually making. Normalising the null arm
/// by λ — the king feature for any search-skip gate — makes one constant transfer
/// across content AND the whole QP ladder, and in the SAFE direction: the feature is
/// small exactly where the 16×16 null arm is already good, so easy content skips more.
///
/// Harvested over 36 k gated macroblocks (4 clips): at T = 400 the split search is
/// skipped on 2.9–22.5% of them while keeping **100.00%** of the achievable cost gain
/// on every clip; T = 600 skips 11–79% for 93–99% kept. `RFF_SPLIT_T=0` disables.
pub(crate) static DEFER_SUBPEL: core::sync::atomic::AtomicU32 =
    core::sync::atomic::AtomicU32::new(0);

pub(crate) static SPLIT_T: core::sync::atomic::AtomicU32 =
    core::sync::atomic::AtomicU32::new(u32::MAX);

/// Observe-only HARVEST for the sub-8x8 dispatch fit (Great Gate P3.3 gate —
/// docs/gate-ledger.md sub8x8-split). One CSV row per P_8x8 quad decision:
/// the 8x8 arm's J, the best SPLIT arm's J (tracked even when 8x8 wins), the
/// chosen sub_mb_type, lme (for margin normalization — the null-arm-over-λ
/// king-feature law), and the MB's variance. `RFF_SUB8_HARVEST=<path>`.
/// R1 PRE-CHECK instrument (docs/gate-repair-plan.md): the SIGNED RD regret of the
/// SATD split decision, per macroblock, in lambda units.
///
/// The census says RD overturns the SATD split pick on 33.8-81.4% of the macroblocks
/// where SATD chose to split. A RATE cannot justify refitting the proxy: `prom_av1e004`
/// was a 3x more accurate cost model that measured DEAD NEUTRAL because its error was
/// rank-invariant near the argmin. What decides R1 is the MAGNITUDE of the disagreement,
/// and the existing harvest throws it away -- `split_gain` is recorded only when the
/// split is KEPT and zeroed on revert.
///
/// So record one signed number:
///
/// ```text
/// dj = (j_split - j_flat) / lambda
///
///   dj > 0  RD reverted: following SATD would have cost `dj` lambda-units. REGRET.
///   dj < 0  RD kept it:  the split saved `-dj`. GAIN.
/// ```
///
/// If the regret mass sits near zero, SATD's false positives are near-ties, the RD pass
/// is expensive insurance against nothing, and R1 closes without touching the proxy.
/// A fat regret tail is the only thing that justifies a refit.
///
///   RFF_SUB8_REGRET=<path>   zero cost when unset (OnceLock + Option, as sub8_harvest)
mod sub8_regret {
    #[allow(unused_imports)]
    use alloc::{
        boxed::Box,
        format,
        string::{String, ToString},
        vec,
        vec::Vec,
    };
    #[allow(unused_imports)]
    use rusty_h264_common::once::OnceLock;
    #[cfg(feature = "std")]
    #[cfg(feature = "std")]
    use std::io::Write;
    #[cfg(feature = "std")]
    #[cfg(feature = "std")]
    use std::sync::Mutex;

    #[cfg(not(feature = "std"))]
    #[allow(dead_code)]

    fn sink() -> &'static Option<()> {
        static NONE: Option<()> = None;

        &NONE
    }

    #[cfg(feature = "std")]

    fn sink() -> &'static Option<Mutex<std::fs::File>> {
        static S: OnceLock<Option<Mutex<std::fs::File>>> = OnceLock::new();
        S.get_or_init(|| {
            rusty_h264_common::knob("RFF_SUB8_REGRET").and_then(|p| {
                let mut f = std::fs::File::create(p).ok()?;
                let _ = writeln!(f, "reverted,j_split,j_flat,lambda,dj_lambda,split_quads");
                Some(Mutex::new(f))
            })
        })
    }

    #[inline]
    pub fn enabled() -> bool {
        sink().is_some()
    }

    /// One macroblock's RD trial outcome. `ja`/`jb` are the split and flat J values.
    #[cfg_attr(not(feature = "std"), allow(unused_variables))]
    pub fn record(ja: f64, jb: f64, lambda: f64, split_quads: usize) {
        #[cfg(feature = "std")]
        if let Some(m) = sink() {
            if let Ok(mut f) = m.lock() {
                let dj = (ja - jb) / lambda.max(1e-9);
                let _ = writeln!(
                    f,
                    "{},{:.1},{:.1},{:.4},{:.4},{}",
                    (jb <= ja) as u8,
                    ja,
                    jb,
                    lambda,
                    dj,
                    split_quads
                );
            }
        }
    }
}

mod sub8_harvest {
    #[allow(unused_imports)]
    use alloc::{
        boxed::Box,
        format,
        string::{String, ToString},
        vec,
        vec::Vec,
    };
    #[allow(unused_imports)]
    use rusty_h264_common::once::OnceLock;
    #[cfg(feature = "std")]
    #[cfg(feature = "std")]
    use std::io::Write;
    #[cfg(feature = "std")]
    #[cfg(feature = "std")]
    use std::sync::Mutex;

    #[cfg(not(feature = "std"))]
    #[allow(dead_code)]

    fn sink() -> &'static Option<()> {
        static NONE: Option<()> = None;

        &NONE
    }

    #[cfg(feature = "std")]

    fn sink() -> &'static Option<Mutex<std::fs::File>> {
        static S: OnceLock<Option<Mutex<std::fs::File>>> = OnceLock::new();
        S.get_or_init(|| {
            rusty_h264_common::knob("RFF_SUB8_HARVEST").and_then(|p| {
                let mut f = std::fs::File::create(p).ok()?;
                // `j8_lme` and `mbvar` are PRE-SEARCH: both are known before the
                // 8 extra motion searches this gate would skip. `st`/`jsplit` are
                // post-search (kept for context), `rd_kept` is the label — did the
                // macroblock's RD trial ultimately KEEP the split?
                let _ = writeln!(f, "j8,jsplit,st,lme,mbvar,j8_lme,mvdiv,rd_kept");
                Some(Mutex::new(f))
            })
        })
    }

    /// One quad's row, buffered until the macroblock's RD trial resolves (the
    /// label is only known then).
    #[cfg_attr(not(feature = "std"), allow(dead_code))]
    pub struct Row {
        pub j8: i64,
        pub jsplit: i64,
        pub st: u8,
        pub lme: f64,
        pub mbvar: i64,
        /// PRE-SEARCH motion divergence: |mv_quad - mv16| in quarter-pel. Known
        /// after the ONE 8x8 search we always run, before the EIGHT sub-searches
        /// this gate would skip. Unlike `j8_lme` (difficulty) and `mbvar`
        /// (texture) — both refuted — this measures the thing splitting actually
        /// exploits: a motion BOUNDARY inside the quad. A quad moving with its
        /// parent has nothing to split.
        pub mvdiv: i32,
    }

    #[inline]
    pub fn enabled() -> bool {
        sink().is_some()
    }

    /// Flushes a macroblock's buffered quad rows with the RD outcome attached.
    #[cfg_attr(not(feature = "std"), allow(unused_variables))]
    pub fn flush(rows: &[Row], rd_kept: bool) {
        #[cfg(feature = "std")]
        if let Some(m) = sink() {
            if let Ok(mut f) = m.lock() {
                for r in rows {
                    let jl = r.j8 as f64 / r.lme.max(1e-9);
                    let _ = writeln!(
                        f,
                        "{},{},{},{:.3},{},{:.2},{},{}",
                        r.j8, r.jsplit, r.st, r.lme, r.mbvar, jl, r.mvdiv, rd_kept as u8
                    );
                }
            }
        }
    }
}

/// P3 RD-pricing probe #2: price the INTRA-vs-INTER decision in the RD
/// currency instead of the SATD proxy + fitted `tune_intra_penalty`.
/// `RFF_INTRA_RD=1`.
///
/// Today: `c_intra = best_i16_satd + lme*tune_intra_penalty` vs the inter
/// arm's SATD cost. Both sides are prediction-error proxies, and the penalty
/// constant exists precisely to correct the proxy's bias — a fitted patch over
/// a wrong-sign currency (the same defect the sub-8x8 probe confirmed, where
/// re-pricing moved the worst clip 4.3 points AND improved the winners). The
/// evaluator needed here already existed and had ZERO callers
/// (`trial_intra`): snapshot -> encode into scratch -> real bits + recon SSD
/// -> restore. "Exported != wired", the same law as the SATD asm kernel that
/// sat uncalled for months.
fn intra_rd_on() -> bool {
    rusty_h264_common::cached_knob!(
        bool,
        rusty_h264_common::knob("RFF_INTRA_RD")
            .map(|v| v == "1")
            .unwrap_or(false)
    )
}

/// P3 RD-pricing probe #3: price the PARTITION SHAPE decision (16x16 vs 16x8
/// vs 8x16 vs P_8x8) in the RD currency. `RFF_SHAPE_RD=1`.
///
/// The third and last SATD-priced DEFAULT-ON site. The shapes are compared on
/// `best_part`'s SATD+lambda*mvbits costs today; finer shapes always reduce
/// prediction error, so the same wrong-sign bias that made sub-8x8 a net loser
/// should bias this decision toward over-splitting too — mildly, since 16x8
/// has far less freedom to fit noise than 4x4. Probe, do not assume.
/// Weight applied to chroma SSD inside the inter RD trials. 1.0 = the original
/// equal-weight sum (byte-identical).
fn chroma_ssd_weight() -> f64 {
    rusty_h264_common::cached_knob!(f64, {
        rusty_h264_common::knob("RFF_RD_CHROMA_W")
            .and_then(|v| v.parse().ok())
            .unwrap_or(1.0)
    })
}

/// Texture ceiling above which shape-RD is vetoed (see the call site).
fn shape_rd_tex_max() -> i64 {
    rusty_h264_common::cached_knob!(i64, {
        rusty_h264_common::knob("RFF_SHAPE_RD_TEXMAX")
            .and_then(|v| v.parse().ok())
            // REFIT 1000 -> 2000 on 2026-08-08. The original 1000 was placed in the
            // open gap of a 24-clip 4x4-era table, below mobile_cif (1494) because
            // mobile then LOST +1.99% BD-SSIM with the shape-RD pass on. Re-measured
            // on the current encoder, mobile now WINS -0.90% (I+P) / -0.45% (I+P+B)
            // unvetoed — a ~2.9-point sign flip on the same clip and the same knob.
            // The threshold-transfer law, exactly: a threshold is only valid for the
            // encoder it was fitted against, and this one went stale.
            //
            // NOT removed, because the above-the-line HOLDOUT refuted removal:
            // maxtex_plaid (median_var 2962) regresses +3.50% unvetoed. Both
            // above-the-line clips flipped sign since the fit, in OPPOSITE
            // directions, so `median_var` still ORDERS them — the line was just in
            // the wrong place. 2000 sits in the open gap between them (1499 .. 2962):
            // mobile is released, plaid stays guarded, and every other clip in the
            // corpus is under 1000 and therefore byte-identical either way.
            //
            // The fit rests on TWO above-the-line points. Content landing in the new
            // (1000, 2000] band is unmeasured; re-run bench/gate_refit.py when a
            // clip appears there.
            .unwrap_or(2000)
    })
}

/// Shape-RD override: `Some(true/false)` from `RFF_SHAPE_RD=1/0`, `None` when unset.
///
/// This USED to return `bool` and be consumed as `shape_rd_on() || cfg.tune_shape_rd`
/// — an OR, which meant `RFF_SHAPE_RD=0` could not turn the gate OFF. shape-RD was
/// therefore the one shipped gate with NO ESCAPE HATCH: it could not be neutralised at
/// runtime, could not be Tier-0 tested by `gatecheck` (whose contract is "every gate's
/// neutral setting still reproduces the un-gated bytes"), and could not be A/B'd against
/// the `fast`-preset regression it is the leading suspect for. Returning an Option makes
/// the env an OVERRIDE in both directions.
fn shape_rd_on() -> Option<bool> {
    static ON: rusty_h264_common::once::OnceLock<Option<bool>> =
        rusty_h264_common::once::OnceLock::new();
    *ON.get_or_init(|| rusty_h264_common::knob("RFF_SHAPE_RD").map(|v| v == "1"))
}

/// `RFF_INTRA_RD_ALL=1` removes the grain gate from the intra RD probe (i.e.
/// price EVERY macroblock by RD) — the arm the 1.71x-for-nothing measurement
/// was taken on. Default: gated to grain.
fn intra_rd_grain_gate() -> bool {
    static ON: rusty_h264_common::once::OnceLock<bool> = rusty_h264_common::once::OnceLock::new();
    *ON.get_or_init(|| {
        rusty_h264_common::knob("RFF_INTRA_RD_ALL")
            .map(|v| v != "1")
            .unwrap_or(true)
    })
}

/// ONLINE SPLIT-PAYOFF CENSUS (best_part campaign, centre 2). The per-QUAD skip
/// gate was pruned with three varied pre-search probes — `j8/lambda`
/// (difficulty), `mbvar` (texture) and `mvdiv` (motion boundary) all lose wins
/// at ~the rate they skip searches (ratios 0.87-1.00), because whether a quad
/// benefits from splitting is a property of its RESIDUAL, which does not exist
/// until you split. No cheap signal predicts it.
///
/// But the same harvest shows the payoff varies 2.4x BY CONTENT — the fraction
/// of quads sitting in a macroblock whose split survived the RD trial:
/// harbour 13.7%, bus 27.7%, foreman 32.5%, mobile 33.2%. So the dispatch grain
/// is the FRAME, not the quad: measure the survival rate online over this
/// frame's first macroblocks and stop searching splits for the remainder if the
/// content is not paying. Same shape as the free-skip census that gates RD-skip
/// and greedy-skip, and me_wide's online payoff learner — within-frame, so it
/// stays deterministic under GOP-parallel encode.
///
/// ⚠ VALUE-WEIGHTED, not a count. The first cut of this census gated on the
/// PERCENTAGE of macroblocks whose split survived, and on crowd_run that threw
/// away 78% of a -2.43% BD-SSIM win to buy its speed: a frame where only a tenth
/// of splits survive can still carry a large win if those few save a lot of
/// bits. Counting decisions instead of weighting them by what they are worth is
/// exactly the objective error the suppressor campaign names as cardinal
/// (unit-weighted net gain, never classification accuracy). The census now
/// accumulates the RD J the surviving splits actually SAVE, in lambda units,
/// and requires a mean saving per searched macroblock.
///
/// `RFF_SUB8_MINPAY` = required mean J saved per searched MB, in lambda units
/// (0 disables the census); `RFF_SUB8_LEARN` = MBs observed before it may act.
fn sub8_pay_cfg() -> (usize, usize) {
    static C: rusty_h264_common::once::OnceLock<(usize, usize)> =
        rusty_h264_common::once::OnceLock::new();
    *C.get_or_init(|| {
        let p = rusty_h264_common::knob("RFF_SUB8_MINPAY")
            .and_then(|v| v.parse().ok())
            .unwrap_or(0);
        let l = rusty_h264_common::knob("RFF_SUB8_LEARN")
            .and_then(|v| v.parse().ok())
            .unwrap_or(64);
        (p, l)
    })
}

/// `RFF_SUB8_GRAIN=0` disables the sub-8x8 grain veto (bisection anchor).
fn sub8_grain_veto_on() -> bool {
    static ON: rusty_h264_common::once::OnceLock<bool> = rusty_h264_common::once::OnceLock::new();
    *ON.get_or_init(|| {
        rusty_h264_common::knob("RFF_SUB8_GRAIN")
            .map(|v| v != "0")
            .unwrap_or(true)
    })
}

/// P3.3 gate probe: re-price the SPLIT-vs-8x8 decision in the RD currency
/// (`J = SSD_recon + lambda*bits`) instead of the SATD proxy `best_part`
/// returns. `RFF_SUB8_RD=1`. See docs/gate-ledger.md sub8x8-split: SATD prices
/// PREDICTION error, which always falls as partitions get finer, while the
/// quantizer would have zeroed that detail anyway -- the wrong-sign-proxy law.
/// This trials both arms through the real transform+quantize+reconstruct and
/// keeps the one the CODED macroblock actually prefers.
/// Allow the I_8x8 candidate on INTRA macroblocks inside P/B slices?
///
/// MEASURED AND PRUNED 2026-08-08 — kept as a comparator, do not re-explore.
/// Recovered only 0.03-0.05 BD points on the I+P losers (foreman +0.28 -> +0.23,
/// harbour +0.12 -> +0.08) and COST grain I+P+B 0.21 (-0.45 -> -0.24). The census
/// explains why: it touches 0.001-0.039% of bytes, because intra-8x8-in-P is rare.
///
/// Hypothesis under test: with inter-8x8 already off, the only 8x8 left in a P frame
/// is on intra-in-P macroblocks, and every remaining 8x8 BD loss sits in a structure
/// that contains P frames while all-intra is clean. Intra-in-P blocks are the ones
/// inter prediction failed on, so they skew toward the fine detail 4x4 handles better.
/// `RFF_I8_IN_P=0` withholds the candidate there.
/// How much the I_8x8 candidate must BEAT I_4x4/I_16x16 by, in lambda units, before
/// it is selected — instead of winning on any margin at all.
///
/// MEASURED AND PRUNED 2026-08-08 — margin 0 (any win) is OPTIMAL, do not re-explore.
/// Sweep over the 21-cell table: margin 0 -> 7 losers, sum -6.72%; margin 8 -> 7
/// losers, sum -2.30%; margin 24 -> 12 losers, sum +1.25%. It gives back akiyo's win
/// (-1.90 -> -0.64 -> +0.06) far faster than it recovers foreman (+0.28 -> +0.22 ->
/// +0.23). The premise was wrong: akiyo's win is an accumulation of MANY MARGINAL
/// per-MB picks, not a few decisive ones, so a margin kills the win first. A per-MB
/// RD margin is not a proxy for the clip-level decisiveness of a win.
///
/// Motivated by a measured propagation effect: an 8x8-coded I-frame is a worse
/// PREDICTION SOURCE, and the damage grows monotonically with GOP length. BD of 8x8
/// vs 4x4 at gop 1 / 10 / 30 / 60: foreman -0.06 / +0.01 / +0.28 / +0.84, harbour
/// -0.37 / -0.12 / +0.12 / +0.31, and even akiyo's win decays -1.90 -> -1.26. Every
/// clip moves the same direction, so this is a mechanism and not content noise.
///
/// The margin is the dispatch. Clips where 8x8 wins DECISIVELY (akiyo, -1.90% on the
/// I-frame alone) clear any sane margin and keep their win; clips where it wins by a
/// hair (foreman, -0.06%) are near-ties that flip on quantisation noise and then cost
/// far more downstream than they saved. Cost margin as the axis, no new signal needed.
/// GREAT GATE: withhold the 8x8 transform on SCREEN content for this frame.
///
/// The PPS still advertises `transform_8x8_mode_flag`; clearing it per frame simply
/// means no macroblock sets `transform_size_8x8_flag`, which is legal and keeps the
/// stream decodable by anything that accepted the sequence.
///
/// The win-signature to check when touching this: NATURAL clips must be
/// BYTE-IDENTICAL with the veto compiled in, because it must abstain on them.
fn apply_screen_t8_veto(fe: &mut FrameEncoder, sig: &crate::signals::FrameSignals) {
    if fe.t8_pick && screen_t8_veto_on() && sig.is_screen() {
        // VALUE, not presence: `transform_8x8` stays true so the flag keeps being
        // written (the PPS advertises it); `t8_pick` false makes every macroblock
        // write it as ZERO.
        fe.t8_pick = false;
    }
}

/// `RFF_T8_SCREEN=0` opts out of the screen veto (the comparator arm).
fn screen_t8_veto_on() -> bool {
    static ON: rusty_h264_common::once::OnceLock<bool> = rusty_h264_common::once::OnceLock::new();
    *ON.get_or_init(|| {
        rusty_h264_common::knob("RFF_T8_SCREEN")
            .map(|v| v != "0")
            .unwrap_or(true)
    })
}

fn i8_margin() -> f64 {
    static M: rusty_h264_common::once::OnceLock<f64> = rusty_h264_common::once::OnceLock::new();
    *M.get_or_init(|| {
        rusty_h264_common::knob("RFF_I8_MARGIN")
            .and_then(|v| v.parse().ok())
            .unwrap_or(0.0)
    })
}

fn i8_in_p_on() -> bool {
    static ON: rusty_h264_common::once::OnceLock<bool> = rusty_h264_common::once::OnceLock::new();
    *ON.get_or_init(|| {
        rusty_h264_common::knob("RFF_I8_IN_P")
            .map(|v| v != "0")
            .unwrap_or(true)
    })
}

fn sub8_rd_on() -> bool {
    static ON: rusty_h264_common::once::OnceLock<bool> = rusty_h264_common::once::OnceLock::new();
    *ON.get_or_init(|| {
        rusty_h264_common::knob("RFF_SUB8_RD")
            .map(|v| v == "1")
            .unwrap_or(false)
    })
}

/// Level-aware bit estimate for a planned inter macroblock: the same
/// `sum rdoq_rate(|level|)` currency the inter-8x8-vs-4x4 decision already uses,
/// plus the motion syntax (which does NOT cancel between a split arm and an
/// 8x8 arm -- they carry a different NUMBER of mvds, and that difference is the
/// whole point of the comparison).
fn plan_rate_bits(plan: &InterPlan, sub_types: [u8; 4]) -> f64 {
    let mut r = 0.0f64;
    if plan.t8x8 {
        for b in &plan.q8 {
            for &l in b.iter() {
                if l != 0 {
                    r += rdoq_rate((l as i64).abs());
                }
            }
        }
    } else {
        for b in &plan.q_blocks {
            for &l in b.iter() {
                if l != 0 {
                    r += rdoq_rate((l as i64).abs());
                }
            }
        }
    }
    for c in 0..2 {
        for &l in &plan.c_dc_levels[c] {
            if l != 0 {
                r += rdoq_rate((l as i64).abs());
            }
        }
        for b in &plan.c_q[c] {
            for &l in b.iter() {
                if l != 0 {
                    r += rdoq_rate((l as i64).abs());
                }
            }
        }
    }
    for m in plan.mvds.iter().take(plan.n_mvd) {
        r += (mvd_bits(m.0) + mvd_bits(m.1)) as f64;
    }
    // sub_mb_type bins (1 for 8x8, 2 for 8x4, 3 for 4x8/4x4) + mb_type overhead.
    for &st in &sub_types {
        r += if st == 0 {
            1.0
        } else if st == 1 {
            2.0
        } else {
            3.0
        };
    }
    r + 16.0
}

/// P3.3 opt-in: search 8x4/4x8/4x4 sub-partitions inside P_8x8 (CABAC quality
/// path, single-ref). `RFF_SUB8X8_SPLIT=1` enables; unset/0 = byte-identical.
fn sub8x8_split_on() -> bool {
    static ON: rusty_h264_common::once::OnceLock<bool> = rusty_h264_common::once::OnceLock::new();
    *ON.get_or_init(|| {
        rusty_h264_common::knob("RFF_SUB8X8_SPLIT")
            .map(|s| s == "1")
            .unwrap_or(false)
    })
}

fn split_t() -> f64 {
    let v = SPLIT_T.load(core::sync::atomic::Ordering::Relaxed);
    if v != u32::MAX {
        return v as f64;
    }
    let d: u32 = rusty_h264_common::knob("RFF_SPLIT_T")
        .and_then(|s| s.parse().ok())
        .unwrap_or(0);
    SPLIT_T.store(d, core::sync::atomic::Ordering::Relaxed);
    d as f64
}

/// Observe-only HARVEST for the PARTITION-split gate (U2/U5).
///
/// The 16×16 search is the null arm and runs first; the 2-way splits and P_8x8 are
/// the expensive arm (7 further `best_part` calls, each with its own full-pel search
/// AND sub-pel refinement). Today they are gated by a fixed `split_gate` formula.
/// Records the null-arm cost, the best split cost, and which won, so the
/// skip-rate-vs-gain-kept ceiling can be swept before any threshold is touched.
mod split_harvest {
    #[allow(unused_imports)]
    use alloc::{
        boxed::Box,
        format,
        string::{String, ToString},
        vec,
        vec::Vec,
    };
    #[allow(unused_imports)]
    use rusty_h264_common::once::OnceLock;
    #[cfg(feature = "std")]
    #[cfg(feature = "std")]
    use std::fs::File;
    #[cfg(feature = "std")]
    #[cfg(feature = "std")]
    use std::io::Write;
    #[cfg(feature = "std")]
    #[cfg(feature = "std")]
    use std::sync::Mutex;

    #[cfg(not(feature = "std"))]
    #[allow(dead_code)]

    fn sink() -> &'static Option<()> {
        static NONE: Option<()> = None;

        &NONE
    }

    #[cfg(feature = "std")]

    fn sink() -> &'static Option<Mutex<File>> {
        static S: OnceLock<Option<Mutex<File>>> = OnceLock::new();
        S.get_or_init(|| {
            rusty_h264_common::knob("RFF_SPLIT_HARVEST").and_then(|p| {
                let mut f = File::create(p).ok()?;
                let _ = writeln!(f, "c16,best,lambda,gate,won");
                Some(Mutex::new(f))
            })
        })
    }

    #[inline]
    pub fn enabled() -> bool {
        sink().is_some()
    }

    #[cfg_attr(not(feature = "std"), allow(unused_variables))]

    pub fn record(c16: i64, best: i64, lambda: f64, gate: i64, won: u8) {
        #[cfg(feature = "std")]
        if let Some(m) = sink() {
            if let Ok(mut f) = m.lock() {
                let _ = writeln!(f, "{c16},{best},{lambda:.4},{gate},{won}");
            }
        }
    }
}

/// Descent C escape hatch: `RFF_HPEL_REF=0` restores the copy-then-SATD half-pel path
/// (byte-identical to it either way — this exists as a bisection anchor).
fn hpel_ref_enabled() -> bool {
    static E: rusty_h264_common::once::OnceLock<bool> = rusty_h264_common::once::OnceLock::new();
    *E.get_or_init(|| {
        rusty_h264_common::knob("RFF_HPEL_REF")
            .map(|v| v != "0")
            .unwrap_or(true)
    })
}

/// H-23: smooth (x264-shape) mvd cost table, in quarter-bit units scaled to the
/// same magnitude as the Exp-Golomb model so λ stays calibrated. `RFF_MVCOST=1`.
static MV_COST_TAB: rusty_h264_common::once::OnceLock<Vec<u16>> =
    rusty_h264_common::once::OnceLock::new();
fn build_mv_cost() -> Vec<u16> {
    (0..4096u32)
        .map(|a| {
            let c = 2.0 * rusty_h264_common::fmath::log2((a + 1) as f64)
                + 0.718
                + if a != 0 { 1.0 } else { 0.0 };
            // Round to quarter-bits then express in the caller's integer "bits"
            // domain by keeping 4× resolution — λ is rescaled to match below.
            rusty_h264_common::fmath::round(c * 4.0) as u16
        })
        .collect()
}
/// H-24: 0 = off (Exp-Golomb step, byte-identical), 1 = DISPATCHED per frame by
/// the `b2_mgain` motion probe, 2 = force-on. The BD sign-flip (bus −1.31 /
/// football −0.24 vs foreman +0.23 / akiyo +0.11) tracks MOTION: the smooth
/// curve pays where |mvd| is large enough to leave the first bracket, and only
/// adds noise where every vector already sits inside it.
static MV_SMOOTH: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(u32::MAX);
pub fn set_mv_smooth(on: bool) {
    MV_SMOOTH.store(
        if on { 2 } else { 0 },
        core::sync::atomic::Ordering::Relaxed,
    )
}
pub fn set_mv_smooth_mode(m: u32) {
    MV_SMOOTH.store(m.min(3), core::sync::atomic::Ordering::Relaxed)
}
/// Dispatch threshold on the per-frame mgain probe (`RFF_MVCOST_T`, default 0.10).
/// Calibrated on the DEPLOYED probe: bus min-frame 0.185 and football med 0.208
/// route ON; foreman med 0.164 is the boundary case, akiyo ~0.00 routes OFF.
/// H-26: the measured TRUE table plus a COHERENCE BIAS on every d≠0 entry —
/// the cheap scalar form of the MV-field externality H-25 root-caused (a chosen
/// vector that leaves the predictor degrades the neighbours' medians; truth
/// per-vector under-prices that shared damage). `RFF_MVCOST_BIAS` in bits,
/// read once at first use. DEFAULT 1.0 — mode 3 ships as the H-26 BIASED form
/// (the original doc here said "default 0 = pure truth", contradicting the
/// `unwrap_or(1.0)` shipped in the same commit; the code carries the intent —
/// set `RFF_MVCOST_BIAS=0` explicitly for the H-25 pure-truth table).
static MV_TRUE_BIASED: rusty_h264_common::once::OnceLock<Vec<u16>> =
    rusty_h264_common::once::OnceLock::new();
fn build_true_biased() -> Vec<u16> {
    let bias_q4 = rusty_h264_common::fmath::round(
        rusty_h264_common::knob("RFF_MVCOST_BIAS")
            .and_then(|v| v.parse::<f64>().ok())
            .unwrap_or(1.0)
            * 4.0,
    ) as u16;
    crate::mvd_cost_tab::MVD_TRUE_COST4
        .iter()
        .enumerate()
        .map(|(d, &c)| if d == 0 { c } else { c.saturating_add(bias_q4) })
        .collect()
}

fn mv_smooth_t() -> f64 {
    static T: rusty_h264_common::once::OnceLock<f64> = rusty_h264_common::once::OnceLock::new();
    *T.get_or_init(|| {
        rusty_h264_common::knob("RFF_MVCOST_T")
            .and_then(|v| v.parse().ok())
            .unwrap_or(0.10)
    })
}
/// 0 = step model, 1 = smooth (this frame routed on), 2 = smooth (forced),
/// 3 = the MEASURED true-cost table (H-25) — no dispatch needed if it wins
/// everywhere, since it is the truth both analytic models approximate.
/// `frame_smooth` is the per-frame probe decision, carried on the frame state
/// (like `sadfp`) — a process-global here races under the GOP-parallel encode.
#[inline]
fn mv_cost_kind(frame_smooth: bool) -> u32 {
    match mv_smooth_mode() {
        0 => 0,
        // H-26 verdict: smooth/truth/truth+bias shuffle within ±0.2 BD fit-noise
        // of each other at dispatch (bus prefers truth, football prefers smooth,
        // none dominates), so the dispatch keeps its ORIGINALLY-GATED smooth
        // ON-model; the measured/biased tables remain as modes 2/3 for research.
        1 => frame_smooth as u32,
        2 => 1, // archaeology: the x264 smooth curve, forced
        _ => 2, // the biased-truth table, forced
    }
}
#[inline]
fn mv_smooth_mode() -> u32 {
    match MV_SMOOTH.load(core::sync::atomic::Ordering::Relaxed) {
        m @ 0..=3 => m,
        _ => {
            static E: rusty_h264_common::once::OnceLock<u32> =
                rusty_h264_common::once::OnceLock::new();
            // DEFAULT 1 = DISPATCHED (H-24). Owner's call: mean −0.27% BD is
            // taken over minimax, accepting a known, bounded +0.16-0.18% on
            // foreman-class content. `RFF_MVCOST=0` restores the pre-H-23 bytes.
            *E.get_or_init(|| {
                rusty_h264_common::knob("RFF_MVCOST")
                    .and_then(|v| v.parse().ok())
                    .unwrap_or(1)
            })
        }
    }
}

/// Challenge-1 A3 escape hatch: `RFF_SATD_AVG=0` restores the materialize-then-SATD
/// quarter-pel cost path (byte-identical either way — a bisection anchor, like
/// `RFF_HPEL_REF`).
/// H-14 R3 escape hatch: `RFF_MECTX=0` restores the per-eval safe dispatch
/// (byte-identical either way — MeCtx returns exactly the safe path's values).
#[cfg_attr(not(accel), allow(dead_code))] // caller is accel-gated
fn mectx_enabled() -> bool {
    static E: rusty_h264_common::once::OnceLock<bool> = rusty_h264_common::once::OnceLock::new();
    *E.get_or_init(|| {
        rusty_h264_common::knob("RFF_MECTX")
            .map(|v| v != "0")
            .unwrap_or(true)
    })
}

fn satd_avg_enabled() -> bool {
    static E: rusty_h264_common::once::OnceLock<bool> = rusty_h264_common::once::OnceLock::new();
    *E.get_or_init(|| {
        rusty_h264_common::knob("RFF_SATD_AVG")
            .map(|v| v != "0")
            .unwrap_or(true)
    })
}

/// Track-B B2 (docs/lets-win-optimize.md): run the FULL-PEL phase of the non-fast
/// motion search in the SAD domain (`psadbw`-class, ~3-4× cheaper per candidate) and
/// reprice the winner in SATD before the rescue/sub-pel phases — the cost split
/// every x264 preset uses (SAD fpel, SATD from subme≥2). ⚠ BITSTREAM-CHANGING (a
/// different full-pel winner can emerge), so it ships opt-in until the per-clip
/// 4-QP BD gate clears it. `set_me_sadfp` overrides; unset → `RFF_ME_SADFP` env,
/// default OFF (off = byte-identical to the pre-B2 encoder).
/// Modes: 0 = off (byte-identical), 1 = DISPATCHED per frame by the `b2_mgain`
/// probe (the shipping shape), 2 = force-on everywhere (the truth-table A/B arm).
static ME_SADFP: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(u32::MAX);
pub fn set_me_sadfp(on: bool) {
    // Harness semantics preserved: `true` = the force-on arm truth tables measure.
    ME_SADFP.store(
        if on { 2 } else { 0 },
        core::sync::atomic::Ordering::Relaxed,
    )
}
pub fn set_me_sadfp_mode(m: u32) {
    ME_SADFP.store(m.min(2), core::sync::atomic::Ordering::Relaxed)
}
fn me_sadfp_mode() -> u32 {
    match ME_SADFP.load(core::sync::atomic::Ordering::Relaxed) {
        u32::MAX => {
            static INIT: rusty_h264_common::once::OnceLock<u32> =
                rusty_h264_common::once::OnceLock::new();
            // DEFAULT = 1 (dispatched) since the H-3 gate: 16-clip corpus mean
            // −0.26% BD, wins bus −1.71 / football −1.84 / foreman −0.44 /
            // shields −0.22, every former loss 0.00; residual tail (soccer +0.09,
            // harbour +0.06) is BD-fit noise — it responds NON-monotonically to
            // threshold changes (less B2 made soccer read WORSE, +0.18).
            // `RFF_ME_SADFP=0` is the escape hatch reproducing the pre-B2 bytes.
            *INIT.get_or_init(|| {
                rusty_h264_common::knob("RFF_ME_SADFP")
                    .and_then(|v| v.parse().ok())
                    .unwrap_or(1)
            })
        }
        m => m,
    }
}

/// B2 dispatch threshold on the per-frame `b2_mgain` probe (`RFF_ME_SADT`).
/// Calibrated on the DEPLOYED estimator (recon reference, sampled MBs), not the
/// offline source-frame probe — the recurring R6 law.
fn me_sadt() -> f64 {
    static T: rusty_h264_common::once::OnceLock<f64> = rusty_h264_common::once::OnceLock::new();
    *T.get_or_init(|| {
        rusty_h264_common::knob("RFF_ME_SADT")
            .and_then(|s| s.parse().ok())
            .unwrap_or(0.13)
    })
}
fn me_sadt_dbg() -> bool {
    static D: rusty_h264_common::once::OnceLock<bool> = rusty_h264_common::once::OnceLock::new();
    *D.get_or_init(|| rusty_h264_common::knob("RFF_ME_SADT_DBG").is_some())
}

/// Fixed-centre batched diamond passes on SAD-routed frames (`RFF_ME_FC=0` falls
/// back to the cascading scalar walk — the bisection anchor). Fixed-centre differs
/// from the cascade only when 2+ points improve in one pass, so it rides B2's BD
/// gate; dispatched-OFF frames never take this path and stay byte-identical.
/// ③ Sub-pel ring FC: fixed-centre argmin passes for the HALF-PEL step, batched
/// through `satd_16x16_x4p` (two calls cover the 8-ring; candidates resolve to
/// h/h/v/v and c/c/c/c plane reads from an integer centre). Quarter-step and any
/// declined pass keep the cascading walk. Bitstream-changing → own gate
/// (`AB_SPFC`), `RFF_SP_FC=0` anchor.
static SP_FC: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(u32::MAX);
pub fn set_sp_fc(on: bool) {
    SP_FC.store(on as u32, core::sync::atomic::Ordering::Relaxed)
}
fn sp_fc_enabled() -> bool {
    match SP_FC.load(core::sync::atomic::Ordering::Relaxed) {
        0 => false,
        1 => true,
        _ => {
            static E: rusty_h264_common::once::OnceLock<bool> =
                rusty_h264_common::once::OnceLock::new();
            *E.get_or_init(|| {
                rusty_h264_common::knob("RFF_SP_FC")
                    .map(|v| v != "0")
                    .unwrap_or(false)
            })
        }
    }
}

static ME_FC: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(u32::MAX);
pub fn set_me_fc(on: bool) {
    ME_FC.store(on as u32, core::sync::atomic::Ordering::Relaxed)
}
fn me_fc_enabled() -> bool {
    match ME_FC.load(core::sync::atomic::Ordering::Relaxed) {
        0 => false,
        1 => true,
        _ => {
            static E: rusty_h264_common::once::OnceLock<bool> =
                rusty_h264_common::once::OnceLock::new();
            *E.get_or_init(|| {
                rusty_h264_common::knob("RFF_ME_FC")
                    .map(|v| v != "0")
                    .unwrap_or(true)
            })
        }
    }
}

/// H-13 SPLIT DISPATCH — measured and REFUTED as a free dispatch, shipped as an
/// OPT-IN rung (default 0 = off = byte-identical). The premise "splits buy
/// ~nothing on near-static frames" is FALSE: at T=0.03 akiyo read +2.45% BD,
/// akiyo_qcif +2.02%, FourPeople +2.00% for only 1.10-1.15× — partition splits
/// EARN BD on every measured content class (the third death of the split-gate
/// idea: U2 T=400, the sum-weighted ceiling, now the mgain axis). foreman/bus
/// route ON at any sane T (min frame mgain 0.061/0.185) and stay byte-identical.
/// `RFF_SPLIT_MG` (fraction) / `set_split_mg` (milli): a priced speed rung, not
/// a free lunch.
static SPLIT_MG: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(u32::MAX);
pub fn set_split_mg(milli: u32) {
    SPLIT_MG.store(milli, core::sync::atomic::Ordering::Relaxed)
}
fn split_mg() -> f64 {
    match SPLIT_MG.load(core::sync::atomic::Ordering::Relaxed) {
        u32::MAX => {
            static E: rusty_h264_common::once::OnceLock<f64> =
                rusty_h264_common::once::OnceLock::new();
            *E.get_or_init(|| {
                rusty_h264_common::knob("RFF_SPLIT_MG")
                    .and_then(|v| v.parse().ok())
                    .unwrap_or(0.0)
            })
        }
        m => m as f64 / 1000.0,
    }
}

/// The flash veto: frames whose zero-MV residual is DC-shift-dominated beyond this
/// fraction route OFF even at high mgain (`RFF_ME_SADDC`). Calibrated on the
/// DEPLOYED per-frame values: crew's harmful ON-frames read dc 0.843–0.859 (the
/// camera flashes) while every good ON-frame on bus/football/foreman reads ≤ 0.478
/// — a 1.76× natural gap; 0.6 sits mid-gap with margin both ways.
fn me_sad_dcmax() -> f64 {
    static T: rusty_h264_common::once::OnceLock<f64> = rusty_h264_common::once::OnceLock::new();
    *T.get_or_init(|| {
        rusty_h264_common::knob("RFF_ME_SADDC")
            .and_then(|s| s.parse().ok())
            .unwrap_or(0.6)
    })
}

// The B2 dispatch signal `b2_mgain` (mgain + dcfrac) lives in `crate::signals`
// (Great Gate P1) — read through `FrameSignals::mgain_dc`, whose doc carries the
// 16-clip truth table and the crew-flash dcfrac rationale.

/// Track-B B3: cap on sub-pel ring ITERATIONS per step (`RFF_SP_MAXIT` /
/// `set_sp_maxit`). 0 = unlimited (the default — byte-identical to the walk-to-
/// convergence encoder); N caps each step's walk at N passes, the bounded budget
/// x264's subme levels have always had. Bitstream-changing when set → BD-gated.
static SP_MAXIT: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(u32::MAX);
pub fn set_sp_maxit(n: u32) {
    SP_MAXIT.store(n, core::sync::atomic::Ordering::Relaxed)
}
fn sp_maxit() -> u32 {
    match SP_MAXIT.load(core::sync::atomic::Ordering::Relaxed) {
        u32::MAX => {
            static INIT: rusty_h264_common::once::OnceLock<u32> =
                rusty_h264_common::once::OnceLock::new();
            *INIT.get_or_init(|| {
                rusty_h264_common::knob("RFF_SP_MAXIT")
                    .and_then(|v| v.parse().ok())
                    .unwrap_or(0)
            })
        }
        n => n,
    }
}

/// B2 calibration: λ multiplier for the SAD-domain full-pel phase (`RFF_ME_SADL`,
/// default 1.0). SATD distortion runs ~2× SAD's scale, so λ tuned for SATD weighs
/// the rate term ~2× heavier in the SAD domain — 0.5 restores the SATD-era
/// rate/distortion balance. Read once per process (hoisted per search).
fn me_sadfp_lambda() -> f64 {
    static E: rusty_h264_common::once::OnceLock<f64> = rusty_h264_common::once::OnceLock::new();
    // 0.5 = the calibrated default (SATD ≈ 2× SAD's scale; at 1.0 the rate term
    // weighs double and foreman flips to a BD loss). Rides with the mode-1 default.
    *E.get_or_init(|| {
        rusty_h264_common::knob("RFF_ME_SADL")
            .and_then(|v| v.parse().ok())
            .unwrap_or(0.5)
    })
}

/// Descent D: sub-pel ring census — evals/improvements by (step, ring position) and
/// by loop ITERATION, so a position or an iteration that never pays is visible rather
/// than assumed.
#[cfg(feature = "profile")]
pub mod spstats {
    #[allow(unused_imports)]
    use alloc::{
        boxed::Box,
        format,
        string::{String, ToString},
        vec,
        vec::Vec,
    };
    use core::sync::atomic::Ordering;
    use rusty_h264_common::atomic::AtomicU64;
    #[allow(unused_imports)]
    use rusty_h264_common::once::OnceLock;
    /// [step 0=half,1=quarter][position 0..8][0=evals,1=improvements]
    pub static POS: [AtomicU64; 2 * 8 * 2] = [const { AtomicU64::new(0) }; 32];
    /// [step][iteration 1..=6 clamped][0=evals,1=improvements]
    pub static IT: [AtomicU64; 2 * 6 * 2] = [const { AtomicU64::new(0) }; 24];
    #[inline]
    pub fn ev(st: usize, pos: usize, it: u32) {
        POS[(st * 8 + pos.min(7)) * 2].fetch_add(1, Ordering::Relaxed);
        IT[(st * 6 + (it.max(1) as usize - 1).min(5)) * 2].fetch_add(1, Ordering::Relaxed);
    }
    #[inline]
    pub fn imp(st: usize, pos: usize, it: u32) {
        POS[(st * 8 + pos.min(7)) * 2 + 1].fetch_add(1, Ordering::Relaxed);
        IT[(st * 6 + (it.max(1) as usize - 1).min(5)) * 2 + 1].fetch_add(1, Ordering::Relaxed);
    }
    /// Sub-pel evaluations that re-price an MV already evaluated in the SAME refinement.
    pub static REDUNDANT: AtomicU64 = AtomicU64::new(0);
    #[inline]
    pub fn redundant() {
        REDUNDANT.fetch_add(1, Ordering::Relaxed);
    }
    pub fn reset() {
        for c in POS.iter() {
            c.store(0, Ordering::Relaxed);
        }
        for c in IT.iter() {
            c.store(0, Ordering::Relaxed);
        }
        REDUNDANT.store(0, Ordering::Relaxed);
    }
    pub fn snapshot() -> (Vec<u64>, Vec<u64>) {
        (
            POS.iter().map(|c| c.load(Ordering::Relaxed)).collect(),
            IT.iter().map(|c| c.load(Ordering::Relaxed)).collect(),
        )
    }
    pub fn redundant_count() -> u64 {
        REDUNDANT.load(Ordering::Relaxed)
    }
}

/// Descent B: which path does each ME cost evaluation actually take?
#[cfg(feature = "profile")]
pub mod satdpath {
    #[allow(unused_imports)]
    use alloc::{
        boxed::Box,
        format,
        string::{String, ToString},
        vec,
        vec::Vec,
    };
    use core::sync::atomic::Ordering;
    use rusty_h264_common::atomic::AtomicU64;
    #[allow(unused_imports)]
    use rusty_h264_common::once::OnceLock;
    pub static C: [AtomicU64; 3] = [const { AtomicU64::new(0) }; 3];
    #[inline]
    pub fn bump(i: usize) {
        C[i].fetch_add(1, Ordering::Relaxed);
    }
    pub fn reset() {
        for c in C.iter() {
            c.store(0, Ordering::Relaxed);
        }
    }
    pub fn snapshot() -> Vec<u64> {
        C.iter().map(|c| c.load(Ordering::Relaxed)).collect()
    }
}

/// The coarse-to-fine step ladder. DEFAULT `[16,8,4]` — the 64 and 32 rungs were
/// REMOVED after the per-rung census showed they are ~39% of full-pel evaluations at a
/// 0.05-0.84% hit rate, and the 20-clip 4-QP BD curve showed those rare hits are actively
/// HARMFUL: a coarse jump finds a distant MV with marginally lower SATD, but it costs
/// more mvd bits AND breaks the spatial coherence of the MV field, degrading every
/// downstream neighbour's predictor. `lambda*mvbits` prices the first effect and is blind
/// to the second. Dropping them is mean -0.93% BD-PSNR / -1.09% BD-SSIM with a WORST clip
/// of +0.00%/+0.00% over 20 clips, and 1.15-1.57x fewer ME cost evaluations.
///
/// The 8 rung is load-bearing: `[16,4]` reads marginally better BD but makes football_cif
/// do 1.55x MORE work, because the step-4 walk then has to crawl the distance the 8 rung
/// covered in one hop. Reach and stride both matter; only the useless TOP is removed.
///
/// Bit i of the mask enables rung i of [64,32,16,8,4]. `RFF_DIA_LADDER=64,32,16,8,4`
/// restores the pre-change ladder byte-for-byte; `set_dia_mask` overrides at runtime so a
/// single process can measure several ladders.
pub const DIA_RUNGS: [i32; 5] = [64, 32, 16, 8, 4];
/// Rungs walked by default: `[16,8,4]`.
pub const DIA_DEFAULT: u32 = 0b11100;

/// SUB-PARTITION LADDER (best_part campaign, 2026-08-06). A sub-8x8 partition is
/// seeded with its PARENT's already-converged MV (`extra = [mv16, mv_quad]`), so
/// the coarse rungs a 16x16 block needs — to reach motion no predictor found —
/// are near-pure toll here. Measured on foreman (quality, 30f, `mecost`):
///
/// | rung | reach | share of ALL ME evals | hit rate |
/// |---|---|---|---|
/// | s0 | 4 px | 30.0% | **0.97%** |
/// | s1 | 2 px | 31.6% | 2.20% |
/// | s2 | 1 px | 38.3% | 6.42% |
///
/// and the shares are IDENTICAL with the split search on or off — i.e. every
/// 4x4 walks the same 4-pixel-reach ladder as an unpredicted 16x16. s0 alone is
/// 863k evaluations to change 8,331 answers. The fine rung still reaches any
/// distance (it walks to convergence), just in 1-px hops from a seed that is
/// already right. `RFF_DIA_SUB` overrides (same `a,b,c` rung syntax as
/// `RFF_DIA_LADDER`); `RFF_DIA_SUB=16,8,4` restores the pre-campaign behaviour.
pub static DIA_SUB_MASK: core::sync::atomic::AtomicU32 =
    core::sync::atomic::AtomicU32::new(u32::MAX);

fn dia_sub_mask() -> u32 {
    let m = DIA_SUB_MASK.load(core::sync::atomic::Ordering::Relaxed);
    if m != u32::MAX {
        return m;
    }
    static INIT: rusty_h264_common::once::OnceLock<u32> = rusty_h264_common::once::OnceLock::new();
    *INIT.get_or_init(|| match rusty_h264_common::knob("RFF_DIA_SUB") {
        Some(v) => {
            let want: Vec<i32> = v.split(',').filter_map(|t| t.trim().parse().ok()).collect();
            let mut m = 0u32;
            for (i, r) in DIA_RUNGS.iter().enumerate() {
                if want.contains(r) {
                    m |= 1 << i;
                }
            }
            m
        }
        // Default: the FINE rung alone (1 px, walked to convergence). Swept
        // {[16,8,4], [8,4], [4]} on 6 clips as BD-rate vs no-splits — the short
        // ladder is not a trade, it WINS on quality too, because coarse rungs on
        // a 4x4 chase spurious far matches that fit the tiny block while
        // wrecking the MV field its neighbours predict from (the same mechanism
        // the diagonal-probe note above records, applied to rung REACH):
        //
        //   clip     full [16,8,4]      fine [4]        evals
        //   foreman  -3.48 / -2.14      -3.59 / -2.21   -44.5%
        //   harbour  -0.29 / +0.146     -0.36 / +0.073
        //   mobile   -2.22 / -2.38      -2.42 / -2.57
        //   tempete  -1.15 / -0.77      -1.25 / -0.89
        //   bus      -6.61 / -5.53      -6.63 / -5.49   (tie)
        //   screen   -11.98 / -12.40    -11.97 / -12.09 (gives back 0.31 of 12.4)
        None => 0b10000,
    })
}
pub static DIA_MASK: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(u32::MAX);
pub fn set_dia_mask(m: u32) {
    DIA_MASK.store(m, core::sync::atomic::Ordering::Relaxed)
}
fn dia_mask() -> u32 {
    let m = DIA_MASK.load(core::sync::atomic::Ordering::Relaxed);
    if m != u32::MAX {
        return m;
    }
    static INIT: rusty_h264_common::once::OnceLock<u32> = rusty_h264_common::once::OnceLock::new();
    *INIT.get_or_init(|| match rusty_h264_common::knob("RFF_DIA_LADDER") {
        Some(v) => {
            let want: Vec<i32> = v.split(',').filter_map(|t| t.trim().parse().ok()).collect();
            let mut m = 0u32;
            for (i, r) in DIA_RUNGS.iter().enumerate() {
                if want.contains(r) {
                    m |= 1 << i;
                }
            }
            if m == 0 {
                DIA_DEFAULT
            } else {
                m
            }
        }
        None => DIA_DEFAULT,
    })
}

/// Descent A: per-STEP-SIZE census of the coarse-to-fine diamond. The ladder is
/// [64,32,16,8,4] quarter-pel (i.e. 16,8,4,2,1 full-pel) and each step walks until it
/// stops improving. Counts evaluations AND improvements per step so a step that never
/// pays can be identified rather than assumed.
#[cfg(feature = "profile")]
pub mod diastats {
    #[allow(unused_imports)]
    use alloc::{
        boxed::Box,
        format,
        string::{String, ToString},
        vec,
        vec::Vec,
    };
    use core::sync::atomic::Ordering;
    use rusty_h264_common::atomic::AtomicU64;
    #[allow(unused_imports)]
    use rusty_h264_common::once::OnceLock;
    /// [step_index][0]=evals, [1]=improvements
    pub static C: [AtomicU64; 12] = [const { AtomicU64::new(0) }; 12];
    #[inline]
    pub fn ev(i: usize) {
        C[i * 2].fetch_add(1, Ordering::Relaxed);
    }
    #[inline]
    pub fn imp(i: usize) {
        C[i * 2 + 1].fetch_add(1, Ordering::Relaxed);
    }
    pub fn reset() {
        for c in C.iter() {
            c.store(0, Ordering::Relaxed);
        }
    }
    pub fn snapshot() -> Vec<(u64, u64)> {
        (0..6)
            .map(|i| {
                (
                    C[i * 2].load(Ordering::Relaxed),
                    C[i * 2 + 1].load(Ordering::Relaxed),
                )
            })
            .collect()
    }
}

/// Sub-pel refinement PATTERN (U1). Bit 0 = 4-point diamond ring instead of the
/// 8-point square; bit 1 = single pass instead of walking to convergence.
///
/// Harvested from 280 k real refinements: ~29 evaluations each, but the LAST
/// improvement lands at eval ~14–15 — **half of every refinement is spent confirming
/// an answer already found** — and the first ring alone captures 64–72% of the total
/// gain. An 8-point ring pays 8 evaluations for that confirmation; a 4-point diamond
/// (what x264's subme uses) pays 4.
///
/// `RFF_SUBPEL_PAT`: 0 = 8-point + iterate (the pre-U1 default), 1 = 4-point +
/// iterate, 2 = 8-point single pass, 3 = 4-point single pass.
pub(crate) static SUBPEL_PAT: core::sync::atomic::AtomicU32 =
    core::sync::atomic::AtomicU32::new(u32::MAX);

/// Learning-window size and ring-1 threshold (percent) for the U1 online dispatcher.
/// `RFF_SUBPEL_DISPATCH=0` disables it (pure `RFF_SUBPEL_PAT` behaviour).
pub(crate) static SP_DISPATCH: core::sync::atomic::AtomicU32 =
    core::sync::atomic::AtomicU32::new(u32::MAX);

fn sp_dispatch_cfg() -> (u32, i64) {
    let forced = SP_DISPATCH.load(core::sync::atomic::Ordering::Relaxed);
    if forced == 0 {
        return (0, 0);
    }
    static C: OnceLock<(u32, i64)> = OnceLock::new();
    *C.get_or_init(|| {
        // DEFAULT OFF — measured and refuted (see the U1 entry in
        // docs/WHYS-speed-gap.md). It only delivers speed where a blanket pattern
        // change already would (bus 1.47x) while costing BD where it delivers none
        // (foreman +0.97% for 1.04x, mobile +0.33% for 0.98x), and mixing refinement
        // quality across frames measured WORSE than a uniform cut (bus +0.81%
        // dispatched vs +0.30% pat2-always) — the refinement feeds the reference
        // chain, so per-frame inconsistency propagates.
        let on = rusty_h264_common::knob("RFF_SUBPEL_DISPATCH")
            .map(|s| s != "0")
            .unwrap_or(false);
        if !on {
            return (0, 0);
        }
        let k = rusty_h264_common::knob("RFF_SUBPEL_LEARN")
            .and_then(|s| s.parse().ok())
            .unwrap_or(200);
        let t = rusty_h264_common::knob("RFF_SUBPEL_T")
            .and_then(|s| s.parse().ok())
            .unwrap_or(67);
        (k, t)
    })
}

/// Explicit override only; `None` means "use the preset's default".
fn subpel_pattern_override() -> Option<u32> {
    let v = SUBPEL_PAT.load(core::sync::atomic::Ordering::Relaxed);
    if v != u32::MAX {
        return Some(v);
    }
    if let Some(e) = rusty_h264_common::knob("RFF_SUBPEL_PAT").and_then(|s| s.parse::<u32>().ok()) {
        SUBPEL_PAT.store(e, core::sync::atomic::Ordering::Relaxed);
        return Some(e);
    }
    None
}

/// Observe-only HARVEST for the sub-pel refinement skip-gate (U1).
///
/// `me-subpel` is 141 ms of a 320 ms quality encode — 44% — at 241 candidate
/// evaluations per macroblock. This tap records, per refinement, the NULL-ARM cost
/// (the full-pel winner, i.e. what we would keep if we skipped) against the cost the
/// refinement actually reached, so the skip-rate-vs-gain-kept ceiling can be swept
/// offline before any gate is written. Writes nothing unless `RFF_SUBPEL_HARVEST`
/// names a file.
mod subpel_harvest {
    #[allow(unused_imports)]
    use alloc::{
        boxed::Box,
        format,
        string::{String, ToString},
        vec,
        vec::Vec,
    };
    #[allow(unused_imports)]
    use rusty_h264_common::once::OnceLock;
    #[cfg(feature = "std")]
    #[cfg(feature = "std")]
    use std::fs::File;
    #[cfg(feature = "std")]
    #[cfg(feature = "std")]
    use std::io::Write;
    #[cfg(feature = "std")]
    #[cfg(feature = "std")]
    use std::sync::Mutex;

    #[cfg(not(feature = "std"))]
    #[allow(dead_code)]

    fn sink() -> &'static Option<()> {
        static NONE: Option<()> = None;

        &NONE
    }

    #[cfg(feature = "std")]

    fn sink() -> &'static Option<Mutex<File>> {
        static S: OnceLock<Option<Mutex<File>>> = OnceLock::new();
        S.get_or_init(|| {
            rusty_h264_common::knob("RFF_SUBPEL_HARVEST").and_then(|p| {
                let mut f = File::create(p).ok()?;
                let _ = writeln!(f, "pre,post,lambda,w,h,evals,to_best,ring1");
                Some(Mutex::new(f))
            })
        })
    }

    #[inline]
    pub fn enabled() -> bool {
        sink().is_some()
    }

    #[cfg_attr(not(feature = "std"), allow(unused_variables))]
    pub fn record(
        pre: i64,
        post: i64,
        lambda: f64,
        w: usize,
        h: usize,
        evals: u32,
        to_best: u32,
        ring1: i64,
    ) {
        #[cfg(feature = "std")]
        if let Some(m) = sink() {
            if let Ok(mut f) = m.lock() {
                let _ = writeln!(
                    f,
                    "{pre},{post},{lambda:.4},{w},{h},{evals},{to_best},{ring1}"
                );
            }
        }
    }
}

/// A/B switch for serving B-direct 4×4 MC from the cached half-pel planes
/// (`RFF_BDIRECT_PLANES=0` restores the direct `mc_luma` 6-tap). Byte-identical
/// either way; the knob exists so the arm can be measured in one binary.
fn bdirect_planes_enabled() -> bool {
    static ON: OnceLock<bool> = OnceLock::new();
    *ON.get_or_init(|| {
        rusty_h264_common::knob("RFF_BDIRECT_PLANES")
            .map(|s| s != "0")
            .unwrap_or(true)
    })
}

fn me_batch_enabled() -> bool {
    static ON: OnceLock<bool> = OnceLock::new();
    *ON.get_or_init(|| {
        rusty_h264_common::knob("RFF_ME_BATCH")
            .map(|s| s != "0")
            .unwrap_or(true)
    })
}

/// A 16-byte-aligned 16×16 luma block — the aligned `op1` openh264's SSE2 SAD/SATD
/// kernels require (`movdqa`). Safe to construct (`forbid(unsafe)` holds); the asm
/// FFI that consumes it lives in `rusty_h264-accel`. Only used on the `asm` feature.
#[cfg(accel)]
#[repr(align(16))]
struct AlignedMb([u8; 256]);

/// A B-slice 16×16 inter-coding spec: the prediction direction and the motion it
/// uses. `dir` 1 = `B_L0_16x16`, 2 = `B_L1_16x16`, 3 = `B_Bi_16x16` (spec Table
/// 7-14). List-0 is `refs[0]` (nearest past anchor); `l1` is List-1 (nearest
/// future anchor). `mv0`/`mv1` are the List-0/List-1 motion vectors (quarter-pel).
#[derive(Clone, Copy)]
struct BInter<'a> {
    dir: u8,
    l1: &'a crate::RefFrame,
    mv0: (i32, i32),
    mv1: (i32, i32),
    /// 0 = single 16x16 (use `dir`/`mv0`/`mv1`); 1 = 16x8; 2 = 8x16. When non-zero
    /// `parts2` carries `(pred, mv0, mv1)` per partition with pred 1=L0 / 2=L1 / 3=Bi.
    mvmode: u8,
    parts2: [(u8, (i32, i32), (i32, i32)); 2],
}

/// 16-byte-aligned 256-`i16` DCT/coefficient buffer — the in-place `movdqa` quant
/// kernel (openh264 `WelsQuantFour4x4` heritage) requires aligned coefficients.
#[cfg(accel)]
#[repr(align(16))]
struct AlignedDct([i16; 256]);

/// Luma variance of the 16×16 source MB at (mb_x, mb_y) — the content signal for
/// the adaptive SAD↔SATD cost dispatch (high variance = detail = SAD misprices).
/// `256·variance` scale (the /256 of the mean-square is kept integer); only the
/// RELATIVE ordering matters for the per-frame percentile, so the constant drops.
// `mb_variance` lives in `crate::signals` (Great Gate P1) — imported above; the
// per-MB raster vector is shared through `FrameSignals::mb_vars`.

/// Adaptive-Quantization per-MB QP map: flat (low-variance) macroblocks get a FINER
/// QP (where blocking/banding is visible), busy ones a COARSER QP (where the eye
/// masks error) — moving bits to where they're seen. The shift is `strength ·
/// (log2 var − frame mean log2 var)`, so it's relative to THIS frame's texture
/// distribution (content-invariant), rounded to an integer QP step and clamped.
/// `strength == 0` → uniform base QP (byte-identical: every `mb_qp_delta` is 0).
/// `RFF_AQ_GRAIN=0` disables the grain veto below — the bisection anchor that
/// reproduces the pre-gate bytes exactly.
fn aq_grain_veto_on() -> bool {
    static ON: rusty_h264_common::once::OnceLock<bool> = rusty_h264_common::once::OnceLock::new();
    *ON.get_or_init(|| {
        rusty_h264_common::knob("RFF_AQ_GRAIN")
            .map(|s| s != "0")
            .unwrap_or(true)
    })
}

fn aq_qp_map(sig: &FrameSignals, base_qp: u8, strength: f64) -> Vec<u8> {
    let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncAq);
    const AQ_DQP_MAX: i32 = 4;
    let n = sig.n_mbs();
    if strength == 0.0 || n == 0 {
        return vec![base_qp; n];
    }
    // GRAIN VETO (Great Gate P2 — docs/gate-ledger.md "aq-grain-veto",
    // PROVISIONAL: fitted against one textured-grain exemplar). Grain breaks
    // AQ's premise from the side the lv_spread back-off cannot see: noise is
    // "busy" everywhere (spread stays LOW-to-mid), but it is not maskable
    // texture — coarsening it shifts bits into coding noise (measured
    // +29.45% BD-SSIM on grain_akiyo, the corpus's only catastrophic AQ loss).
    // Three clauses, each grain-physical, ANDed for precision and abstention:
    //   median_var < 200  — the residual is NOT explained by texture (protects
    //                       mobile 1346+, city 259+; grain reads ≤ 128);
    //   grain_floor > 5   — even the best-predicted MBs carry residual;
    //   mgain < 0.1       — a full-pel search cannot reduce it (not motion).
    // "Unexplained temporal residual: not texture, not motion → noise."
    // Per-frame firing on the 24-clip corpus: grain 58/58 frames, ONE frame of
    // one winner (stockholm 1/58); threshold-insensitive across var<150..250.
    // Misses (textured grain, var ≥ 200) fail OPEN to current behaviour.
    // Clause order = cost order: median_var and the probes are memoized in the
    // signal vector, and short-circuiting keeps the mgain probe off almost
    // every non-grain frame.
    let grain = aq_grain_veto_on() && sig.grain_signature();
    signals::census::bump(signals::census::AQ_GRAIN, grain);
    if grain {
        return vec![base_qp; n];
    }
    // Per-MB variance (the bit-cost weight) and its log2 (+1 avoids log2(0) on a flat
    // MB → reads as maximally flat → finest QP) — both read from the shared signal
    // vector (Great Gate P1: one variance walk per frame, N consumers).
    // (`(v+1) as f64` is converted on the fly at both consumers below, in the
    // same order the old materialized `var` Vec was read — identical sums,
    // one n-sized allocation per frame gone.)
    let lvs = sig.log_vars();
    let (lv, mean_lv) = (&lvs.0, lvs.1);
    // CONTENT-ADAPTIVE STRENGTH: back off where the log-variance SPREAD is high. A
    // wide/bimodal spread means synthetic-ish content (flat regions beside detailed
    // patterns) where "busy = maskable" FAILS and the patterns are salient — full AQ
    // there costs PSNR. Natural content's spread is ~1 (keeps full strength); a
    // synthetic pan's is ~6 (heavily reduced). Ramp 1.0→`AQ_SPREAD_MIN` over
    // [`AQ_SPREAD_LO`, `AQ_SPREAD_HI`].
    const AQ_SPREAD_LO: f64 = 1.5;
    const AQ_SPREAD_HI: f64 = 5.0;
    const AQ_SPREAD_MIN: f64 = 0.0; // extreme spread (pathological synthetic) → AQ OFF
    let std_lv = lvs.2; // the shared lv_spread — same formula, computed once
    let factor =
        (1.0 - (std_lv - AQ_SPREAD_LO) / (AQ_SPREAD_HI - AQ_SPREAD_LO)).clamp(AQ_SPREAD_MIN, 1.0);
    let eff_strength = strength * factor;
    // Per-MB QP shift (clamped): busy (log-var above mean) coarser, flat finer.
    // Site 7's ★★ arm (Round 10): the magic-number round frees this loop of
    // its libm `round` call. Ties-EVEN vs libm's ties-AWAY — differs only on
    // exact .5, which the corpus hash gate measures.
    let poly = crate::fastmath::polytier_on();
    let dqp: Vec<i32> = lv
        .iter()
        .map(|&l| {
            let x = eff_strength * (l - mean_lv);
            (if poly {
                crate::fastmath::round_ties_even_fast(x)
            } else {
                rusty_h264_common::fmath::round(x)
            }) as i32
        })
        .map(|d| d.clamp(-AQ_DQP_MAX, AQ_DQP_MAX))
        .collect();
    // RATE COMPENSATION: AQ nets a rate change (coarsening a busy MB saves more bits
    // than fining a flat one adds), so shift the whole frame's QP by `c` to restore
    // the un-AQ rate — keeping `qp` meaningful. Bit model `bits_i ∝ var_i·2^(−qp_i/6)`
    // (variance as the per-MB cost proxy): `c = 6·log2(Σ var·2^(−dqp/6) / Σ var)`.
    let sum_v: f64 = sig.mb_vars().iter().map(|&v| (v + 1) as f64).sum();
    // `dqp` is clamped to [-AQ_DQP_MAX, AQ_DQP_MAX], so 2^(-d/6) has only nine
    // possible values — but it was being recomputed with a `powf` for every
    // macroblock of every frame, and after the per-call table fix it was still
    // NINE powf + divides per FRAME. It is a pure function of constants, so a
    // `OnceLock` evaluates the same expression once per PROCESS — the
    // `build_mv_cost` pattern, bit-identical by construction.
    static QSTEP: rusty_h264_common::once::OnceLock<[f64; (2 * AQ_DQP_MAX + 1) as usize]> =
        rusty_h264_common::once::OnceLock::new();
    let qstep = QSTEP.get_or_init(|| {
        core::array::from_fn(|i| {
            rusty_h264_common::fmath::powf(2f64, -((i as i32 - AQ_DQP_MAX) as f64) / 6.0)
        })
    });
    let sum_vs: f64 = sig
        .mb_vars()
        .iter()
        .map(|&v| (v + 1) as f64)
        .zip(&dqp)
        .map(|(v, &d)| v * qstep[(d + AQ_DQP_MAX) as usize])
        .sum();
    let c = rusty_h264_common::fmath::round(6.0 * rusty_h264_common::fmath::log2(sum_vs / sum_v))
        as i32;
    dqp.iter()
        .map(|&d| (base_qp as i32 + c + d).clamp(0, 51) as u8)
        .collect()
}

/// Mean per-sampled-pixel residual after GLOBAL-motion compensation of `sy` from
/// `ref_y` (coarse ±12 global ME + ±3 refine, subsampled interior). ~0 on a PURE pan
/// (a single MV predicts the whole frame) — precisely the content where the local ME
/// diamond never genuinely STALLS (its seed = the median = the pan MV is already
/// right), so the `me_wide` rescue can only find SPURIOUS MVs that wreck the B-frame
/// spatial-direct predictors. Gates `me_wide` off there — non-uniform content
/// (real stalls, where me_wide wins) reads well above 0.
/// Per-frame HEAD-ROOM probe for the `me_wide` rescue: on a small subsample of
/// blocks, how much does a WIDE full-pel search beat a PREDICTOR-LOCAL one?
///
/// This measures what the rescue actually buys, before the macroblock loop and
/// without committing any vector — unlike the online payoff gate, which scores its
/// own SATD cost-cut *after* committing MVs and so only ever separated static
/// content. Returns the mean relative SAD improvement, in percent.
///
/// Calibrated against the 20-clip per-clip BD truth table (docs/WHYS-speed-gap.md
/// R5): me_wide earns its 1.4–5.1× on high-head-room content (bus +4.57, blue_sky
/// +4.70, football +1.51, park_joy +0.91) and REGRESSES on low-head-room content
/// (foreman_qcif −1.08, foreman_cif −0.16, tempete −0.12, mobile −0.03).
///
/// Deliberately PER-FRAME, not per-clip: cross-frame adaptive state is
/// nondeterministic under the GOP-parallel encode path (a lesson already paid for
/// by the rescue's own learning window).
/// Head-room threshold (percent) for the `me_wide` frame gate. DEFAULT-ON at 16.
///
/// Calibrated on the DEPLOYED estimator (not the offline probe — they differ) and
/// gated on the full 20-clip `video-tests` corpus plus four synthesized boundary
/// clips, 4-QP BD-rate on PSNR and SSIM:
///
/// | | me_wide always-on | gated at 16 |
/// |---|---|---|
/// | real-corpus mean | +0.62% | +0.547% (88% retained) |
/// | **worst clip** | **−1.08%** (foreman_qcif) | **0.00%** |
/// | clips paying 1.1–3.6× for ~nothing | 13 | 0 |
///
/// Wins preserved: blue_sky +4.70, bus +4.37, park_joy +0.94, football +0.64,
/// shields +0.20; synthesized fast-pan +6.73, rotation +1.72, zoom +1.11.
/// Monotone non-regression — no clip is negative — which is what promotes this from
/// a speed trade to a default.
///
/// `RFF_ME_HR=0` disables the gate and reproduces the pre-gate bytes exactly (the
/// escape hatch / bisection anchor). Thresholds 13 and 16 both clear the boundary
/// clip (foreman_cif +0.07 / +0.03); 10 does NOT (−0.23) — the threshold is
/// calibrated on a narrow boundary pair, so treat it as re-tunable, not settled.
fn me_wide_hr_thresh() -> f64 {
    static T: OnceLock<f64> = OnceLock::new();
    *T.get_or_init(|| {
        rusty_h264_common::knob("RFF_ME_HR")
            .and_then(|s| s.parse().ok())
            .unwrap_or(16.0)
    })
}

/// Cached, because it is read per frame — an `env::var` there is its own tax.
fn me_wide_hr_dbg() -> bool {
    static D: OnceLock<bool> = OnceLock::new();
    *D.get_or_init(|| rusty_h264_common::knob("RFF_ME_HR_DBG").is_some())
}

// `me_wide_headroom` and `global_mc_residual` live in `crate::signals` (Great
// Gate P1) — read through `FrameSignals::headroom` / `FrameSignals::gmc_residual`,
// memoized so the CABAC-P driver's two consumers (the lme motion term and the
// me_wide coherence gate) share ONE computation per frame.

/// Adds the mb-tree per-MB QP offset (TEMPORAL AQ — [`crate::mbtree`]) to the
/// spatial-AQ `aq_qp` map in place. An empty `qpo` (mb-tree off) or a length
/// mismatch is a no-op → byte-identical. Shared by the CAVLC and CABAC slice paths.
fn apply_mbtree_qpo(aq_qp: &mut [u8], qpo: &[i32]) {
    if qpo.len() == aq_qp.len() {
        for (q, &o) in aq_qp.iter_mut().zip(qpo) {
            *q = (*q as i32 + o).clamp(0, 51) as u8;
        }
    }
}

/// IMPLICIT bi-prediction weights `(w0, w1)` from POC distances (spec §8.4.2.3.2,
/// `weighted_bipred_idc == 2`), IDENTICAL to the decoder's `implicit_weights`. The
/// closer anchor gets more weight; an equidistant B (`bframes == 1`) yields 32:32,
/// i.e. the plain average. `(32, 32)` fallback for the degenerate/out-of-range cases
/// the decoder also averages (no long-term refs here).
fn implicit_bi_weights(cur_poc: i32, l0_poc: i32, l1_poc: i32) -> (i32, i32) {
    let td = (l1_poc - l0_poc).clamp(-128, 127);
    let tb = (cur_poc - l0_poc).clamp(-128, 127);
    if td == 0 {
        return (32, 32);
    }
    // `(16384 + |td|/2) / td` for every clamped td, tabulated at compile time —
    // the one variable integer divide left in the B coding path (`td == 0` is
    // guarded above; its slot is unused). Same quotients: BIT-IDENTICAL.
    const TX: [i32; 256] = {
        let mut t = [0i32; 256];
        let mut i = 0usize;
        while i < 256 {
            let td = i as i32 - 128;
            if td != 0 {
                t[i] = (16384 + (if td < 0 { -td } else { td }) / 2) / td;
            }
            i += 1;
        }
        t
    };
    let tx = TX[(td + 128) as usize];
    let dsf = ((tb * tx + 32) >> 6).clamp(-1024, 1023);
    let w1 = dsf >> 2;
    if !(-64..=128).contains(&w1) {
        return (32, 32);
    }
    (64 - w1, w1)
}

/// Bi-prediction blend of two motion-compensated samples `p` (List-0) and `q`
/// (List-1) under weights `(w0, w1)` — the decoder's `b_mc` blend. `(32, 32)` is the
/// plain `(p+q+1)>>1` average.
#[inline(always)]
fn bi_blend(p: i32, q: i32, w: (i32, i32)) -> u8 {
    ((p * w.0 + q * w.1 + 32) >> 6).clamp(0, 255) as u8
}

/// Zig-zag scan of a raster i16 4×4 block into scan-order i32 — the fused-path
/// twin of `scan_4x4_dcac(&q_blocks[..])`, reading quantized levels straight from
/// the hot i16 DCT buffer. Byte-identical: the i16→i32 widening of a quant level
/// is exact (levels always fit i16, being the input to the i16 idct kernel).
#[cfg(accel)]
#[inline]
fn scan_4x4_dcac_i16(d: &[i16]) -> [i32; 16] {
    [
        d[0] as i32,
        d[1] as i32,
        d[4] as i32,
        d[8] as i32,
        d[5] as i32,
        d[2] as i32,
        d[3] as i32,
        d[6] as i32,
        d[9] as i32,
        d[12] as i32,
        d[13] as i32,
        d[10] as i32,
        d[7] as i32,
        d[11] as i32,
        d[14] as i32,
        d[15] as i32,
    ]
}

/// Per-frame intra encoder state: reconstructed planes (coded size) and the
/// per-4×4-block non-zero-coefficient counts used for CAVLC context.
pub struct FrameEncoder {
    mb_w: usize,
    mb_h: usize,
    qp: u8,  // the CURRENT macroblock's target QPy (AQ varies it per MB)
    qpc: u8, // chroma QP for `qp`
    /// Running QPy of the last macroblock that coded an `mb_qp_delta` (spec QPY_PREV).
    /// `mb_qp_delta = qp − cur_qp`; a skip / cbp==0 MB codes no delta and inherits it.
    cur_qp: u8,
    /// Implicit bi-prediction weights `(w0, w1)` for the current B-frame (from its
    /// L0/L1 anchor POC distances). `(32, 32)` = plain average (P/I frames, `bframes
    /// == 1`); unequal for `bframes > 1`.
    bi_w: (i32, i32),
    cw: usize,  // coded luma width
    ccw: usize, // coded chroma width
    // 16-byte aligned (the openh264 deblock/MC/intra asm load aligned row chunks).
    rec_y: AlignedBytes,
    rec_u: AlignedBytes,
    rec_v: AlignedBytes,
    nnz_y: Vec<u8>,        // (mb_w*4) x (mb_h*4)
    nnz_c: [Vec<u8>; 2],   // each (mb_w*2) x (mb_h*2)
    modes_y: Vec<u8>,      // intra4x4 mode per 4×4 block (2=DC for I_16x16 blocks)
    coded_y: Vec<bool>,    // whether each 4×4 block is reconstructed (top-right avail)
    mv_y: Vec<(i16, i16)>, // motion vector per 4×4 block (quarter-pel) — List-0
    inter_y: Vec<bool>,    // whether each 4×4 block is inter-coded
    /// Narrowed i32 -> i8 with the decoder's matching grid: a `ref_idx` is
    /// -1..31 by the spec (7.4.5.1), and `BlockInfo::ref_id` is `&[i8]`.
    ref_idx_y: Vec<i8>,   // reference index per 4×4 block (-1 = intra/uncoded) — List-0
    // B-slice List-1 motion field (empty for P/I). B_L1/B_Bi commit here so a later
    // partition's List-1 median predictor sees it, mirroring the decoder's
    // `mv_neighbors_list(.., 1)` over `mv1`/`ref_idx1`.
    mv1_y: Vec<(i16, i16)>,
    ref_idx1_y: Vec<i8>,
    idz: i64, // intra dead-zone divisor: 2 for all-intra, 3 when frames reference each other
    rdoq_strength: f64, // CABAC trellis (RDOQ) strength; 0 = off (hard quantize, CAVLC path)
    // Explicit P weighted prediction (x264-parity weightp): per-reference LUMA
    // (weight, offset) at denom 6 for THIS slice; empty = off/identity. Set by
    // the P slice coders from the header's own table, applied post-MC at every
    // P prediction build — the same integer form, in the same place, as the
    // decoder's `weight_partition`, or decode(encode(x)) drifts on fades.
    wp: Vec<(i32, i32)>,
    transform_8x8: bool, // High-profile 8x8 transform enabled (transform_8x8_mode_flag)
    sub8x8: bool,        // P_8x8 sub-partition motion (four 8x8 MVs per MB)
    me_wide: bool,       // adaptive wide ME grid search rescue (diamond stalls on flat surfaces)
    /// Track-B B2 for THIS frame: SAD-domain full-pel phase. Set at construction
    /// (force mode), or per frame by the `b2_mgain` dispatcher (mode 1).
    sadfp: bool,
    /// H-24 mv-cost SHAPE routing for THIS frame (mv_smooth mode 1), set by the
    /// same `b2_mgain` probe. Per-frame state, NOT a global: the GOP-parallel
    /// encode runs frames concurrently and a global store races across workers.
    mv_smooth: bool,
    /// H-13: search partition splits this frame (routed off on near-static frames).
    do_splits: bool,
    me_wide_var: u64, // per-pixel source variance below which a block is "flat"
    me_rescue: i64,   // per-pixel residual SATD (on a flat block) that flags a diamond stall
    me_wide_coh: f64, // gate me_wide off when the frame's global-MC residual is below this (pure pan)
    me_range: i32, // rescue grid half-range in px (16 = ±16; wider reaches FAST motion the diamond misses)
    me_fast: bool, // also fire the rescue on HIGH-VARIANCE high-residual blocks (fast-motion stalls, not just flat)
    // ONLINE per-frame rescue-payoff gate (adaptive; WITHIN-frame so it stays
    // deterministic under the frame-parallel encode). Run the real rescue on the
    // first `me_learn` stalls of a frame, count how many the fine grid improves by
    // ≥6.25%, and if that fraction is below `me_payoff_pct`% disable the rescue for
    // the rest of the frame. This separates genuine diamond stalls (tsrc/zoom fine
    // grid improves ~33% of fires) from IRREDUCIBLE residual (rotation/fractal ~5-8%)
    // using the ACTUAL neighbour-seeded diamond — the only faithful signal (a cheap
    // SAD proxy from (0,0) inverts it: rot reads as highest-payoff). Frame-level, so
    // no per-block selection concentrates the B-direct-poisoning spurious MVs.
    me_learn: u32,
    me_payoff_pct: u32,
    /// U1 online sub-pel dispatcher (within-frame, so it stays deterministic under
    /// GOP-parallel encode). For the first `SP_LEARN` refinements of a frame we run
    /// the full 8-point+iterate pattern and accumulate how much of the total gain the
    /// FIRST ring captured; once the window fills, a frame whose gain is concentrated
    /// in ring 1 switches to the single-pass pattern for the rest of the frame.
    ///
    /// Harvested justification: ring-1 captures 63.7% of the gain on foreman (which
    /// loses +2.34% BD to a blanket single-pass) against 69.9–71.9% on bus/mobile
    /// (which lose only +0.30/+0.74% and gain 1.08–1.31×). The fraction separates the
    /// content that can afford the cut from the content that cannot.
    sp_single_pass: bool,
    /// U5-struct: when set, `motion_search` returns its FULL-PEL winner and skips
    /// sub-pel refinement entirely. The partition driver uses this to search all
    /// candidate shapes cheaply, pick one, and refine ONLY the winner's sub-blocks.
    /// Measured ceiling: 3.4–6.4× less sub-pel work (the losing shapes' refinements
    /// are pure waste), i.e. ~1.42× whole-encode at 44% sub-pel share.
    sp_defer: core::cell::Cell<bool>,
    sp_learn_n: core::cell::Cell<u32>,
    sp_ring1: core::cell::Cell<i64>,
    sp_total: core::cell::Cell<i64>,
    sp_1pass: core::cell::Cell<bool>,
    resc_n: core::cell::Cell<u32>, // stalls the fine grid ran on this frame (learning phase)
    resc_big: core::cell::Cell<u32>, // of those, how many it improved ≥6.25%
    resc_off: core::cell::Cell<bool>, // rescue disabled for the rest of this frame
    /// May a macroblock CHOOSE the 8x8 transform this frame?
    ///
    /// Deliberately separate from `transform_8x8`, which means "the PPS advertises
    /// transform_8x8_mode_flag, so the flag must be WRITTEN". Those are not the same
    /// thing, and conflating them is a desync: clearing `transform_8x8` per frame
    /// suppresses a bit the decoder is still reading. Presence is the PPS's business;
    /// value is the per-frame gate's. (Third instance of that confusion in one day —
    /// the sub-8x8 and B_Direct flag bugs were the other two.)
    t8_pick: bool,
    /// Margin, in lambda units, that the I_8x8 candidate must BEAT its rivals by.
    /// See `i8_margin()`.
    i8_margin: f64,
    /// True while planning an INTRA macroblock inside a P/B slice. Those blocks are
    /// a different population from intra MBs in an I slice -- they are the ones inter
    /// prediction FAILED on (occlusion, fine detail), which is exactly where 4x4
    /// adapts better than 8x8. Set per call site, never inferred.
    intra_in_p: bool,
    // Inter 8x8-transform dispatch: 0 = off (intra 8x8 only), 1 = always-RD.
    // There is no mode 2: the enum used to document "2 = content-adaptive" and the
    // only test of this field is `!= 0`, so mode 2 was never anything but mode 1 --
    // confirmed by a BD table that came back identical in all 21 cells.
    inter8x8: u8,
    inter8_pen: i64, // extra rate charge (nonzero-equiv) on the inter 8x8 candidate
    fast: bool,      // Preset::Fast — SATD mode decision (no RDO), 16×16/I_16x16 only
    #[cfg_attr(not(accel), allow(dead_code))] // read in accel-gated blocks
    skip_accel_check: bool, // A/B knob: whole-MB psadbw gate in the P_Skip free-check
    coded_path_v2: bool, // A/B knob: route inter coding through encode_inter_mb_v2
    tune_lambda_scale: f64, // tuning knob: scale on the RD λ (1.0 = standard)
    tune_intra_penalty: f64,
    satd_q: f64,           // adaptive: fraction of high-variance MBs routed to SATD cost
    subpel_force: bool,    // force sub-pel refinement even in the fast preset
    me_snap: bool,         // snap the diamond centre to integer-pel (see config)
    me_subpel_iter: bool,  // walk the sub-pel refine to convergence
    greedy_skip: bool,     // quality preset's SAD-thresholded P_Skip (PredictSadSkip)
    greedy_min_free: u32,  // online free-skip % gating greedy_skip on this frame
    rd_skip: bool,         // decide P_Skip by J = SSD + lambda*bits, not exact-zero residual
    rd_skip_min_free: u32, // online free-skip % gating rd_skip on this frame
    rd_skip_fast_t: f64,   // skip-gate on SSD(skip)/lambda; <= 0 prices every candidate
    satd_var_thresh: i64,  // per-frame variance threshold for the routing (set in a pre-pass)
    aq_strength: f64,      // adaptive quantization: per-MB QP modulation strength (0 = off)
    mb_use_satd: bool,     // per-MB: this MB uses the SATD cost this decision
    // Per-MB luma nnz prediction cache (openh264 scan8 style): a padded 5×5 grid,
    // block (lbx,lby) at (lby+1)*5+(lbx+1); row 0 = top neighbours, col 0 = left.
    // Unavailable edges hold the sentinel 0x80, so the nnz predict is branchless.
    nnz_l_cache: [u8; 25],
    // Same, per chroma plane: a padded 3×3 grid for the 2×2 chroma blocks.
    nnz_c_cache: [[u8; 9]; 2],
    // openh264 predicted-SAD skip apparatus (per MB, mb_w×mb_h): the P_Skip
    // prediction's luma SAD, and whether the MB was actually skipped. The greedy
    // skip threshold for an MB is the median of its skip *neighbours'* skip SADs
    // (`PredictSadSkip`) — so skip propagates only from already-skip regions
    // (seeded by free skips) and self-limits, instead of a fixed bound that drifts.
    mb_skip_sad: Vec<u32>,
    mb_was_skip: Vec<bool>,
}

/// A chosen inter coding for a macroblock: `mb_type` and, per partition, the
/// reference index and motion vector.
type InterChoice = (u8, Vec<(i32, (i32, i32))>);

/// EXTERNAL MV SCORING (`RFF_MV_CMP=1`). Holds another encoder's motion field
/// (per frame, 4x4-block raster) so our own coder can price ITS vectors against
/// ours under REAL coded bits instead of SATD — the only way to tell a bad search
/// from a bad cost function.
#[cfg(feature = "std")]
pub static EXT_MV: std::sync::Mutex<Vec<Vec<(i32, i32)>>> = std::sync::Mutex::new(Vec::new());
/// [n, our bits, ext bits, our SSD, ext SSD, ext won on J, MVs differing]
pub static MVCMP: [rusty_h264_common::atomic::AtomicU64; 7] = {
    const Z: rusty_h264_common::atomic::AtomicU64 = rusty_h264_common::atomic::AtomicU64::new(0);
    [Z; 7]
};
pub static MVCMP_FRAME: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0);
/// Replace our chosen vector with the external field's, for EVERY macroblock where
/// that field used a single 16x16 partition. Transplanting one vector in isolation
/// is meaningless — `mvd` is coded against the NEIGHBOURS' vectors, so a lone
/// foreign vector prices against the wrong predictor. Only a whole coherent field
/// can be compared fairly.
#[cfg_attr(not(feature = "std"), allow(dead_code))]
fn mv_force_on() -> bool {
    static ON: rusty_h264_common::once::OnceLock<bool> = rusty_h264_common::once::OnceLock::new();
    *ON.get_or_init(|| rusty_h264_common::knob("RFF_MV_FORCE").is_some_and(|v| v != "0"))
}
#[cfg_attr(not(feature = "std"), allow(dead_code))]
fn mv_cmp_on() -> bool {
    static ON: rusty_h264_common::once::OnceLock<bool> = rusty_h264_common::once::OnceLock::new();
    *ON.get_or_init(|| rusty_h264_common::knob("RFF_MV_CMP").is_some_and(|v| v != "0"))
}

/// [n, sum our cost, sum oracle cost, blocks the oracle beat us on, cost() evals]
pub static ME_PROBE: [rusty_h264_common::atomic::AtomicU64; 7] = {
    const Z: rusty_h264_common::atomic::AtomicU64 = rusty_h264_common::atomic::AtomicU64::new(0);
    [Z; 7]
};

/// Cached — an `env::var` inside the ME loop inflated it 4x when probed naively.
fn me_oracle_on() -> bool {
    static ON: rusty_h264_common::once::OnceLock<bool> = rusty_h264_common::once::OnceLock::new();
    *ON.get_or_init(|| rusty_h264_common::knob("RFF_ME_ORACLE").is_some_and(|v| v != "0"))
}

/// A snapshot of one macroblock's per-block grids and reconstruction region,
/// used to roll back a trial encode during RD mode decision.
///
/// Every field is a `Vec`, so building one from scratch is ten heap allocations.
/// The RD skip decision snapshots on EVERY candidate macroblock, which made that
/// allocation traffic the decision's dominant cost — hence
/// [`save_mb_into`](FrameEncoder::save_mb_into), which refills a reused buffer.
/// Widens a stored motion vector back to the width every MV computation uses.
///
/// The grids store `(i16, i16)` because a motion vector is quarter-pel and
/// level-bounded, but median prediction, mvd arithmetic and temporal scaling all
/// run at `i32`. Rust has no implicit numeric coercion, so this call is the
/// COMPILER's proof that no arithmetic silently happens at the narrow width.
#[inline]
fn wmv(m: (i16, i16)) -> (i32, i32) {
    (i32::from(m.0), i32::from(m.1))
}

/// Narrows a computed motion vector to the grid's storage width. Safe by the
/// spec's MV range (+/-2048 quarter-pel at level 5.1, 16x headroom in `i16`) and
/// already load-bearing: `pack_mb` narrows the same values unconditionally for
/// the deblock path.
#[inline]
fn nmv(m: (i32, i32)) -> (i16, i16) {
    (m.0 as i16, m.1 as i16)
}

#[derive(Default)]
struct MbState {
    rec_y: Vec<u8>,
    rec_u: Vec<u8>,
    rec_v: Vec<u8>,
    nnz_y: Vec<u8>,
    nnz_c: [Vec<u8>; 2],
    mv_y: Vec<(i16, i16)>,
    inter_y: Vec<bool>,
    /// Narrowed i32 -> i8 with the decoder's matching grid: a `ref_idx` is
    /// -1..31 by the spec (7.4.5.1), and `BlockInfo::ref_id` is `&[i8]`.
    ref_idx_y: Vec<i8>,
    coded_y: Vec<bool>,
    modes_y: Vec<u8>,
    /// QPY_PREV. `qp_delta()` MUTATES this as a side effect of coding
    /// `mb_qp_delta`, so a trial encode advances it; without restoring it the
    /// real encode then codes its delta against the wrong predecessor and the
    /// decoder's QP diverges from the encoder's — a silent stream corruption,
    /// not a quality tweak.
    cur_qp: u8,
}

/// Edge-clamped, coded-size source planes (luma, Cb, Cr).
/// Fast-preset pruned I4x4 mode search ({MPM, DC, V, H} instead of all 9 — the
/// x264-ultrafast-style candidate set). DEFAULT ON for the fast preset (gated:
/// +0.5% size at +0.02 dB on all-intra, +17% all-intra speed); RUSTY_FAST_INTRA=0
/// restores the exhaustive 9-mode search (the pre-flip bitstream).
fn fast_intra_enabled() -> bool {
    static ON: rusty_h264_common::once::OnceLock<bool> = rusty_h264_common::once::OnceLock::new();
    *ON.get_or_init(|| rusty_h264_common::knob("RUSTY_FAST_INTRA").is_none_or(|v| v != "0"))
}

fn coded_source<'a>(
    cfg: &EncoderConfig,
    frame: &YuvPlanes<'a>,
) -> (
    alloc::borrow::Cow<'a, [u8]>,
    alloc::borrow::Cow<'a, [u8]>,
    alloc::borrow::Cow<'a, [u8]>,
) {
    let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncSource);
    let cw = cfg.mb_width() * 16;
    let ch = cfg.mb_height() * 16;
    // MB-aligned frame (the common case): the clamp is the identity, and the
    // drivers only READ the planes — borrow them. This was THREE full-plane
    // clones per slice (11.11); Cow keeps the padded path allocation intact.
    if frame.width == cw && frame.height == ch {
        return (
            alloc::borrow::Cow::Borrowed(frame.y),
            alloc::borrow::Cow::Borrowed(frame.u),
            alloc::borrow::Cow::Borrowed(frame.v),
        );
    }
    let y = clamp_plane(frame.y, frame.width, frame.height, cw, ch);
    let u = clamp_plane(
        frame.u,
        frame.chroma_width(),
        frame.chroma_height(),
        cw / 2,
        ch / 2,
    );
    let v = clamp_plane(
        frame.v,
        frame.chroma_width(),
        frame.chroma_height(),
        cw / 2,
        ch / 2,
    );
    (
        alloc::borrow::Cow::Owned(y),
        alloc::borrow::Cow::Owned(u),
        alloc::borrow::Cow::Owned(v),
    )
}

/// 11.11: per-THREAD recycled slice scratch — the encoder twin of the decoder
/// GridPool's `sc_*` fields (D13's finding, never ported to the encoder until
/// now). GOP-parallel workers each hold their own copy, so reuse stays
/// deterministic. Every buffer is reset to its fresh-allocation contents
/// before use — byte-identical by construction.
mod enc_scratch {
    #[allow(unused_imports)]
    use alloc::{
        boxed::Box,
        format,
        string::{String, ToString},
        vec,
        vec::Vec,
    };
    use core::cell::RefCell;
    #[allow(unused_imports)]
    use rusty_h264_common::once::OnceLock;
    thread_local! {
        static CS: RefCell<Option<super::CabacState>> = const { RefCell::new(None) };
        static QPY: RefCell<Vec<u8>> = const { RefCell::new(Vec::new()) };
        static REFID: RefCell<Vec<i8>> = const { RefCell::new(Vec::new()) };
        // The b-pyramid ref-B deblock tail needs BOTH lists' index grids alive
        // at once, so List-1 gets its own recycled slot.
        static REFID1: RefCell<Vec<i8>> = const { RefCell::new(Vec::new()) };
        static PAYLOAD: RefCell<Vec<u8>> = const { RefCell::new(Vec::new()) };
        static BITS: RefCell<super::BitWriter> = RefCell::new(super::BitWriter::new());
        static SNAP_A: RefCell<super::MbState> = RefCell::new(super::MbState::default());
        static SNAP_B: RefCell<super::MbState> = RefCell::new(super::MbState::default());
    }
    pub(super) fn take_cs() -> Option<super::CabacState> {
        CS.with(|c| c.borrow_mut().take())
    }
    pub(super) fn put_cs(v: super::CabacState) {
        CS.with(|c| *c.borrow_mut() = Some(v));
    }
    pub(super) fn take_qpy() -> Vec<u8> {
        QPY.with(|c| core::mem::take(&mut *c.borrow_mut()))
    }
    pub(super) fn put_qpy(v: Vec<u8>) {
        QPY.with(|c| *c.borrow_mut() = v);
    }
    pub(super) fn take_refid() -> Vec<i8> {
        REFID.with(|c| core::mem::take(&mut *c.borrow_mut()))
    }
    pub(super) fn put_refid(v: Vec<i8>) {
        REFID.with(|c| *c.borrow_mut() = v);
    }
    pub(super) fn take_refid1() -> Vec<i8> {
        REFID1.with(|c| core::mem::take(&mut *c.borrow_mut()))
    }
    pub(super) fn put_refid1(v: Vec<i8>) {
        REFID1.with(|c| *c.borrow_mut() = v);
    }
    pub(super) fn take_payload() -> Vec<u8> {
        PAYLOAD.with(|c| core::mem::take(&mut *c.borrow_mut()))
    }
    pub(super) fn put_payload(v: Vec<u8>) {
        PAYLOAD.with(|c| *c.borrow_mut() = v);
    }
    pub(super) fn take_bits() -> super::BitWriter {
        BITS.with(|c| core::mem::take(&mut *c.borrow_mut()))
    }
    pub(super) fn put_bits(v: super::BitWriter) {
        BITS.with(|c| *c.borrow_mut() = v);
    }
    // E11 dig (11.17): `save_mb()` built a DEFAULT MbState — ten empty Vecs
    // whose pushes then heap-allocate — per RD TRIAL. Two recycled slots: A for
    // the driver sites (their lifetimes are sequential), B for `trial_intra`,
    // which nests inside a shape-RD trial that still holds A.
    pub(super) fn take_snap_a() -> super::MbState {
        SNAP_A.with(|c| core::mem::take(&mut *c.borrow_mut()))
    }
    pub(super) fn put_snap_a(v: super::MbState) {
        SNAP_A.with(|c| *c.borrow_mut() = v);
    }
    pub(super) fn take_snap_b() -> super::MbState {
        SNAP_B.with(|c| core::mem::take(&mut *c.borrow_mut()))
    }
    pub(super) fn put_snap_b(v: super::MbState) {
        SNAP_B.with(|c| *c.borrow_mut() = v);
    }
}

/// Edge-extends `plane` from `w`×`h` to the coded `ow`×`oh`, replicating the last
/// row/column — the source form the MB grid needs.
///
/// Row-wise, because the per-pixel form is O(pixels) of scalar `min`+multiply and is
/// the DOMINANT cost of `enc-source-copy`: every frame whose height is not a multiple
/// of 16 takes this path, which includes all 1080p content (1080/16 = 67.5 → coded
/// height 1088). The stage measured 579 ms over the corpus while the three plane
/// clones on the MB-aligned fast path account for only ~135 ms of it.
///
/// Byte-identical to the per-pixel form (`clamp_plane_per_pixel`, kept as the test
/// oracle): `x.min(w-1)` is the identity below `w` and pins to the last column above
/// it, so a row is a `copy_from_slice` plus a `fill`; `y.min(h-1)` makes the
/// overhanging rows copies of the final row. Both lower to memcpy/memset.
fn clamp_plane(plane: &[u8], w: usize, h: usize, ow: usize, oh: usize) -> Vec<u8> {
    let mut out = vec![0u8; ow * oh];
    for y in 0..oh {
        let sy = y.min(h - 1);
        let src = &plane[sy * w..sy * w + w];
        let dst = &mut out[y * ow..y * ow + ow];
        if ow <= w {
            dst.copy_from_slice(&src[..ow]);
        } else {
            dst[..w].copy_from_slice(src);
            dst[w..].fill(src[w - 1]);
        }
    }
    out
}

/// The original per-pixel edge extension — kept as the correctness oracle for
/// [`clamp_plane`], per the scalar-twin discipline.
#[cfg(test)]
fn clamp_plane_per_pixel(plane: &[u8], w: usize, h: usize, ow: usize, oh: usize) -> Vec<u8> {
    let mut out = vec![0u8; ow * oh];
    for y in 0..oh {
        for x in 0..ow {
            out[y * ow + x] = plane[y.min(h - 1) * w + x.min(w - 1)];
        }
    }
    out
}

#[cfg(test)]
mod source_tests {
    use super::*;
    #[allow(unused_imports)]
    use alloc::{
        boxed::Box,
        format,
        string::{String, ToString},
        vec,
        vec::Vec,
    };
    #[allow(unused_imports)]
    use rusty_h264_common::once::OnceLock;

    /// The ref_bits prune in `best_part` breaks out of the reference loop the
    /// moment `λ·ref_bits(r)` alone reaches the incumbent cost — which is only
    /// sound if `ref_bits` never DECREASES with `r`. Pinned here over every
    /// active-reference count the config admits.
    #[test]
    fn ref_bits_is_monotone() {
        for n in 1..=16usize {
            for r in 1..n {
                assert!(
                    ref_bits(r, n) >= ref_bits(r - 1, n),
                    "ref_bits({r},{n}) < ref_bits({},{n})",
                    r - 1
                );
            }
        }
        // And the coding facts the model encodes: no ref_idx with one ref,
        // one flag bit with two.
        assert_eq!(ref_bits(0, 1), 0);
        assert_eq!(ref_bits(0, 2), 1);
        assert_eq!(ref_bits(1, 2), 1);
    }

    /// Bit-exact golden for the AQ map (site 7's function): flat top row
    /// (zero variance → the log2 shortcut arm), textured rest. Pins the whole
    /// pipeline — log_vars through the per-MB round loop and the rate
    /// compensation — at two strengths.
    #[test]
    fn aq_qp_map_golden() {
        let (cw, ch) = (160usize, 112usize);
        let (mb_w, mb_h) = (cw / 16, ch / 16);
        // Texture amplitude grows per MB COLUMN (variance gradient → the dqp
        // map varies), with exactly ONE flat MB (top-left) so the zero-variance
        // arm runs but the lv SPREAD stays inside AQ's active band — a whole
        // flat region reads as "pathological synthetic" and latches AQ off,
        // which is precisely the degenerate map the assert below refuses.
        let mut sy = vec![64u8; cw * ch];
        for j in 0..ch {
            for i in 0..cw {
                let k = 4 + (i / 16) * 12;
                sy[j * cw + i] = 80 + ((i * 7 + j * 13) % k) as u8;
            }
        }
        for j in 0..16 {
            for i in 0..16 {
                sy[j * cw + i] = 64;
            }
        }
        let mut h = 0xcbf2_9ce4_8422_2325u64;
        crate::fastmath::TEST_POLYTIER.with(|c| c.set(Some(false)));
        for strength in [1.0f64, 0.5] {
            let sig = signals::FrameSignals::new(&sy, cw, mb_w, mb_h, None);
            assert!(sig.mb_vars().contains(&0), "no zero-variance MB");
            let map = aq_qp_map(&sig, 26, strength);
            // The gate must prove the tool ran: a veto's early-out would pin
            // a constant map, not the arithmetic.
            assert!(
                map.iter().any(|&q| q != 26),
                "AQ map degenerate at {strength}"
            );
            // Round 10 decision-identity: the whole AQ pipeline under the
            // POLY arm (poly log2 + ties-even round) must produce the
            // IDENTICAL u8 map — the integer decisions have fat margins and
            // exact .5 ties are measure-zero, and this asserts it rather
            // than argues it.
            crate::fastmath::TEST_POLYTIER.with(|c| c.set(Some(true)));
            let sigp = signals::FrameSignals::new(&sy, cw, mb_w, mb_h, None);
            assert_eq!(
                aq_qp_map(&sigp, 26, strength),
                map,
                "poly arm map differs at {strength}"
            );
            crate::fastmath::TEST_POLYTIER.with(|c| c.set(Some(false)));
            for &q in &map {
                h ^= q as u64;
                h = h.wrapping_mul(0x100_0000_01b3);
            }
        }
        crate::fastmath::TEST_POLYTIER.with(|c| c.set(None));
        assert_eq!(h, 8641818917227936570, "aq_qp_map golden");
    }

    #[test]
    fn clamp_plane_matches_per_pixel_oracle() {
        let mut s: u32 = 0xDEAD_BEEF;
        let mut rnd = || {
            s = s.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
            (s >> 24) as u8
        };
        // Real coded geometries plus adversarial ones: width-only overhang,
        // height-only overhang (the 1080p case), both, and neither.
        let cases = [
            (1920usize, 1080usize, 1920usize, 1088usize), // 1080p luma
            (960, 540, 960, 544),                         // 1080p chroma
            (352, 288, 352, 288),                         // exactly aligned
            (100, 100, 112, 112),                         // both axes overhang
            (37, 5, 48, 16),                              // tiny + ragged
            (16, 1, 16, 16),                              // single source row
            (1, 1, 16, 16),                               // single sample
        ];
        for (w, h, ow, oh) in cases {
            let plane: Vec<u8> = (0..w * h).map(|_| rnd()).collect();
            assert_eq!(
                clamp_plane(&plane, w, h, ow, oh),
                clamp_plane_per_pixel(&plane, w, h, ow, oh),
                "clamp mismatch for {w}x{h} -> {ow}x{oh}"
            );
        }
    }
}

/// The constructor's fourteen env knobs, parsed ONCE per process (E20).
/// Fields mirror the exact expressions they replaced; `Option` where a cfg or
/// preset fallback applies, final value where the default is cfg-independent.
struct CtorEnv {
    sub8x8: Option<bool>,
    me_wide: Option<bool>,
    me_wide_var: u64,
    me_rescue: i64,
    me_wide_coh: f64,
    me_range: i32,
    me_fast: bool,
    me_learn: u32,
    me_payoff_pct: u32,
    defer_subpel: bool,
    inter8: u8,
    inter8_pen: i64,
    subpel: Option<bool>,
    rdskip_minfree: Option<u32>,
}
fn ctor_env() -> &'static CtorEnv {
    static E: rusty_h264_common::once::OnceLock<CtorEnv> = rusty_h264_common::once::OnceLock::new();
    E.get_or_init(|| CtorEnv {
        sub8x8: rusty_h264_common::knob("RFF_SUB8X8").map(|s| s == "1"),
        me_wide: rusty_h264_common::knob("RFF_ME_WIDE").map(|s| s == "1"),
        me_wide_var: rusty_h264_common::knob("RFF_ME_WIDE_VAR")
            .and_then(|s| s.parse().ok())
            .unwrap_or(800),
        me_rescue: rusty_h264_common::knob("RFF_ME_RESCUE")
            .and_then(|s| s.parse().ok())
            .unwrap_or(3),
        me_wide_coh: rusty_h264_common::knob("RFF_ME_COH")
            .and_then(|s| s.parse().ok())
            .unwrap_or(4.0),
        me_range: rusty_h264_common::knob("RFF_ME_RANGE")
            .and_then(|s| s.parse().ok())
            .unwrap_or(24),
        me_fast: rusty_h264_common::knob("RFF_ME_FASTMO")
            .map(|s| s != "0")
            .unwrap_or(true),
        me_learn: rusty_h264_common::knob("RFF_ME_LEARN")
            .and_then(|s| s.parse().ok())
            .unwrap_or(40),
        me_payoff_pct: rusty_h264_common::knob("RFF_ME_PAYOFF")
            .and_then(|s| s.parse().ok())
            .unwrap_or(15),
        defer_subpel: rusty_h264_common::knob("RFF_DEFER_SUBPEL")
            .map(|v| v != "0")
            .unwrap_or(false),
        inter8: rusty_h264_common::knob("RFF_INTER8")
            .and_then(|s| s.parse().ok())
            .unwrap_or(0),
        inter8_pen: rusty_h264_common::knob("RFF_INTER8_PEN")
            .and_then(|s| s.parse().ok())
            .unwrap_or(8),
        subpel: rusty_h264_common::knob("RFF_SUBPEL").map(|v| v != "1"),
        rdskip_minfree: rusty_h264_common::knob("RFF_RDSKIP_MINFREE").and_then(|v| v.parse().ok()),
    })
}

impl FrameEncoder {
    fn new(cfg: &EncoderConfig) -> Self {
        // E20 (inline-execution.md 11.10): this constructor ran FOURTEEN raw
        // `std::env::var` reads - an OS query + String alloc + parse each - PER
        // SLICE, while every other knob in the tree caches its env read once.
        // All fourteen are process-constant by the same contract; `ctor_env()`
        // is their one read.
        let ke = ctor_env();
        let (mb_w, mb_h) = (cfg.mb_width(), cfg.mb_height());
        let (cw, ch) = (mb_w * 16, mb_h * 16);
        let (ccw, cch) = (cw / 2, ch / 2);
        Self {
            mb_w,
            mb_h,
            qp: cfg.qp,
            qpc: chroma_qp(cfg.qp),
            cur_qp: cfg.qp,
            bi_w: (32, 32),
            cw,
            ccw,
            rec_y: AlignedBytes::zeroed(cw * ch),
            rec_u: AlignedBytes::zeroed(ccw * cch),
            rec_v: AlignedBytes::zeroed(ccw * cch),
            nnz_y: vec![0; (mb_w * 4) * (mb_h * 4)],
            nnz_c: [
                vec![0; (mb_w * 2) * (mb_h * 2)],
                vec![0; (mb_w * 2) * (mb_h * 2)],
            ],
            modes_y: vec![2; (mb_w * 4) * (mb_h * 4)],
            coded_y: vec![false; (mb_w * 4) * (mb_h * 4)],
            mv_y: vec![(0, 0); (mb_w * 4) * (mb_h * 4)],
            inter_y: vec![false; (mb_w * 4) * (mb_h * 4)],
            ref_idx_y: vec![-1; (mb_w * 4) * (mb_h * 4)],
            mv1_y: vec![(0, 0); (mb_w * 4) * (mb_h * 4)],
            ref_idx1_y: vec![-1; (mb_w * 4) * (mb_h * 4)],
            // All-intra (no inter references) tolerates the larger dead-zone; in
            // an I+P stream the IDR is a reference, so keep the standard offset.
            idz: if cfg.gop_size <= 1 { 2 } else { 3 },
            rdoq_strength: 0.0, // set >0 only in the CABAC slice coders
            wp: Vec::new(),
            transform_8x8: cfg.transform_8x8,
            // sub8x8 stays OPT-IN: the four P_8x8 sub-MVs feed the B-frames'
            // spatial-direct predictor, so on DIVERGENT motion (rotation/zoom/mixed)
            // it regresses with B-frames (mixed +0.24%, rot +0.42%, zoom +0.40%) —
            // a global effect its local RD gate can't see, and no clean dispatch
            // signal separates it yet (unlike me_wide's pure-pan coherence gate).
            // DEFAULT-ON for Quality (net real-content win; a 6-channel discovery
            // harvest proved no cheap gate beats always-on). Quality-only (Fast never
            // runs it). env RFF_SUB8X8 (0/1) > cfg.sub_8x8 (Some) > preset default.
            sub8x8: ke
                .sub8x8
                .or(cfg.sub_8x8)
                .unwrap_or(cfg.preset == crate::config::Preset::Quality),
            // me_wide is DEFAULT-ON for the Quality preset. VALIDATED 2026-07-27 on the
            // full 20-clip `video-tests` Derf corpus (4-QP BD-rate, PSNR+SSIM, anchor =
            // me_wide ON): **mean +0.62% BD-PSNR / +0.69% BD-SSIM**, i.e. turning it off
            // costs that much. Biggest wins blue_sky +4.70, bus +4.57, football +1.51,
            // park_joy +0.91; synthesized boundary content (smooth fast-pan / rotation /
            // zoom) reaches +2.6..+6.7%. The static clips (akiyo, FourPeople) sit at
            // exactly 0.00 at ~1.0x — the online payoff gate correctly disables it there.
            //
            // ⚠ UNFINISHED DISPATCH — the per-clip BD SIGN-FLIPS (+4.70 blue_sky ..
            // -1.08 foreman_qcif), and the cost when it fires is 1.0-5.1x. Worst value:
            // soccer_4cif 1.70x for +0.00, park_joy 5.08x for +0.91. `me_range` is NOT
            // the separating axis — it is a compromise dial (foreman_qcif loses at EVERY
            // range 24/16/8/4 = -1.08/-0.55/-0.50/-0.19 while blue_sky wins at every one
            // = +4.70/+3.10/+0.73), so shrinking it just trades the win away. The real
            // fix is a content signal that predicts the sign; the truth table for it is
            // in docs/WHYS-speed-gap.md.
            //
            // Quality-only (Fast never runs it). Precedence:
            // env RFF_ME_WIDE (0/1, for A/B) > cfg.me_wide (Some) > preset default.
            sadfp: me_sadfp_mode() == 2,
            mv_smooth: false,
            do_splits: true,
            me_wide: ke
                .me_wide
                .or(cfg.me_wide)
                .unwrap_or(cfg.preset == crate::config::Preset::Quality),
            me_wide_var: ke.me_wide_var,
            me_rescue: ke.me_rescue,
            me_wide_coh: ke.me_wide_coh,
            me_range: ke.me_range,
            me_fast: ke.me_fast,
            me_learn: ke.me_learn,
            me_payoff_pct: ke.me_payoff_pct,
            // U3: `balanced` runs SINGLE-PASS sub-pel. Measured on the 4-QP corpus,
            // a single pass captures 95.5–99.4% of the full refinement's BD benefit
            // (foreman −38.14 vs −39.94, mobile −49.38 vs −49.66, akiyo −26.10 vs
            // −26.43) for 1.03–1.31× less time — a straight Pareto improvement on the
            // preset. `RFF_SUBPEL_PAT=0` restores the full walk-to-convergence.
            sp_single_pass: cfg.preset == crate::config::Preset::Balanced,
            sp_defer: core::cell::Cell::new({
                let a = DEFER_SUBPEL.load(core::sync::atomic::Ordering::Relaxed) != 0
                    || ke.defer_subpel;
                // ONLY the Quality preset runs the multi-shape partition driver. On the
                // fast/balanced path there is a single 16×16 candidate, so there is no
                // losing shape to skip — deferring there does not save the refinement,
                // it DELETES it (measured +91..+145% BD before this guard).
                a && cfg.preset == crate::config::Preset::Quality
            }),
            sp_learn_n: core::cell::Cell::new(0),
            sp_ring1: core::cell::Cell::new(0),
            sp_total: core::cell::Cell::new(0),
            sp_1pass: core::cell::Cell::new(false),
            resc_n: core::cell::Cell::new(0),
            resc_big: core::cell::Cell::new(0),
            resc_off: core::cell::Cell::new(false),
            // DEFAULT 0 (intra 8x8 only). Inter 8x8 was the dominant regression in
            // the 8x8 default measurement: it turned foreman I+P from +0.28% to
            // +1.59% and harbour I+P from +0.12% to +1.62% BD-SSIM, the two largest
            // losses in the whole matrix, while its wins were <= 0.44%. `RFF_INTER8=1`
            // restores the always-RD arm as the comparator.
            t8_pick: cfg.transform_8x8,
            i8_margin: i8_margin(),
            intra_in_p: false,
            inter8x8: ke.inter8,
            // ~2 bits per 8x8 luma block (×4) of CAVLC-8x8 overhead the level-aware
            // rate still under-charges (no native 8x8 entropy model in CAVLC). Keeps
            // the per-MB transform RD from over-picking 8x8 on fine-texture MBs where
            // it doesn't compact — content-adaptive: only decisively-favorable MBs win.
            inter8_pen: ke.inter8_pen,
            // Balanced shares Fast's decision path; only sub-pel differs.
            //
            // `RFF_SUBPEL=1` forces the sub-pel path ON for a non-Quality preset. This
            // was the ONE preset-derived field with no override, which is exactly why
            // the fast/quality bisection stalled: the other three (sub8x8, me_wide,
            // shape-RD) are all byte-identical when toggled on `fast`, so this field
            // carries the whole 29% delta by elimination (docs/WHYS-p-frames.md D5d).
            // `== Fast`, NOT `!= Quality`. The old form gave Preset::Balanced
            // fast = true, i.e. NO sub-pel -- flatly contradicting Balanced's own doc
            // ("Fast's decision path plus sub-pel motion refinement, which Fast
            // omits"). Combined with a CLI that never parsed "balanced", the whole
            // preset was unreachable, which is why a speed harness once adopted it as
            // a deliberate NULL ARM.
            //
            // GRAIN CAVEAT, measured and NOT yet fixed: sub-pel interpolates, and on
            // grain it interpolates NOISE -- Balanced is +13.61% BD-SSIM there against
            // Fast (Quality is worse still, +18.79%). A per-frame veto on
            // `grain_signature()` was attempted at all three slice coders and did NOT
            // take: `fe.fast` is consumed before a per-frame mutation lands, so the
            // gate must move to FrameEncoder construction or into the ME itself.
            // Removed rather than shipped half-applied. Balanced is OPT-IN, so this is
            // a documented trade, not a silent regression.
            //
            // Sub-pel is the single biggest compression lever in the encoder:
            // -30.10 (akiyo) / -44.37 (foreman) / -49.52 (mobile) / -34.15 (harbour)
            // BD-SSIM against Fast, for 2.50x Fast's CPU where Quality costs 16.24x.
            fast: ke
                .subpel
                .unwrap_or(cfg.preset == crate::config::Preset::Fast || SeqFastPath::get()),
            skip_accel_check: cfg.tune_skip_accel_check,
            coded_path_v2: cfg.coded_path_v2,
            aq_strength: cfg.aq_strength,
            tune_lambda_scale: cfg.tune_lambda_scale,
            tune_intra_penalty: cfg.tune_intra_penalty,
            satd_q: cfg.tune_satd_q,
            subpel_force: cfg.tune_subpel || cfg.preset == crate::config::Preset::Balanced,
            me_snap: cfg.tune_me_snap,
            me_subpel_iter: cfg.tune_me_subpel_iter,
            greedy_skip: cfg.tune_greedy_skip,
            greedy_min_free: cfg.tune_greedy_skip_min_free.unwrap_or(85),
            rd_skip: cfg.tune_rd_skip,
            rd_skip_fast_t: cfg.tune_rd_skip_fast_t.unwrap_or(0.0),
            // DECOUPLED from the preset by env, because welding it to `cfg.preset` is
            // what made the sub-pel veto's acceptance test unpassable (D5i): a knob that
            // forces integer-pel could never reproduce `--preset fast`, since Fast also
            // moves this threshold 90 -> 60. `RFF_RDSKIP_MINFREE` makes the combination
            // expressible so it can be measured instead of inferred.
            rd_skip_min_free: ke.rdskip_minfree.or(cfg.tune_rd_skip_min_free).unwrap_or(
                if cfg.preset == crate::config::Preset::Fast {
                    60
                } else {
                    90
                },
            ),
            satd_var_thresh: i64::MAX,
            mb_use_satd: false,
            nnz_l_cache: [0x80; 25],
            nnz_c_cache: [[0x80; 9]; 2],
            mb_skip_sad: vec![0; mb_w * mb_h],
            mb_was_skip: vec![false; mb_w * mb_h],
        }
    }

    /// openh264 `PredictSadSkip`: the greedy P_Skip threshold = the median of the
    /// skip SADs of the *skip* neighbours (left A, top B, top-right C, top-left
    /// fallback for C). Non-skip neighbours contribute 0, so with no skip neighbour
    /// the threshold is 0 (no greedy skip). This makes the skip self-calibrating —
    /// it only spreads where a neighbour already skipped at a comparable SAD.
    fn pred_skip_sad(&self, mb_x: usize, mb_y: usize) -> u32 {
        let mbw = self.mb_w;
        let at = |x: isize, y: isize| -> Option<(bool, u32)> {
            if x < 0 || y < 0 || x >= mbw as isize {
                return None;
            }
            let i = y as usize * mbw + x as usize;
            Some((self.mb_was_skip[i], self.mb_skip_sad[i]))
        };
        let a = at(mb_x as isize - 1, mb_y as isize); // left
        let b = at(mb_x as isize, mb_y as isize - 1); // top
        let c = at(mb_x as isize + 1, mb_y as isize - 1) // top-right
            .or_else(|| at(mb_x as isize - 1, mb_y as isize - 1)); // top-left fallback
        let sad = |n: Option<(bool, u32)>| n.filter(|&(s, _)| s).map_or(0, |(_, v)| v);
        let (sa, sb, sc) = (sad(a), sad(b), sad(c));
        // B and C unavailable but A available → A only.
        if b.is_none() && c.is_none() && a.is_some() {
            return sa;
        }
        match (
            a.is_some_and(|(s, _)| s),
            b.is_some_and(|(s, _)| s),
            c.is_some_and(|(s, _)| s),
        ) {
            (true, false, false) => sa,
            (false, true, false) => sb,
            (false, false, true) => sc,
            _ => sb.max(sa.min(sc)).min(sa.max(sc)), // median(sa, sb, sc)
        }
    }

    /// The `mb_qp_delta` for the current macroblock (`qp − cur_qp`) and commits the
    /// running QPy — called ONLY where the syntax actually codes a delta (I_16x16
    /// always; inter / I_4x4 when `cbp != 0`), so a skip / cbp==0 MB leaves `cur_qp`
    /// unchanged and inherits it, exactly as the decoder's `step_qp` does.
    fn qp_delta(&mut self) -> i32 {
        let d = self.qp as i32 - self.cur_qp as i32;
        // Spec §7.4.5: mb_qp_delta shall be in [-26, +25]. Shipped knobs bound
        // |d| well inside that (AQ_DQP_MAX + MBTREE_DQP_MAX), but an external
        // qpo map could exceed it — an illegal bitstream every strict decoder
        // may reject (ours reconstructs it anyway via the §7.4.5 modulo).
        debug_assert!((-26..=25).contains(&d), "mb_qp_delta {d} outside [-26,25]");
        self.cur_qp = self.qp;
        d
    }

    /// MV-predictor neighbors (left, above, above-right) for the 16×16 partition
    /// of macroblock `(mb_x, mb_y)`, read from the per-4×4-block grids.
    fn mv_neighbors(&self, mb_x: usize, mb_y: usize) -> [MvNeighbor; 3] {
        let w4 = self.mb_w * 4;
        let get = |avail: bool, bx: isize, by: isize| {
            if avail {
                let idx = by as usize * w4 + bx as usize;
                MvNeighbor {
                    available: true,
                    mv: wmv(self.mv_y[idx]),
                    ref_idx: i32::from(self.ref_idx_y[idx]),
                }
            } else {
                MvNeighbor::NONE
            }
        };
        let (bx, by) = (mb_x as isize * 4, mb_y as isize * 4);
        let a = get(mb_x > 0, bx - 1, by);
        let b = get(mb_y > 0, bx, by - 1);
        // C = above-right; if unavailable, fall back to D = above-left.
        let c = if mb_y > 0 && mb_x + 1 < self.mb_w {
            get(true, bx + 4, by - 1)
        } else {
            get(mb_x > 0 && mb_y > 0, bx - 1, by - 1)
        };
        [a, b, c]
    }

    /// The `P_Skip` motion vector (spec §8.4.1.1). P_Skip always references
    /// index 0 (the most recent picture).
    fn skip_mv(&self, mb_x: usize, mb_y: usize) -> (i32, i32) {
        let [a, b, c] = self.mv_neighbors(mb_x, mb_y);
        if !a.available
            || !b.available
            || (a.ref_idx == 0 && a.mv == (0, 0))
            || (b.ref_idx == 0 && b.mv == (0, 0))
        {
            (0, 0)
        } else {
            predict_mv(a, b, c, 0)
        }
    }

    /// Records a macroblock's per-4×4-block motion state (`ref` = reference index
    /// for inter, ignored for intra where `inter` is false).
    fn set_mb_mv(&mut self, mb_x: usize, mb_y: usize, mv: (i32, i32), inter: bool, refi: i32) {
        let w4 = self.mb_w * 4;
        for dy in 0..4 {
            for dx in 0..4 {
                let idx = (mb_y * 4 + dy) * w4 + (mb_x * 4 + dx);
                self.mv_y[idx] = nmv(mv);
                self.inter_y[idx] = inter;
                self.ref_idx_y[idx] = if inter { refi as i8 } else { -1 };
            }
        }
    }

    /// Block-level MV-predictor neighbors for a partition whose top-left 4×4
    /// block is `(pbx, pby)` and which is `pwb` blocks wide. Availability uses
    /// the decoded-block grid, so in-macroblock partitions see earlier ones.
    fn mv_neighbors_block(&self, pbx: isize, pby: isize, pwb: isize) -> [MvNeighbor; 3] {
        let (w4, h4) = ((self.mb_w * 4) as isize, (self.mb_h * 4) as isize);
        let get = |bx: isize, by: isize| -> MvNeighbor {
            if bx < 0 || by < 0 || bx >= w4 || by >= h4 || !self.coded_y[(by * w4 + bx) as usize] {
                MvNeighbor::NONE
            } else {
                let idx = (by * w4 + bx) as usize;
                MvNeighbor {
                    available: true,
                    mv: wmv(self.mv_y[idx]),
                    ref_idx: i32::from(self.ref_idx_y[idx]),
                }
            }
        };
        let a = get(pbx - 1, pby);
        let b = get(pbx, pby - 1);
        let mut c = get(pbx + pwb, pby - 1);
        if !c.available {
            c = get(pbx - 1, pby - 1); // D fallback
        }
        [a, b, c]
    }

    /// List-aware block MV-predictor neighbors (`list` 0 or 1), for the B-slice
    /// per-list `mvd` predictor. Identical geometry to [`Self::mv_neighbors_block`]
    /// but reads the List-1 motion grid when `list == 1`, matching the decoder's
    /// `mv_neighbors_list`. A neighbor not coded in this list reads `ref_idx = -1`
    /// (so `predict_partition_mv` treats it as non-matching, exactly as the decoder).
    fn mv_neighbors_block_list(
        &self,
        pbx: isize,
        pby: isize,
        pwb: isize,
        list: usize,
    ) -> [MvNeighbor; 3] {
        let (w4, h4) = ((self.mb_w * 4) as isize, (self.mb_h * 4) as isize);
        let (mvg, refg): (&[(i16, i16)], &[i8]) = if list == 0 {
            (&self.mv_y, &self.ref_idx_y)
        } else {
            (&self.mv1_y, &self.ref_idx1_y)
        };
        let get = |bx: isize, by: isize| -> MvNeighbor {
            if bx < 0 || by < 0 || bx >= w4 || by >= h4 || !self.coded_y[(by * w4 + bx) as usize] {
                MvNeighbor::NONE
            } else {
                let idx = (by * w4 + bx) as usize;
                MvNeighbor {
                    available: true,
                    mv: wmv(mvg[idx]),
                    ref_idx: i32::from(refg[idx]),
                }
            }
        };
        let a = get(pbx - 1, pby);
        let b = get(pbx, pby - 1);
        let mut c = get(pbx + pwb, pby - 1);
        if !c.available {
            c = get(pbx - 1, pby - 1); // D fallback
        }
        [a, b, c]
    }

    /// SATD cost of a motion-compensated `rw`×`rh` luma region against the source —
    /// THE per-candidate ME cost function (Challenge-1 A2 shape: the per-search
    /// invariants arrive as parameters instead of being re-derived per candidate).
    /// `hp` is the already-resolved plane cache (`None` ⇔ the fast preset, whose
    /// SATD path never reads planes), `hr_on` the hoisted `RFF_HPEL_REF` knob,
    /// `src_row` the hoisted source slice base. Dispatch order (interior full-pel →
    /// in-place plane read → fused avg+SATD → materialize → `mc_luma` fallback) is
    /// the historical `mc_satd` order, so the accepted candidate set — and the
    /// bitstream — are byte-identical to it.
    #[inline]
    fn mc_satd_hp(
        &self,
        reference: &crate::RefFrame,
        hp: Option<&rusty_h264_common::inter::HpelPlanes>,
        hr_on: bool,
        // `hr_on && RFF_SATD_AVG` (and accel compiled in) — hoisted per search like
        // `hr_on`, so the fused-kernel gate costs zero OnceLock loads per candidate.
        // Unused (and always false) on non-accel builds.
        sa_on: bool,
        src_row: &[u8],
        lx: usize,
        ly: usize,
        rw: usize,
        rh: usize,
        mv: (i32, i32),
    ) -> i64 {
        #[cfg(not(accel_x86))]
        let _ = sa_on;
        #[cfg(feature = "profile")]
        let _site = rusty_h264_common::inter::mcstats::SiteTag::new(2);
        let ch = self.mb_h * 16;
        let cw = self.cw;
        let (ix0, iy0) = (
            lx as isize + (mv.0 >> 2) as isize,
            ly as isize + (mv.1 >> 2) as isize,
        );
        let interior_fullpel = mv.0 & 3 == 0
            && mv.1 & 3 == 0
            && ix0 >= 0
            && iy0 >= 0
            && ix0 + rw as isize <= cw as isize
            && iy0 + rh as isize <= ch as isize;
        #[cfg(feature = "profile")]
        {
            let fullpel = mv.0 & 3 == 0 && mv.1 & 3 == 0;
            satdpath::bump(if interior_fullpel {
                0
            } else if fullpel {
                1
            } else {
                2
            });
        }
        if interior_fullpel {
            let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::MeCost);
            let (rx0, ry0) = (ix0 as usize, iy0 as usize);
            return satd_px(src_row, cw, &reference.y[ry0 * cw + rx0..], cw, rw, rh);
        }
        if let Some(hp) = hp {
            if hr_on {
                if let Some((plane, base, stride)) =
                    rusty_h264_common::inter::hpel_ref(hp, lx, ly, rw, rh, mv.0, mv.1)
                {
                    let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::MeCost);
                    return satd_px(src_row, cw, &plane[base..], stride, rw, rh);
                }
            }
            // A3: QUARTER-pel — fuse the two-plane (a+b+1)>>1 average into the
            // SATD kernel itself (no 256-byte materialize + reload, no FFI hop).
            // `satd_avg` returns the exact `Σ|H·d|` that `satd_px` computes on
            // the materialized average, so the cost value — and the bitstream —
            // are byte-identical; on non-AVX2 (or a declined size) it returns
            // `None` and the old materialize path below runs unchanged.
            #[cfg(accel_x86)]
            if sa_on {
                if let Some((pa, ba, pb, bb, stride)) =
                    rusty_h264_common::inter::hpel_qpel_refs(hp, lx, ly, rw, rh, mv.0, mv.1)
                {
                    let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::MeCost);
                    if let Some(v) = rusty_h264_accel::satd_avg(
                        src_row,
                        cw,
                        &pa[ba..],
                        &pb[bb..],
                        stride,
                        rw,
                        rh,
                    ) {
                        return v as i64;
                    }
                }
            }
            let mut pred = [0u8; 256];
            if rusty_h264_common::inter::hpel_block(hp, lx, ly, rw, rh, mv.0, mv.1, &mut pred) {
                return satd_px(src_row, cw, &pred, rw, rw, rh);
            }
            mc_luma(&reference.y, cw, ch, lx, ly, rw, rh, mv.0, mv.1, &mut pred);
            return satd_px(src_row, cw, &pred, rw, rw, rh);
        }
        let mut pred = [0u8; 256];
        mc_luma(&reference.y, cw, ch, lx, ly, rw, rh, mv.0, mv.1, &mut pred);
        satd_px(src_row, cw, &pred, rw, rw, rh)
    }

    /// Track-B B2.1: the SAD twin of `mc_satd_hp` — the SAME dispatch ladder
    /// (interior full-pel → in-place plane read → fused avg → materialize →
    /// `mc_luma`), with SAD (`psadbw`-class) distortion. `mc_sad` (the fast
    /// preset's function) had NONE of the SATD path's accumulated wins, so the
    /// first B2 cut measured 61% MORE `mc_luma` fallbacks; this is the parity fix.
    /// Every arm reads the same samples the materializing path would, so the SAD
    /// value — and therefore the B2-on bitstream — is unchanged by this function.
    #[inline]
    fn mc_sad_hp(
        &self,
        reference: &crate::RefFrame,
        hp: Option<&rusty_h264_common::inter::HpelPlanes>,
        hr_on: bool,
        src_row: &[u8],
        lx: usize,
        ly: usize,
        rw: usize,
        rh: usize,
        mv: (i32, i32),
        _asrc: Option<&[u8; 256]>,
    ) -> i64 {
        #[cfg(feature = "profile")]
        let _site = rusty_h264_common::inter::mcstats::SiteTag::new(2);
        let ch = self.mb_h * 16;
        let cw = self.cw;
        let (ix0, iy0) = (
            lx as isize + (mv.0 >> 2) as isize,
            ly as isize + (mv.1 >> 2) as isize,
        );
        let interior_fullpel = mv.0 & 3 == 0
            && mv.1 & 3 == 0
            && ix0 >= 0
            && iy0 >= 0
            && ix0 + rw as isize <= cw as isize
            && iy0 + rh as isize <= ch as isize;
        if interior_fullpel {
            let (rx0, ry0) = (ix0 as usize, iy0 as usize);
            #[cfg(accel)]
            if rw == 16 && rh == 16 {
                if let Some(src) = _asrc {
                    return rusty_h264_accel::sad_16x16(src, 16, &reference.y[ry0 * cw + rx0..], cw)
                        as i64;
                }
            }
            return sad_strided(src_row, cw, &reference.y[ry0 * cw + rx0..], cw, rw, rh);
        }
        if let Some(hp) = hp {
            if hr_on {
                // Single-plane phases (h/v/c half-pel AND edge full-pel via the
                // padded `f` plane — the E-3 move, which `mc_sad` never had).
                if let Some((plane, base, stride)) =
                    rusty_h264_common::inter::hpel_ref(hp, lx, ly, rw, rh, mv.0, mv.1)
                {
                    return sad_strided(src_row, cw, &plane[base..], stride, rw, rh);
                }
                // Quarter-pel: fused (a+b+1)>>1 + SAD, no materialize.
                if let Some((pa, ba, pb, bb, stride)) =
                    rusty_h264_common::inter::hpel_qpel_refs(hp, lx, ly, rw, rh, mv.0, mv.1)
                {
                    return sad_avg_strided(src_row, cw, &pa[ba..], &pb[bb..], stride, rw, rh);
                }
            }
            let mut pred = [0u8; 256];
            if rusty_h264_common::inter::hpel_block(hp, lx, ly, rw, rh, mv.0, mv.1, &mut pred) {
                return sad_strided(src_row, cw, &pred, rw, rw, rh);
            }
            mc_luma(&reference.y, cw, ch, lx, ly, rw, rh, mv.0, mv.1, &mut pred);
            return sad_strided(src_row, cw, &pred, rw, rw, rh);
        }
        let mut pred = [0u8; 256];
        mc_luma(&reference.y, cw, ch, lx, ly, rw, rh, mv.0, mv.1, &mut pred);
        sad_strided(src_row, cw, &pred, rw, rw, rh)
    }

    /// SAD (sum of absolute differences) of a motion-compensated `rw`×`rh` luma
    /// region against the source — the **fast** preset's motion-search cost.
    ///
    /// SAD is far cheaper than SATD (no Hadamard transform), and the inner loop is
    /// written as `Σ a.abs_diff(b)` over `u8` slices, the exact pattern LLVM
    /// auto-vectorizes to the `psadbw` SAD instruction — the same instruction
    /// x264's hand-written assembly uses, but reached without any `unsafe`. (x264's
    /// fast presets use SAD for the full-pel search for precisely this reason.)
    fn mc_sad(
        &self,
        reference: &crate::RefFrame,
        sy: &[u8],
        lx: usize,
        ly: usize,
        rw: usize,
        rh: usize,
        mv: (i32, i32),
        // 16-aligned source MB (built once per search) for the asm SAD; `None`
        // (and unused) on the scalar build.
        _asrc: Option<&[u8; 256]>,
    ) -> i64 {
        // Descent E depth-6: tag WHO is calling mc_luma. The search's edge fallback and
        // reconstruction land in the same `inter-mc` bucket; pricing a recon-side lever
        // against the merged total is pricing the wrong population.
        #[cfg(feature = "profile")]
        let _site = rusty_h264_common::inter::mcstats::SiteTag::new(2);
        let ch = self.mb_h * 16;
        let cw = self.cw;
        let (ix0, iy0) = (
            lx as isize + (mv.0 >> 2) as isize,
            ly as isize + (mv.1 >> 2) as isize,
        );
        let interior_fullpel = mv.0 & 3 == 0
            && mv.1 & 3 == 0
            && ix0 >= 0
            && iy0 >= 0
            && ix0 + rw as isize <= cw as isize
            && iy0 + rh as isize <= ch as isize;
        // Full-pel interior 16×16: openh264's `psadbw` SAD of the aligned source vs
        // the (movdqu) reference block. SAD is exact, so this is byte-identical to the
        // scalar path — a pure ME speedup (~2.4× the kernel).
        #[cfg(accel)]
        if interior_fullpel && rw == 16 && rh == 16 {
            if let Some(src) = _asrc {
                let (rx0, ry0) = (ix0 as usize, iy0 as usize);
                return rusty_h264_accel::sad_16x16(src, 16, &reference.y[ry0 * cw + rx0..], cw)
                    as i64;
            }
        }
        let mut sad = 0u32;
        if interior_fullpel {
            // Direct from the reference (a copy at full-pel) — no interpolation.
            let (rx0, ry0) = (ix0 as usize, iy0 as usize);
            let refy = &reference.y;
            for dy in 0..rh {
                let s = &sy[(ly + dy) * cw + lx..][..rw];
                let r = &refy[(ry0 + dy) * cw + rx0..][..rw];
                sad += s
                    .iter()
                    .zip(r)
                    .map(|(&a, &b)| a.abs_diff(b) as u32)
                    .sum::<u32>();
            }
        } else {
            let mut pred = [0u8; 256];
            // Same plane-cache read (and same preset gate) as `mc_satd`.
            let from_planes = !self.fast
                && rusty_h264_common::inter::hpel_block(
                    reference.hpel(cw, ch),
                    lx,
                    ly,
                    rw,
                    rh,
                    mv.0,
                    mv.1,
                    &mut pred,
                );
            if !from_planes {
                mc_luma(&reference.y, cw, ch, lx, ly, rw, rh, mv.0, mv.1, &mut pred);
            }
            for dy in 0..rh {
                let s = &sy[(ly + dy) * cw + lx..][..rw];
                let p = &pred[dy * rw..][..rw];
                sad += s
                    .iter()
                    .zip(p)
                    .map(|(&a, &b)| a.abs_diff(b) as u32)
                    .sum::<u32>();
            }
        }
        sad as i64
    }

    /// `bi_dist` for an arbitrary rect — the B 16×8 / 8×16 partition search needs
    /// the bi-blend distortion of a half, not of the whole macroblock. Same blend
    /// and same SAD/SATD choice as the 16×16 form.
    fn bi_dist_rect(
        &self,
        l0: &crate::RefFrame,
        l1: &crate::RefFrame,
        sy: &[u8],
        lx: usize,
        ly: usize,
        rw: usize,
        rh: usize,
        mv0: (i32, i32),
        mv1: (i32, i32),
    ) -> i64 {
        let ch = self.mb_h * 16;
        let (mut a, mut b) = ([0u8; 256], [0u8; 256]);
        mc_luma(&l0.y, self.cw, ch, lx, ly, rw, rh, mv0.0, mv0.1, &mut a);
        mc_luma(&l1.y, self.cw, ch, lx, ly, rw, rh, mv1.0, mv1.1, &mut b);
        let n = rw * rh;
        let mut avg = [0u8; 256];
        for i in 0..n {
            avg[i] = bi_blend(a[i] as i32, b[i] as i32, self.bi_w);
        }
        if self.fast && !self.mb_use_satd {
            let mut sad = 0u32;
            for dy in 0..rh {
                let s = &sy[(ly + dy) * self.cw + lx..][..rw];
                let p = &avg[dy * rw..][..rw];
                sad += s
                    .iter()
                    .zip(p)
                    .map(|(&x, &y)| x.abs_diff(y) as u32)
                    .sum::<u32>();
            }
            sad as i64
        } else {
            satd_px(&sy[ly * self.cw + lx..], self.cw, &avg, rw, rw, rh)
        }
    }

    /// Luma distortion of a `B_Bi` 16×16 prediction: motion-compensate `l0`/`l1`,
    /// average `(p+q+1)>>1` (the decoder's `b_mc` blend at `weighted_bipred_idc=0`),
    /// and score vs the source with the SAME metric the per-list searches used —
    /// SAD on the fast path, SATD when this MB is SATD-routed — so `J_bi` compares
    /// directly against `J0`/`J1`.
    fn bi_dist(
        &self,
        l0: &crate::RefFrame,
        l1: &crate::RefFrame,
        sy: &[u8],
        lx: usize,
        ly: usize,
        mv0: (i32, i32),
        mv1: (i32, i32),
    ) -> i64 {
        // Descent E/F: identify this mc_luma population by call site.
        #[cfg(feature = "profile")]
        let _site = rusty_h264_common::inter::mcstats::SiteTag::new(4);
        let ch = self.mb_h * 16;
        let (mut a, mut b) = ([0u8; 256], [0u8; 256]);
        mc_luma(&l0.y, self.cw, ch, lx, ly, 16, 16, mv0.0, mv0.1, &mut a);
        mc_luma(&l1.y, self.cw, ch, lx, ly, 16, 16, mv1.0, mv1.1, &mut b);
        let mut avg = [0u8; 256];
        for i in 0..256 {
            avg[i] = bi_blend(a[i] as i32, b[i] as i32, self.bi_w);
        }
        if self.fast && !self.mb_use_satd {
            let mut sad = 0u32;
            for dy in 0..16 {
                let s = &sy[(ly + dy) * self.cw + lx..][..16];
                let p = &avg[dy * 16..][..16];
                sad += s
                    .iter()
                    .zip(p)
                    .map(|(&x, &y)| x.abs_diff(y) as u32)
                    .sum::<u32>();
            }
            sad as i64
        } else {
            satd_px(&sy[ly * self.cw + lx..], self.cw, &avg, 16, 16, 16)
        }
    }

    /// Distortion of a pre-formed 16×16 luma prediction vs the source (SAD on the
    /// fast path, SATD when SATD-routed) — the mode-decision cost for `B_Direct`,
    /// on the same scale as the per-list search J so they compare directly.
    fn pred_dist(&self, sy: &[u8], lx: usize, ly: usize, pred: &[u8; 256]) -> i64 {
        if self.fast && !self.mb_use_satd {
            let mut sad = 0u32;
            for dy in 0..16 {
                let s = &sy[(ly + dy) * self.cw + lx..][..16];
                let p = &pred[dy * 16..][..16];
                sad += s
                    .iter()
                    .zip(p)
                    .map(|(&a, &b)| a.abs_diff(b) as u32)
                    .sum::<u32>();
            }
            sad as i64
        } else {
            satd_px(&sy[ly * self.cw + lx..], self.cw, pred, 16, 16, 16)
        }
    }

    /// `colZeroFlag` for absolute 4×4 block `(bx, by)` (spec §8.4.1.2.2): true when
    /// the co-located picture `RefPicList1[0]` (`l1`) is short-term (always, here —
    /// we use no long-term refs) and its co-located block uses List-0 reference 0
    /// with a near-zero (|·| ≤ 1) motion vector. Must match the decoder's `col_zero`.
    fn col_zero(&self, l1: &crate::RefFrame, bx: usize, by: usize) -> bool {
        if l1.w4 == 0 {
            return false;
        }
        let idx = by * l1.w4 + bx;
        if idx >= l1.ref_idx.len() {
            return false;
        }
        // b-pyramid: the co-located picture can be a REFERENCE B, whose blocks
        // may be L1-only — `predFlagL0Col == 0` reads List 1 instead (spec
        // §8.4.1.2.2). This is the SAME defect the decoder root-caused on real
        // x264 pyramid streams; the two derivations must mirror or the
        // encoder's spatial-direct picks drift from what the decoder computes.
        // P/I co-located pictures have empty `ref_idx1` and never take this arm.
        if l1.ref_idx[idx] < 0 {
            if let (Some(&r1), Some(&m1)) = (l1.ref_idx1.get(idx), l1.mv1.get(idx)) {
                return r1 == 0 && m1.0.abs() <= 1 && m1.1.abs() <= 1;
            }
            return false;
        }
        l1.ref_idx[idx] == 0 && l1.mv[idx].0.abs() <= 1 && l1.mv[idx].1.abs() <= 1
    }

    /// Bi-predictive MC of one small region into `pred_y`/`c_pred` at MB-relative
    /// offset `(dx, dy)` — the per-4×4 primitive the spatial-direct derivation uses.
    /// Mirrors the decoder's `b_mc` (average `(p+q+1)>>1` for bi, copy for uni).
    fn b_mc_block(
        &self,
        l0: &crate::RefFrame,
        l1: &crate::RefFrame,
        mb_x: usize,
        mb_y: usize,
        dx: usize,
        dy: usize,
        refi0: i32,
        m0: (i32, i32),
        refi1: i32,
        m1: (i32, i32),
        pred_y: &mut [u8; 256],
        c_pred: &mut [[u8; 64]; 2],
    ) {
        // Descent E/F: identify this mc_luma population by call site.
        #[cfg(feature = "profile")]
        let _site = rusty_h264_common::inter::mcstats::SiteTag::new(4);
        let (ch, cch) = (self.mb_h * 16, self.mb_h * 8);
        let (px, py) = (mb_x * 16 + dx, mb_y * 16 + dy);
        let (mut a, mut b) = ([0u8; 16], [0u8; 16]);
        // 4-wide luma exists ONLY here: P partitions bottom out at 8×8, so B-frame
        // spatial-direct is the encoder's only 4×4 MC. With B-frames on it is ~8% of
        // all MC calls and HALF of all sub-pel ones — and `luma_h`/`luma_v` dispatch
        // to asm only at width 16/8, so 4-wide otherwise runs the scalar 6-tap.
        // Serving it from the cached half-pel planes (bit-identical, and they are
        // already built for this reference by the motion search) is strictly better
        // than adding a 4-wide asm kernel.
        let mc4 = |r: &crate::RefFrame, mv: (i32, i32), out: &mut [u8; 16]| {
            if !self.fast
                && bdirect_planes_enabled()
                && rusty_h264_common::inter::hpel_block(
                    r.hpel(self.cw, ch),
                    px,
                    py,
                    4,
                    4,
                    mv.0,
                    mv.1,
                    out,
                )
            {
                return;
            }
            mc_luma(&r.y, self.cw, ch, px, py, 4, 4, mv.0, mv.1, out);
        };
        if refi0 >= 0 {
            mc4(l0, m0, &mut a);
        }
        if refi1 >= 0 {
            mc4(l1, m1, &mut b);
        }
        for yy in 0..4 {
            for xx in 0..4 {
                let i = yy * 4 + xx;
                let v = match (refi0 >= 0, refi1 >= 0) {
                    (true, true) => bi_blend(a[i] as i32, b[i] as i32, self.bi_w),
                    (true, false) => a[i],
                    _ => b[i],
                };
                pred_y[(dy + yy) * 16 + (dx + xx)] = v;
            }
        }
        // Chroma: the co-located 2×2 block at half resolution.
        let (cpx, cpy) = (mb_x * 8 + dx / 2, mb_y * 8 + dy / 2);
        for c in 0..2 {
            let (r0, r1) = if c == 0 {
                (&l0.u, &l1.u)
            } else {
                (&l0.v, &l1.v)
            };
            let (mut ca, mut cb) = ([0u8; 4], [0u8; 4]);
            if refi0 >= 0 {
                mc_chroma(r0, self.ccw, cch, cpx, cpy, 2, 2, m0.0, m0.1, &mut ca);
            }
            if refi1 >= 0 {
                mc_chroma(r1, self.ccw, cch, cpx, cpy, 2, 2, m1.0, m1.1, &mut cb);
            }
            for yy in 0..2 {
                for xx in 0..2 {
                    let i = yy * 2 + xx;
                    let v = match (refi0 >= 0, refi1 >= 0) {
                        (true, true) => bi_blend(ca[i] as i32, cb[i] as i32, self.bi_w),
                        (true, false) => ca[i],
                        _ => cb[i],
                    };
                    c_pred[c][(dy / 2 + yy) * 8 + (dx / 2 + xx)] = v;
                }
            }
        }
    }

    /// Spatial-direct (`direct_spatial_mv_pred_flag == 1`) prediction for a 16×16 B
    /// macroblock — the shared basis of `B_Skip` and `B_Direct_16x16`. Returns the
    /// prediction and the per-4×4 `(refIdxL0, mvL0, refIdxL1, mvL1)` motion the
    /// decoder's `decode_b_direct` derives (so the caller commits identical motion).
    fn b_direct(
        &self,
        l0: &crate::RefFrame,
        l1: &crate::RefFrame,
        mb_x: usize,
        mb_y: usize,
    ) -> (
        [u8; 256],
        [[u8; 64]; 2],
        [(i32, (i32, i32), i32, (i32, i32)); 16],
    ) {
        let (nbx, nby) = ((mb_x * 4) as isize, (mb_y * 4) as isize);
        let n0 = self.mv_neighbors_block_list(nbx, nby, 4, 0);
        let n1 = self.mv_neighbors_block_list(nbx, nby, 4, 1);
        let min_pos = |a: i32, b: i32| {
            if a < 0 {
                b
            } else if b < 0 {
                a
            } else {
                a.min(b)
            }
        };
        let rid = |n: &[MvNeighbor; 3]| min_pos(min_pos(n[0].ref_idx, n[1].ref_idx), n[2].ref_idx);
        let (mut refi0, mut refi1) = (rid(&n0), rid(&n1));
        let direct_zero = refi0 < 0 && refi1 < 0;
        if direct_zero {
            refi0 = 0;
            refi1 = 0;
        }
        let mv0 = if refi0 >= 0 && !direct_zero {
            predict_mv(n0[0], n0[1], n0[2], refi0)
        } else {
            (0, 0)
        };
        let mv1 = if refi1 >= 0 && !direct_zero {
            predict_mv(n1[0], n1[1], n1[2], refi1)
        } else {
            (0, 0)
        };
        let mut pred_y = [0u8; 256];
        let mut c_pred = [[0u8; 64]; 2];
        let mut motion = [(0i32, (0i32, 0i32), 0i32, (0i32, 0i32)); 16];
        for sby in 0..4 {
            for sbx in 0..4 {
                // direct_8x8_inference_flag = 1 (SPS): every 4×4 in an 8×8 takes
                // the 8×8's OUTER-CORNER co-located block — (0,0)(3,0)(0,3)(3,3),
                // the decoder's `col_block` mapping, spec 8.4.1.2.1.
                let (czx, czy) = ((sbx / 2) * 3, (sby / 2) * 3);
                let cz = !direct_zero && self.col_zero(l1, mb_x * 4 + czx, mb_y * 4 + czy);
                let m0 = if refi0 == 0 && cz { (0, 0) } else { mv0 };
                let m1 = if refi1 == 0 && cz { (0, 0) } else { mv1 };
                motion[sby * 4 + sbx] = (refi0, m0, refi1, m1);
                self.b_mc_block(
                    l0,
                    l1,
                    mb_x,
                    mb_y,
                    sbx * 4,
                    sby * 4,
                    refi0,
                    m0,
                    refi1,
                    m1,
                    &mut pred_y,
                    &mut c_pred,
                );
            }
        }
        (pred_y, c_pred, motion)
    }

    /// Commits a spatial-direct MB's per-4×4 motion into the List-0/List-1 grids so
    /// later MBs' neighbor predictors see it (mirrors the decoder's `b_set_motion`).
    fn commit_direct_motion(
        &mut self,
        mb_x: usize,
        mb_y: usize,
        motion: &[(i32, (i32, i32), i32, (i32, i32)); 16],
    ) {
        let w4 = self.mb_w * 4;
        for sby in 0..4 {
            for sbx in 0..4 {
                let (refi0, m0, refi1, m1) = motion[sby * 4 + sbx];
                let idx = (mb_y * 4 + sby) * w4 + (mb_x * 4 + sbx);
                self.inter_y[idx] = true;
                self.coded_y[idx] = true;
                self.mv_y[idx] = nmv(m0);
                self.ref_idx_y[idx] = refi0 as i8;
                self.mv1_y[idx] = nmv(m1);
                self.ref_idx1_y[idx] = refi1 as i8;
            }
        }
    }

    /// Rate-aware motion search for a luma region: full-pel diamond + half/
    /// quarter-pel refinement minimizing `J = SATD + λ·bits(mvd)`, where the
    /// motion cost is measured against `predictors[0]` (the MV predictor the
    /// `mvd` will actually be coded against). The search is seeded from every
    /// entry in `predictors` plus `(0,0)`. Returns the best MV and its `J`.
    ///
    /// The rate term is only a *search heuristic* — whatever MV it picks is still
    /// coded as a correct `mvd`, so this never affects decodability.
    /// ME ORACLE PROBE (`RFF_ME_ORACLE=1`): does our search actually FIND the best
    /// motion vector available to it? Accumulates our chosen cost against an
    /// exhaustive +-24 full-pel search refined by the identical sub-pel pass, so a
    /// gap is attributable to the SEARCH, not to the cost function or precision.
    /// [n, sum(our cost), sum(oracle cost), blocks where oracle won, cost() evals]
    fn motion_search(
        &self,
        reference: &crate::RefFrame,
        sy: &[u8],
        lx: usize,
        ly: usize,
        rw: usize,
        rh: usize,
        predictors: &[(i32, i32)],
        lambda_me: f64,
        // Some(mv) => skip the full-pel search entirely and refine THIS vector. The
        // starting COST is recomputed here rather than passed in, so the baseline the
        // refinement must beat is priced by the same closure as every candidate.
        start: Option<(i32, i32)>,
    ) -> ((i32, i32), i64) {
        // Bit length of `se(d)` (Exp-Golomb), i.e. what an `mvd` component costs.
        // Branchless closed form of the old `while n > 1 { n >>= 1; len += 2 }` loop:
        // that loop yields `len = 1 + 2·floor(log2(codenum+1))`, and for x ≥ 1
        // `floor(log2(x)) == 31 - x.leading_zeros()`. Removes a data-dependent branch
        // from the innermost ME cost — bit-identical (verified over the d range).
        let mvk = mv_cost_kind(self.mv_smooth);
        // Law 5: resolve the cost TABLE once per search. `get_or_init` on an
        // initialized OnceLock is still an acquire load + branch, and `mvbits`
        // is the innermost ME cost — it ran per CANDIDATE per COMPONENT. The
        // empty slice for the step model costs nothing and initializes nothing.
        let mvtab: &[u16] = match mvk {
            1 => MV_COST_TAB.get_or_init(build_mv_cost),
            2 => MV_TRUE_BIASED.get_or_init(build_true_biased),
            _ => &[],
        };
        let mvbits = |d: i32| -> u32 {
            // H-23: the ME rate model. `RFF_MVCOST=1` swaps the Exp-Golomb STEP
            // function for x264's smooth curve `2·log2(|d|+1) + 0.718 + (d!=0)`.
            // The step function is FLAT inside a power-of-two bracket — it prices
            // d=4 and d=7 identically, so the search takes the far end of a
            // bracket for free, inflating |mvd| (and with it the sign+prefix bits
            // the accountant found are ~14% of the payload). λ cannot fix this:
            // scaling a flat region leaves it flat. Table is in WHOLE bits to keep
            // the caller's integer arithmetic; ×4 internally then rounded, so the
            // curve's ordering survives quantization.
            match mvk {
                1 | 2 => {
                    let a = d.unsigned_abs().min(4095) as usize;
                    mvtab[a] as u32
                }
                _ => {
                    let codenum = if d > 0 {
                        (2 * d - 1) as u32
                    } else {
                        (-2 * d) as u32
                    };
                    1 + 2 * (31 - (codenum + 1).leading_zeros())
                }
            }
        };
        let center = predictors[0];
        let probe = me_oracle_on();
        // Track-B B2: the full-pel phase (seeds/snap/diamond) prices candidates in
        // the SAD domain; the winner is repriced in SATD before rescue/sub-pel.
        // Refine-only searches have no full-pel phase, so B2 does not apply there.
        // `self.sadfp` is force-mode at construction or the per-frame `b2_mgain`
        // dispatcher's routing (mode 1).
        let sadfp = !self.fast && start.is_none() && self.sadfp;
        // Build the 16-aligned source MB ONCE per search for the asm SAD path (fast
        // preset — and B2's SAD full-pel phase — full 16×16). Amortized over every
        // candidate's SAD; the reference block stays unaligned (movdqu). Scalar
        // build does no copy.
        #[cfg(accel)]
        let asrc_buf = if (self.fast || sadfp) && rw == 16 && rh == 16 {
            let mut a = AlignedMb([0u8; 256]);
            for dy in 0..16 {
                a.0[dy * 16..dy * 16 + 16].copy_from_slice(&sy[(ly + dy) * self.cw + lx..][..16]);
            }
            Some(a)
        } else {
            None
        };
        #[cfg(accel)]
        let asrc: Option<&[u8; 256]> = asrc_buf.as_ref().map(|a| &a.0);
        #[cfg(not(accel))]
        let asrc: Option<&[u8; 256]> = None;
        // Challenge-1 A2: hoist the SATD path's per-search invariants OUT of the
        // per-candidate closure. `mc_satd` re-derived, for EVERY candidate: the
        // plane-cache OnceLock (an acquire load + branch, twice on the quarter-pel
        // arm), the `RFF_HPEL_REF` OnceLock, and the source-row slice base (a bounds
        // check). All are constant across the ~20-50 evaluations of one search.
        // `mc_satd_hp` is the same dispatch with those values passed in — the same
        // arms in the same order, so the accepted candidate set is byte-identical.
        let use_sad = self.fast && !self.mb_use_satd;
        let cw = self.cw;
        // Every non-fast search sub-pel-refines at the end, so the planes are built
        // for any reference a search touches — hoisting the get_or_init here does not
        // build planes a lazy path would have avoided.
        let hp: Option<&rusty_h264_common::inter::HpelPlanes> = if !self.fast {
            Some(reference.hpel(cw, self.mb_h * 16))
        } else {
            None
        };
        let hr_on = hpel_ref_enabled();
        // A3 gate, hoisted with the rest (`RFF_HPEL_REF=0` restores the FULL pre-C/A3
        // copy path, so the fused kernel rides the same master anchor).
        let sa_on = cfg!(accel_x86) && hr_on && satd_avg_enabled();
        let src_row = &sy[ly * cw + lx..];
        // H-14 R3: the MeCtx fast evaluator — ONE geometry validation per search,
        // then per-eval integer bounds + direct kernel (collects the measured
        // ~23 ns/eval dispatch chain). Values are exactly the safe path's, so a
        // candidate served here cannot change the bitstream; out-of-window
        // candidates fall back to `mc_satd_hp` (equal values there too).
        #[cfg(accel_x86)]
        let mectx = if !use_sad && mectx_enabled() {
            hp.and_then(|p| {
                rusty_h264_accel::MeCtx::new(
                    src_row, cw, &p.f, &p.h, &p.v, &p.c, p.stride, p.pad, p.pw, p.ph, lx, ly, rw,
                    rh,
                )
            })
        } else {
            None
        };
        let cost = |mv: (i32, i32)| -> i64 {
            let rate = mvbits(mv.0 - center.0) + mvbits(mv.1 - center.1);
            // The smooth table carries 4× resolution; fold that into λ so the
            // rate/distortion balance is unchanged and only the SHAPE differs.
            let lam_r = if mvk != 0 {
                lambda_me * 0.25
            } else {
                lambda_me
            };
            // Fast preset: SAD (psadbw — asm kernel on `--features asm`, else auto-vec)
            // — far cheaper than SATD, the single biggest reason x264 fast out-runs us.
            let dist = if use_sad {
                self.mc_sad(reference, sy, lx, ly, rw, rh, mv, asrc)
            } else {
                #[cfg(accel_x86)]
                {
                    match mectx.as_ref().and_then(|c| c.eval(mv.0, mv.1)) {
                        Some(d) => d as i64,
                        None => self
                            .mc_satd_hp(reference, hp, hr_on, sa_on, src_row, lx, ly, rw, rh, mv),
                    }
                }
                #[cfg(not(accel_x86))]
                {
                    self.mc_satd_hp(reference, hp, hr_on, sa_on, src_row, lx, ly, rw, rh, mv)
                }
            };
            dist + (lam_r * rate as f64) as i64
        };
        // B2's full-pel-phase cost: SAD distortion, λ scaled to the SAD domain
        // (`RFF_ME_SADL`, hoisted). Falls through to `cost` (SATD) whenever B2 is
        // off, so every pre-B2 path is untouched.
        let lam_fp = lambda_me * if sadfp { me_sadfp_lambda() } else { 1.0 };
        let cost_fp = |mv: (i32, i32)| -> i64 {
            if !sadfp {
                return cost(mv);
            }
            let rate = mvbits(mv.0 - center.0) + mvbits(mv.1 - center.1);
            self.mc_sad_hp(reference, hp, hr_on, src_row, lx, ly, rw, rh, mv, asrc)
                + (lam_fp * rate as f64) as i64
        };
        // Seed from (0,0) and each predictor; keep the cheapest.
        let refine_only = start.is_some();
        let (mut best, mut best_c) = match start {
            Some(mv) => (mv, cost(mv)),
            None => {
                let mut b = (0, 0);
                let mut bc = cost_fp(b);
                for &p in predictors {
                    let pc = cost_fp(p);
                    if pc < bc {
                        bc = pc;
                        b = p;
                    }
                }
                (b, bc)
            }
        };
        // SNAP THE DIAMOND CENTRE TO INTEGER-PEL. The diamond below steps by whole
        // pels, so a fractional centre makes EVERY candidate fractional and forces
        // all of them through `mc_luma`'s 6-tap filter — measured at 84-90% of all
        // SATD evaluations. Snapping puts the whole full-pel phase on the direct
        // (no-interpolation) SATD path. The pre-snap seed is kept and re-compared
        // after refinement, so this can only change WHERE we search, never make the
        // returned vector worse than the seed we started from.
        let (seed_mv, mut seed_c) = (best, best_c);
        if !refine_only && self.me_snap && (best.0 & 3 != 0 || best.1 & 3 != 0) {
            let snapped = (
                (best.0 + 2).div_euclid(4) * 4,
                (best.1 + 2).div_euclid(4) * 4,
            );
            best_c = cost_fp(snapped);
            best = snapped;
        }
        // Coarse-to-fine full-pel search: a 4-point diamond walked at each step
        // size from 16 px down to 1 px (steps in quarter-pel units: 64,32,…,4).
        // The larger initial steps reach fast motion the predictor missed; the
        // diamond stays orthogonal (no diagonals) — diagonal probes were found to
        // chase equally-good far matches on ambiguous motion, wrecking MV-field
        // coherence and the neighbor predictors.
        // The fast preset trusts the neighbour MV predictor and refines locally
        // (one coarse reach + fine), like x264's `me=dia`; quality sweeps the full
        // coarse-to-fine range. Each step's diamond still walks until no
        // improvement, so even fast reaches far motion — just in smaller hops.
        // Descent A: the coarse rungs are ~76-80% of full-pel evals at a 0.05-1.0% hit
        // rate (near-equal eval counts per rung = the walk almost never walks, so each
        // rung is a flat ~4-eval toll). RFF_DIA_LADDER selects which rungs to pay for.
        let mut ladder = [0i32; 5];
        let mut nladder = 0usize;
        let steps: &[i32] = if self.fast {
            &[16, 4]
        } else {
            // Shape-aware: sub-8x8 partitions inherit a converged parent MV.
            let m = if rw < 8 || rh < 8 {
                dia_sub_mask()
            } else {
                dia_mask()
            };
            for (i, r) in DIA_RUNGS.iter().enumerate() {
                if m & (1 << i) != 0 {
                    ladder[nladder] = *r;
                    nladder += 1;
                }
            }
            &ladder[..nladder]
        };
        let _gd = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::MeDiamond);
        // FC: batch a fixed-centre diamond pass through the x4 kernels when every
        // candidate is an interior full-pel 16×16 read — one source band covers all
        // four candidates. Applies to BOTH cost domains (`sad_16x16_x4` on
        // SAD-routed frames, `satd_16x16_x4` otherwise); the fast preset keeps its
        // own untouched path. Argmin-of-4 replaces the first-improver cascade —
        // measured BD-POSITIVE on the SAD domain (bus −1.71→−2.61) and gated on the
        // corpus for the SATD domain the same way. `RFF_ME_FC=0` restores cascade.
        #[cfg_attr(not(accel_x86), allow(unused_variables))] // consumed by accel_x86-only blocks
        let fc = !self.fast
            && cfg!(accel_x86)
            && me_fc_enabled()
            && matches!((rw, rh), (16, 16) | (16, 8) | (8, 16) | (8, 8));
        #[cfg_attr(not(accel_x86), allow(unused_variables))] // consumed by accel_x86-only blocks
        let ch_px = self.mb_h as isize * 16;
        // `_si` is the rung's index for the `profile` census (diastats::ev/imp).
        #[allow(clippy::unused_enumerate_index)] // index read only under `profile`
        for (_si, &step) in steps.iter().enumerate() {
            if refine_only {
                break;
            }
            loop {
                #[cfg(accel_x86)]
                if fc && best.0 & 3 == 0 && best.1 & 3 == 0 {
                    // All four candidates full-pel; interior iff the ±step box is.
                    let s = (step >> 2) as isize;
                    let (bx, by) = (
                        lx as isize + (best.0 >> 2) as isize,
                        ly as isize + (best.1 >> 2) as isize,
                    );
                    if bx - s >= 0
                        && by - s >= 0
                        && bx + s + rw as isize <= cw as isize
                        && by + s + rh as isize <= ch_px
                    {
                        let offs = [
                            (by * cw as isize + bx + s) as usize,
                            (by * cw as isize + bx - s) as usize,
                            ((by + s) * cw as isize + bx) as usize,
                            ((by - s) * cw as isize + bx) as usize,
                        ];
                        // 16-wide shapes go through the batch kernel; 8-wide ones
                        // measured SLOWER batched than the per-candidate Wels asm
                        // (H-8 speed gate), so they evaluate individually inside the
                        // SAME argmin — identical values, identical comparisons,
                        // identical bitstream.
                        let batch = if rw != 16 {
                            None
                        } else if sadfp {
                            rusty_h264_accel::sad_x4(src_row, cw, &reference.y, offs, cw, rw, rh)
                        } else {
                            rusty_h264_accel::satd_x4(src_row, cw, &reference.y, offs, cw, rw, rh)
                        };
                        {
                            let ring = [(step, 0), (-step, 0), (0, step), (0, -step)];
                            let (mut bi, mut bc) = (usize::MAX, best_c);
                            for (i, &(dx, dy)) in ring.iter().enumerate() {
                                let mv = (best.0 + dx, best.1 + dy);
                                let cc = match batch {
                                    Some(sads) => {
                                        let rate =
                                            mvbits(mv.0 - center.0) + mvbits(mv.1 - center.1);
                                        sads[i] as i64 + (lam_fp * rate as f64) as i64
                                    }
                                    None => cost_fp(mv),
                                };
                                #[cfg(feature = "profile")]
                                diastats::ev(_si);
                                if cc < bc {
                                    bc = cc;
                                    bi = i;
                                }
                            }
                            if bi == usize::MAX {
                                break;
                            }
                            best_c = bc;
                            best = (best.0 + ring[bi].0, best.1 + ring[bi].1);
                            #[cfg(feature = "profile")]
                            diastats::imp(_si);
                            continue;
                        }
                    }
                }
                let mut improved = false;
                for &(dx, dy) in &[(step, 0), (-step, 0), (0, step), (0, -step)] {
                    let c = (best.0 + dx, best.1 + dy);
                    let cc = cost_fp(c);
                    #[cfg(feature = "profile")]
                    diastats::ev(_si);
                    if cc < best_c {
                        best_c = cc;
                        best = c;
                        improved = true;
                        #[cfg(feature = "profile")]
                        diastats::imp(_si);
                    }
                }
                if !improved {
                    break;
                }
            }
        }
        // DIAMOND-STALLED RESCUE (content-adaptive: fires on the FAILURE, not a proxy).
        // The gradient-descent diamond stalls at a plateau on FLAT cost surfaces and
        // never reaches the far-but-better MV that exists within ±16 (measured: ~+22%
        // BD-rate vs x264's simple dia on smooth content). The precise stall signal is
        // the CONJUNCTION: a FLAT source block (low variance) whose diamond match STILL
        // has a high residual — because on a flat surface the RIGHT MV predicts near-
        // perfectly, so a high residual there means the diamond missed it (a stall).
        // (Residual alone fires on busy blocks where a high residual is inherent — that
        // was 3.3× slower on mand for nothing; variance alone fires on flat-but-well-
        // predicted blocks. The AND targets exactly the stalls.) Then a FINE ±16 step-2
        // grid reaches the true minimum. Fires on a fraction of blocks → affordable.
        // Quality preset only.
        drop(_gd);
        // B2: the full-pel phase priced in the SAD domain — reprice the winner AND
        // the pre-snap seed into the SATD domain the rescue + sub-pel phases (and the
        // final seed-vs-refined comparison) trade in. Two SATD evaluations per
        // search, against the ~20-50 candidate evaluations the SAD domain cheapened.
        if sadfp {
            best_c = cost(best);
            seed_c = cost(seed_mv);
        }
        let _gr = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::MeRescue);
        // H-14 R1 brick 1: `me_fast` defaults TRUE, which makes `flat`'s value
        // IRRELEVANT to the gate below on every default search — yet the full
        // rw×rh sum+sum-of-squares walk (256 pixel loads + muls) ran EAGERLY per
        // search. Lazy-evaluate it: same boolean outcome in every case (me_fast
        // short-circuits first), the dead variance pass simply never runs.
        let flat = |sself: &Self| {
            !refine_only && {
                let (mut s, mut ss) = (0u64, 0u64);
                for dy in 0..rh {
                    for dx in 0..rw {
                        let v = sy[(ly + dy) * sself.cw + lx + dx] as u64;
                        s += v;
                        ss += v * v;
                    }
                }
                // `n = rw * rh` is 256/128/64/32/16 for every H.264 partition --
                // always a power of two -- but the compiler cannot prove that of a
                // runtime value, so both of these were real `div` instructions.
                // A shift is EXACT here; the assert is what keeps it exact, because
                // a shift on a non-power-of-2 is silently wrong.
                let n = (rw * rh) as u64;
                debug_assert!(n.is_power_of_two(), "partition area must be a power of two");
                let sh = n.trailing_zeros();
                ((ss - ((s * s) >> sh)) >> sh) < sself.me_wide_var
            }
        };
        // The online payoff gate may have disabled the rescue for the rest of this
        // frame (irreducible-residual content — rotation/fractal — where the fine grid
        // fixes almost nothing; measured 2.25× on rot for a ~0% BD gain). A gated-off
        // frame runs exactly the diamond → identical to me_wide-off → never worse.
        // FAST-MOTION extension: the flat gate targets smooth-surface stalls, but the
        // diamond ALSO stalls on FAST motion (bus/football: an exhaustive ±24 search
        // recovers 6-15% BD) — those blocks are high-VARIANCE (detail) so `flat` misses
        // them. `me_fast` also fires on any high-residual block; the online payoff gate
        // then keeps it only where a wider search actually pays off (fast motion), and
        // disables it on irreducible-residual detail — the same self-tuning as flat.
        if self.me_wide && !self.fast && (self.me_fast || flat(self)) && !self.resc_off.get() {
            // H-14 R1 brick 2: `best` was priced by the SAME `dist + (λ·rate) as
            // i64` formula on every path that can reach here (cost, cost_fp after
            // the B2 reprice, the FC batch with lam_fp == λ off SAD frames), so
            // its distortion is recoverable EXACTLY by subtraction — the extra
            // full SATD kernel call per search was pure recompute (the
            // codec-eliminate-redundancy "return the already-computed value").
            let rate_b = mvbits(best.0 - center.0) + mvbits(best.1 - center.1);
            let dist = best_c - (lambda_me * rate_b as f64) as i64;
            if dist / (rw * rh).max(1) as i64 > self.me_rescue {
                // FINE ±16 step-2 grid + ±1 refine — recover the true minimum the
                // diamond missed. Fires only on flat-block stalls, so it is affordable.
                // SNAP THE GRID CENTRE TO INTEGER-PEL: the diamond seed can be sub-pel
                // (sub-pel neighbour predictors), and since every grid point shares
                // cx&3, a sub-pel centre forces the WHOLE ±16 grid through mc_luma
                // interpolation — measured 89% of zoom's rescue cost. The rescue only
                // needs the right REGION (a far MV the diamond missed); the sub-pel
                // refine that follows recovers the fraction. Integer centre → the grid
                // hits the fast full-pel SATD path (no interpolation).
                let pre_c = best_c;
                let (cx, cy) = (
                    (best.0 + 2).div_euclid(4) * 4,
                    (best.1 + 2).div_euclid(4) * 4,
                );
                let mut gb = best;
                // BATCHED FULL-PEL GRID (accel): now that the grid centre is integer-pel
                // (all points interior full-pel), hoist the interior/bounds check out of
                // the loop and call the AVX2 SATD directly — skipping mc_satd's per-point
                // interior test + satd_px dispatch. BYTE-IDENTICAL to the cost() path
                // (same 2·satd_16x16 + rate), so it is default-on (RFF_ME_BATCH=0 to A/B
                // it off). ~+7% zoom / +4% tsrc on top of the snap; the SATD kernel itself
                // is already AVX2 and its transform can't amortise across the grid, so
                // this per-call-overhead trim is the ceiling for an "asm grid kernel".
                let cw = self.cw;
                let r = self.me_range;
                let batched = rw == 16 && rh == 16 && cfg!(accel) && {
                    let (icdx, icdy) = (cx >> 2, cy >> 2);
                    lx as i32 + icdx >= r
                        && lx as i32 + icdx + r + 16 <= cw as i32
                        && ly as i32 + icdy >= r
                        && ly as i32 + icdy + r + 16 <= (self.mb_h * 16) as i32
                        && me_batch_enabled()
                };
                #[cfg(accel)]
                if batched {
                    let (icdx, icdy) = ((cx >> 2), (cy >> 2));
                    let src = &sy[ly * cw + lx..];
                    let mut dy = -r;
                    while dy <= r {
                        let rby = (ly as i32 + icdy + dy) as usize;
                        let mut dx = -r;
                        while dx <= r {
                            let rbx = (lx as i32 + icdx + dx) as usize;
                            let satd = 2 * rusty_h264_accel::satd_16x16(
                                src,
                                cw,
                                &reference.y[rby * cw + rbx..],
                                cw,
                            ) as i64;
                            let mv = (cx + dx * 4, cy + dy * 4);
                            let rate = mvbits(mv.0 - center.0) + mvbits(mv.1 - center.1);
                            let cc = satd + (lambda_me * rate as f64) as i64;
                            if cc < best_c {
                                best_c = cc;
                                gb = mv;
                            }
                            dx += 2;
                        }
                        dy += 2;
                    }
                }
                if !batched {
                    let mut dy = -r;
                    while dy <= r {
                        let mut dx = -r;
                        while dx <= r {
                            let cc = cost((cx + dx * 4, cy + dy * 4));
                            if cc < best_c {
                                best_c = cc;
                                gb = (cx + dx * 4, cy + dy * 4);
                            }
                            dx += 2;
                        }
                        dy += 2;
                    }
                }
                best = gb;
                for dy in -1..=1 {
                    for dx in -1..=1 {
                        let c = (best.0 + dx * 4, best.1 + dy * 4);
                        let cc = cost(c);
                        if cc < best_c {
                            best_c = cc;
                            best = c;
                        }
                    }
                }
                // LEARNING PHASE: for the first `me_learn` stalls of the frame, tally
                // whether the grid actually paid off (≥6.25% cost cut). Once the window
                // fills, if too few paid off the residual is irreducible on this content
                // → disable the rescue for the rest of the frame. The window's own MVs
                // are committed, but they're a small spatially-clustered set (not
                // improvement-selected), so on net-neutral content (rot) they can't
                // regress — only frame-level on/off avoids the per-block B-direct
                // selection effect.
                let n = self.resc_n.get();
                if n < self.me_learn {
                    self.resc_n.set(n + 1);
                    if best_c * 16 <= pre_c * 15 {
                        self.resc_big.set(self.resc_big.get() + 1);
                    }
                    if n + 1 == self.me_learn
                        && self.resc_big.get() * 100 < self.me_learn * self.me_payoff_pct
                    {
                        self.resc_off.set(true);
                    }
                }
            }
        }
        drop(_gr);
        let _gs = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::MeSubpel);
        // Sub-pel refinement uses the 6-tap/bilinear interpolation — the expensive
        // per-pixel `mc_luma` path that profiling pinned at ~55% of the entire
        // encode. The fast preset skips it (integer-pel only, like x264's fastest
        // presets `subme=0`): ~3× faster, trading a little quality on sub-pixel
        // motion. The quality preset does the full half-pel + quarter-pel rings.
        if probe {
            // Exhaustive +-24 full-pel around the same centre, then the SAME sub-pel
            // pass, so only the full-pel search strategy differs.
            let mut ob = center;
            let mut oc = i64::MAX;
            for gy in -24i32..=24 {
                for gx in -24i32..=24 {
                    let c = (center.0 + gx * 4, center.1 + gy * 4);
                    let cc = cost(c);
                    if cc < oc {
                        oc = cc;
                        ob = c;
                    }
                }
            }
            let fullpel_best = ob;
            for &st in &[2i32, 1] {
                for &(dx, dy) in &[(st, 0), (-st, 0), (0, st), (0, -st)] {
                    let c = (ob.0 + dx, ob.1 + dy);
                    let cc = cost(c);
                    if cc < oc {
                        oc = cc;
                        ob = c;
                    }
                }
            }
            // EXHAUSTIVE sub-pel: every quarter-pel offset in +-3 around the full-pel
            // winner. Our own pass is a single 4-point probe at half then quarter, so
            // this is what separates a sub-pel deficiency from a full-pel one.
            let mut oc_sp = oc;
            for dy in -3i32..=3 {
                for dx in -3i32..=3 {
                    let c = (fullpel_best.0 + dx, fullpel_best.1 + dy);
                    let cc = cost(c);
                    if cc < oc_sp {
                        oc_sp = cc;
                    }
                }
            }
            // our own sub-pel pass has not run yet; replicate it for a fair compare
            let (mut mb_, mut mc_) = (best, best_c);
            for &st in &[2i32, 1] {
                for &(dx, dy) in &[(st, 0), (-st, 0), (0, st), (0, -st)] {
                    let c = (mb_.0 + dx, mb_.1 + dy);
                    let cc = cost(c);
                    if cc < mc_ {
                        mc_ = cc;
                        mb_ = c;
                    }
                }
            }
            use core::sync::atomic::Ordering::Relaxed;
            ME_PROBE[0].fetch_add(1, Relaxed);
            ME_PROBE[1].fetch_add(mc_.max(0) as u64, Relaxed);
            ME_PROBE[2].fetch_add(oc.max(0) as u64, Relaxed);
            ME_PROBE[3].fetch_add((mc_ > oc) as u64, Relaxed);
            ME_PROBE[5].fetch_add(oc_sp.max(0) as u64, Relaxed);
            ME_PROBE[6].fetch_add((mc_ > oc_sp) as u64, Relaxed);
        }
        let subpel: &[i32] =
            if (self.fast && !self.subpel_force) || (self.sp_defer.get() && !refine_only) {
                &[]
            } else {
                &[2, 1]
            };
        // U1 harvest: the null arm is the full-pel winner we would keep on a skip.
        let (hv_pre, mut hv_evals) = (best_c, 0u32);
        // `to_best` = eval index of the LAST improvement; `ring1` = the cost after the
        // first 8-point half-pel ring. Together they answer "how many of these 29
        // evaluations actually matter", which is the ceiling for any cheaper pattern.
        let (mut hv_to_best, mut hv_ring1) = (0u32, i64::MIN);
        // SHAPE-DISPATCHED SUB-PEL PATTERN (best_part campaign). Same reasoning as
        // the sub-partition diamond ladder: a sub-8x8 partition inherits a parent
        // MV that has ALREADY been full-pel searched and sub-pel refined, so the
        // expensive walk-to-convergence 8-point pattern is confirming an answer it
        // was handed. This module's own harvest sized that: ~29 evaluations per
        // refinement, last improvement at ~14-15 (half the work confirms), and the
        // first ring alone carries 64-72% of the gain. Sub-partitions take pattern
        // pattern 2 (8-point ring, SINGLE pass) unless a caller pinned one.
        // Swept all four on BD-rate vs no-splits (fine ladder active):
        //
        //   pat            foreman SSIM   bus SSIM
        //   0 8pt+iterate     -2.21         -5.49
        //   1 4pt+iterate     -2.06         -5.49
        //   2 8pt+single      -2.13         -5.64   <- best on bus, ~full on foreman
        //   3 4pt+single      -2.03         -5.44
        //
        // The RING is what carries the gain; the ITERATION is confirmation. Pattern
        // 2 drops ~29 evals to ~8 and BEATS the full walk on bus — dropping the
        // ring as well (1, 3) is where quality actually goes. `RFF_SUBPEL_SUB`
        // overrides (0-3); `=0` restores the full pattern for sub-partitions.
        let sub_shape = rw < 8 || rh < 8;
        let mut pat = subpel_pattern_override().unwrap_or_else(|| {
            if sub_shape {
                static P: rusty_h264_common::once::OnceLock<u32> =
                    rusty_h264_common::once::OnceLock::new();
                *P.get_or_init(|| {
                    rusty_h264_common::knob("RFF_SUBPEL_SUB")
                        .and_then(|v| v.parse().ok())
                        .unwrap_or(2)
                })
            } else if self.sp_single_pass {
                2
            } else {
                0
            }
        });
        let (sp_learn, sp_t) = sp_dispatch_cfg();
        // Only dispatch when the caller has not pinned a pattern (pat 0 = default).
        let sp_dispatching = sp_learn > 0 && pat == 0 && !subpel.is_empty();
        if sp_dispatching && self.sp_learn_n.get() >= sp_learn && self.sp_1pass.get() {
            pat = 2;
        }
        // Descent D-2 MEMO. The ring walks around a MOVING centre, so iteration N+1's
        // ring necessarily re-contains the previous centre and several previous ring
        // points: 27-44% of sub-pel evaluations re-price an MV this refinement already
        // priced. `cost()` is PURE in `mv` (rate from mv-centre; distortion from the
        // fixed reference/source/block captures), so memoizing is EXACT -- identical
        // costs, identical comparisons, identical chosen MV, byte-identical output.
        // A miss simply recomputes, so the table's hit rate is a SPEED property only.
        //
        // 64-entry direct-mapped on the low bits of the MV, tagged with the full MV so
        // a collision is a miss rather than a wrong answer. Stack-resident (1 KiB) and
        // re-initialized per refinement: measured cheaper than a thread-local + RefCell
        // borrow on every evaluation, since ~60% of lookups miss.
        const SP_MEMO_N: usize = 64;
        #[inline(always)]
        fn sp_slot(mv: (i32, i32)) -> usize {
            ((mv.0 & 7) as usize) | (((mv.1 & 7) as usize) << 3)
        }
        let mut memo_mv = [(i32::MIN, i32::MIN); SP_MEMO_N];
        let mut memo_c = [0i64; SP_MEMO_N];
        if !subpel.is_empty() {
            let s0 = sp_slot(best);
            memo_mv[s0] = best;
            memo_c[s0] = best_c;
        }
        // Descent D-2 census: the ring walks around a MOVING centre, so iteration N+1's ring
        // necessarily re-contains the previous centre and several previous ring points.
        // Count how many sub-pel evaluations price an MV this refinement ALREADY priced
        // -- redundant recompute is byte-identically removable, unlike dropping work.
        #[cfg(feature = "profile")]
        let mut seen: Vec<(i32, i32)> = Vec::with_capacity(64);
        #[cfg(feature = "profile")]
        {
            seen.push(best);
        }
        // Track-B B3: the sub-pel iteration BUDGET. The ring walks until no
        // improvement; Descent D's census says iteration 1 carries 55% of evals at
        // an 11-13% hit rate, iteration 2 another 35-40% at 1.5-2.5%, and the tail
        // past that almost never pays — but under B2's SAD-chosen starts the tail
        // GROWS (+27% ns/search), eating the SAD savings. A cap bounds the walk the
        // way x264's fixed subme budget does. 0 (default) = unlimited =
        // byte-identical; bitstream-changing otherwise → BD-gated, opt-in.
        let sp_cap = sp_maxit();
        // ③: batched fixed-centre half-pel ring (see `sp_fc_enabled`).
        #[cfg_attr(not(accel_x86), allow(unused_variables))] // consumed by accel_x86-only blocks
        let sp_fc = sp_fc_enabled()
            && !self.fast
            && cfg!(accel_x86)
            && matches!((rw, rh), (16, 16) | (16, 8) | (8, 16) | (8, 8));
        for &step in subpel {
            // Snapping starts this refine from an integer centre instead of the
            // seed's own fractional lattice, so a single 8-point pass can leave
            // precision behind. Walk it until it stops improving to compensate —
            // the snap is what pays for the extra probes.
            let ring8 = [
                (step, 0),
                (-step, 0),
                (0, step),
                (0, -step),
                (step, step),
                (-step, -step),
                (step, -step),
                (-step, step),
            ];
            let ring4 = [(step, 0), (-step, 0), (0, step), (0, -step)];
            let ring: &[(i32, i32)] = if pat & 1 != 0 { &ring4 } else { &ring8 };
            let mut _iter = 0u32;
            loop {
                // ③: from an INTEGER centre at step 2, all 8 ring candidates are
                // single-plane reads (h/h/v/v axes, c/c/c/c diagonals) — batch them
                // as two x4 kernel calls and take the argmin (first-wins in ring
                // order). Any decline (edge, half-pel centre, ring4 pattern) falls
                // through to the cascading walk for this pass.
                // ③b: the QUARTER step — every ±1 offset makes a component odd, so
                // all 8 candidates are two-plane average pairs regardless of the
                // centre's phase; two `satd_avg_x4` calls cover the ring.
                #[cfg(accel_x86)]
                if sp_fc && step == 1 && pat & 1 == 0 {
                    _iter += 1;
                    let hp8 = hp.expect("sp_fc implies non-fast, which resolves hp");
                    let ring8 = [
                        (1, 0),
                        (-1, 0),
                        (0, 1),
                        (0, -1),
                        (1, 1),
                        (-1, -1),
                        (1, -1),
                        (-1, 1),
                    ];
                    let mut prs: [Option<(&[u8], usize, &[u8], usize, usize)>; 8] = [None; 8];
                    let mut all = true;
                    for (i, &(dx, dy)) in ring8.iter().enumerate() {
                        prs[i] = rusty_h264_common::inter::hpel_qpel_refs(
                            hp8,
                            lx,
                            ly,
                            rw,
                            rh,
                            best.0 + dx,
                            best.1 + dy,
                        );
                        all &= prs[i].is_some();
                    }
                    if all {
                        let stride = prs[0].unwrap().4;
                        // Batch kernel for 16-wide only (8-wide measured slower
                        // batched than the per-candidate fused path — H-8 gate);
                        // either way the SAME argmin over the SAME values.
                        let pack = |a: usize, b: usize, c2: usize, d: usize| {
                            if rw != 16 {
                                return None;
                            }
                            let g = |i: usize| {
                                let (pa, oa, pb, ob, _) = prs[i].unwrap();
                                (pa, oa, pb, ob)
                            };
                            rusty_h264_accel::satd_avg_x4(
                                src_row,
                                cw,
                                [g(a), g(b), g(c2), g(d)],
                                stride,
                                rw,
                                rh,
                            )
                        };
                        {
                            let (ax, di) = (pack(0, 1, 2, 3), pack(4, 5, 6, 7));
                            let (mut bi, mut bc) = (usize::MAX, best_c);
                            for i in 0..8 {
                                let (dx, dy) = ring8[i];
                                let mv = (best.0 + dx, best.1 + dy);
                                let cc = match (i < 4, &ax, &di) {
                                    (true, Some(ax), _) => {
                                        let rate =
                                            mvbits(mv.0 - center.0) + mvbits(mv.1 - center.1);
                                        ax[i] as i64 + (lambda_me * rate as f64) as i64
                                    }
                                    (false, _, Some(di)) => {
                                        let rate =
                                            mvbits(mv.0 - center.0) + mvbits(mv.1 - center.1);
                                        di[i - 4] as i64 + (lambda_me * rate as f64) as i64
                                    }
                                    _ => cost(mv),
                                };
                                hv_evals += 1;
                                if cc < bc {
                                    bc = cc;
                                    bi = i;
                                }
                            }
                            if hv_ring1 == i64::MIN {
                                hv_ring1 = if bi == usize::MAX { best_c } else { bc };
                            }
                            if bi == usize::MAX
                                || !self.me_subpel_iter
                                || pat & 2 != 0
                                || (sp_cap != 0 && _iter >= sp_cap)
                            {
                                if bi != usize::MAX {
                                    best_c = bc;
                                    best = (best.0 + ring8[bi].0, best.1 + ring8[bi].1);
                                    hv_to_best = hv_evals;
                                }
                                break;
                            }
                            best_c = bc;
                            best = (best.0 + ring8[bi].0, best.1 + ring8[bi].1);
                            hv_to_best = hv_evals;
                            continue;
                        }
                    }
                    _iter -= 1;
                }
                #[cfg(accel_x86)]
                if sp_fc && step == 2 && best.0 & 3 == 0 && best.1 & 3 == 0 && pat & 1 == 0 {
                    _iter += 1;
                    let hp8 = hp.expect("sp_fc implies non-fast, which resolves hp");
                    let ring8 = [
                        (step, 0),
                        (-step, 0),
                        (0, step),
                        (0, -step),
                        (step, step),
                        (-step, -step),
                        (step, -step),
                        (-step, step),
                    ];
                    let mut refs8: [Option<(&[u8], usize, usize)>; 8] = [None; 8];
                    let mut all = true;
                    for (i, &(dx, dy)) in ring8.iter().enumerate() {
                        refs8[i] = rusty_h264_common::inter::hpel_ref(
                            hp8,
                            lx,
                            ly,
                            rw,
                            rh,
                            best.0 + dx,
                            best.1 + dy,
                        );
                        all &= refs8[i].is_some();
                    }
                    if all {
                        let stride = refs8[0].unwrap().2;
                        // 16-wide batches; 8-wide evaluates per candidate (H-8 gate)
                        // — identical values, identical argmin, identical bitstream.
                        let pack = |a: usize, b: usize, c2: usize, d: usize| {
                            if rw != 16 {
                                return None;
                            }
                            let g = |i: usize| {
                                let (p, o, _) = refs8[i].unwrap();
                                (p, o)
                            };
                            rusty_h264_accel::satd_x4p(
                                src_row,
                                cw,
                                [g(a), g(b), g(c2), g(d)],
                                stride,
                                rw,
                                rh,
                            )
                        };
                        {
                            let (ax, di) = (pack(0, 1, 2, 3), pack(4, 5, 6, 7));
                            let (mut bi, mut bc) = (usize::MAX, best_c);
                            for i in 0..8 {
                                let (dx, dy) = ring8[i];
                                let mv = (best.0 + dx, best.1 + dy);
                                let cc = match (i < 4, &ax, &di) {
                                    (true, Some(ax), _) => {
                                        let rate =
                                            mvbits(mv.0 - center.0) + mvbits(mv.1 - center.1);
                                        ax[i] as i64 + (lambda_me * rate as f64) as i64
                                    }
                                    (false, _, Some(di)) => {
                                        let rate =
                                            mvbits(mv.0 - center.0) + mvbits(mv.1 - center.1);
                                        di[i - 4] as i64 + (lambda_me * rate as f64) as i64
                                    }
                                    _ => cost(mv),
                                };
                                hv_evals += 1;
                                if cc < bc {
                                    bc = cc;
                                    bi = i;
                                }
                            }
                            if hv_ring1 == i64::MIN {
                                hv_ring1 = if bi == usize::MAX { best_c } else { bc };
                            }
                            if bi == usize::MAX
                                || !self.me_subpel_iter
                                || pat & 2 != 0
                                || (sp_cap != 0 && _iter >= sp_cap)
                            {
                                if bi != usize::MAX {
                                    best_c = bc;
                                    best = (best.0 + ring8[bi].0, best.1 + ring8[bi].1);
                                    hv_to_best = hv_evals;
                                }
                                break;
                            }
                            best_c = bc;
                            best = (best.0 + ring8[bi].0, best.1 + ring8[bi].1);
                            hv_to_best = hv_evals;
                            continue;
                        }
                    }
                    _iter -= 1; // declined — the cascade pass below re-counts it
                }
                let mut improved = false;
                _iter += 1;
                // `_pi` is the ring position for the `profile` census (spstats::ev/imp).
                #[allow(clippy::unused_enumerate_index)] // index read only under `profile`
                for (_pi, &(dx, dy)) in ring.iter().enumerate() {
                    let c = (best.0 + dx, best.1 + dy);
                    let slot = sp_slot(c);
                    let cc = if memo_mv[slot] == c {
                        memo_c[slot]
                    } else {
                        let v = cost(c);
                        memo_mv[slot] = c;
                        memo_c[slot] = v;
                        v
                    };
                    hv_evals += 1;
                    // Descent D: which ring POSITION and which ITERATION actually pay?
                    // Same census that showed the diamond's coarse rungs were noise,
                    // aimed at the stage that is now 41% of encode.
                    #[cfg(feature = "profile")]
                    {
                        spstats::ev(if step == 2 { 0 } else { 1 }, _pi, _iter);
                        if seen.contains(&c) {
                            spstats::redundant();
                        } else {
                            seen.push(c);
                        }
                    }
                    if cc < best_c {
                        best_c = cc;
                        best = c;
                        improved = true;
                        hv_to_best = hv_evals;
                        #[cfg(feature = "profile")]
                        spstats::imp(if step == 2 { 0 } else { 1 }, _pi, _iter);
                    }
                }
                if hv_ring1 == i64::MIN {
                    hv_ring1 = best_c;
                }
                if !improved
                    || !self.me_subpel_iter
                    || pat & 2 != 0
                    || (sp_cap != 0 && _iter >= sp_cap)
                {
                    break;
                }
            }
        }
        if sp_dispatching {
            let n = self.sp_learn_n.get();
            if n < sp_learn {
                self.sp_learn_n.set(n + 1);
                if hv_ring1 != i64::MIN {
                    self.sp_ring1
                        .set(self.sp_ring1.get() + (hv_pre - hv_ring1).max(0));
                    self.sp_total
                        .set(self.sp_total.get() + (hv_pre - best_c).max(0));
                }
                if n + 1 == sp_learn {
                    let tot = self.sp_total.get();
                    // Concentrated in ring 1 -> the later rings are affordable to drop.
                    self.sp_1pass
                        .set(tot > 0 && self.sp_ring1.get() * 100 >= tot * sp_t);
                }
            }
        }
        if !subpel.is_empty() && subpel_harvest::enabled() {
            subpel_harvest::record(
                hv_pre, best_c, lambda_me, rw, rh, hv_evals, hv_to_best, hv_ring1,
            );
        }
        // The snap moved the search off the seed; if the seed was better after all,
        // keep it. This is what makes the snap safe by construction.
        if self.me_snap && seed_c < best_c {
            best = seed_mv;
            best_c = seed_c;
        }
        (best, best_c)
    }

    /// Encodes macroblock `(mb_x, mb_y)` as an inter macroblock of the given
    /// `mode` (0 = P_L0_16x16, 1 = P_16x8, 2 = P_8x16) with one motion vector
    /// per partition: motion-compensate each partition, code the macroblock
    /// residual, and reconstruct.
    /// Dispatch to the current coded path (`_v1`) or the isolated fused path
    /// (`_v2`), selected by the hidden `coded_path_v2` A/B knob. Both must produce
    /// byte-identical bitstreams (gated by the `coded_path_ab` test); the split
    /// exists so the two run side-by-side in one binary for honest timing.
    fn encode_inter_mb(
        &mut self,
        w: &mut BitWriter,
        refs: &[crate::RefFrame],
        sy: &[u8],
        su: &[u8],
        sv: &[u8],
        mb_x: usize,
        mb_y: usize,
        mode: u8,
        parts: &[(i32, (i32, i32))],
    ) {
        // RDOQ fork at the DISPATCH: `_v2` is the accel-only coefficient-fused
        // path and HARD-quantizes — it has no trellis support in either its
        // luma or chroma quant. Under a non-zero `rdoq_strength` (B slices
        // ship `cabac_rdoq_b = 32` by default; P dispatches on grain/screen)
        // it must yield to `_v1`, or accel and scalar builds emit DIFFERENT
        // inter bitstreams. Found as the SECOND divergent site while gating
        // the first (the `_v1` accel chroma arm) — the twins lesson: the
        // divergence class is "accel quant without an rdoq fork", and it had
        // two members.
        if self.coded_path_v2 && self.rdoq_strength <= 0.0 {
            self.encode_inter_mb_v2(w, refs, sy, su, sv, mb_x, mb_y, mode, parts);
        } else {
            self.encode_inter_mb_v1(w, refs, sy, su, sv, mb_x, mb_y, mode, parts);
        }
    }

    /// Isolated, coefficient-fused inter coding path (A/B twin of `_v1`). The
    /// quantized luma levels stay in the hot 16-byte-aligned i16 DCT buffer for the
    /// whole MB; the i32 form is materialized on demand only for *coded* blocks
    /// (CAVLC scan + recon dequant), so uncoded quads never pay the conversion and
    /// there is no 256-word i32 `q_blocks` round-trip. Byte-identical to `_v1`
    /// (gated by `coded_path_ab`). Accel-only optimization; the scalar build reuses
    /// `_v1` unchanged.
    fn encode_inter_mb_v2(
        &mut self,
        w: &mut BitWriter,
        refs: &[crate::RefFrame],
        sy: &[u8],
        su: &[u8],
        sv: &[u8],
        mb_x: usize,
        mb_y: usize,
        mode: u8,
        parts: &[(i32, (i32, i32))],
    ) {
        // Descent E/F: identify this mc_luma population by call site.
        #[cfg(feature = "profile")]
        let _site = rusty_h264_common::inter::mcstats::SiteTag::new(1);
        #[cfg(not(accel))]
        {
            self.encode_inter_mb_v1(w, refs, sy, su, sv, mb_x, mb_y, mode, parts);
        }
        #[cfg(accel)]
        {
            let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncInterCode);
            let (qp, qpc) = (self.qp, self.qpc);
            let w4 = self.mb_w * 4;
            let (_ch, cch) = (self.mb_h * 16, self.mb_h * 8);

            // ---- per-partition motion compensation + MV prediction (== v1) ----
            let mut pred_y = [0u8; 256];
            let mut c_pred = [[0u8; 64]; 2];
            let mut mvds = [(0i32, 0i32); 4];
            let mut n_mvd = 0;
            let _g_mc = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::PredBuf);
            for (part, &(rx, ry, rw, rh)) in inter_partitions(mode).iter().enumerate() {
                let (refi, mv) = parts[part];
                let reference = &refs[refi as usize];
                let (pbx, pby) = ((mb_x * 4 + rx / 4) as isize, (mb_y * 4 + ry / 4) as isize);
                let [a, b, c] = self.mv_neighbors_block(pbx, pby, (rw / 4) as isize);
                let pmv = predict_partition_mv(mode, part, a, b, c, refi);
                mvds[n_mvd] = (mv.0 - pmv.0, mv.1 - pmv.1);
                n_mvd += 1;
                for by in ry / 4..ry / 4 + rh / 4 {
                    let d = (mb_y * 4 + by) * w4 + mb_x * 4 + rx / 4;
                    let n4 = rw / 4;
                    self.mv_y[d..d + n4].fill(nmv(mv));
                    self.inter_y[d..d + n4].fill(true);
                    self.ref_idx_y[d..d + n4].fill(refi as i8);
                    self.coded_y[d..d + n4].fill(true);
                }
                if rw == 16 && rh == 16 {
                    self.mc_luma_cached(
                        reference,
                        mb_x * 16,
                        mb_y * 16,
                        16,
                        16,
                        mv.0,
                        mv.1,
                        &mut pred_y,
                    );
                    self.wp_luma(refi, &mut pred_y, 0, 0, 16, 16);
                } else {
                    let mut tmp = [0u8; 256];
                    self.mc_luma_cached(
                        reference,
                        mb_x * 16 + rx,
                        mb_y * 16 + ry,
                        rw,
                        rh,
                        mv.0,
                        mv.1,
                        &mut tmp,
                    );
                    // H-17: the per-pixel re-stride was the runtime-width copy trap
                    // (a bounds-checked store per pixel); const-width row copies are
                    // byte-identical and lower to inline moves.
                    if rw == 8 {
                        for dy in 0..rh {
                            pred_y[(ry + dy) * 16 + rx..][..8].copy_from_slice(&tmp[dy * 8..][..8]);
                        }
                    } else {
                        for dy in 0..rh {
                            pred_y[(ry + dy) * 16 + rx..][..16]
                                .copy_from_slice(&tmp[dy * 16..][..16]);
                        }
                    }
                    self.wp_luma(refi, &mut pred_y, rx, ry, rw, rh);
                }
                let (crx, cry, crw, crh) = (rx / 2, ry / 2, rw / 2, rh / 2);
                for cc in 0..2 {
                    let rc = if cc == 0 { &reference.u } else { &reference.v };
                    if crw == 8 && crh == 8 {
                        mc_chroma(
                            rc,
                            self.ccw,
                            cch,
                            mb_x * 8,
                            mb_y * 8,
                            8,
                            8,
                            mv.0,
                            mv.1,
                            &mut c_pred[cc],
                        );
                    } else {
                        let mut tc = [0u8; 64];
                        mc_chroma(
                            rc,
                            self.ccw,
                            cch,
                            mb_x * 8 + crx,
                            mb_y * 8 + cry,
                            crw,
                            crh,
                            mv.0,
                            mv.1,
                            &mut tc,
                        );
                        // H-17: same const-width row-copy fix as luma.
                        if crw == 4 {
                            for dy in 0..crh {
                                c_pred[cc][(cry + dy) * 8 + crx..][..4]
                                    .copy_from_slice(&tc[dy * 4..][..4]);
                            }
                        } else {
                            for dy in 0..crh {
                                c_pred[cc][(cry + dy) * 8 + crx..][..8]
                                    .copy_from_slice(&tc[dy * 8..][..8]);
                            }
                        }
                    }
                }
            }

            // ---- luma residual + quantization: keep levels in the i16 buffer ----
            let mut dctw = AlignedDct([0i16; 256]);
            let dct = &mut dctw.0;
            let mut cbp_luma = 0u32;
            drop(_g_mc);
            let _g_tq = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncTq);
            let base = mb_y * 16 * self.cw + mb_x * 16;
            for (qi, &(qx, qy)) in [(0usize, 0usize), (8, 0), (0, 8), (8, 8)]
                .iter()
                .enumerate()
            {
                rusty_h264_accel::dct_four_t4(
                    &mut dct[qi * 64..qi * 64 + 64],
                    &sy[base + qy * self.cw + qx..],
                    self.cw,
                    &pred_y[qy * 16 + qx..],
                    16,
                );
            }
            let ff = rusty_h264_common::transform::quant_dz_ff(qp, 6);
            let mf = &rusty_h264_common::transform::QUANT_MF_OH[qp as usize];
            for qi in 0..4 {
                rusty_h264_accel::quant_four_4x4(&mut dct[qi * 64..qi * 64 + 64], &ff, mf);
            }
            // cbp per quad straight from the i16 levels (no i32 q_blocks copy).
            for blk in 0..16 {
                if dct[blk * 16..blk * 16 + 16].iter().any(|&v| v != 0) {
                    cbp_luma |= 1 << (blk / 4);
                }
            }

            // ---- chroma residual (identical to v1: c_q stays i32) ----
            let mut c_dc_levels = [[0i32; 4]; 2];
            let mut c_recon_dc = [[0i32; 4]; 2];
            let mut c_q = [[[0i32; 16]; 4]; 2];
            let (mut any_ac, mut any_dc) = (false, false);
            for c in 0..2 {
                let src = if c == 0 { su } else { sv };
                let dc2x2 = {
                    #[repr(align(16))]
                    struct A([i16; 64]);
                    let mut cdct = A([0i16; 64]);
                    rusty_h264_accel::dct_four_t4(
                        &mut cdct.0,
                        &src[(mb_y * 8) * self.ccw + mb_x * 8..],
                        self.ccw,
                        &c_pred[c],
                        8,
                    );
                    let dc = [
                        cdct.0[0] as i32,
                        cdct.0[16] as i32,
                        cdct.0[32] as i32,
                        cdct.0[48] as i32,
                    ];
                    let ffc = rusty_h264_common::transform::quant_dz_ff(qpc, 6);
                    let mfc = &rusty_h264_common::transform::QUANT_MF_OH[qpc as usize];
                    rusty_h264_accel::quant_four_4x4(&mut cdct.0, &ffc, mfc);
                    for i in 0..4 {
                        let q = &mut c_q[c][i];
                        q[0] = 0;
                        for j in 1..16 {
                            let v = cdct.0[i * 16 + j] as i32;
                            q[j] = v;
                            if v != 0 {
                                any_ac = true;
                            }
                        }
                    }
                    dc
                };
                let dl = forward_quant_chroma_dc(&dc2x2, qpc, false);
                if dl.iter().any(|&v| v != 0) {
                    any_dc = true;
                }
                c_recon_dc[c] = inverse_quant_chroma_dc(&dl, qpc);
                c_dc_levels[c] = dl;
            }
            let cbp_chroma: u32 = if any_ac {
                2
            } else if any_dc {
                1
            } else {
                0
            };
            let cbp = cbp_luma | (cbp_chroma << 4);

            // ---- emit syntax (== v1) ----
            drop(_g_tq);
            let _g_syn = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::Syntax);
            w.write_ue(mode as u32);
            // P_8x8 (mb_type 3): sub_mb_type per 8×8 (spec 7.3.5.2) — v1 parity;
            // omitting these desynced every v2 P_8x8 macroblock.
            if mode == 3 {
                for _ in 0..4 {
                    w.write_ue(0);
                }
            }
            let num_refs = refs.len();
            if num_refs > 1 {
                for &(refi, _) in parts {
                    write_ref_idx(w, refi, num_refs);
                }
            }
            for &(mvdx, mvdy) in &mvds[..n_mvd] {
                w.write_se(mvdx);
                w.write_se(mvdy);
            }
            write_cbp_inter(w, cbp);
            // transform_size_8x8_flag (v1 parity): v2 codes 4×4-transform residual
            // only, but the flag must be PRESENT whenever the stream enables the
            // 8×8 transform — the decoder reads it unconditionally for
            // cbp_luma != 0. Every shape v2 emits allows it (P_8x8 subs are all
            // P_L0_8x8), so presence == (cbp_luma > 0 && transform_8x8).
            if cbp_luma > 0 && self.transform_8x8 {
                w.write_bit(false);
            }
            if cbp != 0 {
                w.write_se(self.qp_delta()); // mb_qp_delta (AQ per-MB QPy)
            }
            self.nnz_cache_load(mb_x, mb_y);
            drop(_g_syn);

            // ---- CAVLC: scan straight from the i16 levels for coded blocks ----
            let _g_scan = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::Scatter);
            for (blk, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
                let (bx, by) = (mb_x * 4 + lbx, mb_y * 4 + lby);
                let total = if cbp_luma & (1 << (blk / 4)) != 0 {
                    let nc = self.nc_pred(lbx, lby);
                    let scan16 = scan_4x4_dcac_i16(&dct[blk * 16..blk * 16 + 16]);
                    encode_residual_block(w, &scan16, 16, nc) as u8
                } else {
                    0
                };
                self.nnz_cache_set(lbx, lby, total);
                self.nnz_y[by * w4 + bx] = total;
            }
            if cbp_chroma != 0 {
                for c in 0..2 {
                    encode_residual_block(w, &c_dc_levels[c], 4, -1);
                }
            }
            if cbp_chroma == 2 {
                self.chroma_cache_load(mb_x, mb_y);
                let w2 = self.mb_w * 2;
                for c in 0..2 {
                    for &(bx, by) in &CHROMA_4X4_SCAN_XY {
                        let nc = self.chroma_nc_pred(c, bx, by);
                        let ac = scan_4x4_ac(&c_q[c][by * 2 + bx]);
                        let total = encode_residual_block(w, &ac, 15, nc) as u8;
                        self.chroma_nnz_cache_set(c, bx, by, total);
                        self.nnz_c[c][(mb_y * 2 + by) * w2 + (mb_x * 2 + bx)] = total;
                    }
                }
            }
            drop(_g_scan);

            // ---- reconstruction: dequantize luma straight from the i16 levels ----
            let _g_rec = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::SkipRecon);
            #[repr(align(16))]
            struct Align16([i16; 64]);
            let mut dct_in = Align16([0i16; 64]);
            for (qi, &(qx, qy)) in [(0usize, 0usize), (8, 0), (0, 8), (8, 8)]
                .iter()
                .enumerate()
            {
                let rec_off = base + qy * self.cw + qx;
                if cbp_luma & (1 << qi) == 0 {
                    for r in 0..8 {
                        let (dsti, srci) = (rec_off + r * self.cw, (qy + r) * 16 + qx);
                        self.rec_y[dsti..dsti + 8].copy_from_slice(&pred_y[srci..srci + 8]);
                    }
                    continue;
                }
                for k in 0..4 {
                    let blk = qi * 4 + k;
                    let mut lvl = [0i32; 16];
                    for i in 0..16 {
                        lvl[i] = dct[blk * 16 + i] as i32;
                    }
                    let deq = dequantize(&lvl, qp);
                    for i in 0..16 {
                        dct_in.0[k * 16 + i] = deq[i] as i16;
                    }
                }
                rusty_h264_accel::idct_four_t4_rec(
                    &mut self.rec_y[rec_off..],
                    self.cw,
                    &pred_y[qy * 16 + qx..],
                    16,
                    &dct_in.0,
                );
            }
            // chroma recon (identical to v1)
            for c in 0..2 {
                let base_c = (mb_y * 8) * self.ccw + mb_x * 8;
                let plane = if c == 0 {
                    &mut self.rec_u
                } else {
                    &mut self.rec_v
                };
                if cbp_chroma == 0 {
                    for r in 0..8 {
                        let dsti = base_c + r * self.ccw;
                        plane[dsti..dsti + 8].copy_from_slice(&c_pred[c][r * 8..r * 8 + 8]);
                    }
                } else {
                    #[repr(align(16))]
                    struct A([i16; 64]);
                    let mut d = A([0i16; 64]);
                    for i in 0..4 {
                        let deq = dequantize(&c_q[c][i], qpc);
                        for j in 0..16 {
                            d.0[i * 16 + j] = deq[j] as i16;
                        }
                        d.0[i * 16] = c_recon_dc[c][i] as i16;
                    }
                    rusty_h264_accel::idct_four_t4_rec(
                        &mut plane[base_c..],
                        self.ccw,
                        &c_pred[c],
                        8,
                        &d.0,
                    );
                }
            }
            for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
                self.modes_y[(mb_y * 4 + lby) * w4 + (mb_x * 4 + lbx)] = 2;
            }
        }
    }

    fn encode_inter_mb_v1(
        &mut self,
        w: &mut BitWriter,
        refs: &[crate::RefFrame],
        sy: &[u8],
        su: &[u8],
        sv: &[u8],
        mb_x: usize,
        mb_y: usize,
        mode: u8,
        parts: &[(i32, (i32, i32))],
    ) {
        self.encode_inter_mb_v1_b(w, refs, sy, su, sv, mb_x, mb_y, mode, parts, None);
    }

    /// As [`Self::encode_inter_mb_v1`], but `b_mode` selects B-slice framing: the
    /// macroblock is coded as `B_L0_16x16` (`mb_type == 1`) instead of the P-slice
    /// `mb_type == mode`. Everything else — the single List-0 partition, the median
    /// `mvd_l0` predictor, the residual, and the reconstruction — is byte-identical
    /// to `P_L0_16x16`, so the caller passes `mode == 0`, `refs == &[L0_anchor]`
    /// (length 1 ⇒ no `ref_idx` coded), and `parts == &[(0, mv)]`.
    /// Decide + reconstruct one inter macroblock (motion compensation, residual,
    /// quantize, reconstruct, commit motion grids) — everything except entropy
    /// coding. Returns an [`InterPlan`] coded by either backend, so CAVLC and CABAC
    /// share this whole path bit-for-bit (the P/B analogue of [`plan_mb`]).
    fn plan_inter_mb(
        &mut self,
        refs: &[crate::RefFrame],
        sy: &[u8],
        su: &[u8],
        sv: &[u8],
        mb_x: usize,
        mb_y: usize,
        mode: u8,
        parts: &[(i32, (i32, i32))],
        bspec: Option<BInter>,
        sub_types: [u8; 4],
    ) -> InterPlan {
        crate::signals::census::work(crate::signals::census::W_MB_PLAN);
        // Descent E/F: identify this mc_luma population by call site.
        #[cfg(feature = "profile")]
        let _site = rusty_h264_common::inter::mcstats::SiteTag::new(1);
        let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncInterCode);
        let (qp, qpc) = (self.qp, self.qpc);
        let w4 = self.mb_w * 4;
        let (ch, cch) = (self.mb_h * 16, self.mb_h * 8);

        // ---- per-partition motion compensation + MV prediction ----
        let mut pred_y = [0u8; 256];
        let mut c_pred = [[0u8; 64]; 2];
        let mut mvds = [(0i32, 0i32); 16]; // ≤16 sub-partitions; no per-MB Vec alloc
        let mut plan_refs = [0i32; 4]; // per-partition ref_idx_l0 (0 for B / 1-ref)
        let mut n_mvd = 0;
        let _g_mc = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::PredBuf);
        // The SPLIT test must come FIRST. `mvmode > 0` means a 16x8/8x16 partition
        // beat every 16x16 mode INCLUDING direct, and when it beat direct the caller
        // leaves `dir` at 0 -- so a `dir == 0` test placed ahead of this one claims
        // the macroblock, reconstructs B_Direct, and emits nothing into `mvds`, while
        // `emit_mb_cabac_b` still writes the split mb_type. The decoder then reads a
        // B_Bi_16x8 with two zero mvds where the encoder reconstructed direct motion.
        // Measured: 34 macroblocks per 6 frames, -2.4 dB luma, and INVISIBLE to the
        // conformance matrix -- both decoders agree with each other, they just
        // disagree with the encoder.
        if let Some(b) = bspec.filter(|b| b.mvmode > 0) {
            // ---- B 16x8 / 8x16: two partitions, each L0 / L1 / Bi ----
            // Prediction and commit run PARTITION-major (partition 1 predicts off
            // partition 0's committed motion, exactly as the decoder's recon does);
            // the mvds are then serialised LIST-major for the emit, which is the
            // spec 7.3.5.1 order the decoder parses.
            let (rects, _) = b_part_layout(b.mvmode);
            let (ch, cch) = (self.mb_h * 16, self.mb_h * 8);
            let mut pm: [[(i32, i32); 2]; 2] = [[(0, 0); 2]; 2]; // [part][list] mvd
            for (part, &(rx, ry, rw, rh)) in rects.iter().enumerate() {
                let (pred, mv0, mv1) = b.parts2[part];
                let (u0, u1) = (pred == 1 || pred == 3, pred == 2 || pred == 3);
                let (pbx, pby) = ((mb_x * 4 + rx / 4) as isize, (mb_y * 4 + ry / 4) as isize);
                if u0 {
                    let [a, c0, c1] = self.mv_neighbors_block_list(pbx, pby, (rw / 4) as isize, 0);
                    let p = predict_partition_mv(b.mvmode, part, a, c0, c1, 0);
                    pm[part][0] = (mv0.0 - p.0, mv0.1 - p.1);
                }
                if u1 {
                    let [a, c0, c1] = self.mv_neighbors_block_list(pbx, pby, (rw / 4) as isize, 1);
                    let p = predict_partition_mv(b.mvmode, part, a, c0, c1, 0);
                    pm[part][1] = (mv1.0 - p.0, mv1.1 - p.1);
                }
                // Motion compensation for this rect.
                let (lx, ly) = (mb_x * 16 + rx, mb_y * 16 + ry);
                let (cx, cy) = (mb_x * 8 + rx / 2, mb_y * 8 + ry / 2);
                let (cw2, ch2) = (rw / 2, rh / 2);
                let mut ay = [0u8; 256];
                let mut by_ = [0u8; 256];
                let mut ac = [[0u8; 64]; 2];
                let mut bc = [[0u8; 64]; 2];
                if u0 {
                    mc_luma(
                        &refs[0].y, self.cw, ch, lx, ly, rw, rh, mv0.0, mv0.1, &mut ay,
                    );
                    mc_chroma(
                        &refs[0].u, self.ccw, cch, cx, cy, cw2, ch2, mv0.0, mv0.1, &mut ac[0],
                    );
                    mc_chroma(
                        &refs[0].v, self.ccw, cch, cx, cy, cw2, ch2, mv0.0, mv0.1, &mut ac[1],
                    );
                }
                if u1 {
                    mc_luma(&b.l1.y, self.cw, ch, lx, ly, rw, rh, mv1.0, mv1.1, &mut by_);
                    mc_chroma(
                        &b.l1.u, self.ccw, cch, cx, cy, cw2, ch2, mv1.0, mv1.1, &mut bc[0],
                    );
                    mc_chroma(
                        &b.l1.v, self.ccw, cch, cx, cy, cw2, ch2, mv1.0, mv1.1, &mut bc[1],
                    );
                }
                for r in 0..rh {
                    for c in 0..rw {
                        let d = (ry + r) * 16 + rx + c;
                        let sidx = r * rw + c;
                        pred_y[d] = match (u0, u1) {
                            (true, true) => bi_blend(ay[sidx] as i32, by_[sidx] as i32, self.bi_w),
                            (true, false) => ay[sidx],
                            _ => by_[sidx],
                        };
                    }
                }
                for cc in 0..2 {
                    for r in 0..ch2 {
                        for c in 0..cw2 {
                            let d = (ry / 2 + r) * 8 + rx / 2 + c;
                            let sidx = r * cw2 + c;
                            c_pred[cc][d] = match (u0, u1) {
                                (true, true) => {
                                    bi_blend(ac[cc][sidx] as i32, bc[cc][sidx] as i32, self.bi_w)
                                }
                                (true, false) => ac[cc][sidx],
                                _ => bc[cc][sidx],
                            };
                        }
                    }
                }
                // Commit this partition before the next one predicts.
                let (cmv0, cr0) = (if u0 { mv0 } else { (0, 0) }, if u0 { 0 } else { -1 });
                let (cmv1, cr1) = (if u1 { mv1 } else { (0, 0) }, if u1 { 0 } else { -1 });
                for by2 in ry / 4..(ry + rh) / 4 {
                    let d = (mb_y * 4 + by2) * w4 + mb_x * 4 + rx / 4;
                    let n4 = rw / 4;
                    self.inter_y[d..d + n4].fill(true);
                    self.coded_y[d..d + n4].fill(true);
                    self.mv_y[d..d + n4].fill(nmv(cmv0));
                    self.ref_idx_y[d..d + n4].fill(cr0);
                    self.mv1_y[d..d + n4].fill(nmv(cmv1));
                    self.ref_idx1_y[d..d + n4].fill(cr1);
                }
            }
            // Serialise LIST-major: all L0 mvds, then all L1.
            for list in 0..2 {
                for part in 0..2 {
                    let pred = b.parts2[part].0;
                    let used = if list == 0 {
                        pred == 1 || pred == 3
                    } else {
                        pred == 2 || pred == 3
                    };
                    if used {
                        mvds[n_mvd] = pm[part][list];
                        n_mvd += 1;
                    }
                }
            }
        } else if let Some(b) = bspec.filter(|b| b.dir == 0) {
            // ---- B_Direct_16x16 (mb_type 0): spatial-direct prediction, no mvd ----
            let (dp, dc, motion) = self.b_direct(&refs[0], b.l1, mb_x, mb_y);
            pred_y = dp;
            c_pred = dc;
            self.commit_direct_motion(mb_x, mb_y, &motion);
        } else if let Some(b) = bspec {
            // ---- B 16×16 prediction: List-0 / List-1 / Bi ----
            let use0 = b.dir == 1 || b.dir == 3;
            let use1 = b.dir == 2 || b.dir == 3;
            let (lx, ly) = (mb_x * 16, mb_y * 16);
            let (cx, cy) = (mb_x * 8, mb_y * 8);
            let (pbx, pby) = ((mb_x * 4) as isize, (mb_y * 4) as isize);
            // Per-list `mvd` against the median predictor over that list's neighbors.
            if use0 {
                let [a, c0, c1] = self.mv_neighbors_block_list(pbx, pby, 4, 0);
                let p = predict_partition_mv(0, 0, a, c0, c1, 0);
                mvds[n_mvd] = (b.mv0.0 - p.0, b.mv0.1 - p.1);
                n_mvd += 1;
            }
            if use1 {
                let [a, c0, c1] = self.mv_neighbors_block_list(pbx, pby, 4, 1);
                let p = predict_partition_mv(0, 0, a, c0, c1, 0);
                mvds[n_mvd] = (b.mv1.0 - p.0, b.mv1.1 - p.1);
                n_mvd += 1;
            }
            // Motion compensation. L0/L1 write straight into pred; Bi averages
            // (p+q+1)>>1 — the decoder's `b_mc` blend with weighted_bipred_idc=0.
            let mut a_y = [0u8; 256];
            let mut b_y = [0u8; 256];
            let mut a_c = [[0u8; 64]; 2];
            let mut b_c = [[0u8; 64]; 2];
            if use0 {
                mc_luma(
                    &refs[0].y, self.cw, ch, lx, ly, 16, 16, b.mv0.0, b.mv0.1, &mut a_y,
                );
                mc_chroma(
                    &refs[0].u,
                    self.ccw,
                    cch,
                    cx,
                    cy,
                    8,
                    8,
                    b.mv0.0,
                    b.mv0.1,
                    &mut a_c[0],
                );
                mc_chroma(
                    &refs[0].v,
                    self.ccw,
                    cch,
                    cx,
                    cy,
                    8,
                    8,
                    b.mv0.0,
                    b.mv0.1,
                    &mut a_c[1],
                );
            }
            if use1 {
                mc_luma(
                    &b.l1.y, self.cw, ch, lx, ly, 16, 16, b.mv1.0, b.mv1.1, &mut b_y,
                );
                mc_chroma(
                    &b.l1.u,
                    self.ccw,
                    cch,
                    cx,
                    cy,
                    8,
                    8,
                    b.mv1.0,
                    b.mv1.1,
                    &mut b_c[0],
                );
                mc_chroma(
                    &b.l1.v,
                    self.ccw,
                    cch,
                    cx,
                    cy,
                    8,
                    8,
                    b.mv1.0,
                    b.mv1.1,
                    &mut b_c[1],
                );
            }
            match (use0, use1) {
                (true, true) => {
                    for i in 0..256 {
                        pred_y[i] = bi_blend(a_y[i] as i32, b_y[i] as i32, self.bi_w);
                    }
                    for c in 0..2 {
                        for i in 0..64 {
                            c_pred[c][i] = bi_blend(a_c[c][i] as i32, b_c[c][i] as i32, self.bi_w);
                        }
                    }
                }
                (true, false) => {
                    pred_y = a_y;
                    c_pred = a_c;
                }
                _ => {
                    pred_y = b_y;
                    c_pred = b_c;
                }
            }
            // Commit per-list motion so later MBs' per-list predictors see it.
            // Row fills (E1, 11.16): every value is CONSTANT across the MB.
            let (cmv0, cr0) = (if use0 { b.mv0 } else { (0, 0) }, if use0 { 0 } else { -1 });
            let (cmv1, cr1) = (if use1 { b.mv1 } else { (0, 0) }, if use1 { 0 } else { -1 });
            for by in 0..4 {
                let d = (mb_y * 4 + by) * w4 + mb_x * 4;
                self.inter_y[d..d + 4].fill(true);
                self.coded_y[d..d + 4].fill(true);
                self.mv_y[d..d + 4].fill(nmv(cmv0));
                self.ref_idx_y[d..d + 4].fill(cr0);
                self.mv1_y[d..d + 4].fill(nmv(cmv1));
                self.ref_idx1_y[d..d + 4].fill(cr1);
            }
        } else if mode == 3 && sub_types != [0u8; 4] {
            // ---- P_8x8 with sub-partitions (Great Gate P3.3) ------------------
            // Mirrors the decoder's `decode_p8x8` EXACTLY: per sub-partition in
            // decode order, median-predict from the COMMITTED grid (plain
            // `predict_mv` -- the 16x8/8x16 directional rules do not apply to
            // sub-partitions), derive the mvd, commit, then motion-compensate.
            // `parts` is FLAT in decode order; `ref_idx` is per 8x8 (spec: its
            // sub-partitions share it).
            let mut k = 0usize;
            for p8 in 0..4usize {
                let (b8x, b8y) = ((p8 % 2) * 8, (p8 / 2) * 8);
                plan_refs[p8] = parts[k].0;
                for &(srx, sry, srw, srh) in sub_mb_partitions_p(sub_types[p8]) {
                    let (refi, mv) = parts[k];
                    debug_assert_eq!(refi, plan_refs[p8], "ref_idx_l0 is per 8x8");
                    k += 1;
                    let (px, py) = (b8x + srx, b8y + sry);
                    let (pbx, pby) = ((mb_x * 4 + px / 4) as isize, (mb_y * 4 + py / 4) as isize);
                    let [a, b, c] = self.mv_neighbors_block(pbx, pby, (srw / 4) as isize);
                    let pmv = predict_mv(a, b, c, refi);
                    mvds[n_mvd] = (mv.0 - pmv.0, mv.1 - pmv.1);
                    n_mvd += 1;
                    // Commit BEFORE the next sub-partition predicts (chaining --
                    // exactly the decoder's order).
                    for by in py / 4..(py + srh) / 4 {
                        for bx in px / 4..(px + srw) / 4 {
                            let idx = (mb_y * 4 + by) * w4 + (mb_x * 4 + bx);
                            self.mv_y[idx] = nmv(mv);
                            self.inter_y[idx] = true;
                            self.ref_idx_y[idx] = refi as i8;
                            self.coded_y[idx] = true;
                        }
                    }
                    // Luma + chroma MC into the sub-region (parametric kernels --
                    // 4-wide takes the scalar fall-through; the 4-wide kernel
                    // brick is the recorded default-on precondition).
                    let reference = &refs[refi as usize];
                    let mut tmp = [0u8; 256];
                    mc_luma(
                        &reference.y,
                        self.cw,
                        ch,
                        mb_x * 16 + px,
                        mb_y * 16 + py,
                        srw,
                        srh,
                        mv.0,
                        mv.1,
                        &mut tmp,
                    );
                    for dy in 0..srh {
                        pred_y[(py + dy) * 16 + px..][..srw]
                            .copy_from_slice(&tmp[dy * srw..][..srw]);
                    }
                    self.wp_luma(refi, &mut pred_y, px, py, srw, srh);
                    let (crx, cry, crw, crh) = (px / 2, py / 2, srw / 2, srh / 2);
                    for cc in 0..2 {
                        let rc = if cc == 0 { &reference.u } else { &reference.v };
                        let mut tc = [0u8; 64];
                        mc_chroma(
                            rc,
                            self.ccw,
                            cch,
                            mb_x * 8 + crx,
                            mb_y * 8 + cry,
                            crw,
                            crh,
                            mv.0,
                            mv.1,
                            &mut tc,
                        );
                        for dy in 0..crh {
                            c_pred[cc][(cry + dy) * 8 + crx..][..crw]
                                .copy_from_slice(&tc[dy * crw..][..crw]);
                        }
                    }
                }
            }
        } else {
            for (part, &(rx, ry, rw, rh)) in inter_partitions(mode).iter().enumerate() {
                let (refi, mv) = parts[part];
                plan_refs[part] = refi; // per-partition ref_idx_l0 → carried to the CABAC emit
                let reference = &refs[refi as usize];
                let (pbx, pby) = ((mb_x * 4 + rx / 4) as isize, (mb_y * 4 + ry / 4) as isize);
                let [a, b, c] = self.mv_neighbors_block(pbx, pby, (rw / 4) as isize);
                let pmv = predict_partition_mv(mode, part, a, b, c, refi);
                mvds[n_mvd] = (mv.0 - pmv.0, mv.1 - pmv.1);
                n_mvd += 1;
                // Commit this partition's motion so later partitions can predict from it.
                for by in ry / 4..ry / 4 + rh / 4 {
                    for bx in rx / 4..rx / 4 + rw / 4 {
                        let idx = (mb_y * 4 + by) * w4 + (mb_x * 4 + bx);
                        self.mv_y[idx] = nmv(mv);
                        self.inter_y[idx] = true;
                        self.ref_idx_y[idx] = refi as i8;
                        self.coded_y[idx] = true;
                    }
                }
                // Luma MC into the partition's sub-region. A full-MB (16×16) partition is
                // the whole `pred_y`, so MC straight into it — no scratch + repack copy.
                if rw == 16 && rh == 16 {
                    self.mc_luma_cached(
                        reference,
                        mb_x * 16,
                        mb_y * 16,
                        16,
                        16,
                        mv.0,
                        mv.1,
                        &mut pred_y,
                    );
                } else {
                    let mut tmp = [0u8; 256];
                    self.mc_luma_cached(
                        reference,
                        mb_x * 16 + rx,
                        mb_y * 16 + ry,
                        rw,
                        rh,
                        mv.0,
                        mv.1,
                        &mut tmp,
                    );
                    // H-17: const-width row copies (see the v2 twin).
                    if rw == 8 {
                        for dy in 0..rh {
                            pred_y[(ry + dy) * 16 + rx..][..8].copy_from_slice(&tmp[dy * 8..][..8]);
                        }
                    } else {
                        for dy in 0..rh {
                            pred_y[(ry + dy) * 16 + rx..][..16]
                                .copy_from_slice(&tmp[dy * 16..][..16]);
                        }
                    }
                }
                self.wp_luma(refi, &mut pred_y, rx, ry, rw, rh);
                // Chroma MC (half-resolution region); 8×8 = the whole plane prediction.
                let (crx, cry, crw, crh) = (rx / 2, ry / 2, rw / 2, rh / 2);
                for cc in 0..2 {
                    let rc = if cc == 0 { &reference.u } else { &reference.v };
                    if crw == 8 && crh == 8 {
                        mc_chroma(
                            rc,
                            self.ccw,
                            cch,
                            mb_x * 8,
                            mb_y * 8,
                            8,
                            8,
                            mv.0,
                            mv.1,
                            &mut c_pred[cc],
                        );
                    } else {
                        let mut tc = [0u8; 64];
                        mc_chroma(
                            rc,
                            self.ccw,
                            cch,
                            mb_x * 8 + crx,
                            mb_y * 8 + cry,
                            crw,
                            crh,
                            mv.0,
                            mv.1,
                            &mut tc,
                        );
                        // H-17: const-width row copies (see the v2 twin).
                        if crw == 4 {
                            for dy in 0..crh {
                                c_pred[cc][(cry + dy) * 8 + crx..][..4]
                                    .copy_from_slice(&tc[dy * 4..][..4]);
                            }
                        } else {
                            for dy in 0..crh {
                                c_pred[cc][(cry + dy) * 8 + crx..][..8]
                                    .copy_from_slice(&tc[dy * 8..][..8]);
                            }
                        }
                    }
                }
            }
        } // end P per-partition formation (else of the B branch)

        // ---- luma residual + quantization ----
        let mut q_blocks = [[0i32; 16]; 16]; // raster, levels
        let mut cbp_luma = 0u32;
        // Inter 8x8-transform candidate (High profile, scalar path). Filled by the
        // per-MB 4x4-vs-8x8 RD below; false/zero means the 4x4 residual is used.
        #[allow(unused_mut)]
        let mut t8x8 = false;
        #[allow(unused_mut)]
        let mut q8 = [[0i32; 64]; 4];
        drop(_g_mc);
        let _g_tq = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncTq);
        #[cfg(accel)]
        {
            // openh264 `WelsDctFourT4_sse2` (fused residual+DCT) → i16, then
            // `WelsQuantFour4x4_sse2` in place — the whole DCT→quant chain stays in i16,
            // no i32 round-trip. Quant is openh264's structure carrying OUR deadzone
            // (`quant_dz_ff` + `QUANT_MF_OH`), so levels are bit-identical to `quantize`.
            let mut dctw = AlignedDct([0i16; 256]);
            let dct = &mut dctw.0;
            let base = mb_y * 16 * self.cw + mb_x * 16;
            for (qi, &(qx, qy)) in [(0usize, 0usize), (8, 0), (0, 8), (8, 8)]
                .iter()
                .enumerate()
            {
                rusty_h264_accel::dct_four_t4(
                    &mut dct[qi * 64..qi * 64 + 64],
                    &sy[base + qy * self.cw + qx..],
                    self.cw,
                    &pred_y[qy * 16 + qx..],
                    16,
                );
            }
            if self.rdoq_strength > 0.0 {
                // Trellis for inter (Great Gate P2 — mirrors the I16 site): scalar
                // RDOQ from the asm DCT output instead of the asm hard quantizer.
                // Inter codes DC in the 4×4 (first=0) and uses the /6 deadzone —
                // exactly the scalar arm's `rdoq(&coeffs, qp, 6, strength, 0)`.
                for (blk, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
                    let coeffs: [i32; 16] = core::array::from_fn(|i| dct[blk * 16 + i] as i32);
                    let q = rdoq(&coeffs, qp, 6, self.rdoq_strength, 0);
                    if q.iter().any(|&v| v != 0) {
                        cbp_luma |= 1 << (blk / 4);
                    }
                    q_blocks[lby * 4 + lbx] = q;
                }
            } else {
                let ff = rusty_h264_common::transform::quant_dz_ff(qp, 6);
                let mf = &rusty_h264_common::transform::QUANT_MF_OH[qp as usize];
                for qi in 0..4 {
                    rusty_h264_accel::quant_four_4x4(&mut dct[qi * 64..qi * 64 + 64], &ff, mf);
                }
                for (blk, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
                    let mut nz = false;
                    for i in 0..16 {
                        let v = dct[blk * 16 + i] as i32;
                        q_blocks[lby * 4 + lbx][i] = v;
                        nz |= v != 0;
                    }
                    if nz {
                        cbp_luma |= 1 << (blk / 4);
                    }
                }
            } // end hard-quantize arm (rdoq_strength == 0)
        }
        #[cfg(not(accel))]
        {
            // Scalar/`wide`: gather all 16 residual blocks, batched forward-DCT, quantize.
            let mut res_blocks = [[0i32; 16]; 16]; // raster
            for lby in 0..4 {
                for lbx in 0..4 {
                    let b = &mut res_blocks[lby * 4 + lbx];
                    for dy in 0..4 {
                        let row =
                            &sy[(mb_y * 16 + lby * 4 + dy) * self.cw + mb_x * 16 + lbx * 4..][..4];
                        for dx in 0..4 {
                            b[dy * 4 + dx] = row[dx] as i32
                                - pred_y[(lby * 4 + dy) * 16 + (lbx * 4 + dx)] as i32;
                        }
                    }
                }
            }
            let mut coeffs = [[0i32; 16]; 16];
            forward_dct_blocks(&res_blocks, &mut coeffs);
            for (blk, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
                let q = rdoq(&coeffs[lby * 4 + lbx], qp, 6, self.rdoq_strength, 0);
                if q.iter().any(|&v| v != 0) {
                    cbp_luma |= 1 << (blk / 4);
                }
                q_blocks[lby * 4 + lbx] = q;
            }
        }

        // Per-MB transform-size RD (runs in scalar AND accel builds — q_blocks +
        // cbp_luma are filled by whichever quant path ran; the 8x8 candidate + its
        // recon are pure Rust). One 8x8 DCT per 8x8 block vs four 4x4s.
        // Content-adaptive by construction — the winner is chosen per MB.
        //
        // `sub_types == [0;4]` IS the spec's `noSubMbPartSizeLessThan8x8Flag`
        // (7.3.5): transform_size_8x8_flag is FORBIDDEN when any sub-partition is
        // smaller than 8x8. The comment this replaces asserted "every inter
        // partition here is >= 8x8", which was true when it was written and stopped
        // being true when sub-8x8 shipped -- `sub_types` is a parameter of this very
        // function. A stale invariant in a comment is not a guard.
        {
            // `allow_t8` is the spec's condition for transform_size_8x8_flag being
            // PERMITTED at all, evaluated here rather than at emit time: if the plan
            // picked 8x8 for a macroblock that may not signal it, the flag would be
            // suppressed while our reconstruction still used the 8x8 transform --
            // encoder/decoder drift, which our own conformance gate cannot see.
            //   * `sub_types == [0;4]` is noSubMbPartSizeLessThan8x8Flag.
            //   * B_Direct_16x16 is permitted since direct_8x8_inference_flag = 1
            //     in our SPS (its direct partitions count as 8×8).
            let allow_t8 = sub_types == [0u8; 4];
            if self.t8_pick && self.inter8x8 != 0 && allow_t8 {
                let lambda = 0.85 * self.tune_lambda_scale * crate::fastmath::lambda_qp(qp);
                let mut ssd4 = 0i64;
                let mut rate4 = 0f64;
                for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
                    let mut predb = [0i32; 16];
                    for dy in 0..4 {
                        for dx in 0..4 {
                            predb[dy * 4 + dx] =
                                pred_y[(lby * 4 + dy) * 16 + (lbx * 4 + dx)] as i32;
                        }
                    }
                    let deq = dequantize(&q_blocks[lby * 4 + lbx], qp);
                    let s = reconstruct_4x4(&deq, &predb);
                    for dy in 0..4 {
                        for dx in 0..4 {
                            let sx = mb_x * 16 + lbx * 4 + dx;
                            let syy = mb_y * 16 + lby * 4 + dy;
                            let d = s[dy * 4 + dx] as i64 - sy[syy * self.cw + sx] as i64;
                            ssd4 += d * d;
                        }
                    }
                    for &l in &q_blocks[lby * 4 + lbx] {
                        if l != 0 {
                            rate4 += rdoq_rate((l as i64).abs());
                        }
                    }
                }
                let (q8c, cbp8, rate8, _rec8, ssd8) =
                    plan_inter8_luma(sy, self.cw, mb_x, mb_y, &pred_y, qp);
                // Both candidates priced with the SAME level-aware rate (Σ rdoq_rate);
                // `inter8_pen` is an optional extra bias (default 0) on the 8x8 flag.
                let j4 = ssd4 as f64 + lambda * (rate4 + 16.0);
                let j8 = ssd8 as f64 + lambda * (rate8 + 16.0 + self.inter8_pen as f64);
                if cbp8 > 0 && j8 < j4 {
                    t8x8 = true;
                    cbp_luma = cbp8;
                    q8 = q8c;
                }
            }
        }

        // ---- chroma residual (prediction already built per partition) ----
        let mut c_dc_levels = [[0i32; 4]; 2];
        let mut c_recon_dc = [[0i32; 4]; 2];
        let mut c_q = [[[0i32; 16]; 4]; 2];
        let (mut any_ac, mut any_dc) = (false, false);
        for c in 0..2 {
            let src = if c == 0 { su } else { sv };
            // Fast path: one dct_four_t4 covers the whole 8x8 chroma region (all 4
            // blocks, residual+DCT fused straight from the planes); block b's pre-quant
            // DC is dct[b*16] (quad z-scan == 2x2 raster); quant_four_4x4 with our
            // FF/MF is bit-identical to scalar `quantize`. Same pairing the P_Skip
            // free-check proved byte-identical over the corpus.
            #[cfg(accel)]
            let (mut dc2x2, applied) = if self.rdoq_strength > 0.0 {
                // RDOQ fork — the SAME fork every other accel quant site
                // carries (inter luma, I16 AC, intra chroma AC). The fused
                // dct+quant kernel HARD-quantizes, so under a non-zero trellis
                // strength it must yield to the scalar twin below. Without
                // this, accel and scalar builds emitted DIFFERENT bitstreams
                // for inter chroma AC the day `cabac_rdoq_b` shipped
                // default-on — the one path where the accel/scalar
                // byte-identity invariant silently did not hold.
                ([0i32; 4], false)
            } else {
                #[repr(align(16))]
                struct A([i16; 64]);
                let mut dct = A([0i16; 64]);
                rusty_h264_accel::dct_four_t4(
                    &mut dct.0,
                    &src[(mb_y * 8) * self.ccw + mb_x * 8..],
                    self.ccw,
                    &c_pred[c],
                    8,
                );
                let dc = [
                    dct.0[0] as i32,
                    dct.0[16] as i32,
                    dct.0[32] as i32,
                    dct.0[48] as i32,
                ];
                let ffc = rusty_h264_common::transform::quant_dz_ff(qpc, 6);
                let mfc = &rusty_h264_common::transform::QUANT_MF_OH[qpc as usize];
                rusty_h264_accel::quant_four_4x4(&mut dct.0, &ffc, mfc);
                for i in 0..4 {
                    let q = &mut c_q[c][i];
                    q[0] = 0;
                    for j in 1..16 {
                        let v = dct.0[i * 16 + j] as i32;
                        q[j] = v;
                        if v != 0 {
                            any_ac = true;
                        }
                    }
                }
                (dc, true)
            };
            #[cfg(not(accel))]
            let (mut dc2x2, applied) = ([0i32; 4], false);
            if !applied {
                // Scalar/`wide` twin: gather, batch forward DCT, quantize per block.
                let mut res_blocks = [[0i32; 16]; 4];
                for by in 0..2 {
                    for bx in 0..2 {
                        let b = &mut res_blocks[by * 2 + bx];
                        for dy in 0..4 {
                            let row = &src
                                [(mb_y * 8 + by * 4 + dy) * self.ccw + mb_x * 8 + bx * 4..][..4];
                            for dx in 0..4 {
                                b[dy * 4 + dx] = row[dx] as i32
                                    - c_pred[c][(by * 4 + dy) * 8 + (bx * 4 + dx)] as i32;
                            }
                        }
                    }
                }
                let mut coeffs = [[0i32; 16]; 4];
                forward_dct_blocks(&res_blocks, &mut coeffs);
                for i in 0..4 {
                    dc2x2[i] = coeffs[i][0];
                    let mut q = rdoq(&coeffs[i], qpc, 6, self.rdoq_strength, 1);
                    q[0] = 0;
                    if q[1..].iter().any(|&v| v != 0) {
                        any_ac = true;
                    }
                    c_q[c][i] = q;
                }
            }
            let dl = forward_quant_chroma_dc(&dc2x2, qpc, false);
            if dl.iter().any(|&v| v != 0) {
                any_dc = true;
            }
            c_recon_dc[c] = inverse_quant_chroma_dc(&dl, qpc);
            c_dc_levels[c] = dl;
        }
        let cbp_chroma: u32 = if any_ac {
            2
        } else if any_dc {
            1
        } else {
            0
        };
        let cbp = cbp_luma | (cbp_chroma << 4);

        drop(_g_tq);
        let _g_rec = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::SkipRecon);
        // ---- reconstruction (luma) ----
        #[cfg(accel)]
        if t8x8 {
            // 8x8-transform recon is pure Rust (no asm 8x8 kernels yet); inverse of
            // the decoder's t8x8 inter path. Same code as the scalar branch below.
            let weight = [16i32; 64];
            for b8 in 0..4usize {
                let (b8x, b8y) = (b8 % 2, b8 / 2);
                let res_r = inverse_quant_8x8(&q8[b8], qp, &weight);
                let predb: [i32; 64] = core::array::from_fn(|i| {
                    pred_y[(b8y * 8 + i / 8) * 16 + (b8x * 8 + i % 8)] as i32
                });
                let recon = add_residual_8x8(&res_r, &predb);
                for dy in 0..8 {
                    let d = (mb_y * 16 + b8y * 8 + dy) * self.cw + mb_x * 16 + b8x * 8;
                    self.rec_y[d..d + 8].copy_from_slice(&recon[dy * 8..dy * 8 + 8]);
                }
            }
        } else {
            // Dequantize all 16 blocks into the 4-quadrant int16 layout (16-byte
            // aligned — the kernel uses movdqa coeff loads), then inverse-DCT + add
            // prediction + clip per quadrant via openh264. The inverse butterfly +
            // (x+32)>>6 is bit-identical to reconstruct_4x4 (verified in accel).
            // An 8x8 quad whose cbp bit is clear has ZERO residual: reconstruction
            // IS the prediction (the decoder's own uncoded-region fast path) — a row
            // copy replaces dequant + convert + idct for that quad. Byte-identical:
            // idct of an all-zero block adds (0+32)>>6 = 0 to pred, clip is identity.
            #[repr(align(16))]
            struct Align16([i16; 64]);
            let mut dct_in = Align16([0i16; 64]);
            let base = mb_y * 16 * self.cw + mb_x * 16;
            for (qi, &(qx, qy)) in [(0usize, 0usize), (8, 0), (0, 8), (8, 8)]
                .iter()
                .enumerate()
            {
                let rec_off = base + qy * self.cw + qx;
                if cbp_luma & (1 << qi) == 0 {
                    for r in 0..8 {
                        let (dsti, srci) = (rec_off + r * self.cw, (qy + r) * 16 + qx);
                        self.rec_y[dsti..dsti + 8].copy_from_slice(&pred_y[srci..srci + 8]);
                    }
                    continue;
                }
                for k in 0..4 {
                    let blk = qi * 4 + k;
                    let (lbx, lby) = LUMA_4X4_SCAN_XY[blk];
                    let deq = dequantize(&q_blocks[lby * 4 + lbx], qp);
                    for i in 0..16 {
                        dct_in.0[k * 16 + i] = deq[i] as i16;
                    }
                }
                rusty_h264_accel::idct_four_t4_rec(
                    &mut self.rec_y[rec_off..],
                    self.cw,
                    &pred_y[qy * 16 + qx..],
                    16,
                    &dct_in.0,
                );
            }
        }
        #[cfg(not(accel))]
        if t8x8 {
            // 8x8-transform reconstruction (inverse of the decoder's t8x8 inter path).
            let weight = [16i32; 64];
            for b8 in 0..4usize {
                let (b8x, b8y) = (b8 % 2, b8 / 2);
                let res_r = inverse_quant_8x8(&q8[b8], qp, &weight);
                let predb: [i32; 64] = core::array::from_fn(|i| {
                    pred_y[(b8y * 8 + i / 8) * 16 + (b8x * 8 + i % 8)] as i32
                });
                let recon = add_residual_8x8(&res_r, &predb);
                for dy in 0..8 {
                    let d = (mb_y * 16 + b8y * 8 + dy) * self.cw + mb_x * 16 + b8x * 8;
                    self.rec_y[d..d + 8].copy_from_slice(&recon[dy * 8..dy * 8 + 8]);
                }
            }
        } else {
            for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
                let deq = dequantize(&q_blocks[lby * 4 + lbx], qp);
                // Fused recon straight from the u8 prediction (E1 r2; same
                // transplant as plan_i4x4's) — no i32 predb, no temp, no store.
                reconstruct_4x4_into(
                    &deq,
                    &pred_y,
                    (lby * 4) * 16 + lbx * 4,
                    16,
                    &mut self.rec_y,
                    (mb_y * 16 + lby * 4) * self.cw + mb_x * 16 + lbx * 4,
                    self.cw,
                );
            }
        }
        for c in 0..2 {
            // Fast path: dequantize into the quad i16 layout (raster == the kernel's
            // z-order for a 2x2) with the Hadamard DC injected, then ONE
            // idct+add-pred+clip kernel writes the 8x8 straight into the plane —
            // bit-identical to the scalar tail below (verified kernel pairing).
            #[cfg(accel)]
            {
                let base = (mb_y * 8) * self.ccw + mb_x * 8;
                let plane = if c == 0 {
                    &mut self.rec_u
                } else {
                    &mut self.rec_v
                };
                if cbp_chroma == 0 {
                    // No chroma residual at all: recon = prediction (row copies).
                    for r in 0..8 {
                        let dsti = base + r * self.ccw;
                        plane[dsti..dsti + 8].copy_from_slice(&c_pred[c][r * 8..r * 8 + 8]);
                    }
                } else {
                    #[repr(align(16))]
                    struct A([i16; 64]);
                    let mut d = A([0i16; 64]);
                    for i in 0..4 {
                        let deq = dequantize(&c_q[c][i], qpc);
                        for j in 0..16 {
                            d.0[i * 16 + j] = deq[j] as i16;
                        }
                        d.0[i * 16] = c_recon_dc[c][i] as i16;
                    }
                    rusty_h264_accel::idct_four_t4_rec(
                        &mut plane[base..],
                        self.ccw,
                        &c_pred[c],
                        8,
                        &d.0,
                    );
                }
            }
            #[cfg(not(accel))]
            {
                // Dequantize the 4 blocks (raster, DC overridden by the 2×2-Hadamard
                // recon), then batch the inverse DCT and share the add+clip tail.
                let mut deq_blocks = [[0i32; 16]; 4];
                for i in 0..4 {
                    deq_blocks[i] = dequantize(&c_q[c][i], qpc);
                    deq_blocks[i][0] = c_recon_dc[c][i];
                }
                let mut res = [[0i32; 16]; 4];
                inverse_dct_blocks(&deq_blocks, &mut res);
                let plane = if c == 0 {
                    &mut self.rec_u
                } else {
                    &mut self.rec_v
                };
                for by in 0..2 {
                    for bx in 0..2 {
                        let mut predb = [0i32; 16];
                        for dy in 0..4 {
                            for dx in 0..4 {
                                predb[dy * 4 + dx] =
                                    c_pred[c][(by * 4 + dy) * 8 + (bx * 4 + dx)] as i32;
                            }
                        }
                        let s = add_residual_4x4(&res[by * 2 + bx], &predb);
                        store(plane, self.ccw, mb_x * 8 + bx * 4, mb_y * 8 + by * 4, &s);
                    }
                }
            }
        }
        // MV grid + coded flags were set per partition; mark modes as DC.
        for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
            self.modes_y[(mb_y * 4 + lby) * w4 + (mb_x * 4 + lbx)] = 2;
        }
        InterPlan {
            mvds,
            plan_refs,
            n_mvd,
            cbp,
            q_blocks,
            c_dc_levels,
            c_q,
            t8x8,
            q8,
            sub_types,
        }
    }

    /// Code one planned inter macroblock as CAVLC (the original `encode_inter_mb_v1_b`
    /// tail). `plan_inter_mb` already committed the reconstruction + motion grids.
    fn encode_inter_mb_v1_b(
        &mut self,
        w: &mut BitWriter,
        refs: &[crate::RefFrame],
        sy: &[u8],
        su: &[u8],
        sv: &[u8],
        mb_x: usize,
        mb_y: usize,
        mode: u8,
        parts: &[(i32, (i32, i32))],
        bspec: Option<BInter>,
    ) {
        let plan = self.plan_inter_mb(refs, sy, su, sv, mb_x, mb_y, mode, parts, bspec, [0u8; 4]);
        let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncEmit);
        self.emit_inter_cavlc(w, refs.len(), mb_x, mb_y, mode, parts, bspec, &plan);
    }

    /// CAVLC entropy coding for a planned inter macroblock.
    fn emit_inter_cavlc(
        &mut self,
        w: &mut BitWriter,
        num_refs: usize,
        mb_x: usize,
        mb_y: usize,
        mode: u8,
        parts: &[(i32, (i32, i32))],
        bspec: Option<BInter>,
        plan: &InterPlan,
    ) {
        let w4 = self.mb_w * 4;
        let (cbp, cbp_luma, cbp_chroma) = (plan.cbp, plan.cbp & 15, plan.cbp >> 4);
        // Same rule as the CABAC path (and as `plan_inter_mb`'s `allow_t8`):
        // with direct_8x8_inference_flag = 1 every shape we emit (incl.
        // B_Direct_16x16) may carry transform_size_8x8_flag.
        let allow8 = true;
        let _ = &bspec; // shape still consumed below for mb_type
                        // mb_pred order (spec 7.3.5.1): mb_type, then all ref_idx_l0, then all mvd_l0.
                        // B-slice mb_type = the B direction 1/2/3; P-slice uses `mode`. ref_idx coded
                        // only when >1 reference is active.
        w.write_ue(bspec.map_or(mode as u32, |b| b.dir as u32)); // inter mb_type
                                                                 // P_8x8 (mb_type 3): sub_mb_type per 8×8 (spec 7.3.5.2, before ref_idx/mvd).
                                                                 // 0 = P_L0_8x8 (one MV) — the only shape emitted for now.
        if mode == 3 {
            for _ in 0..4 {
                w.write_ue(0);
            }
        }
        if num_refs > 1 {
            for &(refi, _) in parts {
                write_ref_idx(w, refi, num_refs);
            }
        }
        for &(mvdx, mvdy) in &plan.mvds[..plan.n_mvd] {
            w.write_se(mvdx);
            w.write_se(mvdy);
        }
        write_cbp_inter(w, cbp);
        // transform_size_8x8_flag: after cbp, before mb_qp_delta, present only when
        // luma has coefficients, the 8x8 transform is enabled, and the macroblock is
        // ALLOWED to signal it. Omitting `allow8` wrote the flag on B_Direct_16x16
        // macroblocks, which is exactly why CAVLC B + 8x8 produced an invalid slice
        // ("mb_type N in B slice too large") -- the defect the old blanket refusal
        // in `lib.rs` was masking rather than the missing feature it claimed.
        if cbp_luma > 0 && self.transform_8x8 && allow8 {
            w.write_bit(plan.t8x8);
        }
        if cbp != 0 {
            w.write_se(self.qp_delta()); // mb_qp_delta (AQ per-MB QPy)
        }
        self.nnz_cache_load(mb_x, mb_y);
        if plan.t8x8 {
            // 8x8 residual: four interleaved 4x4 CAVLC sub-blocks per 8x8 block
            // (coeff k of sub s -> 8x8 scan position 4k+s), the inverse of the
            // decoder's t8x8 inter luma read. nnz set per 4x4 sub-block.
            for b8 in 0..4usize {
                let (b8x, b8y) = (b8 % 2, b8 / 2);
                let scan8 = scan_8x8_fwd(&plan.q8[b8]);
                for sub in 0..4usize {
                    let (cx, cy) = (b8x * 2 + sub % 2, b8y * 2 + sub / 2);
                    let (bx, by) = (mb_x * 4 + cx, mb_y * 4 + cy);
                    let total = if cbp_luma & (1 << b8) != 0 {
                        let nc = self.nc_pred(cx, cy);
                        let blk: [i32; 16] = core::array::from_fn(|k| scan8[4 * k + sub]);
                        encode_residual_block(w, &blk, 16, nc) as u8
                    } else {
                        0
                    };
                    self.nnz_cache_set(cx, cy, total);
                    self.nnz_y[by * w4 + bx] = total;
                }
            }
        } else {
            for (blk, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
                let (bx, by) = (mb_x * 4 + lbx, mb_y * 4 + lby);
                let total = if cbp_luma & (1 << (blk / 4)) != 0 {
                    let nc = self.nc_pred(lbx, lby);
                    let scan16 = scan_4x4_dcac(&plan.q_blocks[lby * 4 + lbx]);
                    encode_residual_block(w, &scan16, 16, nc) as u8
                } else {
                    0
                };
                self.nnz_cache_set(lbx, lby, total);
                self.nnz_y[by * w4 + bx] = total;
            }
        }
        if cbp_chroma != 0 {
            for c in 0..2 {
                encode_residual_block(w, &plan.c_dc_levels[c], 4, -1);
            }
        }
        if cbp_chroma == 2 {
            self.chroma_cache_load(mb_x, mb_y);
            let w2 = self.mb_w * 2;
            for c in 0..2 {
                for &(bx, by) in &CHROMA_4X4_SCAN_XY {
                    let nc = self.chroma_nc_pred(c, bx, by);
                    let ac = scan_4x4_ac(&plan.c_q[c][by * 2 + bx]);
                    let total = encode_residual_block(w, &ac, 15, nc) as u8;
                    self.chroma_nnz_cache_set(c, bx, by, total);
                    self.nnz_c[c][(mb_y * 2 + by) * w2 + (mb_x * 2 + bx)] = total;
                }
            }
        }
    }

    /// Descent F: reconstruction / skip-check MC through the cached half-pel planes
    /// instead of the per-pixel 6-tap. `hpel_block` is proven bit-identical to `mc_luma`
    /// (`hpel_block_matches_mc_luma_exactly`) and the `f` plane is the padded,
    /// edge-replicated reference, so both paths are BYTE-IDENTICAL; anything outside the
    /// padded plane still falls back to `mc_luma`.
    ///
    /// Census that motivated it: with the search's edge fallback fixed, `mc_luma` is
    /// 3.8-5.2% of encode and splits recon ~56-67% / skip-check ~24-35%, the latter at a
    /// content-independent one call per macroblock.
    #[inline]
    fn mc_luma_cached(
        &self,
        reference: &crate::RefFrame,
        x0: usize,
        y0: usize,
        bw: usize,
        bh: usize,
        mvx: i32,
        mvy: i32,
        out: &mut [u8],
    ) {
        let ch = self.mb_h * 16;
        let cw = self.cw;
        if !self.fast {
            let p = reference.hpel(cw, ch);
            if rusty_h264_common::inter::hpel_block(p, x0, y0, bw, bh, mvx, mvy, out) {
                return;
            }
            if let Some((plane, base, stride)) =
                rusty_h264_common::inter::hpel_ref(p, x0, y0, bw, bh, mvx, mvy)
            {
                for r in 0..bh {
                    out[r * bw..r * bw + bw].copy_from_slice(&plane[base + r * stride..][..bw]);
                }
                return;
            }
        }
        mc_luma(&reference.y, cw, ch, x0, y0, bw, bh, mvx, mvy, out);
    }

    /// Motion-compensates the `P_Skip` prediction (luma + both chroma) from
    /// reference 0 at the skip MV.
    /// Luma half of the P_Skip prediction. Split out so the fast path can test the
    /// luma residual first and only motion-compensate chroma when luma is free —
    /// for the majority of (non-free) macroblocks the chroma MC is never needed.
    /// Applies this slice's explicit LUMA weight for reference `r` to a region
    /// of a 16-stride prediction buffer — the encoder-side twin of the
    /// decoder's `weight_partition` luma arm, same integer form (denom 6:
    /// `((p·w + 32) >> 6) + o`, clamped). Identity/off skips in O(1).
    #[inline]
    fn wp_luma(&self, r: i32, buf: &mut [u8; 256], x0: usize, y0: usize, rw: usize, rh: usize) {
        let Some(&(lw, lo)) = self.wp.get(r.max(0) as usize) else {
            return;
        };
        if lw == 64 && lo == 0 {
            return;
        }
        for dy in 0..rh {
            for p in buf[(y0 + dy) * 16 + x0..][..rw].iter_mut() {
                *p = (((*p as i32 * lw + 32) >> 6) + lo).clamp(0, 255) as u8;
            }
        }
    }

    fn skip_predict_luma(
        &self,
        refs: &[crate::RefFrame],
        mb_x: usize,
        mb_y: usize,
        mv: (i32, i32),
    ) -> [u8; 256] {
        // Descent E/F: identify this mc_luma population by call site.
        #[cfg(feature = "profile")]
        let _site = rusty_h264_common::inter::mcstats::SiteTag::new(3);
        let reference = &refs[0]; // P_Skip always references index 0
        let mut pred_y = [0u8; 256];
        self.mc_luma_cached(
            reference,
            mb_x * 16,
            mb_y * 16,
            16,
            16,
            mv.0,
            mv.1,
            &mut pred_y,
        );
        // P_Skip references index 0 — weighted like any other P prediction
        // (the decoder weights its P_Skip recon too).
        self.wp_luma(0, &mut pred_y, 0, 0, 16, 16);
        pred_y
    }

    /// Chroma half of the P_Skip prediction (see [`Self::skip_predict_luma`]).
    fn skip_predict_chroma(
        &self,
        refs: &[crate::RefFrame],
        mb_x: usize,
        mb_y: usize,
        mv: (i32, i32),
    ) -> [[u8; 64]; 2] {
        let reference = &refs[0];
        let cch = self.mb_h * 8;
        let mut pred_c = [[0u8; 64]; 2];
        for c in 0..2 {
            let rc = if c == 0 { &reference.u } else { &reference.v };
            mc_chroma(
                rc,
                self.ccw,
                cch,
                mb_x * 8,
                mb_y * 8,
                8,
                8,
                mv.0,
                mv.1,
                &mut pred_c[c],
            );
        }
        pred_c
    }

    /// Whether the luma half of the P_Skip prediction has an all-zero quantized
    /// residual. Tested first and independently so the caller can defer the chroma
    /// MC + test for the common case where luma already disqualifies the skip (a
    /// "free", exact P_Skip costs no bits and is strictly beneficial).
    fn skip_luma_is_free(&self, sy: &[u8], mb_x: usize, mb_y: usize, pred_y: &[u8; 256]) -> bool {
        let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncFree);
        let qp = self.qp;
        // Fast path (deployment): the SAME asm kernels the coding path uses —
        // `dct_four_t4` computes the 4x4 DCTs of (src - pred) for an 8x8 quad
        // STRAIGHT FROM THE PLANES (no scalar gather), `quant_four_4x4` quantizes
        // with the identical FF/MF math as scalar `quantize` (bit-identical), and
        // "free" = all 64 levels zero, which is order-independent. Per-quad early
        // exit. The knob interleaves this against the scalar twin for A/B.
        #[cfg(accel)]
        if self.skip_accel_check {
            #[repr(align(16))]
            struct Align16([i16; 64]);
            let mut dct = Align16([0i16; 64]);
            let ff = rusty_h264_common::transform::quant_dz_ff(qp, 6);
            let mf = &rusty_h264_common::transform::QUANT_MF_OH[qp as usize];
            for &(qx, qy) in &[(0usize, 0usize), (8, 0), (0, 8), (8, 8)] {
                rusty_h264_accel::dct_four_t4(
                    &mut dct.0,
                    &sy[(mb_y * 16 + qy) * self.cw + mb_x * 16 + qx..],
                    self.cw,
                    &pred_y[qy * 16 + qx..],
                    16,
                );
                rusty_h264_accel::quant_four_4x4(&mut dct.0, &ff, mf);
                if dct.0.iter().any(|&v| v != 0) {
                    return false;
                }
            }
            return true;
        }
        // Exact quantize-to-zero bounds (mirrors `quantize`: level != 0 iff
        // (|c| + ff[p])·mf_oh[p] >= 2^16). With |C_ij| <= 4·SAD (max |H| entry = 2)
        // and C_DC = Σres, most blocks are decided by one SAD/sum pass — the full
        // scalar DCT+quant proof only runs for the rare undecided middle band.
        // BIT-EXACT: both shortcuts are sufficient conditions of the exact check.
        let ff = rusty_h264_common::transform::quant_dz_ff(qp, 6);
        // NINE integer divides per skip test became a table lookup.
        // `(65536 + mf - 1) / mf` is `ceil(65536 / mf)`, a pure function of
        // `mf = QUANT_MF_OH[qp][p]` -- 52 x 8 values, so the table is EXACT.
        // `t_dc` also re-divided the `p == 0` case the loop had just computed.
        let ceil8 = &rusty_h264_common::transform::CEIL_65536_MF[qp as usize];
        let mut t_min = i32::MAX;
        for p in 0..8 {
            t_min = t_min.min(ceil8[p] - ff[p] as i32);
        }
        let t_dc = ceil8[0] - ff[0] as i32;
        // Whole-MB gate: SAD(any 4x4) <= SAD(MB), so 4*SAD_MB < T_min proves all 16
        // blocks quantize to zero from ONE (psadbw) SAD. On skip-heavy content most
        // free MBs are exact/near-exact copies (SAD_MB ~ 0) - they skip the whole
        // per-block walk. Not-free MBs pay one extra SAD (~2% of their check).
        for by in 0..4 {
            for bx in 0..4 {
                let mut res = [0i32; 16];
                let (mut sad, mut dc) = (0i32, 0i32);
                for dy in 0..4 {
                    let row = &sy[(mb_y * 16 + by * 4 + dy) * self.cw + mb_x * 16 + bx * 4..][..4];
                    for dx in 0..4 {
                        let d = row[dx] as i32 - pred_y[(by * 4 + dy) * 16 + (bx * 4 + dx)] as i32;
                        res[dy * 4 + dx] = d;
                        sad += d.abs();
                        dc += d;
                    }
                }
                if 4 * sad < t_min {
                    continue; // every |C| <= 4·SAD < T_min → all levels zero
                }
                if dc.abs() >= t_dc {
                    return false; // DC level provably nonzero
                }
                if quantize(&forward_core(&res), qp, 6).iter().any(|&v| v != 0) {
                    return false;
                }
            }
        }
        true
    }

    /// Chroma half of [`Self::skip_is_free`].
    fn skip_chroma_is_free(
        &self,
        su: &[u8],
        sv: &[u8],
        mb_x: usize,
        mb_y: usize,
        pred_c: &[[u8; 64]; 2],
    ) -> bool {
        let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncFree);
        let qpc = self.qpc;
        // Fast path: one dct_four_t4 covers the whole 8x8 chroma plane region (all 4
        // blocks, residual+DCT fused, no scalar gather). Block order is the quad's
        // z-scan == raster for 2x2, so block b's DC (pre-quant) sits at dct[b*16] —
        // exactly the dc2x2 the Hadamard check needs. quant_four_4x4 with our FF/MF
        // is bit-identical to scalar `quantize`; AC-free = positions 1..16 all zero.
        #[cfg(accel)]
        if self.skip_accel_check {
            #[repr(align(16))]
            struct Align16C([i16; 64]);
            let mut dct = Align16C([0i16; 64]);
            let ff = rusty_h264_common::transform::quant_dz_ff(qpc, 6);
            let mf = &rusty_h264_common::transform::QUANT_MF_OH[qpc as usize];
            for c in 0..2 {
                let src = if c == 0 { su } else { sv };
                rusty_h264_accel::dct_four_t4(
                    &mut dct.0,
                    &src[(mb_y * 8) * self.ccw + mb_x * 8..],
                    self.ccw,
                    &pred_c[c],
                    8,
                );
                let dc2x2 = [
                    dct.0[0] as i32,
                    dct.0[16] as i32,
                    dct.0[32] as i32,
                    dct.0[48] as i32,
                ];
                rusty_h264_accel::quant_four_4x4(&mut dct.0, &ff, mf);
                for b in 0..4 {
                    if dct.0[b * 16 + 1..b * 16 + 16].iter().any(|&v| v != 0) {
                        return false;
                    }
                }
                if forward_quant_chroma_dc(&dc2x2, qpc, false)
                    .iter()
                    .any(|&v| v != 0)
                {
                    return false;
                }
            }
            return true;
        }
        for c in 0..2 {
            let src = if c == 0 { su } else { sv };
            let mut dc2x2 = [0i32; 4];
            for &(bx, by) in &CHROMA_4X4_SCAN_XY {
                let mut res = [0i32; 16];
                // Row slices (11.15): the LUMA twin got this shape in the
                // transcendentals round (23 bc -> 2); the chroma twin was missed.
                for dy in 0..4 {
                    let row = &src[(mb_y * 8 + by * 4 + dy) * self.ccw + mb_x * 8 + bx * 4..][..4];
                    for dx in 0..4 {
                        res[dy * 4 + dx] =
                            row[dx] as i32 - pred_c[c][(by * 4 + dy) * 8 + (bx * 4 + dx)] as i32;
                    }
                }
                let coeffs = forward_core(&res);
                dc2x2[by * 2 + bx] = coeffs[0];
                if quantize(&coeffs, qpc, 6)[1..].iter().any(|&v| v != 0) {
                    return false;
                }
            }
            if forward_quant_chroma_dc(&dc2x2, qpc, false)
                .iter()
                .any(|&v| v != 0)
            {
                return false;
            }
        }
        true
    }

    /// SSD between the source and a macroblock prediction (luma + chroma).
    fn pred_ssd(
        &self,
        sy: &[u8],
        su: &[u8],
        sv: &[u8],
        mb_x: usize,
        mb_y: usize,
        pred_y: &[u8; 256],
        pred_c: &[[u8; 64]; 2],
    ) -> i64 {
        let mut ssd = 0i64;
        for dy in 0..16 {
            let i = (mb_y * 16 + dy) * self.cw + mb_x * 16;
            for (a, b) in sy[i..i + 16].iter().zip(&pred_y[dy * 16..dy * 16 + 16]) {
                let d = *a as i64 - *b as i64;
                ssd += d * d;
            }
        }
        for c in 0..2 {
            let src = if c == 0 { su } else { sv };
            for dy in 0..8 {
                let i = (mb_y * 8 + dy) * self.ccw + mb_x * 8;
                for (a, b) in src[i..i + 8].iter().zip(&pred_c[c][dy * 8..dy * 8 + 8]) {
                    let d = *a as i64 - *b as i64;
                    ssd += d * d;
                }
            }
        }
        ssd
    }

    /// SSD between the *reconstructed* macroblock and the source.
    fn mb_ssd(&self, sy: &[u8], su: &[u8], sv: &[u8], mb_x: usize, mb_y: usize) -> i64 {
        let mut ssd = 0i64;
        // TWO bounds checks per pixel (source and recon) became two per ROW.
        for dy in 0..16 {
            let i = (mb_y * 16 + dy) * self.cw + mb_x * 16;
            for (a, b) in sy[i..i + 16].iter().zip(&self.rec_y[i..i + 16]) {
                let d = *a as i64 - *b as i64;
                ssd += d * d;
            }
        }
        // CHROMA WEIGHT. Chroma SSD is summed at 1:1 with luma here, but every
        // metric this decision is graded on is LUMA (`ssim_y`, Y-PSNR), and a
        // 4:2:0 macroblock carries half as many chroma samples as luma, so an
        // equal-weight sum lets chroma steer a decision the grader cannot see.
        // Worst on the most chroma-rich content in the corpus. Default 1.0 =
        // byte-identical; the sweep lives in the gate ledger.
        let cw_ = chroma_ssd_weight();
        if cw_ != 0.0 {
            let mut cssd = 0i64;
            for c in 0..2 {
                let (src, rec) = if c == 0 {
                    (su, &self.rec_u)
                } else {
                    (sv, &self.rec_v)
                };
                for dy in 0..8 {
                    let i = (mb_y * 8 + dy) * self.ccw + mb_x * 8;
                    for (a, b) in src[i..i + 8].iter().zip(&rec[i..i + 8]) {
                        let d = *a as i64 - *b as i64;
                        cssd += d * d;
                    }
                }
            }
            ssd += if cw_ == 1.0 {
                cssd
            } else {
                (cssd as f64 * cw_) as i64
            };
        }
        ssd
    }

    /// Reconstructs a `P_Skip` macroblock (reconstruction *is* the prediction —
    /// no residual coded) and records its motion state.
    /// Writes a macroblock's prediction into the recon planes verbatim, with NO
    /// motion-state side effects (the B path commits motion via
    /// `commit_direct_motion`). B_Skip used to skip this entirely — legal while
    /// B recon had no reader, but intra-in-B predicts from these planes, and a
    /// stale neighbour is encoder-recon drift the two-decoder gate cannot see
    /// (it shows up as a quality drop at unchanged bytes).
    fn commit_pred_pixels(
        &mut self,
        mb_x: usize,
        mb_y: usize,
        pred_y: &[u8; 256],
        pred_c: &[[u8; 64]; 2],
    ) {
        let base = mb_y * 16 * self.cw + mb_x * 16;
        for r in 0..16 {
            let d = base + r * self.cw;
            self.rec_y[d..d + 16].copy_from_slice(&pred_y[r * 16..r * 16 + 16]);
        }
        let cbase = mb_y * 8 * self.ccw + mb_x * 8;
        for c in 0..2 {
            let plane = if c == 0 {
                &mut self.rec_u
            } else {
                &mut self.rec_v
            };
            for r in 0..8 {
                let d = cbase + r * self.ccw;
                plane[d..d + 8].copy_from_slice(&pred_c[c][r * 8..r * 8 + 8]);
            }
        }
    }

    fn commit_skip(
        &mut self,
        mb_x: usize,
        mb_y: usize,
        mv: (i32, i32),
        pred_y: &[u8; 256],
        pred_c: &[[u8; 64]; 2],
    ) {
        let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::MvGrid);
        // Skip recon = the prediction verbatim: straight row copies (byte-identical
        // to the old per-4x4 gather + store scatter, ~5x fewer ops).
        let base = mb_y * 16 * self.cw + mb_x * 16;
        for r in 0..16 {
            let d = base + r * self.cw;
            self.rec_y[d..d + 16].copy_from_slice(&pred_y[r * 16..r * 16 + 16]);
        }
        let cbase = mb_y * 8 * self.ccw + mb_x * 8;
        for c in 0..2 {
            let plane = if c == 0 {
                &mut self.rec_u
            } else {
                &mut self.rec_v
            };
            for r in 0..8 {
                let d = cbase + r * self.ccw;
                plane[d..d + 8].copy_from_slice(&pred_c[c][r * 8..r * 8 + 8]);
            }
        }
        self.set_mb_mv(mb_x, mb_y, mv, true, 0);
        let w4 = self.mb_w * 4;
        for row in 0..4 {
            let st = (mb_y * 4 + row) * w4 + mb_x * 4;
            self.modes_y[st..st + 4].fill(2);
            self.coded_y[st..st + 4].fill(true);
        }
    }

    /// Trial-encodes an inter macroblock to measure its rate-distortion cost
    /// `(SSD, bits)` without committing: snapshot the macroblock's grid + recon
    /// region, run the real `encode_inter_mb` into a scratch writer, read the
    /// bit count and reconstruction SSD, then restore. Neighbor CAVLC context is
    /// read (not mutated), so the bit count is accurate.
    #[cfg_attr(not(feature = "std"), allow(dead_code))]
    fn trial_inter(
        &mut self,
        refs: &[crate::RefFrame],
        sy: &[u8],
        su: &[u8],
        sv: &[u8],
        mb_x: usize,
        mb_y: usize,
        mode: u8,
        parts: &[(i32, (i32, i32))],
    ) -> (i64, usize) {
        let snap = self.save_mb(mb_x, mb_y);
        let mut scratch = BitWriter::new();
        self.encode_inter_mb(&mut scratch, refs, sy, su, sv, mb_x, mb_y, mode, parts);
        let bits = scratch.bit_len();
        let ssd = self.mb_ssd(sy, su, sv, mb_x, mb_y);
        self.load_mb(mb_x, mb_y, &snap);
        (ssd, bits)
    }

    /// Trial-encodes the macroblock as **intra** (`encode_mb` runs its own
    /// I_16x16-vs-I_4x4 decision), measuring `(SSD, bits)` without committing —
    /// the intra candidate for the RD mode decision.
    /// Trial-encodes THIS macroblock as coded (`plan_mb` picks intra vs the
    /// already-chosen inter) and returns `(recon SSD, real bits)`, restoring
    /// every grid it touched. The RD currency for a mode decision — see
    /// `intra_rd_on`.
    fn trial_intra(
        &mut self,
        sy: &[u8],
        su: &[u8],
        sv: &[u8],
        mb_x: usize,
        mb_y: usize,
        is_p: bool,
    ) -> (i64, usize) {
        let mut snap = enc_scratch::take_snap_b();
        self.save_mb_into(mb_x, mb_y, &mut snap);
        // E2 r2: a fresh BitWriter (heap Vec) was built PER RD TRIAL; recycle
        // this thread's — `clear()` keeps the grown allocation.
        let mut scratch = enc_scratch::take_bits();
        scratch.clear();
        encode_mb(self, &mut scratch, mb_x, mb_y, sy, su, sv, is_p);
        let bits = scratch.bit_len();
        enc_scratch::put_bits(scratch);
        let ssd = self.mb_ssd(sy, su, sv, mb_x, mb_y);
        self.load_mb(mb_x, mb_y, &snap);
        enc_scratch::put_snap_b(snap);
        (ssd, bits)
    }

    /// Best `(ref_idx, mv, cost)` for one partition by `SATD + λ·bits`, searched
    /// across every reference (`cost` is that SATD-domain rate-distortion cost).
    /// `extra` seeds the search with already-found MVs (e.g. the 16×16 result when
    /// refining a sub-partition).
    #[inline]
    fn best_part(
        &self,
        refs: &[crate::RefFrame],
        sy: &[u8],
        nb: &[MvNeighbor; 3],
        num_refs: usize,
        rx: usize,
        ry: usize,
        rw: usize,
        rh: usize,
        extra: &[(i32, i32)],
        lme: f64,
    ) -> (i32, (i32, i32), i64) {
        crate::signals::census::work(crate::signals::census::W_BEST_PART);
        let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncMe);
        let [a, b, c] = *nb;
        let (mut br, mut bmv, mut bc) = (0i32, (0, 0), i64::MAX);
        for r in 0..num_refs {
            // EXACT ref_bits prune (multiref campaign): reference `r`'s total
            // cost is `motion_cost + λ·ref_bits(r)` with `motion_cost >= 0`
            // (SATD plus a nonnegative mvd rate), so it is bounded below by
            // the λ·ref_bits term alone. Once the incumbent `bc` is at or
            // under that bound, reference `r` cannot win the strict `<` — and
            // `ref_bits` is NONDECREASING in `r` (Exp-Golomb length; pinned by
            // `ref_bits_is_monotone`), so neither can any later reference:
            // `break`, byte-identically. Fires exactly where multi-ref costs
            // the most for the least — near-perfect ref-0 matches on cheap
            // partitions at high λ.
            let rb = (lme * ref_bits(r, num_refs) as f64) as i64;
            if rb >= bc {
                break;
            }
            crate::signals::census::work(crate::signals::census::W_REF_SEARCH);
            // STACK seeds. This was `vec![..]` + `extend_from_slice` — a heap
            // allocation on EVERY search, and the sub-8x8 split arm took the call
            // count from 50k to 208k per clip, so it is ~208k allocations whose
            // payload is at most three MVs. `extra` is [] / [mv16] / [mv16, mv]
            // at every call site; the assert pins that rather than trusting it.
            debug_assert!(extra.len() <= 3, "seed budget");
            let mut sbuf = [(0i32, 0i32); 4];
            sbuf[0] = predict_mv(a, b, c, r as i32);
            let n = 1 + extra.len().min(3);
            sbuf[1..n].copy_from_slice(&extra[..n - 1]);
            let seeds = &sbuf[..n];
            let (mv, cost) = self.motion_search(&refs[r], sy, rx, ry, rw, rh, seeds, lme, None);
            let cost = cost + rb;
            if cost < bc {
                bc = cost;
                br = r as i32;
                bmv = mv;
            }
        }
        (br, bmv, bc)
    }

    /// Sub-pel-refines ONE already-chosen partition, reusing `motion_search`'s cost
    /// closure via its `start` hook so the rate term and predictor centre are exactly
    /// the ones the full search used. Companion to `best_part` under `sp_defer`.
    fn refine_part(
        &self,
        refs: &[crate::RefFrame],
        sy: &[u8],
        nb: &[MvNeighbor; 3],
        num_refs: usize,
        rx: usize,
        ry: usize,
        rw: usize,
        rh: usize,
        lme: f64,
        r: i32,
        mv: (i32, i32),
    ) -> ((i32, i32), i64) {
        let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncMe);
        let [a, b, c] = *nb;
        let rb = (lme * ref_bits(r as usize, num_refs) as f64) as i64;
        let seeds = [predict_mv(a, b, c, r)];
        let (m, cc) =
            self.motion_search(&refs[r as usize], sy, rx, ry, rw, rh, &seeds, lme, Some(mv));
        (m, cc + rb)
    }

    /// Cheapest `I_16x16` prediction's SAD over the four whole-block modes, using
    /// the already-reconstructed top/left neighbours — the intra candidate's cost
    /// in the fast (SAD) mode decision, without the full `I_4x4` search.
    fn best_i16_sad(&self, sy: &[u8], mb_x: usize, mb_y: usize) -> i64 {
        let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncIntraCost);
        let (lx, ly) = (mb_x * 16, mb_y * 16);
        let (avail_top, avail_left) = (mb_y > 0, mb_x > 0);
        let mut top = [0u8; 16];
        let mut left = [0u8; 16];
        if avail_top {
            let r = (ly - 1) * self.cw + lx;
            top.copy_from_slice(&self.rec_y[r..r + 16]);
        }
        if avail_left {
            // E2 r2 EXPERIMENT: a bounded column slice — `i * cw` is provably
            // inside `15*cw + 1` for i < 16, so LLVM can retire the per-element
            // checks the strided gather carried. (Kept only if the bc count
            // dropped; see 11.12a.)
            let col = &self.rec_y[ly * self.cw + lx - 1..(ly + 15) * self.cw + lx];
            for i in 0..16 {
                left[i] = col[i * self.cw];
            }
        }
        let corner = if avail_top && avail_left {
            self.rec_y[(ly - 1) * self.cw + lx - 1]
        } else {
            0
        };
        let mut best = i64::MAX;
        for mode in [
            I16Mode::Dc,
            I16Mode::Vertical,
            I16Mode::Horizontal,
            I16Mode::Plane,
        ] {
            if !mode.available(avail_top, avail_left) {
                continue;
            }
            let pred = i16_pred(
                self, mode, avail_top, avail_left, &top, &left, corner, lx, ly,
            );
            best = best.min(sad_16x16(sy, self.cw, lx, ly, &pred));
        }
        best
    }

    /// SATD sibling of [`Self::best_i16_sad`] — the intra candidate's cost in the
    /// quality preset's SATD mode decision (openh264's `WelsMdI16x16`).
    fn best_i16_satd(&self, sy: &[u8], mb_x: usize, mb_y: usize) -> i64 {
        let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncIntraCost);
        let (lx, ly) = (mb_x * 16, mb_y * 16);
        let (avail_top, avail_left) = (mb_y > 0, mb_x > 0);
        let mut top = [0u8; 16];
        let mut left = [0u8; 16];
        if avail_top {
            let r = (ly - 1) * self.cw + lx;
            top.copy_from_slice(&self.rec_y[r..r + 16]);
        }
        if avail_left {
            // E2 r2 EXPERIMENT: a bounded column slice — `i * cw` is provably
            // inside `15*cw + 1` for i < 16, so LLVM can retire the per-element
            // checks the strided gather carried. (Kept only if the bc count
            // dropped; see 11.12a.)
            let col = &self.rec_y[ly * self.cw + lx - 1..(ly + 15) * self.cw + lx];
            for i in 0..16 {
                left[i] = col[i * self.cw];
            }
        }
        let corner = if avail_top && avail_left {
            self.rec_y[(ly - 1) * self.cw + lx - 1]
        } else {
            0
        };
        let mut best = i64::MAX;
        for mode in [
            I16Mode::Dc,
            I16Mode::Vertical,
            I16Mode::Horizontal,
            I16Mode::Plane,
        ] {
            if !mode.available(avail_top, avail_left) {
                continue;
            }
            let pred = i16_pred(
                self, mode, avail_top, avail_left, &top, &left, corner, lx, ly,
            );
            best = best.min(satd_16x16(sy, self.cw, lx, ly, &pred));
        }
        best
    }

    /// Snapshots the per-block grids and reconstruction for one macroblock, so a
    /// trial encode can be rolled back.
    fn save_mb(&self, mb_x: usize, mb_y: usize) -> MbState {
        let mut d = MbState::default();
        self.save_mb_into(mb_x, mb_y, &mut d);
        d
    }

    /// [`save_mb`](Self::save_mb) into an existing buffer, reusing its allocations.
    /// The per-macroblock region is a fixed size, so after the first call every
    /// `Vec` already has the capacity it needs and refilling is a pure copy.
    fn save_mb_into(&self, mb_x: usize, mb_y: usize, d: &mut MbState) {
        let w4 = self.mb_w * 4;
        let w2 = self.mb_w * 2;
        // REFUTED, do not retry: the `load_mb` row-slice shape does NOT mirror onto
        // this function. Rewriting these as `$o.clear()` + `extend_from_slice` per
        // row does retire all 15 bounds checks, but `Vec::extend_from_slice` carries
        // a capacity check and a `memcpy` call of its own, and the function grew
        // 1030 -> 1771 instructions (+72%) to buy them. A write into a
        // pre-sized destination (`load_mb`) and an append into a growable one are
        // not the same shape; only the former is a clean win.
        macro_rules! reg4 {
            ($v:expr, $o:expr) => {{
                $o.clear();
                for dy in 0..4 {
                    for dx in 0..4 {
                        $o.push($v[(mb_y * 4 + dy) * w4 + mb_x * 4 + dx]);
                    }
                }
            }};
        }
        macro_rules! regn {
            ($v:expr, $o:expr, $n:expr, $ox:expr, $oy:expr, $stride:expr) => {{
                $o.clear();
                for dy in 0..$n {
                    for dx in 0..$n {
                        $o.push($v[($oy + dy) * $stride + $ox + dx]);
                    }
                }
            }};
        }
        regn!(self.rec_y, d.rec_y, 16, mb_x * 16, mb_y * 16, self.cw);
        regn!(self.rec_u, d.rec_u, 8, mb_x * 8, mb_y * 8, self.ccw);
        regn!(self.rec_v, d.rec_v, 8, mb_x * 8, mb_y * 8, self.ccw);
        reg4!(self.nnz_y, d.nnz_y);
        regn!(self.nnz_c[0], d.nnz_c[0], 2, mb_x * 2, mb_y * 2, w2);
        regn!(self.nnz_c[1], d.nnz_c[1], 2, mb_x * 2, mb_y * 2, w2);
        reg4!(self.mv_y, d.mv_y);
        reg4!(self.inter_y, d.inter_y);
        reg4!(self.ref_idx_y, d.ref_idx_y);
        reg4!(self.coded_y, d.coded_y);
        reg4!(self.modes_y, d.modes_y);
        d.cur_qp = self.cur_qp;
    }

    /// Restores a macroblock's grids + reconstruction from a [`save_mb`] snapshot.
    fn load_mb(&mut self, mb_x: usize, mb_y: usize, s: &MbState) {
        let w4 = self.mb_w * 4;
        let w2 = self.mb_w * 2;
        // One bounds check PER ROW instead of per element. `$n` is a literal at
        // every call site, so both slices have a compile-time extent and LLVM
        // folds the length equality `copy_from_slice` would otherwise check --
        // this is the shape that WORKS, as distinct from the runtime-extent row
        // slice the decoder campaign measured and REFUTED.
        macro_rules! put4 {
            ($v:expr, $src:expr) => {
                for dy in 0..4 {
                    let d = (mb_y * 4 + dy) * w4 + mb_x * 4;
                    $v[d..d + 4].copy_from_slice(&$src[dy * 4..dy * 4 + 4]);
                }
            };
        }
        macro_rules! putn {
            ($v:expr, $src:expr, $n:expr, $ox:expr, $oy:expr, $stride:expr) => {
                for dy in 0..$n {
                    let d = ($oy + dy) * $stride + $ox;
                    $v[d..d + $n].copy_from_slice(&$src[dy * $n..dy * $n + $n]);
                }
            };
        }
        putn!(self.rec_y, s.rec_y, 16, mb_x * 16, mb_y * 16, self.cw);
        putn!(self.rec_u, s.rec_u, 8, mb_x * 8, mb_y * 8, self.ccw);
        putn!(self.rec_v, s.rec_v, 8, mb_x * 8, mb_y * 8, self.ccw);
        put4!(self.nnz_y, s.nnz_y);
        putn!(self.nnz_c[0], s.nnz_c[0], 2, mb_x * 2, mb_y * 2, w2);
        putn!(self.nnz_c[1], s.nnz_c[1], 2, mb_x * 2, mb_y * 2, w2);
        put4!(self.mv_y, s.mv_y);
        put4!(self.inter_y, s.inter_y);
        put4!(self.ref_idx_y, s.ref_idx_y);
        put4!(self.coded_y, s.coded_y);
        put4!(self.modes_y, s.modes_y);
        self.cur_qp = s.cur_qp;
    }

    /// Loads the per-MB luma nnz prediction cache (openh264 `scan8` style): the top
    /// row from the macroblock above and the left column from the macroblock to the
    /// left (both already in `nnz_y`), with `0x80` at the picture edges. After this,
    /// neighbour nnz reads are branchless cache indexing — no bounds-checked `Option`.
    fn nnz_cache_load(&mut self, mb_x: usize, mb_y: usize) {
        let w4 = self.mb_w * 4;
        for lbx in 0..4 {
            self.nnz_l_cache[1 + lbx] = if mb_y == 0 {
                0x80
            } else {
                self.nnz_y[(mb_y * 4 - 1) * w4 + (mb_x * 4 + lbx)]
            };
        }
        for lby in 0..4 {
            self.nnz_l_cache[(lby + 1) * 5] = if mb_x == 0 {
                0x80
            } else {
                self.nnz_y[(mb_y * 4 + lby) * w4 + (mb_x * 4 - 1)]
            };
        }
    }

    /// Branchless nnz prediction (`nC`) for luma block `(lbx,lby)` from the cache —
    /// the `0x80` sentinel + `& 0x7f` mask collapse the four availability cases
    /// (matches the scalar nnz predict). Call after the block's left/top are cached.
    #[inline]
    fn nc_pred(&self, lbx: usize, lby: usize) -> i32 {
        let left = self.nnz_l_cache[(lby + 1) * 5 + lbx] as i32; // (lbx-1)+1
        let top = self.nnz_l_cache[lby * 5 + (lbx + 1)] as i32; // (lby-1)+1
        let r = left + top;
        if r < 0x80 {
            (r + 1) >> 1
        } else {
            r & 0x7f
        }
    }

    /// Records a luma block's nnz into the per-MB cache (for later neighbour reads).
    #[inline]
    fn nnz_cache_set(&mut self, lbx: usize, lby: usize, total: u8) {
        self.nnz_l_cache[(lby + 1) * 5 + (lbx + 1)] = total;
    }

    /// Loads the per-MB chroma nnz prediction cache (both planes) from the chroma
    /// blocks above/left, `0x80` at the picture edges — the chroma analogue of
    /// [`Self::nnz_cache_load`] (2×2 blocks → padded 3×3 grid).
    fn chroma_cache_load(&mut self, mb_x: usize, mb_y: usize) {
        let w2 = self.mb_w * 2;
        for c in 0..2 {
            for bx in 0..2 {
                self.nnz_c_cache[c][1 + bx] = if mb_y == 0 {
                    0x80
                } else {
                    self.nnz_c[c][(mb_y * 2 - 1) * w2 + (mb_x * 2 + bx)]
                };
            }
            for by in 0..2 {
                self.nnz_c_cache[c][(by + 1) * 3] = if mb_x == 0 {
                    0x80
                } else {
                    self.nnz_c[c][(mb_y * 2 + by) * w2 + (mb_x * 2 - 1)]
                };
            }
        }
    }

    /// Branchless chroma nnz prediction (`nC`) for plane `c`, block `(bx,by)`.
    #[inline]
    fn chroma_nc_pred(&self, c: usize, bx: usize, by: usize) -> i32 {
        let left = self.nnz_c_cache[c][(by + 1) * 3 + bx] as i32;
        let top = self.nnz_c_cache[c][by * 3 + (bx + 1)] as i32;
        let r = left + top;
        if r < 0x80 {
            (r + 1) >> 1
        } else {
            r & 0x7f
        }
    }

    /// Records a chroma block's nnz into the per-MB cache.
    #[inline]
    fn chroma_nnz_cache_set(&mut self, c: usize, bx: usize, by: usize, total: u8) {
        self.nnz_c_cache[c][(by + 1) * 3 + (bx + 1)] = total;
    }
}

/// Encodes a slice's macroblocks then RBSP trailing bits, returning the
/// **deblocked** reconstruction to serve as the next frame's reference.
///
/// `is_p` selects P-slice framing (`mb_skip_run` prefix + intra `mb_type` +5
/// offset). In phase 4a every macroblock is still coded intra; motion-compensated
/// macroblocks arrive in 4b (using `reference`).
/// Boundary strengths for one macroblock, derived from the encoder's own grids
/// the moment it finishes coding.
///
/// `ref_idx_y` holds raw indices (-1 for intra) rather than the deblocker's
/// `NO_REF` sentinel; safe because reference identity is only compared between
/// two INTER blocks, which always carry a valid index.
// NOT inlined: this sits at three exits of the hottest loop in the encoder, and
// inlining it there costs more in I-cache and register pressure on the
// surrounding code than the call saves (measured: the loop grew ~2x the
// derivation's own cost).
#[inline(never)]
fn derive_mb_bs_from(
    fe: &FrameEncoder,
    mb_x: usize,
    mb_y: usize,
    kind: rusty_h264_common::deblock::MbKind,
) -> rusty_h264_common::deblock::MbBs {
    let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncBs);
    let view = rusty_h264_common::deblock::BlockInfo {
        inter: &fe.inter_y,
        nnz: &fe.nnz_y,
        mv: &fe.mv_y,
        ref_id: &fe.ref_idx_y,
        mv1: &[],
        ref_id1: &[],
        w4: fe.mb_w * 4,
        t8x8: &[],
        poc0: &[],
        poc1: &[],
        bs: &[],
        kind: &[],
    };
    rusty_h264_common::deblock::derive_mb_kind(&view, mb_x, mb_y, kind)
}

pub(crate) fn encode_slice_data(
    w: &mut BitWriter,
    cfg: &EncoderConfig,
    frame: &YuvPlanes<'_>,
    qp: u8,
    is_p: bool,
    refs: &[crate::RefFrame],
    qpo: &[i32],
    aq_probe: Option<&YuvPlanes<'_>>,
    wp: &[(i32, i32)],
) -> crate::RefFrame {
    let _g_prep = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncPrep);
    let mut fe = FrameEncoder::new(cfg);
    fe.wp = wp.to_vec();
    let precomp = rusty_h264_common::deblock::precomputed_bs_enabled();
    let mut bs_grid =
        vec![rusty_h264_common::deblock::MbBs::UNSET; if precomp { fe.mb_w * fe.mb_h } else { 0 }];
    fe.qp = qp;
    fe.qpc = chroma_qp(qp);
    fe.cur_qp = qp;
    if cfg.cabac_dz_div > 0 {
        fe.idz = cfg.cabac_dz_div; // CABAC-specific dead-zone override
    } // QPY_PREV starts at the slice QP so the first mb_qp_delta is 0
    let (sy, su, sv) = coded_source(cfg, frame);
    // Great Gate P1: ONE lazy signal vector per frame; every gate below reads
    // through it, so no probe runs twice and unused signals cost nothing.
    // On an IDR (no refs) the previous SOURCE frame stands in as the temporal
    // reference — the AQ grain veto needs it (docs/gate-ledger.md aq-grain-veto);
    // every ME gate below still keys on `refs`, not on the signal vector.
    let probe_y: Option<Vec<u8>> = if refs.is_empty() {
        aq_probe.map(|f| coded_source(cfg, f).0.into_owned())
    } else {
        None
    };
    let sig = FrameSignals::new(
        &sy,
        fe.cw,
        fe.mb_w,
        fe.mb_h,
        refs.first().map(|r| &r.y[..]).or(probe_y.as_deref()),
    );
    apply_screen_t8_veto(&mut fe, &sig);
    let lambda = 0.85 * fe.tune_lambda_scale * crate::fastmath::lambda_qp(qp);
    let num_refs = refs.len();
    // me_wide CONTENT GATE: on a pure PAN the global-MC residual ≈ 0, so the diamond's
    // seed (median = pan MV) is already right and the wide rescue only over-fits
    // (spurious MVs that hurt the B-frames' spatial-direct — the panc regression).
    // Gate it off there; non-uniform content (real stalls) reads well above 0.
    if is_p && fe.me_wide && !refs.is_empty() && sig.gmc_residual() < fe.me_wide_coh {
        fe.me_wide = false;
    }
    // me_wide HEAD-ROOM GATE (the dispatcher the truth table asked for). The rescue
    // only pays where a wide search actually beats a predictor-local one; measure
    // that directly per frame and route the frame. `RFF_ME_HR` sets the threshold
    // (percent); 0 disables the gate and restores the always-on behaviour.
    // Skip the probe entirely when the gate is disabled: it must not tax the
    // default path (`RFF_ME_HR=0`), which stays byte-identical to pre-gate output.
    if fe.me_wide && !refs.is_empty() && (me_wide_hr_thresh() > 0.0 || me_wide_hr_dbg()) {
        let hr = sig.headroom();
        if me_wide_hr_dbg() {
            eprintln!("ME_HR qp{qp} headroom={hr:.2}");
        }
        if me_wide_hr_thresh() > 0.0 && hr < me_wide_hr_thresh() {
            fe.me_wide = false;
        }
    }
    // Track-B B2 DISPATCH (WHYS H-2): SAD full-pel wins where a plain full-pel
    // translational search actually improves on zero motion (`b2_mgain`) and loses
    // on flash/fine-detail content. Probe per frame, route the frame — per-frame,
    // not cross-frame, so it stays deterministic under GOP-parallel encode.
    if me_sadfp_mode() == 1 && !fe.fast && !refs.is_empty() {
        let (mg, dc) = sig.mgain_dc();
        if me_sadt_dbg() {
            eprintln!("B2_MG qp{qp} mgain={mg:.3} dcfrac={dc:.3}");
        }
        fe.sadfp = mg >= me_sadt() && dc <= me_sad_dcmax();
        // H-24: the mv-cost SHAPE rides the same probe (its BD sign-flip tracks
        // motion for the same physical reason B2's does).
        if mv_smooth_mode() == 1 {
            // dcfrac veto mirrors B2's: crew-class FLASH frames satisfy the mgain
            // test but SAD/mvd statistics mislead there (H-13/H-26).
            fe.mv_smooth = mg >= mv_smooth_t() && dc <= me_sad_dcmax();
        }
        // H-13: near-static frames skip the split searches entirely.
        let smg = split_mg();
        if smg > 0.0 {
            fe.do_splits = mg >= smg;
        }
    }
    // Content-adaptive cost-function dispatch (codec-content-adaptive-dispatch): the
    // fast preset prices modes by cheap SAD, which is rate-blind on detailed MBs;
    // route the top `satd_q` fraction of highest-VARIANCE MBs to the rate-faithful
    // SATD cost. A per-frame PERCENTILE threshold makes the routed fraction — hence
    // the speed/quality split — content-invariant (same q → same fraction on any
    // clip). `satd_q == 0` leaves the threshold at MAX (pure SAD, byte-identical).
    if is_p && fe.satd_q > 0.0 {
        fe.satd_var_thresh = sig.var_percentile_thresh(fe.satd_q);
    }
    // Adaptive Quantization: per-MB target QPy from content (finer on flat MBs,
    // coarser on busy ones). `mb_qpy` records each MB's ACTUAL QPy (a skip / cbp==0
    // MB inherits `cur_qp`), for the deblock filter. `strength 0` → uniform → the
    // mb_qp_delta stays 0, byte-identical.
    let mut aq_qp = aq_qp_map(&sig, qp, fe.aq_strength);
    apply_mbtree_qpo(&mut aq_qp, qpo); // mb-tree temporal AQ (empty = byte-identical)
    signals::harvest(
        &sig,
        if is_p { 'P' } else { 'I' },
        qp,
        &signals::GateDecisions {
            me_wide: fe.me_wide,
            sadfp: fe.sadfp,
            mv_smooth: fe.mv_smooth,
            do_splits: fe.do_splits,
            lme_scale: 1.0,
            satd_thresh: fe.satd_var_thresh,
        },
    );
    let mut mb_qpy = enc_scratch::take_qpy();
    mb_qpy.clear();
    mb_qpy.resize(fe.mb_w * fe.mb_h, qp);
    let mut skip_run = 0u32;
    // ---- adaptive RD-skip gate -------------------------------------------
    // RD P_Skip is a large win on temporally redundant content and a large LOSS
    // on detailed content (SSIM: akiyo -13.1%, FourPeople -5.6% vs in_to_tree
    // +34.0%, stockholm +95.7%). The separating signal is the content's own
    // FREE-skip rate — how much of it is already exactly redundant — and the gap
    // is wide (winners >=58.7%, losers <=6.4%). Measure it ONLINE over the first
    // slice of the frame and enable RD skip for the remainder only if it clears
    // the bar. Within-frame, so it stays deterministic under GOP-parallel encode.
    if is_p && mv_cmp_on() {
        MVCMP_FRAME.fetch_add(1, core::sync::atomic::Ordering::Relaxed);
    }
    // Reused across every RD-skip candidate — see `MbState`.
    let mut rdskip_snap = MbState::default();
    let mut rdskip_free = 0usize;
    let mut rdskip_seen = 0usize;
    let mut rdskip_on = false;
    let mut greedy_on = fe.greedy_min_free == 0; // 0 = ungated (historic behaviour)
    let rdskip_learn = (fe.mb_w * fe.mb_h / 8).max(64);
    let rdskip_min_free = fe.rd_skip_min_free as usize;

    drop(_g_prep);
    let _g_loop = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncMbLoop);
    for mb_y in 0..fe.mb_h {
        for mb_x in 0..fe.mb_w {
            let mb_idx = mb_y * fe.mb_w + mb_x;
            let mb_qp = aq_qp[mb_idx];
            fe.qp = mb_qp;
            fe.qpc = chroma_qp(mb_qp);
            // P_Skip: motion-compensate from the most-recent reference; accept if free.
            // Chosen inter coding: (mb_type, per-partition (ref_idx, mv)).
            let mut inter: Option<InterChoice> = None;
            // Bits of an inter macroblock already encoded by the skip decision
            // below. When present the emit path splices them instead of encoding
            // the same macroblock a second time.
            let mut coded: Option<BitWriter> = None;
            if is_p {
                if num_refs > 0 {
                    // P_Skip prediction (reference 0). A free skip (zero residual) is
                    // taken immediately; the quality preset also takes a greedy P_Skip
                    // when its SAD is below the neighbour-predicted bound (below).
                    let _g_skip =
                        rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncSkip);
                    let _g_smc =
                        rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::Neighbors);
                    rdskip_seen += 1;
                    if rdskip_seen >= rdskip_learn {
                        rdskip_on = rdskip_free * 100 >= rdskip_seen * rdskip_min_free;
                        greedy_on = fe.greedy_min_free == 0
                            || rdskip_free * 100 >= rdskip_seen * fe.greedy_min_free as usize;
                    }
                    let mv_skip = fe.skip_mv(mb_x, mb_y);
                    let skip_y = fe.skip_predict_luma(refs, mb_x, mb_y, mv_skip);
                    drop(_g_smc);
                    let luma_free = fe.skip_luma_is_free(&sy, mb_x, mb_y, &skip_y);
                    // Chroma MC only when it can matter: luma already free (so the
                    // skip might be taken) or the quality path needs it below.
                    let skip_c = if luma_free || !fe.fast {
                        fe.skip_predict_chroma(refs, mb_x, mb_y, mv_skip)
                    } else {
                        [[0u8; 64]; 2]
                    };
                    let is_free =
                        luma_free && fe.skip_chroma_is_free(&su, &sv, mb_x, mb_y, &skip_c);
                    // Skip-prediction luma SAD (the quality preset's predicted-SAD apparatus).
                    let skip_sad = if fe.fast {
                        0
                    } else {
                        let (lx, ly) = (mb_x * 16, mb_y * 16);
                        let mut s = 0u32;
                        for dy in 0..16 {
                            let src = &sy[(ly + dy) * fe.cw + lx..][..16];
                            let p = &skip_y[dy * 16..][..16];
                            s += src
                                .iter()
                                .zip(p)
                                .map(|(&a, &b)| a.abs_diff(b) as u32)
                                .sum::<u32>();
                        }
                        s
                    };
                    if is_free {
                        fe.commit_skip(mb_x, mb_y, mv_skip, &skip_y, &skip_c);
                        if !fe.fast {
                            fe.mb_was_skip[mb_idx] = true;
                            fe.mb_skip_sad[mb_idx] = skip_sad;
                        }
                        mb_qpy[mb_idx] = fe.cur_qp; // skip inherits QPy
                        rdskip_free += 1;
                        if precomp {
                            bs_grid[mb_idx] = derive_mb_bs_from(
                                &fe,
                                mb_x,
                                mb_y,
                                rusty_h264_common::deblock::MbKind::Skip,
                            );
                        }
                        skip_run += 1;
                        continue;
                    }
                    drop(_g_skip);
                    let (lx, ly) = (mb_x * 16, mb_y * 16);
                    let nb = {
                        let _g = rusty_h264_common::prof::scope(
                            rusty_h264_common::prof::Stage::EncMvPred,
                        );
                        fe.mv_neighbors_block(mb_x as isize * 4, mb_y as isize * 4, 4)
                    };
                    let lme = rusty_h264_common::fmath::sqrt(lambda);

                    if fe.fast {
                        // Fast preset: pick the cheapest *prediction* by SATD (no
                        // trial-encoding), then always code its residual — P_16x16 vs
                        // I_16x16 only, no sub-partitions. Crucially it does NOT make a
                        // SATD skip-vs-code decision: P_Skip is taken only for a truly
                        // free (zero-residual) macroblock, handled above. Pricing skip
                        // by SATD would drop residual the QP wants coded and tank PSNR;
                        // like x264's fast presets, fast trades *efficiency* (more bits)
                        // for speed, not quality. The faster ME is what makes it fast.
                        // Adaptive dispatch: high-variance MBs price by SATD (both
                        // inter — via `mb_use_satd` in `best_part` — and intra), the
                        // rest by cheap SAD. Set the per-MB flag before best_part.
                        // Dedup (E15 W9): memoized twin, already built when satd_q > 0.
                        fe.mb_use_satd = fe.satd_q > 0.0
                            && sig.mb_vars()[mb_y * fe.mb_w + mb_x] >= fe.satd_var_thresh;
                        let (r16, mv16, cost_inter) =
                            fe.best_part(refs, &sy, &nb, num_refs, lx, ly, 16, 16, &[], lme);
                        let cost_intra = if fe.mb_use_satd {
                            fe.best_i16_satd(&sy, mb_x, mb_y)
                        } else {
                            fe.best_i16_sad(&sy, mb_x, mb_y)
                        } + (lme * fe.tune_intra_penalty) as i64;
                        inter = if cost_intra < cost_inter {
                            None // intra wins → encode_mb below
                        } else {
                            Some((0, vec![(r16, mv16)]))
                        };
                    } else {
                        // Quality preset: openh264's mode-decision model — SATD + λ·mvbits
                        // cost ESTIMATE (no per-candidate trial-encode); modes are ranked
                        // by that cost and only the winner is encoded (once) below. This
                        // removes ~the 93%-of-quality re-encode cost.

                        // Greedy P_Skip (openh264 `PredictSadSkip`): take the skip when its
                        // luma SAD is below the neighbour-predicted skip SAD. The threshold
                        // is what skip neighbours achieved, so the skip propagates from the
                        // free skips and self-limits — no fixed bound, no inter-chain drift.
                        if fe.greedy_skip && greedy_on && skip_sad < fe.pred_skip_sad(mb_x, mb_y) {
                            fe.commit_skip(mb_x, mb_y, mv_skip, &skip_y, &skip_c);
                            fe.mb_was_skip[mb_idx] = true;
                            fe.mb_skip_sad[mb_idx] = skip_sad;
                            mb_qpy[mb_idx] = fe.cur_qp; // skip inherits QPy
                            if precomp {
                                bs_grid[mb_idx] = derive_mb_bs_from(
                                    &fe,
                                    mb_x,
                                    mb_y,
                                    rusty_h264_common::deblock::MbKind::Skip,
                                );
                            }
                            skip_run += 1;
                            continue;
                        }

                        // 16×16 baseline (SATD + λ·bits, with sub-pel refinement).
                        let (r16, mv16, c16) =
                            fe.best_part(refs, &sy, &nb, num_refs, lx, ly, 16, 16, &[], lme);
                        let mut best_c = c16;
                        let mut pick: Option<InterChoice> = Some((0, vec![(r16, mv16)]));

                        // Sub-partitions, ranked by SATD, gated on a heavy 16×16 (a likely
                        // motion boundary — the 4 sub-pel searches are the expensive part).
                        const QSTEP16: [i64; 6] = [10, 11, 13, 14, 16, 18];
                        let qstep16 = QSTEP16[(fe.qp % 6) as usize] << (fe.qp / 6);
                        let split_gate = ((30 * (qstep16 + 160)) >> 3) * 2;
                        let split_t = split_t();
                        if fe.do_splits
                            && c16 > split_gate
                            && (split_t <= 0.0 || (c16 as f64) >= split_t * lme)
                        {
                            let (rt, mvt, ct) =
                                fe.best_part(refs, &sy, &nb, num_refs, lx, ly, 16, 8, &[mv16], lme);
                            let (rb, mvb, cb) = fe.best_part(
                                refs,
                                &sy,
                                &nb,
                                num_refs,
                                lx,
                                ly + 8,
                                16,
                                8,
                                &[mv16],
                                lme,
                            );
                            let (rl, mvl, cl) =
                                fe.best_part(refs, &sy, &nb, num_refs, lx, ly, 8, 16, &[mv16], lme);
                            let (rr, mvr, cr) = fe.best_part(
                                refs,
                                &sy,
                                &nb,
                                num_refs,
                                lx + 8,
                                ly,
                                8,
                                16,
                                &[mv16],
                                lme,
                            );
                            if ct + cb < best_c {
                                best_c = ct + cb;
                                pick = Some((1u8, vec![(rt, mvt), (rb, mvb)]));
                            }
                            if cl + cr < best_c {
                                best_c = cl + cr;
                                pick = Some((2u8, vec![(rl, mvl), (rr, mvr)]));
                            }

                            // P_8x8: four independent 8×8 sub-partitions (finer motion
                            // granularity — the win on complex/boundary motion). Each 8×8
                            // seeded by the 16×16 MV; the exact chained MVD is computed in
                            // plan_inter_mb. Same heavy-16×16 gate as the 2-way splits.
                            if fe.sub8x8 {
                                let mut c8 = (lme * 4.0) as i64; // ~4 sub_mb_type bits
                                let mut p8 = Vec::with_capacity(4);
                                for &(qx, qy) in &[(0usize, 0usize), (8, 0), (0, 8), (8, 8)] {
                                    let (r, mv, c) = fe.best_part(
                                        refs,
                                        &sy,
                                        &nb,
                                        num_refs,
                                        lx + qx,
                                        ly + qy,
                                        8,
                                        8,
                                        &[mv16],
                                        lme,
                                    );
                                    c8 += c;
                                    p8.push((r, mv));
                                }
                                if c8 < best_c {
                                    best_c = c8;
                                    pick = Some((3u8, p8));
                                }
                            }
                        }

                        // U5-struct: everything above searched FULL-PEL only when
                        // `sp_defer` is set. Now that a shape has won, refine just its
                        // sub-blocks — the losing shapes' refinements were the waste
                        // (measured 3.4–6.4× more refinement than necessary).
                        if fe.sp_defer.get() {
                            if let Some((mode, parts)) = pick.as_mut() {
                                let regions: &[(usize, usize, usize, usize)] = match mode {
                                    1 => &[(0, 0, 16, 8), (0, 8, 16, 8)],
                                    2 => &[(0, 0, 8, 16), (8, 0, 8, 16)],
                                    3 => &[(0, 0, 8, 8), (8, 0, 8, 8), (0, 8, 8, 8), (8, 8, 8, 8)],
                                    _ => &[(0, 0, 16, 16)],
                                };
                                let mut tot = if *mode == 3 { (lme * 4.0) as i64 } else { 0 };
                                for (i, &(qx, qy, pw, ph)) in regions.iter().enumerate() {
                                    let (r, mv) = parts[i];
                                    let (m2, c2) = fe.refine_part(
                                        refs,
                                        &sy,
                                        &nb,
                                        num_refs,
                                        lx + qx,
                                        ly + qy,
                                        pw,
                                        ph,
                                        lme,
                                        r,
                                        mv,
                                    );
                                    parts[i] = (r, m2);
                                    tot += c2;
                                }
                                best_c = tot;
                            }
                        }
                        if split_harvest::enabled() {
                            let won = match pick.as_ref().map(|p| p.0) {
                                Some(0) | None => 0u8,
                                Some(m) => m,
                            };
                            split_harvest::record(c16, best_c, lme, split_gate, won);
                        }
                        // Intra is ALWAYS a candidate (textured / occluded content):
                        // I_16x16 SATD + λ·mode bits.
                        let c_intra = fe.best_i16_satd(&sy, mb_x, mb_y)
                            + (lme * fe.tune_intra_penalty) as i64;
                        inter = if c_intra < best_c { None } else { pick };
                        fe.mb_was_skip[mb_idx] = false;
                        fe.mb_skip_sad[mb_idx] = skip_sad;
                    }

                    // ---- RD P_Skip ----------------------------------------
                    // The default criterion skips only when the residual quantizes
                    // to EXACTLY zero. That matches x264 at both extremes (akiyo
                    // 72.5% vs 73.6%, mobile 1.0% vs 1.4%) but falls 17-23 points
                    // short in the middle (foreman 6.4% vs 23.6%), because x264
                    // also skips macroblocks whose residual is small-but-nonzero.
                    // Decide it properly: trial-encode the chosen mode for real
                    // bits + reconstruction SSD, and compare J = SSD + lambda*R
                    // against the skip. Raw-SAD versions of this comparison fail
                    // badly (coding REPAIRS the residual, skipping keeps it), so
                    // the distortion term has to come from the reconstruction.
                    if fe.rd_skip && rdskip_on && inter.is_some() {
                        let skip_cp = fe.skip_predict_chroma(refs, mb_x, mb_y, mv_skip);
                        // A P_Skip carries no residual, so its RECONSTRUCTION *is*
                        // its prediction — the skip SSD needs no state mutation at
                        // all. The commit / mb_ssd / restore round trip this
                        // replaces cost a full macroblock save+restore on every
                        // candidate, including the ones that go on to code.
                        let ssd_s = fe.pred_ssd(&sy, &su, &sv, mb_x, mb_y, &skip_y, &skip_cp);
                        debug_assert_eq!(
                            ssd_s,
                            {
                                let snap = fe.save_mb(mb_x, mb_y);
                                fe.commit_skip(mb_x, mb_y, mv_skip, &skip_y, &skip_cp);
                                let v = fe.mb_ssd(&sy, &su, &sv, mb_x, mb_y);
                                fe.load_mb(mb_x, mb_y, &snap);
                                v
                            },
                            "skip prediction SSD must equal the committed-skip reconstruction SSD"
                        );
                        // A skip inside a run costs ~1 bit of mb_skip_run.
                        let j_skip = ssd_s as f64 + lambda;
                        // Search-skip gate: when the null arm is this cheap it
                        // almost always wins, so take it without pricing the coded
                        // arm at all. This is where the decision's remaining cost
                        // lives — the coded arm is encoded and then discarded on
                        // 55-80% of candidates.
                        let take_skip = if fe.rd_skip_fast_t > 0.0
                            && (ssd_s as f64) <= lambda * fe.rd_skip_fast_t
                        {
                            true
                        } else {
                            // Otherwise encode ONCE, into scratch, and KEEP the state.
                            // If the skip loses, those are the real bits and they splice
                            // straight into the slice. The previous shape trial-encoded,
                            // threw the result away, and then encoded again — paying
                            // twice on the path that actually codes.
                            fe.save_mb_into(mb_x, mb_y, &mut rdskip_snap);
                            // E11 dig (11.17): pooled — was a fresh Vec per coded-arm trial.
                            let mut scratch = enc_scratch::take_bits();
                            scratch.clear();
                            // `inter.is_some()` was checked ~40 lines up, which is
                            // far enough that neither clippy nor a reader should
                            // have to take it on trust. Bind it here instead.
                            if let Some((m, p)) = inter.as_ref() {
                                fe.encode_inter_mb(
                                    &mut scratch,
                                    refs,
                                    &sy,
                                    &su,
                                    &sv,
                                    mb_x,
                                    mb_y,
                                    *m,
                                    p,
                                );
                            }
                            let bits_c = scratch.bit_len();
                            let ssd_c = fe.mb_ssd(&sy, &su, &sv, mb_x, mb_y);
                            let won = j_skip <= ssd_c as f64 + lambda * bits_c as f64;
                            if won {
                                fe.load_mb(mb_x, mb_y, &rdskip_snap); // undo it; take the skip
                                enc_scratch::put_bits(scratch); // recycle the unused coded arm
                                true
                            } else {
                                coded = Some(scratch); // keep it — no second encode
                                false
                            }
                        };
                        if take_skip {
                            fe.commit_skip(mb_x, mb_y, mv_skip, &skip_y, &skip_cp);
                            if !fe.fast {
                                fe.mb_was_skip[mb_idx] = true;
                                fe.mb_skip_sad[mb_idx] = skip_sad;
                            }
                            mb_qpy[mb_idx] = fe.cur_qp;
                            if precomp {
                                bs_grid[mb_idx] = derive_mb_bs_from(
                                    &fe,
                                    mb_x,
                                    mb_y,
                                    rusty_h264_common::deblock::MbKind::Skip,
                                );
                            }
                            skip_run += 1;
                            continue;
                        }
                    }
                }
                w.write_ue(skip_run); // run of skipped macroblocks before this one
                skip_run = 0;
            }
            #[cfg(feature = "std")]
            if mv_force_on() && is_p && inter.is_some() {
                let fi = MVCMP_FRAME.load(core::sync::atomic::Ordering::Relaxed);
                let ext = EXT_MV.lock().unwrap();
                if let Some(field) = ext.get(fi) {
                    let w4 = fe.mb_w * 4;
                    let b0 = (mb_y * 4) * w4 + mb_x * 4;
                    // uniform 16x16 only: a sub-partitioned macroblock has no single
                    // vector to transplant, so leave those to our own decision
                    let uniform =
                        (0..4).all(|r| (0..4).all(|c| field.get(b0 + r * w4 + c) == field.get(b0)));
                    if uniform {
                        if let Some(&emv) = field.get(b0) {
                            inter = Some((0, vec![(0, emv)]));
                            MVCMP[6].fetch_add(1, core::sync::atomic::Ordering::Relaxed);
                        }
                    }
                }
            }
            #[cfg(feature = "std")]
            if mv_cmp_on() && is_p {
                if let Some((mode, parts)) = inter.as_ref() {
                    let fi = MVCMP_FRAME.load(core::sync::atomic::Ordering::Relaxed);
                    let ext = EXT_MV.lock().unwrap();
                    if let Some(field) = ext.get(fi) {
                        let bidx = (mb_y * 4) * (fe.mb_w * 4) + mb_x * 4;
                        if let Some(&emv) = field.get(bidx) {
                            let (mode, parts) = (*mode, parts.clone());
                            drop(ext);
                            // Both priced through the SAME pipeline: MC, transform,
                            // quantize, CAVLC. Real bits, real reconstruction SSD.
                            let (so, bo) =
                                fe.trial_inter(refs, &sy, &su, &sv, mb_x, mb_y, mode, &parts);
                            let (se, be) =
                                fe.trial_inter(refs, &sy, &su, &sv, mb_x, mb_y, 0, &[(0, emv)]);
                            let jo = so as f64 + lambda * bo as f64;
                            let je = se as f64 + lambda * be as f64;
                            use core::sync::atomic::Ordering::Relaxed;
                            MVCMP[0].fetch_add(1, Relaxed);
                            MVCMP[1].fetch_add(bo as u64, Relaxed);
                            MVCMP[2].fetch_add(be as u64, Relaxed);
                            MVCMP[3].fetch_add(so.max(0) as u64, Relaxed);
                            MVCMP[4].fetch_add(se.max(0) as u64, Relaxed);
                            MVCMP[5].fetch_add((je < jo) as u64, Relaxed);
                            MVCMP[6].fetch_add((parts[0].1 != emv) as u64, Relaxed);
                        }
                    }
                }
            }
            // Capture the kind before `inter` is consumed: the deblocking
            // strengths of an intra macroblock are pure constants.
            let mb_kind = match &inter {
                // A single partition covers the whole macroblock with one
                // (ref, mv), which collapses the internal derivation to nnz.
                Some((_, parts)) if parts.len() == 1 => {
                    rusty_h264_common::deblock::MbKind::InterUniform
                }
                Some(_) => rusty_h264_common::deblock::MbKind::Inter,
                None => rusty_h264_common::deblock::MbKind::Intra,
            };
            match inter {
                Some((mode, parts)) => match coded {
                    // Encoded already, during the skip decision — splice the bits in
                    // rather than encoding this macroblock for a second time.
                    Some(sc) => {
                        w.append(&sc);
                        enc_scratch::put_bits(sc);
                    }
                    None => fe.encode_inter_mb(w, refs, &sy, &su, &sv, mb_x, mb_y, mode, &parts),
                },
                None => encode_mb(&mut fe, w, mb_x, mb_y, &sy, &su, &sv, is_p),
            }
            mb_qpy[mb_idx] = fe.cur_qp; // ACTUAL QPy (updated iff an mb_qp_delta was coded)
            if precomp {
                bs_grid[mb_idx] = derive_mb_bs_from(&fe, mb_x, mb_y, mb_kind);
            }
        }
    }
    debug_assert!(
        !precomp
            || bs_grid
                .iter()
                .all(|b| *b != rusty_h264_common::deblock::MbBs::UNSET),
        "a macroblock loop exit failed to store its boundary strengths"
    );
    if is_p && skip_run > 0 {
        w.write_ue(skip_run); // trailing skipped macroblocks
    }
    w.rbsp_trailing_bits();

    // Deblock the reconstruction; the result is the inter reference. Baseline: the
    // intra mask is `!inter_y` (passed directly, no alloc); no B (List-1 empty); no
    // 8×8 transform (t8x8 empty). ref_id is each block's List-0 ref index.
    drop(_g_loop);
    let _g_fin = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncFinal);
    // No NO_REF-mapping collect: it ran over every 4x4 block every frame (~1.9 MB
    // of allocation + map at 1080p) to produce a grid that is only ever read for
    // INTER-vs-INTER comparisons, where the encoder's raw indices are already
    // equivalent. Intra blocks short-circuit before reference identity is touched.
    let info = rusty_h264_common::deblock::BlockInfo {
        inter: &fe.inter_y,
        nnz: &fe.nnz_y,
        mv: &fe.mv_y,
        ref_id: &fe.ref_idx_y,
        mv1: &[],
        ref_id1: &[],
        w4: fe.mb_w * 4,
        t8x8: &[],
        bs: &bs_grid,
        poc0: &[],
        poc1: &[],
        kind: &[],
    };
    // Per-MB actual QPy (AQ varies it; `mb_qp_delta`-driven). With `aq_strength 0`
    // this is uniform, reproducing the old scalar-QP filtering exactly.
    drop(_g_fin);
    rusty_h264_common::deblock::filter_frame(
        &mut fe.rec_y,
        &mut fe.rec_u,
        &mut fe.rec_v,
        fe.mb_w,
        fe.mb_h,
        &mb_qpy,
        0, // chroma_qp_index_offset — the encoder emits 0
        0, // slice_alpha_c0_offset — the encoder always signals zero offsets
        0, // slice_beta_offset
        &info,
    );
    enc_scratch::put_qpy(mb_qpy);
    let w4 = fe.mb_w * 4;
    crate::RefFrame {
        y: fe.rec_y,
        u: fe.rec_u,
        v: fe.rec_v,
        poc: 0,       // set by the caller (it knows the display order)
        frame_num: 0, // set by the caller
        // List-0 motion field, for a later B-frame's spatial-direct colZeroFlag.
        mv: fe.mv_y,
        ref_idx: fe.ref_idx_y,
        mv1: Vec::new(),
        ref_idx1: Vec::new(),
        w4,
        // Filtered lazily on first sub-pel search use (see `RefFrame::hpel`).
        hpel: crate::HpelOnce::new(),
    }
}

/// Codes a B-slice's macroblock layer. B-frames are **non-reference**, so the
/// reconstruction is computed (the CAVLC nnz predictor needs it) but discarded.
///
/// This brick: every MB is coded `B_L0_16x16` (`mb_type == 1`) — a real
/// motion-compensated prediction from `l0` (the nearest PAST anchor, List-0 index
/// 0) plus a coded residual. Because every MB is List-0-only with `ref_idx`
/// inferred 0, the per-4×4 List-0 motion field and its median `mvd` predictor are
/// byte-identical to the P-slice `P_L0_16x16` path — so this reuses
/// [`FrameEncoder::encode_inter_mb_v1_b`] verbatim, differing from P only in the
/// `mb_type` value. `l1` (nearest future anchor) is unused until `B_Bi` lands.
pub(crate) fn encode_slice_data_b(
    w: &mut BitWriter,
    cfg: &EncoderConfig,
    frame: &YuvPlanes<'_>,
    qp: u8,
    poc: i32,
    l0: &crate::RefFrame,
    l1: &crate::RefFrame,
    _qpo: &[i32],
) {
    let mut fe = FrameEncoder::new(cfg);
    fe.qp = qp;
    fe.qpc = chroma_qp(qp);
    fe.cur_qp = qp;
    if cfg.cabac_dz_div > 0 {
        fe.idz = cfg.cabac_dz_div; // CABAC-specific dead-zone override
    } // QPY_PREV starts at the slice QP so the first mb_qp_delta is 0
      // Implicit bi-prediction weights from the anchor POC distances (matches the
      // decoder). Equidistant B (bframes==1) → 32:32 (plain average); unequal → weighted.
    fe.bi_w = implicit_bi_weights(poc, l0.poc, l1.poc);
    let (sy, su, sv) = coded_source(cfg, frame);
    // Great Gate P1: the shared per-frame signal vector (List-0 anchor as ref).
    let sig = FrameSignals::new(&sy, fe.cw, fe.mb_w, fe.mb_h, Some(&l0.y[..]));
    apply_screen_t8_veto(&mut fe, &sig);
    let lambda = 0.85 * fe.tune_lambda_scale * crate::fastmath::lambda_qp(qp);
    let lme = rusty_h264_common::fmath::sqrt(lambda);
    let refs = core::slice::from_ref(l0); // List-0 = [nearest past anchor]
                                          // Same content-adaptive SAD→SATD dispatch as the P path (codec-content-adaptive-
                                          // dispatch): the top `satd_q` fraction of highest-variance MBs price by SATD.
    if fe.satd_q > 0.0 {
        fe.satd_var_thresh = sig.var_percentile_thresh(fe.satd_q);
    }
    signals::harvest(
        &sig,
        'b',
        qp,
        &signals::GateDecisions {
            satd_thresh: fe.satd_var_thresh,
            ..Default::default()
        },
    );
    let mut skip_run = 0u32; // run of consecutive B_Skip MBs pending a coded MB
    for mb_y in 0..fe.mb_h {
        for mb_x in 0..fe.mb_w {
            let (lx, ly) = (mb_x * 16, mb_y * 16);
            let (pbx, pby) = (mb_x as isize * 4, mb_y as isize * 4);
            // Dedup (E15 W9): memoized twin, already built when satd_q > 0.
            fe.mb_use_satd =
                fe.satd_q > 0.0 && sig.mb_vars()[mb_y * fe.mb_w + mb_x] >= fe.satd_var_thresh;
            // Per-list median MV predictors — the search-rate center AND the actual
            // `mvd` predictor (identical to the decoder's `predict_partition_mv`).
            let n0 = fe.mv_neighbors_block_list(pbx, pby, 4, 0);
            let n1 = fe.mv_neighbors_block_list(pbx, pby, 4, 1);
            let pmv0 = predict_partition_mv(0, 0, n0[0], n0[1], n0[2], 0);
            let pmv1 = predict_partition_mv(0, 0, n1[0], n1[1], n1[2], 0);
            // Independent List-0 / List-1 motion searches (their J already includes
            // the mvd rate against the matching predictor, so J0/J1 compare directly).
            // Spatial-direct prediction (basis of B_Skip and B_Direct_16x16).
            let (dp, dc, dmotion) = fe.b_direct(l0, l1, mb_x, mb_y);
            // B_Skip: take the direct prediction with NO coded residual (~1 bit in
            // the mb_skip_run) only when it is truly FREE — its residual quantizes to
            // zero at the B QP, so skipping loses nothing. (A looser SATD-threshold
            // skip was measured strictly WORSE: on B's derived prediction the SATD
            // proxy over-values the skip, dropping residual the quantizer wanted —
            // the same proxy-vs-quantization gap seen on sub-pel. So skip only when
            // provably free; the rest goes through the L0/L1/Bi/Direct RD decision.)
            if fe.skip_luma_is_free(&sy, mb_x, mb_y, &dp)
                && fe.skip_chroma_is_free(&su, &sv, mb_x, mb_y, &dc)
            {
                fe.commit_direct_motion(mb_x, mb_y, &dmotion);
                fe.commit_pred_pixels(mb_x, mb_y, &dp, &dc);
                skip_run += 1;
                continue;
            }
            let d_direct = fe.pred_dist(&sy, lx, ly, &dp);
            let (mv0, j0) = fe.motion_search(l0, &sy, lx, ly, 16, 16, &[pmv0], lme, None);
            let (mv1, j1) = fe.motion_search(l1, &sy, lx, ly, 16, 16, &[pmv1], lme, None);
            // Bi: average the two winners' predictions; rate = both mvds.
            let d_bi = fe.bi_dist(l0, l1, &sy, lx, ly, mv0, mv1);
            let r_bi = mvd_bits(mv0.0 - pmv0.0)
                + mvd_bits(mv0.1 - pmv0.1)
                + mvd_bits(mv1.0 - pmv1.0)
                + mvd_bits(mv1.1 - pmv1.1);
            let j_bi = d_bi + (lme * r_bi as f64) as i64;
            // B_Direct (mb_type 0): spatial-direct prediction, NO coded MV — so its
            // J (d_direct, computed above) carries zero mvd rate and it wins wherever
            // the derived motion predicts as well as an explicit vector.
            // Pick the cheapest of {0=Direct, 1=L0, 2=L1, 3=Bi}; Direct wins ties.
            let (mut dir, mut best) = (0u8, d_direct);
            if j0 < best {
                dir = 1;
                best = j0;
            }
            if j1 < best {
                dir = 2;
                best = j1;
            }
            if j_bi < best {
                dir = 3;
                best = j_bi;
            }
            let _ = best; // CAVLC B has no partition search to price against it
            w.write_ue(skip_run); // run of B_Skips preceding this coded MB
            skip_run = 0;
            let bspec = BInter {
                dir,
                l1,
                mv0,
                mv1,
                mvmode: 0,
                parts2: [(0, (0, 0), (0, 0)); 2],
            };
            fe.encode_inter_mb_v1_b(w, refs, &sy, &su, &sv, mb_x, mb_y, 0, &[], Some(bspec));
        }
    }
    if skip_run > 0 {
        w.write_ue(skip_run); // trailing B_Skip run
    }
    w.rbsp_trailing_bits();
}

/// `se(d)` Exp-Golomb bit length — the `mvd`-component rate for the B mode
/// decision. Same closed form as `motion_search`'s private `mvbits` (kept separate
/// so the P search's heuristic — and thus P output — is untouched).
#[inline(always)]
fn mvd_bits(d: i32) -> u32 {
    let codenum = if d > 0 {
        (2 * d - 1) as u32
    } else {
        (-2 * d) as u32
    };
    1 + 2 * (31 - (codenum + 1).leading_zeros())
}

/// Reads a 4×4 residual block (source minus a raster prediction block).
/// Writes `ref_idx_l0` (spec: `te(v)` when two references are active — a single
/// flag — else `ue(v)`). Only called when more than one reference is active.
fn write_ref_idx(w: &mut BitWriter, refi: i32, num_refs: usize) {
    if num_refs == 2 {
        w.write_bit(refi == 0); // te(v): value = !bit
    } else {
        w.write_ue(refi as u32);
    }
}

/// Approximate bit cost of coding `ref_idx = r` with `num_refs` active, for the
/// motion-estimation rate term. Zero with a single reference (no `ref_idx` coded).
fn ref_bits(r: usize, num_refs: usize) -> u32 {
    if num_refs <= 1 {
        0
    } else if num_refs == 2 {
        1
    } else {
        let mut n = r as u32 + 1;
        let mut len = 1;
        while n > 1 {
            n >>= 1;
            len += 2;
        }
        len
    }
}

fn residual(src: &[u8], stride: usize, x0: usize, y0: usize, pred: &[i32; 16]) -> [i32; 16] {
    let mut r = [0i32; 16];
    for dy in 0..4 {
        let row = &src[(y0 + dy) * stride + x0..][..4];
        for dx in 0..4 {
            r[dy * 4 + dx] = row[dx] as i32 - pred[dy * 4 + dx];
        }
    }
    r
}

/// Writes reconstructed samples back into a plane.
#[cfg_attr(not(feature = "std"), allow(dead_code))]
#[allow(dead_code)]
fn store(plane: &mut [u8], stride: usize, x0: usize, y0: usize, s: &[u8; 16]) {
    for dy in 0..4 {
        let d = (y0 + dy) * stride + x0;
        plane[d..d + 4].copy_from_slice(&s[dy * 4..dy * 4 + 4]);
    }
}

/// Extracts the 4×4 raster prediction block at `(bx, by)` from a 16×16 (256-sample)
/// luma prediction.
#[cfg(not(accel))]
fn pred_block(pred: &[u8; 256], bx: usize, by: usize) -> [i32; 16] {
    let mut p = [0i32; 16];
    for dy in 0..4 {
        let row = &pred[(by * 4 + dy) * 16 + bx * 4..][..4];
        for dx in 0..4 {
            p[dy * 4 + dx] = row[dx] as i32;
        }
    }
    p
}

/// Sum of absolute transformed differences over a 16×16 luma macroblock — the
/// mode-decision cost (correlates with coded bits better than plain SAD).
/// SATD of a `w`×`h` luma block: `src` (stride `ss`) vs `pred` (stride `ps`).
///
/// With the `accel` cfg and a supported size this is `2 · rusty_h264_accel::satd_*`
/// (the portable SIMD kernels that replaced openh264's asm, same contract), which
/// is **byte-identical** to the scalar `Σ|H·d|` Hadamard: the kernel returns
/// `(Σ+1)>>1`, and `Σ` is always even (every 4×4 Hadamard coefficient shares the block
/// sum's parity, so 16 of them sum even), so `×2` recovers `Σ` exactly — proven over
/// 20 k random blocks at 4×4/8×8/16×16 in `tests/satd_asm_compare.rs`. Without accel (or
/// for an unsupported size) it falls back to the scalar Hadamard — the original path.
#[inline]
pub(crate) fn satd_px(src: &[u8], ss: usize, pred: &[u8], ps: usize, w: usize, h: usize) -> i64 {
    #[cfg(accel)]
    {
        let asm = match (w, h) {
            (16, 16) => Some(rusty_h264_accel::satd_16x16(src, ss, pred, ps)),
            (16, 8) => Some(rusty_h264_accel::satd_16x8(src, ss, pred, ps)),
            (8, 16) => Some(rusty_h264_accel::satd_8x16(src, ss, pred, ps)),
            (8, 8) => Some(rusty_h264_accel::satd_8x8(src, ss, pred, ps)),
            (4, 4) => Some(rusty_h264_accel::satd_4x4(src, ss, pred, ps)),
            // CENSUS #8, closed by COMPOSITION rather than new intrinsics. The
            // sub-8x8 split arm made 8x4 and 4x8 hot, and openh264 ships no
            // kernel for either — but SATD here is DEFINED as the sum of 4x4
            // Hadamards (see the scalar arm below), so both are exactly two
            // `satd_4x4` calls. Each wrapper returns (Σ+1)>>1 and every 4x4 Σ is
            // even, so summing the halves and doubling once is bit-identical to
            // the scalar path — verified by hash, not argued.
            (8, 4) => Some(
                rusty_h264_accel::satd_4x4(src, ss, pred, ps)
                    + rusty_h264_accel::satd_4x4(&src[4..], ss, &pred[4..], ps),
            ),
            (4, 8) => Some(
                rusty_h264_accel::satd_4x4(src, ss, pred, ps)
                    + rusty_h264_accel::satd_4x4(&src[4 * ss..], ss, &pred[4 * ps..], ps),
            ),
            _ => None,
        };
        if let Some(v) = asm {
            return 2 * v as i64;
        }
    }
    // Scalar Hadamard (also the no-asm path): Σ over the 4×4 sub-blocks.
    let (nbx, nby) = (w / 4, h / 4);
    let mut blocks = [[0i32; 16]; 16];
    let mut bi = 0;
    for by in 0..nby {
        for bx in 0..nbx {
            let blk = &mut blocks[bi];
            for dy in 0..4 {
                for dx in 0..4 {
                    blk[dy * 4 + dx] = src[(by * 4 + dy) * ss + bx * 4 + dx] as i32
                        - pred[(by * 4 + dy) * ps + bx * 4 + dx] as i32;
                }
            }
            bi += 1;
        }
    }
    satd_4x4_sum(&blocks[..nbx * nby])
}

/// SAD of a `w`×`h` block: `src` (stride `ss`) vs a strided region `r` (stride `rs`)
/// — the openh264 `psadbw` kernels for the shapes that ship them (they take strides,
/// so in-place plane reads need NO materialize), scalar `Σ abs_diff` rows otherwise
/// (LLVM lowers the idiom to `psadbw` for contiguous rows).
#[inline]
fn sad_strided(src: &[u8], ss: usize, r: &[u8], rs: usize, w: usize, h: usize) -> i64 {
    #[cfg(accel)]
    {
        match (w, h) {
            (16, 16) => return rusty_h264_accel::sad_16x16(src, ss, r, rs) as i64,
            (16, 8) => return rusty_h264_accel::sad_16x8(src, ss, r, rs) as i64,
            (8, 16) => return rusty_h264_accel::sad_8x16(src, ss, r, rs) as i64,
            _ => {}
        }
    }
    let mut sad = 0u32;
    for dy in 0..h {
        let a = &src[dy * ss..][..w];
        let b = &r[dy * rs..][..w];
        sad += a
            .iter()
            .zip(b)
            .map(|(&x, &y)| x.abs_diff(y) as u32)
            .sum::<u32>();
    }
    sad as i64
}

/// Fused `SAD(src, (a+b+1)>>1)` — the quarter-pel SAD without materializing the
/// average (the B2 sibling of the A3 `satd_avg` kernel, scalar because the avg+SAD
/// idiom auto-vectorizes and quarter-phase SAD evals are seed-frequency only).
#[inline]
fn sad_avg_strided(
    src: &[u8],
    ss: usize,
    a: &[u8],
    b: &[u8],
    rs: usize,
    w: usize,
    h: usize,
) -> i64 {
    let mut sad = 0u32;
    for dy in 0..h {
        let s = &src[dy * ss..][..w];
        let pa = &a[dy * rs..][..w];
        let pb = &b[dy * rs..][..w];
        for i in 0..w {
            let p = ((pa[i] as u16 + pb[i] as u16 + 1) >> 1) as u8;
            sad += s[i].abs_diff(p) as u32;
        }
    }
    sad as i64
}

fn satd_16x16(src: &[u8], stride: usize, lx: usize, ly: usize, pred: &[u8; 256]) -> i64 {
    satd_px(&src[ly * stride + lx..], stride, pred, 16, 16, 16)
}

/// SAD over a 16×16 luma macroblock against a prediction — the fast preset's
/// intra cost, kept in the same (SAD) domain as its inter cost. `Σ a.abs_diff(b)`
/// over `u8` slices auto-vectorizes to `psadbw`.
fn sad_16x16(src: &[u8], stride: usize, lx: usize, ly: usize, pred: &[u8; 256]) -> i64 {
    let mut sad = 0u32;
    for dy in 0..16 {
        let s = &src[(ly + dy) * stride + lx..][..16];
        let p = &pred[dy * 16..][..16];
        sad += s
            .iter()
            .zip(p)
            .map(|(&a, &b)| a.abs_diff(b) as u32)
            .sum::<u32>();
    }
    sad as i64
}

/// SATD over an 8×8 chroma block (four 4×4 sub-blocks) against a prediction.
fn satd_8x8(src: &[u8], stride: usize, x0: usize, y0: usize, pred: &[u8; 64]) -> i64 {
    satd_px(&src[y0 * stride + x0..], stride, pred, 8, 8, 8)
}

/// SATD of one 4×4 luma block against a prediction.
fn satd_4x4(src: &[u8], stride: usize, px: usize, py: usize, pred: &[u8; 16]) -> i64 {
    satd_px(&src[py * stride + px..], stride, pred, 4, 4, 4)
}

/// Whether an `Intra_4x4` mode is usable given top/left neighbor availability.
fn i4_mode_available(mode: u8, top: bool, left: bool) -> bool {
    match mode {
        0 | 3 | 7 => top, // vertical, diag-down-left, vertical-left
        1 | 8 => left,    // horizontal, horizontal-up
        2 => true,        // DC
        _ => top && left, // diag-down-right, vertical-right, horizontal-down
    }
}

/// Result of planning an I_4x4 macroblock (luma). Reconstruction has already
/// been written into the frame's `rec_y` and `coded_y` by [`plan_i4x4`].
struct I4Plan {
    modes: [u8; 16],    // per-block intra4x4 mode, raster [lby*4+lbx]
    q: [[i32; 16]; 16], // per-block quantized coefficients (full, raster)
    cbp_luma: u32,      // 4-bit coded-block-pattern (one bit per 8×8 region)
    nonzero: i64,       // total non-zero coefficients (rate proxy)
}

/// A fully-decided intra macroblock: the mode decision, the quantized coefficients,
/// and the committed reconstruction. Produced by [`plan_mb`] (which reuses the
/// entire mode-decision + transform + reconstruct path), then consumed by an
/// entropy backend — `emit_mb_cavlc` or `emit_mb_cabac` — so the two coders share
/// every non-entropy decision bit-for-bit (the bringup-encoder reuse guarantee).
struct MbPlan {
    use_i4: bool,
    // I_16x16 (when !use_i4): prediction mode, whether any AC is coded (cbp_luma=15),
    // luma DC levels (block order), per-4×4 quantized AC (raster).
    i16_mode: I16Mode,
    i16_cbp15: bool,
    i16_dc_levels: [i32; 16],
    i16_q: [[i32; 16]; 16],
    // I_4x4 (when use_i4 && i8 is None): the sub-plan, already reconstructed.
    i4: Option<I4Plan>,
    // I_8x8 (High profile; when use_i4 && i8 is Some): the sub-plan, already
    // reconstructed. use_i4 means "I_NxN"; i8 present disambiguates 8x8 from 4x4.
    i8: Option<I8Plan>,
    // Chroma (shared by both luma types).
    chroma_mode: u8,
    cbp_chroma: u32,
    c_dc_levels: [[i32; 4]; 2],
    c_q_blocks: [[[i32; 16]; 4]; 2],
}

/// A fully-decided inter macroblock: the per-partition motion residuals, coded
/// block pattern, and quantized residual, with the reconstruction + motion grids
/// already committed. Produced by [`FrameEncoder::plan_inter_mb`] (which reuses the
/// whole MC + residual + reconstruct path), then coded by `emit_inter_cavlc` or
/// `emit_inter_cabac` — so the two entropy backends share every non-entropy
/// decision bit-for-bit (the P/B analogue of [`MbPlan`]).
struct InterPlan {
    // Per-partition mvd (P: mvd_l0; B: mvd_l0 then mvd_l1). 16 slots: P_8x8 with
    // 4x4 sub-partitions carries up to 16 (Great Gate P3.3); every other mode <= 4.
    mvds: [(i32, i32); 16],
    plan_refs: [i32; 4], // per-partition ref_idx_l0 (multi-ref P; 0 for B / single-ref)
    /// P_8x8 only: `sub_mb_type` per 8x8 quad ([0;4] = all 8x8 = the pre-P3.3
    /// shape, byte-identical emission). Ignored by every other mode.
    sub_types: [u8; 4],
    n_mvd: usize,
    cbp: u32,
    q_blocks: [[i32; 16]; 16], // luma quantized levels (raster) — used when !t8x8
    c_dc_levels: [[i32; 4]; 2],
    c_q: [[[i32; 16]; 4]; 2],
    t8x8: bool,         // transform_size_8x8_flag (High profile, 8x8 luma residual)
    q8: [[i32; 64]; 4], // per-8x8-block quantized levels (raster) — used when t8x8
}

/// Gathers the 4×4 luma intra neighbors at pixel `(px, py)` from `rec_y`.
fn gather_i4(
    fe: &FrameEncoder,
    px: usize,
    py: usize,
    avail_top: bool,
    avail_left: bool,
    bx: usize,
    by: usize,
) -> ([u8; 8], [u8; 4], u8) {
    let (cw, w4) = (fe.cw, fe.mb_w * 4);
    let mut top = [0u8; 8];
    let mut left = [0u8; 4];
    let mut corner = 0;
    if avail_top {
        let r = (py - 1) * cw + px;
        top[..4].copy_from_slice(&fe.rec_y[r..r + 4]);
        let tr_avail = bx + 1 < w4 && fe.coded_y[(by - 1) * w4 + (bx + 1)];
        if tr_avail {
            top[4..8].copy_from_slice(&fe.rec_y[r + 4..r + 8]);
        } else {
            let t3 = top[3];
            top[4..8].fill(t3);
        }
    }
    if avail_left {
        let col = &fe.rec_y[py * cw + px - 1..(py + 3) * cw + px];
        for i in 0..4 {
            left[i] = col[i * cw];
        }
    }
    if avail_top && avail_left {
        corner = fe.rec_y[(py - 1) * cw + px - 1];
    }
    (top, left, corner)
}

/// Plans an I_4x4 macroblock: picks a mode per 4×4 block (lowest-SATD available
/// mode), quantizes, and reconstructs serially into `rec_y` so each block can
/// predict from the previous one.
/// Neighbour 4x4 block intra mode for the MPM candidate: in-MB blocks read the
/// in-progress local `modes`; blocks in earlier MBs read `fe.modes_y`. (bx, by)
/// are the current block's absolute 4x4 grid coords.
#[inline]
fn modes_at(
    fe: &FrameEncoder,
    modes: &[u8; 16],
    lbx: usize,
    lby: usize,
    dx: isize,
    dy: isize,
    bx: usize,
    by: usize,
) -> u8 {
    let (nx, ny) = (lbx as isize + dx, lby as isize + dy);
    if (0..4).contains(&nx) && (0..4).contains(&ny) {
        modes[ny as usize * 4 + nx as usize]
    } else {
        let w4 = fe.mb_w * 4;
        let gx = (bx as isize + dx) as usize;
        let gy = (by as isize + dy) as usize;
        fe.modes_y[gy * w4 + gx]
    }
}

fn plan_i4x4(fe: &mut FrameEncoder, sy: &[u8], mb_x: usize, mb_y: usize, qp: u8) -> I4Plan {
    let w4 = fe.mb_w * 4;
    let mut modes = [2u8; 16];
    let mut q = [[0i32; 16]; 16];
    let mut cbp_luma = 0u32;
    let mut nonzero = 0i64;

    for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
        let (bx, by) = (mb_x * 4 + lbx, mb_y * 4 + lby);
        let (px, py) = (bx * 4, by * 4);
        let avail_top = by > 0;
        let avail_left = bx > 0;
        let (top, left, corner) = gather_i4(fe, px, py, avail_top, avail_left, bx, by);

        // Pick the lowest-SATD available mode. RUSTY_FAST_INTRA prunes the
        // candidate set to {MPM, DC, V, H} (x264-ultrafast-style); the H.264
        // predicted mode (min of left/top block modes, DC on the edge) keeps the
        // 1-bit prev_intra4x4_pred_mode signalling cheap for the common winner.
        let mut best_m = 2u8;
        let mut best_cost = i64::MAX;
        // E2 W16: keep the winner's prediction from the search loop; it was
        // re-predicted below after the decision.
        let mut best_pred = [0u8; 16];
        if fe.fast && fast_intra_enabled() {
            let lm = if bx > 0 {
                modes_at(fe, &modes, lbx, lby, -1, 0, bx, by)
            } else {
                2
            };
            let tm = if by > 0 {
                modes_at(fe, &modes, lbx, lby, 0, -1, bx, by)
            } else {
                2
            };
            let mpm = lm.min(tm);
            let mut cands = [mpm, 2u8, 0, 1];
            for i in 1..4 {
                for j in 0..i {
                    if cands[i] == cands[j] {
                        cands[i] = 255;
                    }
                }
            }
            for &m in cands.iter() {
                if m == 255 || !i4_mode_available(m, avail_top, avail_left) {
                    continue;
                }
                let pred = intra4x4_pred(m, avail_top, avail_left, &top, &left, corner);
                let cost = satd_4x4(sy, fe.cw, px, py, &pred);
                if cost < best_cost {
                    best_cost = cost;
                    best_m = m;
                    best_pred = pred;
                }
            }
        } else {
            for m in 0..9u8 {
                if !i4_mode_available(m, avail_top, avail_left) {
                    continue;
                }
                let pred = intra4x4_pred(m, avail_top, avail_left, &top, &left, corner);
                let cost = satd_4x4(sy, fe.cw, px, py, &pred);
                if cost < best_cost {
                    best_cost = cost;
                    best_m = m;
                    best_pred = pred;
                }
            }
        }

        // Quantize + reconstruct with the chosen mode (the search's own winner).
        let pred = best_pred;
        let mut predb = [0i32; 16];
        for i in 0..16 {
            predb[i] = pred[i] as i32;
        }
        let res = residual(sy, fe.cw, px, py, &predb);
        let qb = rdoq(&forward_core(&res), qp, fe.idz, fe.rdoq_strength, 0); // full 16 incl DC
                                                                             // E2 W17: the decoder's fused recon (11.6a win 1 executed) — no [u8;16]
                                                                             // temp, no separate `store` walk; abl/prof parity is inside the callee.
        reconstruct_4x4_into(
            &dequantize(&qb, qp),
            &pred,
            0,
            4,
            &mut fe.rec_y,
            py * fe.cw + px,
            fe.cw,
        );
        fe.coded_y[by * w4 + bx] = true;

        let nz = qb.iter().filter(|&&v| v != 0).count();
        if nz > 0 {
            cbp_luma |= 1 << ((lby / 2) * 2 + (lbx / 2));
        }
        nonzero += nz as i64;
        modes[lby * 4 + lbx] = best_m;
        q[lby * 4 + lbx] = qb;
    }
    I4Plan {
        modes,
        q,
        cbp_luma,
        nonzero,
    }
}

/// A planned I_8x8 macroblock (High profile): one intra8x8 mode + one 8x8 DCT per
/// 8x8 block. Reconstructed serially into `rec_y` (each block predicts from the
/// previous), and `modes_y` written per block so later blocks' MPM sees earlier.
struct I8Plan {
    modes: [u8; 4],    // per-8x8-block intra8x8 mode (raster b8 0..3)
    q: [[i32; 64]; 4], // per-8x8-block quantized levels (raster)
    cbp_luma: u32,     // 4-bit coded-block-pattern (one bit per 8x8 block)
    nonzero: i64,      // rate proxy
}

/// Forward zig-zag scan of a raster 8x8 block: `scan[i] = raster[ZIGZAG_8X8[i]]`
/// (the inverse of the decoder's `un_scan_8x8`).
const ZIGZAG_8X8: [usize; 64] = [
    0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, 20,
    13, 6, 7, 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, 44, 51, 58, 59,
    52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63,
];

#[inline]
fn scan_8x8_fwd(raster: &[i32; 64]) -> [i32; 64] {
    core::array::from_fn(|i| raster[ZIGZAG_8X8[i]])
}

/// Gather the 8x8 intra reference samples (top[16] incl top-right, left[8], corner)
/// from `rec_y` — the encoder counterpart of the decoder's `gather_i8`.
fn gather_i8_enc(
    fe: &FrameEncoder,
    px: usize,
    py: usize,
    avail_top: bool,
    avail_left: bool,
    bx: usize,
    by: usize,
) -> ([u8; 16], [u8; 8], u8, bool) {
    let (cw, w4) = (fe.cw, fe.mb_w * 4);
    let mut top = [0u8; 16];
    let mut left = [0u8; 8];
    let mut corner = 0;
    if avail_top {
        let r = (py - 1) * cw + px;
        top[..8].copy_from_slice(&fe.rec_y[r..r + 8]);
        let tr_avail = bx + 2 < w4 && fe.coded_y[(by - 1) * w4 + (bx + 2)];
        if tr_avail {
            top[8..16].copy_from_slice(&fe.rec_y[r + 8..r + 16]);
        } else {
            let t7 = top[7];
            top[8..16].fill(t7);
        }
    }
    if avail_left {
        let col = &fe.rec_y[py * cw + px - 1..(py + 7) * cw + px];
        for i in 0..8 {
            left[i] = col[i * cw];
        }
    }
    let avail_corner = avail_top && avail_left;
    if avail_corner {
        corner = fe.rec_y[(py - 1) * cw + px - 1];
    }
    (top, left, corner, avail_corner)
}

/// Plans an I_8x8 macroblock: per 8x8 block, picks the lowest-SATD intra8x8 mode,
/// 8x8-forward-transforms + quantizes, and reconstructs serially into `rec_y`.
fn plan_i8x8(fe: &mut FrameEncoder, sy: &[u8], mb_x: usize, mb_y: usize, qp: u8) -> I8Plan {
    let w4 = fe.mb_w * 4;
    let mut modes = [2u8; 4];
    let mut q = [[0i32; 64]; 4];
    let mut cbp_luma = 0u32;
    let mut nonzero = 0i64;
    let weight = [16i32; 64];

    for b8 in 0..4usize {
        let (b8x, b8y) = (b8 % 2, b8 / 2);
        let (px, py) = (mb_x * 16 + b8x * 8, mb_y * 16 + b8y * 8);
        let (bx, by) = (mb_x * 4 + b8x * 2, mb_y * 4 + b8y * 2); // top-left 4x4 cell
        let avail_top = b8y > 0 || mb_y > 0;
        let avail_left = b8x > 0 || mb_x > 0;
        let (top, left, corner, avail_corner) =
            gather_i8_enc(fe, px, py, avail_top, avail_left, bx, by);

        // Mode decision: lowest-SATD available intra8x8 mode (same 9 modes / avail
        // rules as intra4x4). The MPM (predict_i4_mode on the top-left 4x4) keeps the
        // 1-bit prev-mode signalling cheap; a small penalty biases toward it.
        let predicted = predict_i4_mode(fe, bx, by);
        let mut best_m = 2u8;
        let mut best_cost = i64::MAX;
        // E2 W18: keep the winner's prediction; it was re-predicted below.
        let mut best_pred = [0u8; 64];
        for m in 0..9u8 {
            if !i4_mode_available(m, avail_top, avail_left) {
                continue;
            }
            let pred = intra8x8_pred(m, avail_top, avail_left, avail_corner, &top, &left, corner);
            let mut cost = satd_8x8(sy, fe.cw, px, py, &pred);
            if m != predicted {
                cost += 4 * fe.qp as i64; // ~mode-signal penalty (rem vs prev flag)
            }
            if cost < best_cost {
                best_cost = cost;
                best_m = m;
                best_pred = pred;
            }
        }
        modes[b8] = best_m;

        // Forward 8x8 transform + quantize + reconstruct (shared decoder primitives).
        let pred = best_pred;
        let mut res = [0i32; 64];
        // E2 W19: row slices (the proven plan_inter8_luma shape).
        for dy in 0..8 {
            let row = &sy[(py + dy) * fe.cw + px..][..8];
            for dx in 0..8 {
                res[dy * 8 + dx] = row[dx] as i32 - pred[dy * 8 + dx] as i32;
            }
        }
        let levels = quantize_8x8(&forward_core_8x8(&res), qp, &weight, fe.idz);
        let nz = levels.iter().filter(|&&v| v != 0).count();
        if nz > 0 {
            cbp_luma |= 1 << b8;
        }
        nonzero += nz as i64;
        q[b8] = levels;

        let res_r = inverse_quant_8x8(&levels, qp, &weight);
        let predb: [i32; 64] = core::array::from_fn(|i| pred[i] as i32);
        let recon = add_residual_8x8(&res_r, &predb);
        for dy in 0..8 {
            let d = (py + dy) * fe.cw + px;
            fe.rec_y[d..d + 8].copy_from_slice(&recon[dy * 8..dy * 8 + 8]);
        }
        // Publish the mode into all four 4x4 cells + mark coded — so the next 8x8
        // block's MPM (and later MBs' neighbours) see it, exactly as the decoder does.
        for sry in 0..2 {
            let d = (by + sry) * w4 + bx;
            fe.modes_y[d..d + 2].fill(best_m);
            fe.coded_y[d..d + 2].fill(true);
        }
    }
    I8Plan {
        modes,
        q,
        cbp_luma,
        nonzero,
    }
}

/// Inter 8×8-transform luma candidate. Forward-8×8 + quantize + reconstruct each of
/// the four 8×8 blocks of the motion-compensated residual `(source − pred_y)`, the
/// pure inverse of the decoder's t8x8 inter luma path (`inv_quant8` ∘ `un_scan_8x8`
/// ∘ `add_residual_8x8`). Returns the quantized levels, `cbp_luma`, a LEVEL-AWARE rate
/// estimate (Σ `rdoq_rate(|level|)` — charges the 8×8's fewer-but-larger coeffs at
/// their true bit cost, not a blind count), the 256-sample reconstruction, and its
/// SSD vs source. Inter deadzone `dz_div = 6`; scaling list flat (16).
fn plan_inter8_luma(
    sy: &[u8],
    cw: usize,
    mb_x: usize,
    mb_y: usize,
    pred_y: &[u8; 256],
    qp: u8,
) -> ([[i32; 64]; 4], u32, f64, [u8; 256], i64) {
    let weight = [16i32; 64];
    let mut q8 = [[0i32; 64]; 4];
    let mut cbp = 0u32;
    let mut rate = 0f64;
    let mut rec = [0u8; 256];
    let mut ssd = 0i64;
    for b8 in 0..4usize {
        let (b8x, b8y) = (b8 % 2, b8 / 2);
        let mut res = [0i32; 64];
        for dy in 0..8 {
            let row = &sy[(mb_y * 16 + b8y * 8 + dy) * cw + mb_x * 16 + b8x * 8..][..8];
            for dx in 0..8 {
                let p = pred_y[(b8y * 8 + dy) * 16 + (b8x * 8 + dx)] as i32;
                res[dy * 8 + dx] = row[dx] as i32 - p;
            }
        }
        let levels = quantize_8x8(&forward_core_8x8(&res), qp, &weight, 6);
        let mut nz = false;
        for &l in &levels {
            if l != 0 {
                nz = true;
                rate += rdoq_rate((l as i64).abs());
            }
        }
        if nz {
            cbp |= 1 << b8;
        }
        q8[b8] = levels;

        let res_r = inverse_quant_8x8(&levels, qp, &weight);
        let predb: [i32; 64] =
            core::array::from_fn(|i| pred_y[(b8y * 8 + i / 8) * 16 + (b8x * 8 + i % 8)] as i32);
        let recon = add_residual_8x8(&res_r, &predb);
        for dy in 0..8 {
            let row = &sy[(mb_y * 16 + b8y * 8 + dy) * cw + mb_x * 16 + b8x * 8..][..8];
            for dx in 0..8 {
                let ri = (b8y * 8 + dy) * 16 + (b8x * 8 + dx);
                rec[ri] = recon[dy * 8 + dx];
                let d = recon[dy * 8 + dx] as i64 - row[dx] as i64;
                ssd += d * d;
            }
        }
    }
    (q8, cbp, rate, rec, ssd)
}

/// 16×16 luma intra prediction. For interior MBs (both neighbors available) this
/// dispatches to accel's `i16x16_luma_pred` (bit-identical to the spec
/// predictor); edge MBs (partial availability → C-only DC variants) use the scalar
/// path. The scalar `top`/`left`/`corner` are gathered by the caller regardless.
#[inline]
fn i16_pred(
    fe: &FrameEncoder,
    mode: I16Mode,
    avail_top: bool,
    avail_left: bool,
    top: &[u8; 16],
    left: &[u8; 16],
    corner: u8,
    lx: usize,
    ly: usize,
) -> [u8; 256] {
    #[cfg(accel)]
    if avail_top && avail_left {
        let mode_n = match mode {
            I16Mode::Vertical => 0,
            I16Mode::Horizontal => 1,
            I16Mode::Dc => 2,
            I16Mode::Plane => 3,
        };
        let mut p = AlignedMb([0; 256]);
        rusty_h264_accel::i16x16_luma_pred(mode_n, &mut p.0, &fe.rec_y[..], ly * fe.cw + lx, fe.cw);
        return p.0;
    }
    let _ = (fe, lx, ly);
    luma16x16_pred(mode, avail_top, avail_left, top, left, corner)
}

/// 8×8 chroma intra prediction. Interior MBs use accel's `chroma8x8_pred`
/// for the V/Plane modes (bit-identical); DC/Horizontal (C-only in openh264) and edge MBs
/// use the scalar path.
#[inline]
fn chroma_pred(
    fe: &FrameEncoder,
    mode: u8,
    avail_top: bool,
    avail_left: bool,
    c: usize,
    top: &[u8; 8],
    left: &[u8; 8],
    corner: u8,
    cx: usize,
    cy: usize,
) -> [u8; 64] {
    #[cfg(accel)]
    if avail_top && avail_left && (mode == 2 || mode == 3) {
        let plane = if c == 0 { &fe.rec_u } else { &fe.rec_v };
        // E2 W20: a 64-byte aligned buffer — the 256-byte AlignedMb zeroed 192
        // dead bytes and then paid a second 64-byte copy-out.
        #[repr(align(16))]
        struct A64([u8; 64]);
        let mut p = A64([0; 64]);
        rusty_h264_accel::chroma8x8_pred(mode, &mut p.0, &plane[..], cy * fe.ccw + cx, fe.ccw);
        return p.0;
    }
    let _ = (fe, c, cx, cy);
    chroma8x8_pred(mode, avail_top, avail_left, top, left, corner)
}

/// Predicted `Intra_4x4` mode for the block at absolute coords `(bx, by)` —
/// `min` of the left/top neighbor modes, or DC if either is unavailable.
fn predict_i4_mode(fe: &FrameEncoder, bx: usize, by: usize) -> u8 {
    if bx == 0 || by == 0 {
        return 2;
    }
    let w4 = fe.mb_w * 4;
    fe.modes_y[by * w4 + (bx - 1)].min(fe.modes_y[(by - 1) * w4 + bx])
}

/// Zig-zag scan: block (raster 4×4) index at scan position i.
const RDOQ_ZZ: [usize; 16] = [0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15];

/// Approximate CABAC bit cost of coding one residual coefficient at magnitude
/// `level`: significant_coeff_flag (~1) + coeff_abs_level_minus1 bins (gt1 + UEG0)
/// + sign (~1); `level == 0` is significant_coeff_flag = 0 (~1). A coarse model —
/// the transform-norm/bin-to-bit scaling is absorbed into the calibrated strength.
#[inline]
fn rdoq_rate(level: i64) -> f64 {
    // The closed form (level 0 → 1, 1 → 3, k ≥ 2 → 3 + min(k−1, 13), saturating
    // at 16 from level 14 up) tabulated: two branches + a min + an int→f64
    // convert become one clamped load. Same 16 values, BIT-IDENTICAL.
    const LUT: [f64; 16] = [
        1.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 16.0,
    ];
    LUT[(level.min(15)) as usize]
}

/// `65536 / QUANT_MF_OH[qp][k]` for every (qp, k) — [`rdoq`]'s dequant steps.
/// The reciprocals are pure functions of two const tables, so they are computed
/// ONCE per process instead of eight `divsd` per trellis call (once per 4×4
/// residual block). IEEE division is deterministic: same operands, same bits.
fn rdoq_qstep(qp: u8) -> &'static [f64; 8] {
    static T: rusty_h264_common::once::OnceLock<[[f64; 8]; 52]> =
        rusty_h264_common::once::OnceLock::new();
    &T.get_or_init(|| {
        core::array::from_fn(|q| {
            core::array::from_fn(|k| {
                65536.0 / rusty_h264_common::transform::QUANT_MF_OH[q][k] as f64
            })
        })
    })[qp as usize]
}

/// Rate-distortion optimized quantization (CABAC trellis, RDOQ) for one 4×4 residual
/// block. Refines the hard-decision levels toward min over {|q|, |q|-1} of
/// `SSD_coef + λ·R_cabac` per coefficient (coefficient-domain distortion
/// `(|coeff| - level·deq_step)²`; `λ = strength·2^((qp-12)/3)`). `strength == 0`
/// returns the hard quantization unchanged (the CAVLC path). `first` = 1 skips the
/// DC (AC-only categories: I_16x16 AC, chroma AC), else 0.
fn rdoq(coeffs: &[i32; 16], qp: u8, dz_div: i64, strength: f64, first: usize) -> [i32; 16] {
    let mut q = quantize(coeffs, qp, dz_div);
    if strength <= 0.0 {
        return q;
    }
    let lambda = strength * crate::fastmath::lambda_qp(qp);
    // Distortion is measured in the QUANTIZER-INPUT (forward-transform) domain, where
    // level L reconstructs to L·qstep, qstep = 2^16 / MF (the inverse of the forward
    // quant scale). The transform norm (forward↔pixel) folds into `strength`.
    const POS: [usize; 16] = [0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7];
    // Eight reciprocals per call became a process-lifetime table ([`rdoq_qstep`]):
    // the hoist-to-call-scope round left 8 `divsd` per trellis invocation; both
    // inputs are const tables, so the divides now happen once per process.
    let qstep = rdoq_qstep(qp);
    let dist = |p: usize, level: i64| -> f64 {
        let e = coeffs[p].unsigned_abs() as f64 - level as f64 * qstep[POS[p]];
        e * e
    };
    // Pass 1: per-coefficient level lowering (|q| → |q|-1) minimizing D + λ·R.
    for i in first..16 {
        let p = RDOQ_ZZ[i];
        let m = q[p].unsigned_abs() as i64;
        if m == 0 {
            continue;
        }
        let j_keep = dist(p, m) + lambda * rdoq_rate(m);
        let j_down = dist(p, m - 1) + lambda * rdoq_rate(m - 1);
        if j_down < j_keep {
            let nl = (m - 1) as i32;
            q[p] = if q[p] < 0 { -nl } else { nl };
        }
    }
    // Pass 2: last-significant-position trimming. Zeroing the trailing significant
    // coefficient frees its own bits AND the last_significant flag + every sig=0 flag
    // between it and the previous significant coefficient (positions past the new last
    // aren't coded at all) — the dominant RDOQ gain on sparse (inter) residuals.
    while let Some(li) = (first..16).rev().find(|&i| q[RDOQ_ZZ[i]] != 0) {
        let p = RDOQ_ZZ[li];
        let m = q[p].unsigned_abs() as i64;
        let prev = (first..li).rev().find(|&i| q[RDOQ_ZZ[i]] != 0);
        let base = prev.map_or(first, |j| j + 1);
        let bits = rdoq_rate(m) + 1.0 + (li - base) as f64; // coeff + last-flag + freed sig=0
        let d_add = dist(p, 0) - dist(p, m);
        if d_add < lambda * bits {
            q[p] = 0;
        } else {
            break;
        }
    }
    q
}

/// Decide one intra macroblock (I_16x16 vs I_4x4, prediction modes, chroma),
/// forward-transform + quantize, and commit the reconstruction + neighbour mode
/// state — everything except entropy coding. The returned [`MbPlan`] is coded by
/// either entropy backend, so CAVLC and CABAC share this whole path bit-for-bit.
fn plan_mb(
    fe: &mut FrameEncoder,
    mb_x: usize,
    mb_y: usize,
    sy: &[u8],
    su: &[u8],
    sv: &[u8],
) -> MbPlan {
    let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncIntraCode);
    let qp = fe.qp;
    let qpc = fe.qpc;
    // Lagrangian λ for rate-distortion decisions (standard H.264 form).
    let lambda = 0.85 * fe.tune_lambda_scale * crate::fastmath::lambda_qp(qp);

    // ---------------- luma ----------------
    let (lx, ly) = (mb_x * 16, mb_y * 16);
    let avail_top = mb_y > 0;
    let avail_left = mb_x > 0;
    let mut top = [0u8; 16];
    let mut left = [0u8; 16];
    if avail_top {
        let r = (ly - 1) * fe.cw + lx;
        top.copy_from_slice(&fe.rec_y[r..r + 16]);
    }
    if avail_left {
        let col = &fe.rec_y[ly * fe.cw + lx - 1..(ly + 15) * fe.cw + lx];
        for i in 0..16 {
            left[i] = col[i * fe.cw];
        }
    }
    let corner = if avail_top && avail_left {
        fe.rec_y[(ly - 1) * fe.cw + lx - 1]
    } else {
        0
    };

    let w4 = fe.mb_w * 4;

    // ============ I_16x16 plan (reconstruct into a local buffer) ============
    let mut i16_mode = I16Mode::Dc;
    let mut best_pred = i16_pred(
        fe,
        I16Mode::Dc,
        avail_top,
        avail_left,
        &top,
        &left,
        corner,
        lx,
        ly,
    );
    let mut best_cost = satd_16x16(sy, fe.cw, lx, ly, &best_pred);
    for mode in [I16Mode::Vertical, I16Mode::Horizontal, I16Mode::Plane] {
        if !mode.available(avail_top, avail_left) {
            continue;
        }
        let pred = i16_pred(fe, mode, avail_top, avail_left, &top, &left, corner, lx, ly);
        let cost = satd_16x16(sy, fe.cw, lx, ly, &pred);
        if cost < best_cost {
            best_cost = cost;
            i16_mode = mode;
            best_pred = pred;
        }
    }
    // I_16x16 blocks are independent (one fixed whole-MB prediction), so batch the
    // forward DCT (`forward_dct_blocks` → SIMD), bit-identical to `forward_core`.
    let mut dc4x4 = [0i32; 16];
    let mut i16_q = [[0i32; 16]; 16];
    // Fast path: forward DCT of (src - pred) straight from the planes per 8x8 quad,
    // quantize with the identical FF/MF math (deadzone = fe.idz), recon via the
    // bit-identical idct+add+clip kernel — the same pairing encode_inter_mb and the
    // P_Skip free-check already use, byte-identical to the scalar twin below.
    #[cfg(accel)]
    let (i16_dc_levels, _i16_recon_dc, recon16) = {
        #[repr(align(16))]
        struct A([i16; 256]);
        let mut dct = A([0i16; 256]);
        let base = ly * fe.cw + lx;
        for (qi, &(qx, qy)) in [(0usize, 0usize), (8, 0), (0, 8), (8, 8)]
            .iter()
            .enumerate()
        {
            rusty_h264_accel::dct_four_t4(
                &mut dct.0[qi * 64..qi * 64 + 64],
                &sy[base + qy * fe.cw + qx..],
                fe.cw,
                &best_pred[qy * 16 + qx..],
                16,
            );
        }
        for (blk, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
            dc4x4[lby * 4 + lbx] = dct.0[blk * 16] as i32;
        }
        if fe.rdoq_strength > 0.0 {
            // Trellis (all-intra only): scalar RDOQ from the asm DCT output instead of
            // the asm hard quantizer. dct.0 keeps the raw DCT here; the recon loop below
            // overwrites it with the dequantized RDOQ levels.
            for (blk, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
                let coeffs: [i32; 16] = core::array::from_fn(|i| dct.0[blk * 16 + i] as i32);
                let mut q = rdoq(&coeffs, qp, fe.idz, fe.rdoq_strength, 1);
                q[0] = 0;
                i16_q[lby * 4 + lbx] = q;
            }
        } else {
            let ff = rusty_h264_common::transform::quant_dz_ff(qp, fe.idz);
            let mf = &rusty_h264_common::transform::QUANT_MF_OH[qp as usize];
            for qi in 0..4 {
                rusty_h264_accel::quant_four_4x4(&mut dct.0[qi * 64..qi * 64 + 64], &ff, mf);
            }
            for (blk, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
                let q = &mut i16_q[lby * 4 + lbx];
                q[0] = 0;
                for i in 1..16 {
                    q[i] = dct.0[blk * 16 + i] as i32;
                }
            }
        }
        let i16_dc_levels = forward_quant_luma_dc(&dc4x4, qp, true);
        let i16_recon_dc = inverse_quant_luma_dc(&i16_dc_levels, qp);
        // Recon: dequantize (DC injected from the Hadamard) back into quad layout,
        // then idct+add-pred+clip into the trial buffer.
        let mut recon16 = [0u8; 256];
        for (blk, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
            let mut deq = dequantize(&i16_q[lby * 4 + lbx], qp);
            deq[0] = i16_recon_dc[lby * 4 + lbx];
            for i in 0..16 {
                dct.0[blk * 16 + i] = deq[i] as i16;
            }
        }
        for (qi, &(qx, qy)) in [(0usize, 0usize), (8, 0), (0, 8), (8, 8)]
            .iter()
            .enumerate()
        {
            rusty_h264_accel::idct_four_t4_rec(
                &mut recon16[qy * 16 + qx..],
                16,
                &best_pred[qy * 16 + qx..],
                16,
                &dct.0[qi * 64..qi * 64 + 64],
            );
        }
        (i16_dc_levels, i16_recon_dc, recon16)
    };
    #[cfg(not(accel))]
    let (i16_dc_levels, _i16_recon_dc, recon16) = {
        let mut res_blocks = [[0i32; 16]; 16];
        for by in 0..4 {
            for bx in 0..4 {
                let predb = pred_block(&best_pred, bx, by);
                res_blocks[by * 4 + bx] = residual(sy, fe.cw, lx + bx * 4, ly + by * 4, &predb);
            }
        }
        let mut coeffs = [[0i32; 16]; 16];
        forward_dct_blocks(&res_blocks, &mut coeffs);
        for i in 0..16 {
            dc4x4[i] = coeffs[i][0];
            let mut q = rdoq(&coeffs[i], qp, fe.idz, fe.rdoq_strength, 1);
            q[0] = 0;
            i16_q[i] = q;
        }
        let i16_dc_levels = forward_quant_luma_dc(&dc4x4, qp, true);
        let i16_recon_dc = inverse_quant_luma_dc(&i16_dc_levels, qp);
        let mut recon16 = [0u8; 256];
        let mut deq_blocks = [[0i32; 16]; 16];
        for i in 0..16 {
            deq_blocks[i] = dequantize(&i16_q[i], qp);
            deq_blocks[i][0] = i16_recon_dc[i];
        }
        let mut idct = [[0i32; 16]; 16];
        inverse_dct_blocks(&deq_blocks, &mut idct);
        for by in 0..4 {
            for bx in 0..4 {
                let s = add_residual_4x4(&idct[by * 4 + bx], &pred_block(&best_pred, bx, by));
                for dy in 0..4 {
                    for dx in 0..4 {
                        recon16[(by * 4 + dy) * 16 + (bx * 4 + dx)] = s[dy * 4 + dx];
                    }
                }
            }
        }
        (i16_dc_levels, i16_recon_dc, recon16)
    };
    let i16_cbp15 = i16_q.iter().any(|b| b[1..].iter().any(|&c| c != 0));
    let i16_dc_nz = i16_dc_levels.iter().filter(|&&v| v != 0).count() as i64;
    let i16_ac_nz: i64 = i16_q
        .iter()
        .map(|b| b[1..].iter().filter(|&&v| v != 0).count() as i64)
        .sum();
    // I_16x16 AC is all-or-nothing: any AC ⇒ all 16 blocks pay a coeff_token.
    let i16_rate = i16_dc_nz + i16_ac_nz + if i16_cbp15 { 16 } else { 0 };
    // Reconstruction distortion (SSD) for the rate-distortion decision.
    let mut ssd16 = 0i64;
    for dy in 0..16 {
        let sr = (ly + dy) * fe.cw + lx;
        for (r, sv2) in recon16[dy * 16..dy * 16 + 16].iter().zip(&sy[sr..sr + 16]) {
            let d = *r as i64 - *sv2 as i64;
            ssd16 += d * d;
        }
    }

    // ============ chroma (shared by both luma types; commit immediately) ============
    let (cx, cy) = (mb_x * 8, mb_y * 8);
    // Gather both components' neighbors, then pick a chroma mode by combined SATD.
    let mut ntop = [[0u8; 8]; 2];
    let mut nleft = [[0u8; 8]; 2];
    let mut ncorner = [0u8; 2];
    for c in 0..2 {
        let rec_c = if c == 0 { &fe.rec_u } else { &fe.rec_v };
        if avail_top {
            let r = (cy - 1) * fe.ccw + cx;
            ntop[c].copy_from_slice(&rec_c[r..r + 8]);
        }
        if avail_left {
            let col = &rec_c[cy * fe.ccw + cx - 1..(cy + 7) * fe.ccw + cx];
            for i in 0..8 {
                nleft[c][i] = col[i * fe.ccw];
            }
        }
        if avail_top && avail_left {
            ncorner[c] = rec_c[(cy - 1) * fe.ccw + cx - 1];
        }
    }
    let mut chroma_mode = 0u8;
    let mut best_c_cost = i64::MAX;
    // E2 W11 (inline-execution.md 11.12): the WINNING mode's two prediction
    // planes were re-predicted after the search; keep them from the loop.
    let mut best_pred8 = [[0u8; 64]; 2];
    for m in 0..4u8 {
        if !chroma_mode_available(m, avail_top, avail_left) {
            continue;
        }
        let mut cost = 0i64;
        let mut cur = [[0u8; 64]; 2];
        for c in 0..2 {
            let src = if c == 0 { su } else { sv };
            cur[c] = chroma_pred(
                fe, m, avail_top, avail_left, c, &ntop[c], &nleft[c], ncorner[c], cx, cy,
            );
            cost += satd_8x8(src, fe.ccw, cx, cy, &cur[c]);
        }
        if cost < best_c_cost {
            best_c_cost = cost;
            chroma_mode = m;
            best_pred8 = cur;
        }
    }

    let mut c_dc_levels = [[0i32; 4]; 2];
    let mut c_q_blocks = [[[0i32; 16]; 4]; 2];
    let mut any_chroma_ac = false;
    let mut any_chroma_dc = false;
    for c in 0..2 {
        let src = if c == 0 { su } else { sv };
        let pred8 = best_pred8[c]; // the search's winner, not a re-prediction
        #[cfg_attr(accel, allow(unused_variables))] // non-accel scalar path only
        let pblk = |bx: usize, by: usize| -> [i32; 16] {
            let mut predb = [0i32; 16];
            for dy in 0..4 {
                for dx in 0..4 {
                    predb[dy * 4 + dx] = pred8[(by * 4 + dy) * 8 + (bx * 4 + dx)] as i32;
                }
            }
            predb
        };
        // Fast path: forward DCT of (src - pred8) straight from the planes, quantize
        // with identical FF/MF (idz deadzone), recon via one idct+add+clip kernel —
        // bit-identical to the scalar twin below (proven kernel pairings).
        let mut dc2x2 = [0i32; 4];
        let mut qbs = [[0i32; 16]; 4];
        #[cfg(accel)]
        let recon_dc = {
            #[repr(align(16))]
            struct A([i16; 64]);
            let mut d = A([0i16; 64]);
            rusty_h264_accel::dct_four_t4(&mut d.0, &src[cy * fe.ccw + cx..], fe.ccw, &pred8, 8);
            for i in 0..4 {
                dc2x2[i] = d.0[i * 16] as i32;
            }
            if fe.rdoq_strength > 0.0 {
                // Trellis (all-intra only): scalar RDOQ from the asm chroma DCT.
                for i in 0..4 {
                    let coeffs: [i32; 16] = core::array::from_fn(|j| d.0[i * 16 + j] as i32);
                    let mut q = rdoq(&coeffs, qpc, fe.idz, fe.rdoq_strength, 1);
                    q[0] = 0;
                    if q[1..].iter().any(|&v| v != 0) {
                        any_chroma_ac = true;
                    }
                    qbs[i] = q;
                }
            } else {
                let ff = rusty_h264_common::transform::quant_dz_ff(qpc, fe.idz);
                let mf = &rusty_h264_common::transform::QUANT_MF_OH[qpc as usize];
                rusty_h264_accel::quant_four_4x4(&mut d.0, &ff, mf);
                for i in 0..4 {
                    let q = &mut qbs[i];
                    q[0] = 0;
                    for j in 1..16 {
                        let v = d.0[i * 16 + j] as i32;
                        q[j] = v;
                        if v != 0 {
                            any_chroma_ac = true;
                        }
                    }
                }
            }
            let dl = forward_quant_chroma_dc(&dc2x2, qpc, true);
            if dl.iter().any(|&v| v != 0) {
                any_chroma_dc = true;
            }
            let recon_dc = inverse_quant_chroma_dc(&dl, qpc);
            for i in 0..4 {
                let deq = dequantize(&qbs[i], qpc);
                for j in 0..16 {
                    d.0[i * 16 + j] = deq[j] as i16;
                }
                d.0[i * 16] = recon_dc[i] as i16;
            }
            let plane = if c == 0 { &mut fe.rec_u } else { &mut fe.rec_v };
            rusty_h264_accel::idct_four_t4_rec(
                &mut plane[cy * fe.ccw + cx..],
                fe.ccw,
                &pred8,
                8,
                &d.0,
            );
            c_dc_levels[c] = dl;
            recon_dc
        };
        #[cfg(not(accel))]
        let recon_dc = {
            let mut res_blocks = [[0i32; 16]; 4];
            for by in 0..2 {
                for bx in 0..2 {
                    res_blocks[by * 2 + bx] =
                        residual(src, fe.ccw, cx + bx * 4, cy + by * 4, &pblk(bx, by));
                }
            }
            let mut coeffs = [[0i32; 16]; 4];
            forward_dct_blocks(&res_blocks, &mut coeffs);
            for i in 0..4 {
                dc2x2[i] = coeffs[i][0];
                let mut q = rdoq(&coeffs[i], qpc, fe.idz, fe.rdoq_strength, 1);
                q[0] = 0;
                qbs[i] = q;
                if q[1..].iter().any(|&v| v != 0) {
                    any_chroma_ac = true;
                }
            }
            let dl = forward_quant_chroma_dc(&dc2x2, qpc, true);
            if dl.iter().any(|&v| v != 0) {
                any_chroma_dc = true;
            }
            let recon_dc = inverse_quant_chroma_dc(&dl, qpc);
            let mut deq_blocks = [[0i32; 16]; 4];
            for i in 0..4 {
                deq_blocks[i] = dequantize(&qbs[i], qpc);
                deq_blocks[i][0] = recon_dc[i];
            }
            let mut idct = [[0i32; 16]; 4];
            inverse_dct_blocks(&deq_blocks, &mut idct);
            let plane = if c == 0 { &mut fe.rec_u } else { &mut fe.rec_v };
            for by in 0..2 {
                for bx in 0..2 {
                    let s = add_residual_4x4(&idct[by * 2 + bx], &pblk(bx, by));
                    store(plane, fe.ccw, cx + bx * 4, cy + by * 4, &s);
                }
            }
            c_dc_levels[c] = dl;
            recon_dc
        };
        let _ = recon_dc;
        c_q_blocks[c] = qbs;
    }
    let cbp_chroma: u32 = if any_chroma_ac {
        2
    } else if any_chroma_dc {
        1
    } else {
        0
    };

    // ============ I_NxN plan + RD: I_16x16 vs I_4x4 vs (High profile) I_8x8 ============
    // I_4x4 and I_8x8 both reconstruct serially into rec_y, but each block predicts
    // only from NEIGHBOURS + earlier blocks it fills itself — never the stale MB
    // content — so running I_8x8 after I_4x4 needs no restore. J = SSD + λ·R picks the
    // per-MB transform (the content-adaptive win: 8x8 on smooth, 4x4 on detail).
    let base = ly * fe.cw + lx;
    let i4 = if i16_rate > 2 {
        Some(plan_i4x4(fe, sy, mb_x, mb_y, qp))
    } else {
        None
    };
    let (j4, i4_recon) = match &i4 {
        Some(p) => {
            let mut ssd = 0i64;
            let mut rec = [0u8; 256];
            for dy in 0..16 {
                let sr = base + dy * fe.cw;
                let row = &fe.rec_y[sr..sr + 16];
                rec[dy * 16..dy * 16 + 16].copy_from_slice(row);
                for (a, b) in row.iter().zip(&sy[sr..sr + 16]) {
                    let d = *a as i64 - *b as i64;
                    ssd += d * d;
                }
            }
            (ssd as f64 + lambda * (p.nonzero + 16) as f64, Some(rec))
        }
        None => (f64::INFINITY, None),
    };
    // STRUCTURAL gate (candidate): offer the I_8x8 candidate at all only when the
    // macroblock's slice type says it is worth offering. See `i8_in_p_on`.
    let i8 = if fe.t8_pick && (!fe.intra_in_p || i8_in_p_on()) {
        Some(plan_i8x8(fe, sy, mb_x, mb_y, qp))
    } else {
        None
    };
    let j8 = match &i8 {
        Some(p) => {
            let mut ssd = 0i64;
            for dy in 0..16 {
                let sr = base + dy * fe.cw;
                for (a, b) in fe.rec_y[sr..sr + 16].iter().zip(&sy[sr..sr + 16]) {
                    let d = *a as i64 - *b as i64;
                    ssd += d * d;
                }
            }
            ssd as f64 + lambda * (p.nonzero + 16) as f64
        }
        None => f64::INFINITY,
    };
    let j16 = ssd16 as f64 + lambda * i16_rate as f64;

    // ============ commit the RD winner's reconstruction + neighbour modes ============
    // The 8x8 candidate must clear its rivals BY A MARGIN (default 0 = any win, the
    // pre-margin behaviour). `i8_margin` is in lambda units so it is QP-invariant.
    let m8 = fe.i8_margin * lambda;
    let (use_i4, i4, i8) = if i8.is_some() && j8 + m8 <= j4 && j8 + m8 <= j16 {
        // I_8x8: plan_i8x8 already committed rec_y AND modes_y (per 8x8 block).
        (true, None, i8)
    } else if i4.is_some() && j4 < j16 {
        // I_4x4: restore its reconstruction (I_8x8 may have overwritten rec_y), publish modes.
        let rec = i4_recon.unwrap();
        for dy in 0..16 {
            let d = base + dy * fe.cw;
            fe.rec_y[d..d + 16].copy_from_slice(&rec[dy * 16..dy * 16 + 16]);
        }
        let modes = i4.as_ref().unwrap().modes;
        for lby in 0..4 {
            let d = (mb_y * 4 + lby) * w4 + mb_x * 4;
            fe.modes_y[d..d + 4].copy_from_slice(&modes[lby * 4..lby * 4 + 4]);
        }
        (true, i4, None)
    } else {
        // I_16x16: commit its reconstruction, mark modes DC.
        for dy in 0..16 {
            let d = (ly + dy) * fe.cw + lx;
            fe.rec_y[d..d + 16].copy_from_slice(&recon16[dy * 16..dy * 16 + 16]);
        }
        for lby in 0..4 {
            let d = (mb_y * 4 + lby) * w4 + mb_x * 4;
            fe.modes_y[d..d + 4].fill(2);
        }
        (false, None, None)
    };
    // Mark all luma blocks coded for the next macroblock's top-right availability.
    for lby in 0..4 {
        let d = (mb_y * 4 + lby) * w4 + mb_x * 4;
        fe.coded_y[d..d + 4].fill(true);
    }

    MbPlan {
        use_i4,
        i16_mode,
        i16_cbp15,
        i16_dc_levels,
        i16_q,
        i4,
        i8,
        chroma_mode,
        cbp_chroma,
        c_dc_levels,
        c_q_blocks,
    }
}

/// Emit one planned intra macroblock as CAVLC (the original `encode_mb` tail). Reads
/// only the decided values from `plan`; `plan_mb` already committed recon + modes.
fn encode_mb(
    fe: &mut FrameEncoder,
    w: &mut BitWriter,
    mb_x: usize,
    mb_y: usize,
    sy: &[u8],
    su: &[u8],
    sv: &[u8],
    is_p: bool,
) {
    fe.intra_in_p = is_p;
    let plan = plan_mb(fe, mb_x, mb_y, sy, su, sv);
    // In a P-slice, intra macroblock types are offset by 5 (0..4 are inter).
    let mb_type_offset = if is_p { 5 } else { 0 };
    let w4 = fe.mb_w * 4;
    let cbp_chroma = plan.cbp_chroma;

    // ============ emit luma ============
    if let Some(i8) = plan.i8.as_ref().filter(|_| plan.use_i4) {
        // ---- I_8x8 (High profile): mb_type = I_NxN, transform_size_8x8_flag = 1, then
        // one intra8x8 mode per 8x8 block, cbp, mb_qp_delta, and the 8x8 residual as
        // four interleaved 4x4 CAVLC sub-blocks (coeff k of sub s -> 8x8 scan 4k+s). ----
        let cbp = i8.cbp_luma | (cbp_chroma << 4);
        w.write_ue(mb_type_offset); // mb_type = I_NxN
        w.write_bit(true); // transform_size_8x8_flag = 1
        for b8 in 0..4usize {
            let (bx, by) = (mb_x * 4 + (b8 % 2) * 2, mb_y * 4 + (b8 / 2) * 2);
            let predicted = predict_i4_mode(fe, bx, by);
            let actual = i8.modes[b8];
            if actual == predicted {
                w.write_bit(true);
            } else {
                w.write_bit(false);
                let rem = if actual < predicted {
                    actual
                } else {
                    actual - 1
                };
                w.write_bits(rem as u32, 3);
            }
        }
        w.write_ue(plan.chroma_mode as u32); // intra_chroma_pred_mode
        write_cbp_intra(w, cbp);
        if cbp != 0 {
            w.write_se(fe.qp_delta());
        }
        fe.nnz_cache_load(mb_x, mb_y);
        for b8 in 0..4usize {
            let (b8x, b8y) = (b8 % 2, b8 / 2);
            let scan8 = scan_8x8_fwd(&i8.q[b8]);
            for sub in 0..4usize {
                let (cx, cy) = (b8x * 2 + sub % 2, b8y * 2 + sub / 2);
                let (bx, by) = (mb_x * 4 + cx, mb_y * 4 + cy);
                let total = if i8.cbp_luma & (1 << b8) != 0 {
                    let nc = fe.nc_pred(cx, cy);
                    let blk: [i32; 16] = core::array::from_fn(|k| scan8[4 * k + sub]);
                    encode_residual_block(w, &blk, 16, nc) as u8
                } else {
                    0
                };
                fe.nnz_cache_set(cx, cy, total);
                fe.nnz_y[by * w4 + bx] = total;
            }
        }
    } else if plan.use_i4 {
        let i4 = plan.i4.as_ref().unwrap();
        let cbp = i4.cbp_luma | (cbp_chroma << 4);
        w.write_ue(mb_type_offset); // mb_type = I_4x4 (+5 in P-slices)
        if fe.transform_8x8 {
            w.write_bit(false); // transform_size_8x8_flag = 0 (this I_NxN is 4x4)
        }
        for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
            let (bx, by) = (mb_x * 4 + lbx, mb_y * 4 + lby);
            let predicted = predict_i4_mode(fe, bx, by);
            let actual = i4.modes[lby * 4 + lbx];
            if actual == predicted {
                w.write_bit(true);
            } else {
                w.write_bit(false);
                let rem = if actual < predicted {
                    actual
                } else {
                    actual - 1
                };
                w.write_bits(rem as u32, 3);
            }
        }
        w.write_ue(plan.chroma_mode as u32); // intra_chroma_pred_mode
        write_cbp_intra(w, cbp);
        if cbp != 0 {
            w.write_se(fe.qp_delta()); // mb_qp_delta (AQ per-MB QPy)
        }
        fe.nnz_cache_load(mb_x, mb_y);
        for (blk, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
            let (bx, by) = (mb_x * 4 + lbx, mb_y * 4 + lby);
            let total = if i4.cbp_luma & (1 << (blk / 4)) != 0 {
                let nc = fe.nc_pred(lbx, lby);
                let scan16 = scan_4x4_dcac(&i4.q[lby * 4 + lbx]);
                encode_residual_block(w, &scan16, 16, nc) as u8
            } else {
                0
            };
            fe.nnz_cache_set(lbx, lby, total);
            fe.nnz_y[by * w4 + bx] = total;
        }
    } else {
        let mb_type =
            1 + plan.i16_mode as u32 + 4 * cbp_chroma + if plan.i16_cbp15 { 12 } else { 0 };
        w.write_ue(mb_type + mb_type_offset);
        w.write_ue(plan.chroma_mode as u32); // intra_chroma_pred_mode
        w.write_se(fe.qp_delta()); // mb_qp_delta (I_16x16 always codes it; AQ per-MB QPy)
        fe.nnz_cache_load(mb_x, mb_y);
        let nc_dc = fe.nc_pred(0, 0);
        let dc_scan = scan_4x4_dcac(&plan.i16_dc_levels);
        encode_residual_block(w, &dc_scan, 16, nc_dc);
        for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
            fe.nnz_cache_set(lbx, lby, 0);
            fe.nnz_y[(mb_y * 4 + lby) * w4 + (mb_x * 4 + lbx)] = 0;
        }
        if plan.i16_cbp15 {
            for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
                let (bx, by) = (mb_x * 4 + lbx, mb_y * 4 + lby);
                let nc = fe.nc_pred(lbx, lby);
                let ac = scan_4x4_ac(&plan.i16_q[lby * 4 + lbx]);
                let total = encode_residual_block(w, &ac, 15, nc) as u8;
                fe.nnz_cache_set(lbx, lby, total);
                fe.nnz_y[by * w4 + bx] = total;
            }
        }
    }

    // ============ emit chroma residual (shared) ============
    if cbp_chroma != 0 {
        for c in 0..2 {
            encode_residual_block(w, &plan.c_dc_levels[c], 4, -1);
        }
    }
    if cbp_chroma == 2 {
        fe.chroma_cache_load(mb_x, mb_y);
        let w2 = fe.mb_w * 2;
        for c in 0..2 {
            for &(bx, by) in &CHROMA_4X4_SCAN_XY {
                let nc = fe.chroma_nc_pred(c, bx, by);
                let ac = scan_4x4_ac(&plan.c_q_blocks[c][by * 2 + bx]);
                let total = encode_residual_block(w, &ac, 15, nc) as u8;
                fe.chroma_nnz_cache_set(c, bx, by, total);
                fe.nnz_c[c][(mb_y * 2 + by) * w2 + (mb_x * 2 + bx)] = total;
            }
        }
    }
}

// ============================================================================
// CABAC I-slice entropy coding — the exact forward inverse of the decoder's
// `decode_slice_data_cabac` I-slice path (rusty_h264-decoder mb16.rs). Every
// binarization + context-selection here mirrors a `parse_*_cabac` there; the
// neighbour state (nzc cache, cbf_dc, cat, cmode, mb_cbp, last_delta_qp) is
// reconstructed identically so the contexts evolve bit-for-bit. Reuses `plan_mb`
// for the entire mode-decision/transform/recon (shared with CAVLC).
// ============================================================================

// Res-property tables: the ONE copy shared with the decoder via `cabac_tables`
// (they were byte-copies here; a single divergent entry desyncs the coders).
use rusty_h264_common::cabac_tables::{
    CACHE30 as CB_CACHE30, G_SCAN4 as CB_G_SCAN4, NZC_CACHE as CB_NZC_CACHE, RES_CBF as CB_RES_CBF,
    RES_MAP as CB_RES_MAP, RES_MAXC2 as CB_RES_MAXC2, RES_MAXPOS as CB_RES_MAXPOS,
    RES_ONE as CB_RES_ONE,
};
const CB_RP_I16_DC: usize = 1;
const CB_RP_I16_AC: usize = 2;
const CB_RP_LUMA_4X4: usize = 3;
const CB_RP_CHROMA_DC: usize = 7;
const CB_RP_CHROMA_AC: usize = 9;
/// Luma 8x8 (ctxBlockCat 5). Mirrors the decoder's `RP_LUMA_8X8`.
const CB_RP_LUMA_8X8: usize = 6;

/// Inverse of `cabac_unary(ctx, off)`: bin0 at `ctx`; for value >= 1, `value-1` ones
/// then a terminating 0, all at `ctx+off`.
fn cb_unary(cab: &mut CabacEncoder, ctx: usize, off: usize, value: u32) {
    if value == 0 {
        cab.encode_decision(ctx, 0);
        return;
    }
    cab.encode_decision(ctx, 1);
    for _ in 0..value - 1 {
        cab.encode_decision(ctx + off, 1);
    }
    cab.encode_decision(ctx + off, 0);
}

/// Exp-Golomb order-`k` in bypass — inverse of `cabac_exp_bypass(k)`.
fn cb_exp_bypass(cab: &mut CabacEncoder, mut k: i32, mut n: u32) {
    while n >= (1 << k) {
        cab.encode_bypass(1);
        n -= 1 << k;
        k += 1;
    }
    cab.encode_bypass(0);
    // The suffix tail is exactly `encode_bypass_bits` (MSB-first, k bits) — the
    // API CABAC-4 reserved and then reimplemented inline here (H9). Same call
    // sequence, byte-identical.
    cab.encode_bypass_bits(n, k as u32);
}

/// UEG0 coeff-level suffix — inverse of `cabac_ueg_level(ctx)` (TU prefix <=13 at
/// `ctx`, then an EG0 bypass suffix).
fn cb_ueg_level(cab: &mut CabacEncoder, ctx: usize, value: u32) {
    if value == 0 {
        cab.encode_decision(ctx, 0);
        return;
    }
    let ones = value.min(13);
    for _ in 0..ones {
        cab.encode_decision(ctx, 1);
    }
    if value < 13 {
        cab.encode_decision(ctx, 0);
    } else {
        cb_exp_bypass(cab, 0, value - 13);
    }
}

/// `mb_qp_delta` — inverse of `parse_mb_qp_delta_cabac` (ctxIdxOffset 60).
pub fn cb_mb_qp_delta(cab: &mut CabacEncoder, last_delta_qp: &mut i32, delta: i32) {
    const O: usize = 60;
    let ctx_inc = (*last_delta_qp != 0) as usize;
    if delta == 0 {
        cab.encode_decision(O + ctx_inc, 0);
    } else {
        cab.encode_decision(O + ctx_inc, 1);
        // code = 2|d| - (d>0); the decode's cabac_unary sees code-1.
        let code = 2 * delta.unsigned_abs() - (delta > 0) as u32;
        cb_unary(cab, O + 2, 1, code - 1);
    }
    *last_delta_qp = delta;
}

/// `intra_chroma_pred_mode` (TU cMax=3) — inverse of `parse_intra_chroma_pred_mode_cabac`.
fn cb_chroma_pred_mode(cab: &mut CabacEncoder, ctx_inc: usize, mode: u8) {
    const C: usize = 64;
    if mode == 0 {
        cab.encode_decision(C + ctx_inc, 0);
        return;
    }
    cab.encode_decision(C + ctx_inc, 1);
    if mode == 1 {
        cab.encode_decision(C + 3, 0);
    } else if mode == 2 {
        cab.encode_decision(C + 3, 1);
        cab.encode_decision(C + 3, 0);
    } else {
        cab.encode_decision(C + 3, 1);
        cab.encode_decision(C + 3, 1);
    }
}

/// I-slice `mb_type` — inverse of `parse_mb_type_i_cabac` (ctxIdxOffset 3).
fn cb_mb_type_i(
    cab: &mut CabacEncoder,
    ctx_inc: usize,
    use_i4: bool,
    i16_mode: u32,
    cbp_chroma: u32,
    cbp_luma15: bool,
) {
    const O: usize = 3;
    if use_i4 {
        cab.encode_decision(O + ctx_inc, 0); // I_NxN
        return;
    }
    cab.encode_decision(O + ctx_inc, 1);
    cab.encode_terminate(false); // not I_PCM
    cab.encode_decision(O + 3, cbp_luma15 as u32);
    if cbp_chroma != 0 {
        cab.encode_decision(O + 4, 1);
        cab.encode_decision(O + 5, (cbp_chroma == 2) as u32);
    } else {
        cab.encode_decision(O + 4, 0);
    }
    cab.encode_decision(O + 6, (i16_mode >> 1) & 1);
    cab.encode_decision(O + 7, i16_mode & 1);
}

/// One `Intra_4x4` pred-mode — inverse of `parse_intra4x4_pred_mode_cabac` (ctx 68).
fn cb_intra4x4_pred_mode(cab: &mut CabacEncoder, predicted: u8, actual: u8) {
    const IPR: usize = 68;
    if actual == predicted {
        cab.encode_decision(IPR, 1);
    } else {
        cab.encode_decision(IPR, 0);
        let rem = if actual < predicted {
            actual
        } else {
            actual - 1
        } as u32;
        cab.encode_decision(IPR + 1, rem & 1);
        cab.encode_decision(IPR + 1, (rem >> 1) & 1);
        cab.encode_decision(IPR + 1, (rem >> 2) & 1);
    }
}

/// `coded_block_pattern` — inverse of `parse_cbp_cabac` (ctxIdxOffset 73).
pub fn cb_cbp(cab: &mut CabacEncoder, top: Option<u8>, left: Option<u8>, cbp: u32) {
    const CBP: usize = 73;
    let t = |m: u32| top.map_or(0u32, |c| ((c as u32 & m) == 0) as u32);
    let l = |m: u32| left.map_or(0u32, |c| ((c as u32 & m) == 0) as u32);
    let nb = |x: u32| (x == 0) as u32;
    let b0 = cbp & 1;
    let b1 = (cbp >> 1) & 1;
    let b2 = (cbp >> 2) & 1;
    let b3 = (cbp >> 3) & 1;
    cab.encode_decision(CBP + (l(1 << 1) + (t(1 << 2) << 1)) as usize, b0);
    cab.encode_decision(CBP + (nb(b0) + (t(1 << 3) << 1)) as usize, b1);
    cab.encode_decision(CBP + (l(1 << 3) + (nb(b0) << 1)) as usize, b2);
    cab.encode_decision(CBP + (nb(b2) + (nb(b1) << 1)) as usize, b3);
    let cbp_chroma = cbp >> 4;
    let ct = top.map_or(0u32, |c| ((c >> 4) != 0) as u32);
    let cl = left.map_or(0u32, |c| ((c >> 4) != 0) as u32);
    cab.encode_decision(
        CBP + 4 + (cl + (ct << 1)) as usize,
        (cbp_chroma != 0) as u32,
    );
    if cbp_chroma != 0 {
        let ct2 = top.map_or(0u32, |c| ((c >> 4) == 2) as u32);
        let cl2 = left.map_or(0u32, |c| ((c >> 4) == 2) as u32);
        cab.encode_decision(
            CBP + 8 + (cl2 + (ct2 << 1)) as usize,
            (cbp_chroma == 2) as u32,
        );
    }
}

/// One residual block — inverse of `parse_residual_cabac`. `coeffs` is scan-order
/// (len >= maxPos+1). Returns totalCoeffNum (for the nzc cache + deblock nnz).
fn cb_residual(
    cab: &mut CabacEncoder,
    nzc: &mut [u8; 48],
    cbf_dc: &mut u16,
    iz: usize,
    rp: usize,
    is_intra: bool,
    ndc: (Option<u16>, Option<u16>),
    coeffs: &[i32],
) -> u32 {
    // ctxBlockCat 5 is the ONLY category with no coded_block_flag: presence is inferred
    // from CodedBlockPatternLuma, so emitting one here would desync the decoder. Same
    // asymmetry the decoder's reader documents.
    let is8 = rp == CB_RP_LUMA_8X8;
    let is_dc = rp == CB_RP_I16_DC || rp == CB_RP_CHROMA_DC || rp == CB_RP_CHROMA_DC + 1;
    let (mut na, mut nb) = (is_intra as u8, is_intra as u8);
    let scan = CB_NZC_CACHE[iz.min(23)];
    if is_dc {
        if let Some(t) = ndc.0 {
            nb = ((t >> rp) & 1) as u8;
        }
        if let Some(l) = ndc.1 {
            na = ((l >> rp) & 1) as u8;
        }
    } else {
        if nzc[scan - 8] != 0xff {
            nb = (nzc[scan - 8] != 0) as u8;
        }
        if nzc[scan - 1] != 0xff {
            na = (nzc[scan - 1] != 0) as u8;
        }
    }
    let maxpos = CB_RES_MAXPOS[rp] as usize;
    let coeff_num = coeffs[..=maxpos].iter().filter(|&&c| c != 0).count() as u32;
    let cbf = coeff_num != 0;
    if !is8 {
        cab.encode_decision(85 + CB_RES_CBF[rp] + (na + (nb << 1)) as usize, cbf as u32);
        if !cbf {
            if !is_dc {
                nzc[scan] = 0;
            }
            return 0;
        }
    } else if !cbf {
        // Caller must not invoke cat 5 for an all-zero block: with no cbf to carry the
        // "empty" signal, the decoder would read a significance map that was never
        // written. CBP is what suppresses it, upstream.
        debug_assert!(
            false,
            "cat 5 called with an all-zero block; CBP should have gated it"
        );
        nzc[scan] = 0;
        return 0;
    }
    if is_dc {
        *cbf_dc |= 1 << rp;
    }
    // significance map. For the 4x4 categories ctxIdxInc IS the scan position; cat 5
    // folds 63 positions onto 15 (sig) / 9 (last) contexts via the Table 9-43 maps,
    // at absolute bases rather than `105/166 + off`.
    let map = 105 + CB_RES_MAP[rp];
    let last = 166 + CB_RES_MAP[rp];
    let lastnz = (0..=maxpos).rev().find(|&i| coeffs[i] != 0).unwrap();
    for i in 0..maxpos {
        let s = coeffs[i] != 0;
        let (sig_ctx, last_ctx) = if is8 {
            (
                rusty_h264_common::cabac_tables::CAT5_SIG_BASE
                    + rusty_h264_common::cabac_tables::SIG8X8[i] as usize,
                rusty_h264_common::cabac_tables::CAT5_LAST_BASE
                    + rusty_h264_common::cabac_tables::LAST8X8[i] as usize,
            )
        } else {
            (map + i, last + i)
        };
        cab.encode_decision(sig_ctx, s as u32);
        if s {
            let is_last = i == lastnz;
            cab.encode_decision(last_ctx, is_last as u32);
            if is_last {
                break;
            }
        }
    }
    // levels (reverse scan)
    let one = 227 + CB_RES_ONE[rp];
    let abs = 232 + CB_RES_ONE[rp];
    let maxc2 = CB_RES_MAXC2[rp];
    let (mut c1, mut c2) = (1i32, 0i32);
    for i in (0..=maxpos).rev() {
        if coeffs[i] != 0 {
            let av = coeffs[i].unsigned_abs();
            let gt1 = av > 1;
            cab.encode_decision(one + c1 as usize, gt1 as u32);
            if gt1 {
                cb_ueg_level(cab, abs + c2 as usize, av - 2);
                c2 = (c2 + 1).min(maxc2);
                c1 = 0;
            } else if c1 != 0 {
                c1 = (c1 + 1).min(4);
            }
            cab.encode_bypass((coeffs[i] < 0) as u32);
        }
    }
    if is8 {
        // One 8x8 covers four consecutive z-order 4x4 cells. Every later
        // coded_block_flag ctxIdxInc reads this cache, so all four must carry the
        // count -- writing only `scan` would corrupt the NEXT macroblock's contexts.
        // Byte-for-byte the decoder's rule; the two must agree or the stream desyncs.
        for k in 0..4 {
            nzc[CB_NZC_CACHE[(iz + k).min(23)]] = coeff_num as u8;
        }
    } else if !is_dc {
        nzc[scan] = coeff_num as u8;
    }
    coeff_num
}

/// Build the 48-entry padded nzc cache from the top/left neighbour MB exports
/// (openh264 `WelsFillCacheNonZeroCount`) — identical to the decoder.
fn cb_build_nzc(mb_nzc: &[[u8; 24]], top: Option<usize>, left: Option<usize>) -> [u8; 48] {
    let mut nzc = [0xffu8; 48];
    if let Some(t) = top {
        let tn = mb_nzc[t];
        nzc[1..5].copy_from_slice(&tn[12..16]);
        (nzc[0], nzc[5], nzc[29]) = (0, 0, 0);
        (nzc[6], nzc[7]) = (tn[20], tn[21]);
        (nzc[30], nzc[31]) = (tn[22], tn[23]);
    }
    if let Some(l) = left {
        let ln = mb_nzc[l];
        (nzc[8], nzc[16], nzc[24], nzc[32]) = (ln[3], ln[7], ln[11], ln[15]);
        (nzc[13], nzc[21], nzc[37], nzc[45]) = (ln[17], ln[21], ln[19], ln[23]);
    }
    nzc
}

/// Extract the 24-entry per-MB nzc (raster luma + chroma) for future neighbours.
fn cb_export_nzc(nzc: &[u8; 48]) -> [u8; 24] {
    let mut mn = [0u8; 24];
    // Four CONTIGUOUS runs of four, not a strided gather: copy them as slices so
    // the compiler emits moves rather than sixteen bounds-checked element loads.
    mn[0..4].copy_from_slice(&nzc[9..13]);
    mn[4..8].copy_from_slice(&nzc[17..21]);
    mn[8..12].copy_from_slice(&nzc[25..29]);
    mn[12..16].copy_from_slice(&nzc[33..37]);
    (mn[16], mn[17], mn[20], mn[21]) = (nzc[14], nzc[15], nzc[22], nzc[23]);
    (mn[18], mn[19], mn[22], mn[23]) = (nzc[38], nzc[39], nzc[46], nzc[47]);
    for v in mn.iter_mut() {
        if *v == 0xff {
            *v = 0;
        }
    }
    mn
}

/// Per-frame CABAC neighbour state (I-slice): one entry per macroblock, mirroring
/// the arrays the decoder's `decode_slice_data_cabac` maintains.
/// Saturate to the only thing the mvd ctxInc can distinguish: |mvd| capped at
/// 33. Its reader computes `|nb| + |na|` and compares against 3 and 32, so if
/// either neighbour is >= 33 the sum already exceeds 32 both ways, and if both
/// are < 33 the cap is a no-op. Same argument as the decoder's `sat_mvd`.
fn cb_sat_mvd(m: [[i16; 2]; 16]) -> [[u8; 2]; 16] {
    core::array::from_fn(|i| core::array::from_fn(|c| m[i][c].unsigned_abs().min(33) as u8))
}

struct CabacState {
    cat: Vec<u8>,          // 2 = I_16x16, 0 = I_NxN, 100 = inter (mb_type / skip ctxInc)
    // 0..=3 with the read being `(1..=3).contains(..)`: two bits in four bytes.
    cmode: Vec<i8>,        // per-MB chroma mode (chroma-pred ctxInc)
    mb_cbp: Vec<u8>,       // per-MB cbp byte (cbp ctxInc)
    cbf_dc: Vec<u16>,      // per-MB DC coded_block_flag mask (residual DC ctxInc)
    mb_nzc: Vec<[u8; 24]>, // per-MB nzc export (residual AC ctxInc)
    // Inter (P/B) neighbour state — mirrors the decoder's WelsFillCacheInterCabac.
    // SATURATED-ABSOLUTE u8: the ctxInc reads |mvd| against 3 and 32, so neither
    // the sign nor any magnitude above 33 is reachable (see `cb_sat_mvd`).
    mb_mvd: Vec<[[u8; 2]; 16]>, // per-MB per-4x4 List-0 mvd (raster), for the mvd ctxInc cache
    mb_ref: Vec<[i8; 16]>,       // per-MB per-4x4 List-0 ref idx (raster); -1 = unavailable
    mb_mvd1: Vec<[[u8; 2]; 16]>, // B: per-MB per-4x4 List-1 mvd
    mb_ref1: Vec<[i8; 16]>,      // B: per-MB per-4x4 List-1 ref idx
    mb_skip: Vec<bool>,          // per-MB mb_skip_flag (skip ctxInc)
    mb_direct: Vec<bool>,        // B: per-MB B_Direct/B_Skip (B mb_type ctxInc)
    mb_t8x8: Vec<bool>,          // per-MB transform_size_8x8_flag (ctxIdxOffset 399 ctxInc)
    last_delta_qp: i32,
}

impl CabacState {
    /// Reset a recycled state to `new(n)`'s exact contents on retained
    /// capacity (D13's encoder twin, 11.11).
    fn refill(&mut self, n: usize) {
        fn rf<T: Clone>(v: &mut Vec<T>, n: usize, val: T) {
            v.clear();
            v.resize(n, val);
        }
        rf(&mut self.cat, n, 0);
        rf(&mut self.cmode, n, 0);
        rf(&mut self.mb_cbp, n, 0);
        rf(&mut self.cbf_dc, n, 0);
        rf(&mut self.mb_nzc, n, [0u8; 24]);
        rf(&mut self.mb_mvd, n, [[0u8; 2]; 16]);
        rf(&mut self.mb_ref, n, [-1i8; 16]);
        rf(&mut self.mb_mvd1, n, [[0u8; 2]; 16]);
        rf(&mut self.mb_ref1, n, [-1i8; 16]);
        rf(&mut self.mb_skip, n, false);
        rf(&mut self.mb_direct, n, false);
        rf(&mut self.mb_t8x8, n, false);
        self.last_delta_qp = 0;
    }

    /// Pooled form of `new(n)`: ~13 vecs (hundreds of KB per slice) recycled
    /// from this thread's previous slice.
    fn pooled(n: usize) -> Self {
        let mut cs = enc_scratch::take_cs().unwrap_or_else(|| CabacState::new(0));
        cs.refill(n);
        cs
    }

    fn new(n: usize) -> Self {
        CabacState {
            cat: vec![0; n],
            cmode: vec![0; n],
            mb_cbp: vec![0; n],
            cbf_dc: vec![0; n],
            mb_nzc: vec![[0u8; 24]; n],
            mb_mvd: vec![[[0u8; 2]; 16]; n],
            mb_ref: vec![[-1i8; 16]; n],
            mb_mvd1: vec![[[0u8; 2]; 16]; n],
            mb_ref1: vec![[-1i8; 16]; n],
            mb_skip: vec![false; n],
            mb_direct: vec![false; n],
            mb_t8x8: vec![false; n],
            last_delta_qp: 0,
        }
    }
}

/// Emit one planned intra macroblock as CABAC (I-slice). Mirrors the decoder's
/// I-slice MB body exactly: `mb_type`, then per luma-type the intra modes / cbp /
/// `mb_qp_delta` / residual in spec order, maintaining `cs` and `fe.nnz_y`.
fn emit_mb_cabac_i(
    fe: &mut FrameEncoder,
    cab: &mut CabacEncoder,
    cs: &mut CabacState,
    plan: &MbPlan,
    mb_x: usize,
    mb_y: usize,
) {
    let mb_w = fe.mb_w;
    let addr = mb_y * mb_w + mb_x;
    let top = if mb_y > 0 { Some(addr - mb_w) } else { None };
    let left = if mb_x > 0 { Some(addr - 1) } else { None };

    // ---- mb_type (I-slice prefix; carries I_16x16 pred-mode/cbp) ----
    let li = left.map_or(0, |a| (cs.cat[a] >= 2) as usize);
    let ti = top.map_or(0, |a| (cs.cat[a] >= 2) as usize);
    let acct = crate::bitacct::enabled();
    let t0 = if acct { cab.pos() } else { 0 };
    if plan.use_i4 {
        cb_mb_type_i(cab, li + ti, true, 0, 0, false);
    } else {
        cb_mb_type_i(
            cab,
            li + ti,
            false,
            plan.i16_mode as u32,
            plan.cbp_chroma,
            plan.i16_cbp15,
        );
    }
    if acct {
        crate::bitacct::add(crate::bitacct::B::MbType, cab.pos() - t0);
    }
    let t1 = if acct { cab.pos() } else { 0 };
    emit_intra_body_cabac(fe, cab, cs, plan, mb_x, mb_y, addr, top, left);
    if acct {
        crate::bitacct::add(crate::bitacct::B::IntraBody, cab.pos() - t1);
    }
}

/// The intra macroblock body (chroma pred mode, intra modes, cbp, mb_qp_delta,
/// residual) shared by I-slice intra and P/B-slice intra — everything AFTER the
/// slice-specific `mb_type` prefix (which already carries the I_16x16 pred-mode/cbp).
/// Commits a raster-local per-block nnz record into the frame grid as four
/// row copies (E28, inline-execution.md 11.16): the emit loops run in SCAN
/// order for the CABAC contexts, so they collect locally and commit here —
/// one bounds check per row instead of one per block.
#[inline]
fn nnz_commit_rows(fe: &mut FrameEncoder, mb_x: usize, mb_y: usize, loc: &[u8; 16]) {
    let w4 = fe.mb_w * 4;
    for lby in 0..4 {
        let d = (mb_y * 4 + lby) * w4 + mb_x * 4;
        fe.nnz_y[d..d + 4].copy_from_slice(&loc[lby * 4..lby * 4 + 4]);
    }
}

fn emit_intra_body_cabac(
    fe: &mut FrameEncoder,
    cab: &mut CabacEncoder,
    cs: &mut CabacState,
    plan: &MbPlan,
    mb_x: usize,
    mb_y: usize,
    addr: usize,
    top: Option<usize>,
    left: Option<usize>,
) {
    let cbp_chroma = plan.cbp_chroma;
    // chroma-pred-mode ctxInc from neighbour chroma modes (1..=3).
    let cci = left.map_or(0, |a| (1..=3).contains(&cs.cmode[a]) as usize)
        + top.map_or(0, |a| (1..=3).contains(&cs.cmode[a]) as usize);

    let mut nzc;
    let mut cbfdc = 0u16;
    let ndc = (top.map(|a| cs.cbf_dc[a]), left.map(|a| cs.cbf_dc[a]));

    if !plan.use_i4 {
        // ---- I_16x16 ----
        cb_chroma_pred_mode(cab, cci, plan.chroma_mode);
        cs.cmode[addr] = plan.chroma_mode as i8;
        cs.cat[addr] = 2;
        cs.mb_cbp[addr] = ((cbp_chroma as u8) << 4) | if plan.i16_cbp15 { 15 } else { 0 };
        nzc = cb_build_nzc(&cs.mb_nzc, top, left);

        let delta = fe.qp_delta();
        cb_mb_qp_delta(cab, &mut cs.last_delta_qp, delta);

        // luma DC
        let dc_scan = scan_4x4_dcac(&plan.i16_dc_levels);
        cb_residual(
            cab,
            &mut nzc,
            &mut cbfdc,
            0,
            CB_RP_I16_DC,
            true,
            ndc,
            &dc_scan,
        );
        // luma AC
        let mut nnz_loc = [0u8; 16];
        for (iz, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
            let total = if plan.i16_cbp15 {
                let ac = scan_4x4_ac(&plan.i16_q[lby * 4 + lbx]);
                cb_residual(cab, &mut nzc, &mut cbfdc, iz, CB_RP_I16_AC, true, ndc, &ac)
            } else {
                nzc[CB_NZC_CACHE[iz]] = 0;
                0
            };
            nnz_loc[lby * 4 + lbx] = total as u8;
        }
        nnz_commit_rows(fe, mb_x, mb_y, &nnz_loc);
        cb_emit_chroma_residual(
            cab,
            fe,
            &mut nzc,
            &mut cbfdc,
            ndc,
            true,
            plan.cbp_chroma,
            &plan.c_dc_levels,
            &plan.c_q_blocks,
            mb_x,
            mb_y,
        );
    } else if let Some(i8) = plan.i8.as_ref() {
        // ---- I_NxN with transform_size_8x8_flag = 1 (I_8x8, High profile) ----
        // ORDER IS LOAD-BEARING: for I_NxN the flag precedes the intra pred modes
        // (spec 7.3.5), because the modes themselves are per-8x8 when it is set.
        // ctxIdx = 399 + condTermFlagA + condTermFlagB, each 1 when that neighbour
        // MB carries the flag -- the exact mirror of the decoder's read.
        let ta = left.map_or(0, |x| cs.mb_t8x8[x] as usize);
        let tb = top.map_or(0, |x| cs.mb_t8x8[x] as usize);
        cab.encode_decision(399 + ta + tb, 1);
        cs.mb_t8x8[addr] = true;
        for b8 in 0..4usize {
            let (bx, by) = (mb_x * 4 + (b8 % 2) * 2, mb_y * 4 + (b8 / 2) * 2);
            let predicted = predict_i4_mode(fe, bx, by);
            cb_intra4x4_pred_mode(cab, predicted, i8.modes[b8]);
        }
        cb_chroma_pred_mode(cab, cci, plan.chroma_mode);
        cs.cmode[addr] = plan.chroma_mode as i8;
        cs.cat[addr] = 0;
        let cbp = i8.cbp_luma | (cbp_chroma << 4);
        cb_cbp(
            cab,
            top.map(|a| cs.mb_cbp[a]),
            left.map(|a| cs.mb_cbp[a]),
            cbp,
        );
        cs.mb_cbp[addr] = cbp as u8;
        nzc = cb_build_nzc(&cs.mb_nzc, top, left);

        if cbp == 0 {
            cs.last_delta_qp = 0;
            nnz_commit_rows(fe, mb_x, mb_y, &[0u8; 16]);
        } else {
            let delta = fe.qp_delta();
            cb_mb_qp_delta(cab, &mut cs.last_delta_qp, delta);
            let mut nnz_loc = [0u8; 16];
            for b8 in 0..4usize {
                let (b8x, b8y) = (b8 % 2, b8 / 2);
                // Unlike CAVLC -- which has no 8x8 entropy model and must split the
                // block into four interleaved 4x4 sub-blocks -- CABAC codes the 8x8
                // as ONE 64-coefficient ctxBlockCat-5 block.
                let total = if i8.cbp_luma & (1 << b8) != 0 {
                    let scan8 = scan_8x8_fwd(&i8.q[b8]);
                    cb_residual(
                        cab,
                        &mut nzc,
                        &mut cbfdc,
                        b8 * 4,
                        CB_RP_LUMA_8X8,
                        true,
                        ndc,
                        &scan8,
                    )
                } else {
                    for k in 0..4 {
                        nzc[CB_NZC_CACHE[b8 * 4 + k]] = 0;
                    }
                    0
                };
                for sy in 0..2 {
                    for sx in 0..2 {
                        nnz_loc[(b8y * 2 + sy) * 4 + b8x * 2 + sx] = total as u8;
                    }
                }
            }
            nnz_commit_rows(fe, mb_x, mb_y, &nnz_loc);
            cb_emit_chroma_residual(
                cab,
                fe,
                &mut nzc,
                &mut cbfdc,
                ndc,
                true,
                plan.cbp_chroma,
                &plan.c_dc_levels,
                &plan.c_q_blocks,
                mb_x,
                mb_y,
            );
        }
    } else {
        // ---- I_NxN (I_4x4) ----
        let i4 = plan.i4.as_ref().unwrap();
        if fe.transform_8x8 {
            let ta = left.map_or(0, |x| cs.mb_t8x8[x] as usize);
            let tb = top.map_or(0, |x| cs.mb_t8x8[x] as usize);
            cab.encode_decision(399 + ta + tb, 0);
        }
        for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
            let (bx, by) = (mb_x * 4 + lbx, mb_y * 4 + lby);
            let predicted = predict_i4_mode(fe, bx, by);
            cb_intra4x4_pred_mode(cab, predicted, i4.modes[lby * 4 + lbx]);
        }
        cb_chroma_pred_mode(cab, cci, plan.chroma_mode);
        cs.cmode[addr] = plan.chroma_mode as i8;
        cs.cat[addr] = 0;
        let cbp = i4.cbp_luma | (cbp_chroma << 4);
        cb_cbp(
            cab,
            top.map(|a| cs.mb_cbp[a]),
            left.map(|a| cs.mb_cbp[a]),
            cbp,
        );
        cs.mb_cbp[addr] = cbp as u8;
        nzc = cb_build_nzc(&cs.mb_nzc, top, left);

        if cbp == 0 {
            cs.last_delta_qp = 0;
            nnz_commit_rows(fe, mb_x, mb_y, &[0u8; 16]);
        } else {
            let delta = fe.qp_delta();
            cb_mb_qp_delta(cab, &mut cs.last_delta_qp, delta);
            let mut nnz_loc = [0u8; 16];
            for id8 in 0..4usize {
                for id4 in 0..4usize {
                    let iz = id8 * 4 + id4;
                    let (lbx, lby) = LUMA_4X4_SCAN_XY[iz];
                    let total = if i4.cbp_luma & (1 << id8) != 0 {
                        let sc = scan_4x4_dcac(&i4.q[lby * 4 + lbx]);
                        cb_residual(
                            cab,
                            &mut nzc,
                            &mut cbfdc,
                            iz,
                            CB_RP_LUMA_4X4,
                            true,
                            ndc,
                            &sc,
                        )
                    } else {
                        nzc[CB_NZC_CACHE[iz]] = 0;
                        0
                    };
                    nnz_loc[lby * 4 + lbx] = total as u8;
                }
            }
            nnz_commit_rows(fe, mb_x, mb_y, &nnz_loc);
            cb_emit_chroma_residual(
                cab,
                fe,
                &mut nzc,
                &mut cbfdc,
                ndc,
                true,
                plan.cbp_chroma,
                &plan.c_dc_levels,
                &plan.c_q_blocks,
                mb_x,
                mb_y,
            );
        }
    }

    cs.cbf_dc[addr] = cbfdc;
    cs.mb_nzc[addr] = cb_export_nzc(&nzc);
}

/// Chroma DC + AC residual (shared by intra I_16x16/I_NxN and inter) — matches the
/// decoder's chroma residual order. `is_intra` selects the coded_block_flag default
/// (nA=nB default to is_intra). Populates the chroma nnz grid for deblock.
fn cb_emit_chroma_residual(
    cab: &mut CabacEncoder,
    fe: &mut FrameEncoder,
    nzc: &mut [u8; 48],
    cbfdc: &mut u16,
    ndc: (Option<u16>, Option<u16>),
    is_intra: bool,
    cbp_chroma: u32,
    c_dc_levels: &[[i32; 4]; 2],
    c_q: &[[[i32; 16]; 4]; 2],
    mb_x: usize,
    mb_y: usize,
) {
    let w2 = fe.mb_w * 2;
    if cbp_chroma >= 1 {
        for i in 0..2usize {
            cb_residual(
                cab,
                nzc,
                cbfdc,
                16 + i * 4,
                CB_RP_CHROMA_DC + i,
                is_intra,
                ndc,
                &c_dc_levels[i],
            );
        }
    }
    if cbp_chroma == 2 {
        for i in 0..2usize {
            for (id4, &(bx, by)) in CHROMA_4X4_SCAN_XY.iter().enumerate() {
                let ac = scan_4x4_ac(&c_q[i][by * 2 + bx]);
                let total = cb_residual(
                    cab,
                    nzc,
                    cbfdc,
                    16 + i * 4 + id4,
                    CB_RP_CHROMA_AC + i,
                    is_intra,
                    ndc,
                    &ac,
                );
                fe.nnz_c[i][(mb_y * 2 + by) * w2 + (mb_x * 2 + bx)] = total as u8;
            }
        }
    }
}

/// CABAC all-intra slice-data coder (IDR / I-slice). Mirrors `encode_slice_data`'s
/// setup + deblock + `RefFrame` construction, but codes every MB via `plan_mb` +
/// `emit_mb_cabac_i` into a CABAC bitstream. `w` already holds the byte-aligned
/// slice header; the CABAC bytes are appended after `cabac_alignment_one_bit`.
pub(crate) fn encode_slice_data_cabac_intra(
    w: &mut BitWriter,
    cfg: &EncoderConfig,
    frame: &YuvPlanes<'_>,
    qp: u8,
    qpo: &[i32],
    aq_probe: Option<&YuvPlanes<'_>>,
) -> crate::RefFrame {
    let mut fe = FrameEncoder::new(cfg);
    fe.qp = qp;
    fe.qpc = chroma_qp(qp);
    fe.cur_qp = qp;
    if cfg.cabac_dz_div > 0 {
        fe.idz = cfg.cabac_dz_div; // CABAC-specific dead-zone override
    }
    let (sy, su, sv) = coded_source(cfg, frame);
    // Great Gate P1: the shared per-frame signal vector. Intra has no coding
    // reference, but the batch path hands the previous SOURCE frame as the AQ
    // grain probe (docs/gate-ledger.md aq-grain-veto) — without it the veto
    // fails open and the temporal signals stay cold.
    let probe_y: Option<Vec<u8>> = aq_probe.map(|f| coded_source(cfg, f).0.into_owned());
    let sig = FrameSignals::new(&sy, fe.cw, fe.mb_w, fe.mb_h, probe_y.as_deref());
    apply_screen_t8_veto(&mut fe, &sig);
    let mut aq_qp = aq_qp_map(&sig, qp, fe.aq_strength);
    apply_mbtree_qpo(&mut aq_qp, qpo); // mb-tree temporal AQ (empty = byte-identical)
    signals::harvest(&sig, 'I', qp, &signals::GateDecisions::default());
    let mut mb_qpy = enc_scratch::take_qpy();
    mb_qpy.clear();
    mb_qpy.resize(fe.mb_w * fe.mb_h, qp);

    // CABAC trellis (RDOQ): structure-adaptive. ON only for ALL-INTRA streams
    // (gop_size<=1), where each IDR is independent so trading a little distortion for
    // rate is a clean −0.5..−1.3% BD-rate win. OFF inside a GOP: there the I-frame is
    // a REFERENCE, and degrading it costs the dependent P-frames more than the I-frame
    // saves (measured ~+0.1% net) — so the safe end is a true no-op (never regresses).
    fe.rdoq_strength = if cfg.gop_size <= 1 {
        cfg.cabac_rdoq
    } else {
        0.0
    };
    // Contexts init from SliceQPY (the slice qp), init_idc unused for I, is_i = true.
    let mut cab = CabacEncoder::new_with_out(qp as i32, 0, true, enc_scratch::take_payload());
    let mut cs = CabacState::pooled(fe.mb_w * fe.mb_h);
    let total = fe.mb_w * fe.mb_h;
    // E15 round 2: slice-constant accountant knob (twin of the P/B hoists).
    let acct = crate::bitacct::enabled();

    for mb_y in 0..fe.mb_h {
        for mb_x in 0..fe.mb_w {
            let mb_idx = mb_y * fe.mb_w + mb_x;
            let mb_qp = aq_qp[mb_idx];
            fe.qp = mb_qp;
            fe.qpc = chroma_qp(mb_qp);
            let plan = plan_mb(&mut fe, mb_x, mb_y, &sy, &su, &sv);
            emit_mb_cabac_i(&mut fe, &mut cab, &mut cs, &plan, mb_x, mb_y);
            mb_qpy[mb_idx] = fe.cur_qp;
            // end_of_slice_flag (EncodeTerminate): 1 on the last MB, else 0.
            cb_term_acct(&mut cab, mb_idx + 1 == total, acct);
        }
    }
    enc_scratch::put_cs(cs);

    // Append CABAC slice data after cabac_alignment_one_bit (pad header with 1-bits).
    while !w.is_byte_aligned() {
        w.write_bit(true);
    }
    let payload = cab.into_bytes();
    w.write_aligned_bytes(&payload);
    enc_scratch::put_payload(payload);

    // Deblock the reconstruction (all-intra: BS derives from intra-ness) -> reference.
    let mut ref_id = enc_scratch::take_refid();
    ref_id.clear();
    ref_id.extend(
        fe.ref_idx_y
            .iter()
            .map(|&r| if r >= 0 { r } else { i8::MIN }),
    );
    let info = rusty_h264_common::deblock::BlockInfo {
        inter: &fe.inter_y,
        nnz: &fe.nnz_y,
        mv: &fe.mv_y,
        ref_id: &ref_id,
        mv1: &[],
        ref_id1: &[],
        w4: fe.mb_w * 4,
        t8x8: &[],
        poc0: &[],
        poc1: &[],
        bs: &[],
        kind: &[],
    };
    rusty_h264_common::deblock::filter_frame(
        &mut fe.rec_y,
        &mut fe.rec_u,
        &mut fe.rec_v,
        fe.mb_w,
        fe.mb_h,
        &mb_qpy,
        0,
        0,
        0,
        &info,
    );
    enc_scratch::put_qpy(mb_qpy);
    enc_scratch::put_refid(ref_id);
    let w4 = fe.mb_w * 4;
    crate::RefFrame {
        y: fe.rec_y,
        u: fe.rec_u,
        v: fe.rec_v,
        poc: 0,
        frame_num: 0,
        mv: fe.mv_y,
        ref_idx: fe.ref_idx_y,
        mv1: Vec::new(),
        ref_idx1: Vec::new(),
        w4,
        // Filtered lazily on first sub-pel search use (see `RefFrame::hpel`).
        hpel: crate::HpelOnce::new(),
    }
}

// ============================================================================
// CABAC P-slice entropy coding — the forward inverse of the decoder's
// decode_slice_data_cabac P-slice path. mb_skip_flag / mb_type_p / mvd (UEG3) /
// inter residual, plus intra-in-P (the shared intra body under a P mb_type prefix).
// Scope: 1 reference (no ref_idx), P_16x16/16x8/8x16 (no P_8x8/sub_mb_type) — the
// modes the encoder's decision produces.
// ============================================================================

/// UEG3 mvd suffix — inverse of `decode_ueg_mv(base)` (TU prefix at base+{0,1,2,3,3..},
/// cMax 7, then EG3 bypass). `v` is the value decode_ueg_mv returns.
fn cb_ueg_mv(cab: &mut CabacEncoder, base: usize, v: u32) {
    const P2C: [usize; 8] = [0, 1, 2, 3, 3, 3, 3, 3];
    if v == 0 {
        cab.encode_decision(base, 0);
        return;
    }
    cab.encode_decision(base, 1);
    if v <= 7 {
        // (v-1) ones then a terminating 0, at base+P2C[count] for count = 1..
        let mut count = 1;
        for _ in 0..v - 1 {
            cab.encode_decision(base + P2C[count], 1);
            count += 1;
        }
        cab.encode_decision(base + P2C[count], 0);
    } else {
        // prefix maxes out: 7 ones (count 1..7) then EG3(v-8).
        for count in 1..8 {
            cab.encode_decision(base + P2C[count], 1);
        }
        let acct = crate::bitacct::enabled();
        let tb = if acct { cab.pos() } else { 0 };
        cb_exp_bypass(cab, 3, v - 8);
        if acct {
            crate::bitacct::add(crate::bitacct::B::MvdBypass, cab.pos() - tb);
        }
    }
}

/// One `mvd` component — inverse of `parse_mvd_cabac(comp, ctx_inc)` (ctxIdxOffset
/// 40 for x, 47 for y).
fn cb_mvd(cab: &mut CabacEncoder, comp: usize, ctx_inc: usize, d: i32) {
    // One accountant-knob read per call (was three; runs per MVD component).
    let acct = crate::bitacct::enabled();
    let th = if acct { cab.pos() } else { u64::MAX };
    let base = 40 + comp * 7;
    if d == 0 {
        cab.encode_decision(base + ctx_inc, 0);
        if th != u64::MAX {
            crate::bitacct::add_mvd_sample(0, cab.pos() - th);
        }
        return;
    }
    cab.encode_decision(base + ctx_inc, 1);
    cb_ueg_mv(cab, base + 3, d.unsigned_abs() - 1); // decode adds 1 back
    let ts = if acct { cab.pos() } else { 0 };
    cab.encode_bypass((d < 0) as u32);
    if acct {
        crate::bitacct::add(crate::bitacct::B::MvdSign, cab.pos() - ts);
    }
    if th != u64::MAX {
        crate::bitacct::add_mvd_sample(d.unsigned_abs(), cab.pos() - th);
    }
}

/// `mb_skip_flag` — inverse of `parse_mb_skip_cabac` (ctx 11 P + neighbour-not-skip).
fn cb_mb_skip(cab: &mut CabacEncoder, ctx_inc: usize, skip: bool) {
    cab.encode_decision(ctx_inc, skip as u32);
}

/// `ref_idx_l0` (P) — inverse of `parse_ref_idx_cabac`. Unary binarization,
/// ctxIdxOffset 54: binIdx 0 → `ctx0` (condTermFlagA + 2·condTermFlagB, condTermFlagN =
/// neighbour partition's ref_idx > 0), binIdx 1 → 4, binIdx ≥2 → 5 (spec 9.3.3.1.1.6).
pub fn cb_ref_idx(cab: &mut CabacEncoder, ctx0: usize, r: u32) {
    const B: usize = 54;
    let mut v = r;
    let mut bin_idx = 0u32;
    loop {
        let bin = (v > 0) as u32;
        let ctx = match bin_idx {
            0 => ctx0,
            1 => 4,
            _ => 5,
        };
        cab.encode_decision(B + ctx, bin);
        if bin == 0 {
            break;
        }
        v -= 1;
        bin_idx += 1;
    }
}

/// P-slice inter `mb_type` (0/1/2 = P_L0_16x16 / P_16x8 / P_8x16) — inverse of the
/// inter branch of `parse_mb_type_p_cabac` (ctx base 11).
fn cb_mb_type_p_inter(cab: &mut CabacEncoder, mode: u8) {
    const S: usize = 11;
    cab.encode_decision(S + 3, 0); // inter (prefix bit 0)
    match mode {
        0 => {
            cab.encode_decision(S + 4, 0);
            cab.encode_decision(S + 5, 0);
        }
        3 => {
            // P_8x8 (bins "0 0 1")
            cab.encode_decision(S + 4, 0);
            cab.encode_decision(S + 5, 1);
        }
        1 => {
            cab.encode_decision(S + 4, 1);
            cab.encode_decision(S + 6, 1);
        }
        _ => {
            // mode == 2 (P_8x16)
            cab.encode_decision(S + 4, 1);
            cab.encode_decision(S + 6, 0);
        }
    }
}

/// P `sub_mb_type` CABAC — inverse of `parse_sub_mb_type_p_cabac` (ctx base 21).
/// Only 0 = P_L0_8x8 (bin "1") is emitted (8×8 sub-partitions only).
fn cb_sub_mb_type_p(cab: &mut CabacEncoder, sub_type: u8) {
    const S: usize = 21;
    // Inverse of `parse_sub_mb_type_p_cabac`: b(S)=1 → 0 (8×8);
    // b(S)=0, b(S+1)=0 → 1 (8×4); b(S)=0, b(S+1)=1 → 3−b(S+2) (2 = 4×8, 3 = 4×4).
    match sub_type {
        0 => cab.encode_decision(S, 1),
        1 => {
            cab.encode_decision(S, 0);
            cab.encode_decision(S + 1, 0);
        }
        2 => {
            cab.encode_decision(S, 0);
            cab.encode_decision(S + 1, 1);
            cab.encode_decision(S + 2, 1);
        }
        3 => {
            cab.encode_decision(S, 0);
            cab.encode_decision(S + 1, 1);
            cab.encode_decision(S + 2, 0);
        }
        _ => unreachable!("invalid P sub_mb_type"),
    }
}

/// P-slice intra `mb_type` prefix — inverse of the intra branch of
/// `parse_mb_type_p_cabac` (ctx base 11). Carries the I_16x16 pred-mode/cbp exactly
/// like the I-slice mb_type, so the shared intra body re-emits neither.
fn cb_mb_type_p_intra(cab: &mut CabacEncoder, plan: &MbPlan) {
    const S: usize = 11;
    cab.encode_decision(S + 3, 1); // intra (prefix bit 1)
    if plan.use_i4 {
        cab.encode_decision(S + 6, 0); // I_4x4
        return;
    }
    cab.encode_decision(S + 6, 1); // I_16x16
    cab.encode_terminate(false); // not I_PCM
    cab.encode_decision(S + 7, plan.i16_cbp15 as u32);
    if plan.cbp_chroma != 0 {
        cab.encode_decision(S + 8, 1);
        cab.encode_decision(S + 8, (plan.cbp_chroma == 2) as u32);
    } else {
        cab.encode_decision(S + 8, 0);
    }
    cab.encode_decision(S + 9, (plan.i16_mode as u32 >> 1) & 1);
    cab.encode_decision(S + 9, plan.i16_mode as u32 & 1);
}

/// P-slice partition layout: `(part_idx, z-blocks)` per motion partition (matches
/// the decoder's `part!` invocations). part_idx = the partition's top-left z-block
/// (its `CACHE30` slot for the mvd ctxInc); z-blocks = every 4x4 it covers.
fn p_partition_layout(mode: u8) -> &'static [(usize, &'static [usize])] {
    match mode {
        1 => &[
            (0, &[0, 1, 2, 3, 4, 5, 6, 7]),
            (8, &[8, 9, 10, 11, 12, 13, 14, 15]),
        ],
        2 => &[
            (0, &[0, 1, 2, 3, 8, 9, 10, 11]),
            (4, &[4, 5, 6, 7, 12, 13, 14, 15]),
        ],
        // P_8x8: four 8×8 quads (z-order 4×4 blocks), part order == inter_partitions(3).
        3 => &[
            (0, &[0, 1, 2, 3]),
            (4, &[4, 5, 6, 7]),
            (8, &[8, 9, 10, 11]),
            (12, &[12, 13, 14, 15]),
        ],
        _ => &[(0, &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15])],
    }
}

/// Sub-partition geometry per `sub_mb_type` (P): pixel rects within an 8×8, in
/// decode order. VERBATIM the decoder's `sub_mb_partitions` (spec Table 7-17) —
/// the decoder's parse is the contract this encoder inverts (Great Gate P3.3).
fn sub_mb_partitions_p(sub_type: u8) -> &'static [(usize, usize, usize, usize)] {
    match sub_type {
        0 => &[(0, 0, 8, 8)],
        1 => &[(0, 0, 8, 4), (0, 4, 8, 4)],
        2 => &[(0, 0, 4, 8), (4, 0, 4, 8)],
        _ => &[(0, 0, 4, 4), (4, 0, 4, 4), (0, 4, 4, 4), (4, 4, 4, 4)],
    }
}

/// Per-sub-partition `(part_idx, zblocks)` for the mvd cache/context emission —
/// the sub-partition refinement of `p_partition_layout(3)`'s quads. `part_idx` =
/// the sub-partition's first 4×4 in MB z-order (its top-left), `zblocks` = the
/// 4×4s it covers; quad `p8`'s blocks are `4·p8 ..= 4·p8+3` in z-order
/// (0=TL, 1=TR, 2=BL, 3=BR within the quad).
fn p_sub_partition_layout(p8: usize, sub_type: u8) -> &'static [(usize, &'static [usize])] {
    const T: [[&[(usize, &[usize])]; 4]; 4] = [
        [
            &[(0, &[0, 1, 2, 3])],
            &[(0, &[0, 1]), (2, &[2, 3])],
            &[(0, &[0, 2]), (1, &[1, 3])],
            &[(0, &[0]), (1, &[1]), (2, &[2]), (3, &[3])],
        ],
        [
            &[(4, &[4, 5, 6, 7])],
            &[(4, &[4, 5]), (6, &[6, 7])],
            &[(4, &[4, 6]), (5, &[5, 7])],
            &[(4, &[4]), (5, &[5]), (6, &[6]), (7, &[7])],
        ],
        [
            &[(8, &[8, 9, 10, 11])],
            &[(8, &[8, 9]), (10, &[10, 11])],
            &[(8, &[8, 10]), (9, &[9, 11])],
            &[(8, &[8]), (9, &[9]), (10, &[10]), (11, &[11])],
        ],
        [
            &[(12, &[12, 13, 14, 15])],
            &[(12, &[12, 13]), (14, &[14, 15])],
            &[(12, &[12, 14]), (13, &[13, 15])],
            &[(12, &[12]), (13, &[13]), (14, &[14]), (15, &[15])],
        ],
    ];
    T[p8][sub_type as usize]
}

/// Emit one motion partition's `mvd` (x,y) and splat it into the 30-entry cache +
/// per-MB raster mvd/ref grids — inverse of the decoder's `parse_mvd_partition`.
fn cb_emit_mvd_partition(
    cab: &mut CabacEncoder,
    part_idx: usize,
    zblocks: &[usize],
    mvdc: &mut [[i16; 2]; 30],
    refc: &mut [i8; 30],
    mmvd: &mut [[i16; 2]; 16],
    mref: &mut [i8; 16],
    mvd: (i32, i32),
    ref_idx: i8, // the partition's ref_idx_l0 (0 for single-ref) — stored for neighbour context
) {
    let s = CB_CACHE30[part_idx];
    let ctx = |comp: usize| -> usize {
        let mut a = 0i32;
        if refc[s - 6] >= 0 {
            a += mvdc[s - 6][comp].unsigned_abs() as i32;
        }
        if refc[s - 1] >= 0 {
            a += mvdc[s - 1][comp].unsigned_abs() as i32;
        }
        if a >= 3 {
            1 + (a > 32) as usize
        } else {
            0
        }
    };
    cb_mvd(cab, 0, ctx(0), mvd.0);
    cb_mvd(cab, 1, ctx(1), mvd.1);
    let (mx, my) = (mvd.0 as i16, mvd.1 as i16);
    for &zb in zblocks {
        mvdc[CB_CACHE30[zb]] = [mx, my];
        refc[CB_CACHE30[zb]] = ref_idx;
        mmvd[CB_G_SCAN4[zb]] = [mx, my];
        mref[CB_G_SCAN4[zb]] = ref_idx;
    }
}

/// Emit one planned INTER macroblock as CABAC (P-slice, mb_skip_flag already coded
/// as 0). `mode`/`parts` + `plan` from `plan_inter_mb`. 1-ref: no ref_idx.
fn emit_mb_cabac_p_inter(
    fe: &mut FrameEncoder,
    cab: &mut CabacEncoder,
    cs: &mut CabacState,
    mode: u8,
    plan: &InterPlan,
    mb_x: usize,
    mb_y: usize,
    num_refs: usize,
) {
    let mb_w = fe.mb_w;
    let addr = mb_y * mb_w + mb_x;
    let top = if mb_y > 0 { Some(addr - mb_w) } else { None };
    let left = if mb_x > 0 { Some(addr - 1) } else { None };

    // Bit accountant (instrument #6): each tap is a `pos()` delta — exact coded
    // bits for that element — behind an atomic-bool check when disabled.
    let acct = crate::bitacct::enabled();
    let mut t0 = if acct { cab.pos() } else { 0 };
    cb_mb_type_p_inter(cab, mode);
    // P_8x8: four sub_mb_type, spec order before ref_idx/mvd ([0;4] = all 8×8 =
    // the pre-P3.3 emission, byte-identical).
    if mode == 3 {
        for &st in &plan.sub_types {
            cb_sub_mb_type_p(cab, st);
        }
    }
    if acct {
        crate::bitacct::add(crate::bitacct::B::MbType, cab.pos() - t0);
        t0 = cab.pos();
    }

    // ---- mb_pred (spec 7.3.5.1): all ref_idx_l0 FIRST, then all mvd_l0 ----
    let mut mvdc = [[0i16; 2]; 30];
    let mut refc = [-1i8; 30];
    cb_fill_inter_cache(
        &cs.mb_ref, &cs.mb_mvd, &mut refc, &mut mvdc, top, left, addr, mb_w,
    );
    let mut mmvd = [[0i16; 2]; 16];
    let mut mref = [0i8; 16];
    let layout = p_partition_layout(mode);
    // Phase 1: ref_idx_l0 per partition, only when the slice has >1 active reference.
    // Update refc after each so a later partition's ref context sees the earlier one.
    if num_refs > 1 {
        for (part, &(part_idx, zblocks)) in layout.iter().enumerate() {
            let r = plan.plan_refs[part];
            let s = CB_CACHE30[part_idx];
            let ctx0 = (refc[s - 1] > 0) as usize + 2 * (refc[s - 6] > 0) as usize;
            cb_ref_idx(cab, ctx0, r as u32);
            for &zb in zblocks {
                refc[CB_CACHE30[zb]] = r as i8;
            }
        }
    }
    if acct {
        crate::bitacct::add(crate::bitacct::B::RefIdx, cab.pos() - t0);
        t0 = cab.pos();
    }
    // Phase 2: mvd per partition (carries the ref into refc/mref for neighbour context).
    if mode == 3 && plan.sub_types != [0u8; 4] {
        // Sub-partitioned P_8x8: one mvd per sub-partition, decode order, the
        // quad ref for context (P3.3 -- layout from `p_sub_partition_layout`).
        let mut k = 0usize;
        for p8 in 0..4usize {
            for &(part_idx, zblocks) in p_sub_partition_layout(p8, plan.sub_types[p8]) {
                cb_emit_mvd_partition(
                    cab,
                    part_idx,
                    zblocks,
                    &mut mvdc,
                    &mut refc,
                    &mut mmvd,
                    &mut mref,
                    plan.mvds[k],
                    plan.plan_refs[p8] as i8,
                );
                k += 1;
            }
        }
    } else {
        for (part, &(part_idx, zblocks)) in layout.iter().enumerate() {
            cb_emit_mvd_partition(
                cab,
                part_idx,
                zblocks,
                &mut mvdc,
                &mut refc,
                &mut mmvd,
                &mut mref,
                plan.mvds[part],
                plan.plan_refs[part] as i8,
            );
        }
    }
    if acct {
        crate::bitacct::add(crate::bitacct::B::Mvd, cab.pos() - t0);
    }
    cs.mb_mvd[addr] = cb_sat_mvd(mmvd);
    cs.mb_ref[addr] = mref;
    cs.cat[addr] = 100;
    let allow8 = plan.sub_types == [0u8; 4];
    cb_emit_inter_residual(fe, cab, cs, plan, mb_x, mb_y, addr, top, left, allow8);
}

/// Inter cbp + residual (is_intra = false) — shared by P and B inter MBs. Maintains
/// cs.mb_cbp/cbf_dc/mb_nzc/last_delta_qp + fe.nnz_y.
fn cb_emit_inter_residual(
    fe: &mut FrameEncoder,
    cab: &mut CabacEncoder,
    cs: &mut CabacState,
    plan: &InterPlan,
    mb_x: usize,
    mb_y: usize,
    addr: usize,
    top: Option<usize>,
    left: Option<usize>,
    allow8: bool,
) {
    let w4 = fe.mb_w * 4;
    let cbp = plan.cbp;
    let (cbp_luma, cbp_chroma) = (cbp & 15, cbp >> 4);
    let acct = crate::bitacct::enabled();
    let mut t0 = if acct { cab.pos() } else { 0 };
    cb_cbp(
        cab,
        top.map(|a| cs.mb_cbp[a]),
        left.map(|a| cs.mb_cbp[a]),
        cbp,
    );
    if acct {
        crate::bitacct::add(crate::bitacct::B::Cbp, cab.pos() - t0);
    }
    cs.mb_cbp[addr] = cbp as u8;
    // transform_size_8x8_flag, INTER position: after cbp, before mb_qp_delta, and
    // only when luma carries coefficients (spec 7.3.5). Contrast the I_NxN position,
    // which is before the pred modes -- the two are different points in the syntax,
    // which is why this needs its own write rather than a shared helper.
    // `plan_inter_mb` enforces the noSubMbPartSizeLessThan8x8Flag half of the
    // condition by never selecting t8x8 alongside a sub-8x8 split.
    // `allow8` is the spec's noSubMbPartSizeLessThan8x8Flag, mirroring the decoder's
    // own `allow8`. It gates the flag's PRESENCE, not just its value: omitting it wrote
    // one extra bin on every sub-8x8-split P_8x8 with luma coefficients and desynced
    // the stream some macroblocks later (ffmpeg reported it as a bogus intra mode).
    let t8_present = cbp_luma > 0 && fe.transform_8x8 && allow8;
    if t8_present {
        let ta = left.map_or(0, |x| cs.mb_t8x8[x] as usize);
        let tb = top.map_or(0, |x| cs.mb_t8x8[x] as usize);
        cab.encode_decision(399 + ta + tb, plan.t8x8 as u32);
    }
    // ABSENT means INFERRED ZERO, and the decoder stores that zero as the neighbour
    // ctxIdxInc for later macroblocks. Storing `plan.t8x8` here regardless of
    // presence would drift our context from the decoder's on any MB where the flag
    // was suppressed -- a desync that only shows up MBs later.
    cs.mb_t8x8[addr] = t8_present && plan.t8x8;
    let mut nzc = cb_build_nzc(&cs.mb_nzc, top, left);
    let mut cbfdc = 0u16;
    let ndc = (top.map(|a| cs.cbf_dc[a]), left.map(|a| cs.cbf_dc[a]));

    if cbp == 0 {
        cs.last_delta_qp = 0;
        for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
            fe.nnz_y[(mb_y * 4 + lby) * w4 + (mb_x * 4 + lbx)] = 0;
        }
    } else {
        let delta = fe.qp_delta();
        if acct {
            t0 = cab.pos();
        }
        cb_mb_qp_delta(cab, &mut cs.last_delta_qp, delta);
        if acct {
            crate::bitacct::add(crate::bitacct::B::QpDelta, cab.pos() - t0);
            t0 = cab.pos();
        }
        if plan.t8x8 {
            // ctxBlockCat 5: ONE 64-coefficient block per 8x8, no coded_block_flag.
            for b8 in 0..4usize {
                let (b8x, b8y) = (b8 % 2, b8 / 2);
                let total = if cbp_luma & (1 << b8) != 0 {
                    let scan8 = scan_8x8_fwd(&plan.q8[b8]);
                    cb_residual(
                        cab,
                        &mut nzc,
                        &mut cbfdc,
                        b8 * 4,
                        CB_RP_LUMA_8X8,
                        false,
                        ndc,
                        &scan8,
                    )
                } else {
                    for k in 0..4 {
                        nzc[CB_NZC_CACHE[b8 * 4 + k]] = 0;
                    }
                    0
                };
                for sy in 0..2 {
                    for sx in 0..2 {
                        fe.nnz_y[(mb_y * 4 + b8y * 2 + sy) * w4 + (mb_x * 4 + b8x * 2 + sx)] =
                            total as u8;
                    }
                }
            }
        } else {
            for id8 in 0..4usize {
                for id4 in 0..4usize {
                    let iz = id8 * 4 + id4;
                    let (lbx, lby) = LUMA_4X4_SCAN_XY[iz];
                    let (bx, by) = (mb_x * 4 + lbx, mb_y * 4 + lby);
                    let total = if cbp_luma & (1 << id8) != 0 {
                        let sc = scan_4x4_dcac(&plan.q_blocks[lby * 4 + lbx]);
                        cb_residual(
                            cab,
                            &mut nzc,
                            &mut cbfdc,
                            iz,
                            CB_RP_LUMA_4X4,
                            false,
                            ndc,
                            &sc,
                        )
                    } else {
                        nzc[CB_NZC_CACHE[iz]] = 0;
                        0
                    };
                    fe.nnz_y[by * w4 + bx] = total as u8;
                }
            }
        }
        if acct {
            crate::bitacct::add(crate::bitacct::B::ResidLuma, cab.pos() - t0);
            t0 = cab.pos();
        }
        cb_emit_chroma_residual(
            cab,
            fe,
            &mut nzc,
            &mut cbfdc,
            ndc,
            false,
            cbp_chroma,
            &plan.c_dc_levels,
            &plan.c_q,
            mb_x,
            mb_y,
        );
        if acct {
            crate::bitacct::add(crate::bitacct::B::ResidChroma, cab.pos() - t0);
        }
    }
    cs.cbf_dc[addr] = cbfdc;
    cs.mb_nzc[addr] = cb_export_nzc(&nzc);
}

/// Emit one planned INTRA macroblock inside a P-slice: the P mb_type prefix (which
/// carries the I_16x16 pred-mode/cbp) then the shared intra body.
fn emit_mb_cabac_p_intra(
    fe: &mut FrameEncoder,
    cab: &mut CabacEncoder,
    cs: &mut CabacState,
    plan: &MbPlan,
    mb_x: usize,
    mb_y: usize,
) {
    let mb_w = fe.mb_w;
    let addr = mb_y * mb_w + mb_x;
    let top = if mb_y > 0 { Some(addr - mb_w) } else { None };
    let left = if mb_x > 0 { Some(addr - 1) } else { None };
    let acct = crate::bitacct::enabled();
    let t0 = if acct { cab.pos() } else { 0 };
    cb_mb_type_p_intra(cab, plan);
    emit_intra_body_cabac(fe, cab, cs, plan, mb_x, mb_y, addr, top, left);
    if acct {
        // Whole intra MB (mb_type + modes + its residual) — intra MBs are ~5% of
        // P-frame MBs; splitting them further is a separate tap set.
        crate::bitacct::add(crate::bitacct::B::IntraBody, cab.pos() - t0);
    }
}

/// Emit a P_Skip macroblock's `mb_skip_flag = 1` and update neighbour state. The
/// motion grid was committed by `commit_skip`; the mvd/ref cache is LEFT at its
/// init (-1 ref) — matching the decoder, which does not touch mb_mvd/mb_ref for a
/// P_Skip (so a skip neighbour contributes nothing to a later mvd ctxInc).
fn emit_p_skip_cabac(
    cab: &mut CabacEncoder,
    cs: &mut CabacState,
    addr: usize,
    top: Option<usize>,
    left: Option<usize>,
) {
    let sctx = 11
        + left.map_or(0, |a| (!cs.mb_skip[a]) as usize)
        + top.map_or(0, |a| (!cs.mb_skip[a]) as usize);
    let acct = crate::bitacct::enabled();
    let t0 = if acct { cab.pos() } else { 0 };
    cb_mb_skip(cab, sctx, true);
    if acct {
        crate::bitacct::add(crate::bitacct::B::SkipFlag, cab.pos() - t0);
    }
    cs.mb_skip[addr] = true;
    cs.cat[addr] = 100;
    cs.last_delta_qp = 0;
}

/// CABAC P-slice data coder. Mirrors `encode_slice_data`'s decision (P_Skip check +
/// fast/quality inter-vs-intra RD) exactly — only the emit differs (per-MB
/// mb_skip_flag + CABAC syntax + per-MB end_of_slice terminate).
/// Median source macroblock variance for a frame — the TEXTURE dispatch signal for
/// the ME lambda scale.
///
/// Raising the ME rate term biases the search toward cheaper motion vectors, which
/// costs texture detail. SSIM is texture-sensitive where PSNR is not, so on maximum-
/// texture content a higher lambda improves BD-PSNR while REGRESSING BD-SSIM.
/// Measured median MB variance vs BD-SSIM at lme 1.8:
///   akiyo 61 (-0.20), foreman 219 (-0.61), city 300 (-0.89), bus 454 (-0.01),
///   football 583 (-1.03), **mobile 1554 (+0.45 LOSS)**
/// The one loser carries 2.7x the texture of the next clip, so the split is wide.
///
/// Computed from the SOURCE, so it is available in-slice for BOTH P and B and needs
/// no cross-frame state — unlike the B-only direct-win rate, which cannot gate a knob
/// that both encoders read and whose carry-forward would be nondeterministic under
/// frame-parallel encode. Subsampled 2x2 (16x fewer loads) — a median over ~400
/// macroblocks does not need every pixel.
// `frame_median_mb_var` lives in `crate::signals` (Great Gate P1) — read through
// `FrameSignals::median_var`. NOTE its estimator caveat there: it is deliberately
// a DIFFERENT formula from `mb_variance` (the lme clip table was calibrated on it).

/// Texture-dispatched ME lambda scale: the calibrated high value on normal content,
/// the conservative shipped value on maximum-texture content where it costs SSIM.
/// Cached `RFF_LME_Q` env override for [`EncoderConfig::tune_lme_q`] (one binary,
/// N sweep arms — and never an `env::var` in a per-frame path).
fn lme_q_env() -> Option<f64> {
    static E: rusty_h264_common::once::OnceLock<Option<f64>> =
        rusty_h264_common::once::OnceLock::new();
    *E.get_or_init(|| rusty_h264_common::knob("RFF_LME_Q").and_then(|v| v.parse().ok()))
}

fn me_lambda_scale(cfg: &EncoderConfig, sig: &FrameSignals, per_mb_tex: bool) -> f64 {
    let hi = match cfg.tune_lme_hi {
        Some(v) if v > 0.0 => v,
        _ => return cfg.cabac_lambda_scale,
    };
    // TWO terms, and each is justified by a DIFFERENT clip it must classify —
    // neither alone is sufficient, which is why the texture-only version shipped
    // disabled.
    //
    //   clip      global-MC resid   median var   wants hi lme?
    //   akiyo          1.51             61          yes
    //   foreman        9.59            219          yes
    //   city          12.44            300          yes
    //   football      24.83            583          yes
    //   TEMPETE       10.52            746          NO  <- caught by TEXTURE (650)
    //   MOBILE        19.50           1554          NO  <- caught by TEXTURE
    //   BUS           27.47            454          NO  <- caught by MOTION (20)
    //
    // mobile is maximum texture: a higher ME rate term biases toward cheaper MVs,
    // costing texture detail, and SSIM is texture-sensitive where PSNR is not.
    // bus is fast GLOBAL motion (a pan): its cost surface is dominated by one
    // global vector, so pushing the rate term drags MVs off it. football is
    // chaotic LOCAL motion at similar texture and WANTS the high value, so texture
    // cannot separate the two — the global-MC residual can, and in the opposite
    // direction, which is exactly why the pair works where either alone fails.
    // Great Gate P1 (`tune_lme_q`): when the caller applies the texture veto PER MB
    // by percentile, skip the frame-median form here so the two never stack — the
    // motion veto below stays frame-level in both forms.
    if !per_mb_tex && sig.median_var() >= cfg.tune_lme_tex_thresh.unwrap_or(650) {
        return cfg.cabac_lambda_scale;
    }
    if sig.has_ref() {
        let mot = cfg.tune_lme_motion_thresh.unwrap_or(26.0);
        if sig.gmc_residual() >= mot {
            return cfg.cabac_lambda_scale;
        }
    }
    hi
}

pub(crate) fn encode_slice_data_cabac_p(
    w: &mut BitWriter,
    cfg: &EncoderConfig,
    frame: &YuvPlanes<'_>,
    qp: u8,
    refs: &[crate::RefFrame],
    qpo: &[i32],
    wp: &[(i32, i32)],
) -> crate::RefFrame {
    let mut fe = FrameEncoder::new(cfg);
    fe.wp = wp.to_vec();
    fe.qp = qp;
    fe.qpc = chroma_qp(qp);
    fe.cur_qp = qp;
    if cfg.cabac_dz_div > 0 {
        fe.idz = cfg.cabac_dz_div; // CABAC-specific dead-zone override
    }
    // Inter trellis (opt-in, Great Gate P2): P slices are REFERENCES — see
    // `cabac_rdoq_p`'s structure-adaptive caveat. 0 = off, byte-identical.
    fe.rdoq_strength = 0.0; // set below, once `sig` exists (content-gated)
                            // RD P_Skip threshold arm (P3 item 2): `RFF_RDSKIP_T` overrides for sweep
                            // arms and CLI conformance runs, mirroring `RFF_BSKIP_T`. Unset = config.
    {
        static T: rusty_h264_common::once::OnceLock<Option<f64>> =
            rusty_h264_common::once::OnceLock::new();
        if let Some(t) =
            *T.get_or_init(|| rusty_h264_common::knob("RFF_RDSKIP_T").and_then(|v| v.parse().ok()))
        {
            fe.rd_skip = t > 0.0;
            fe.rd_skip_fast_t = t;
        }
    }
    let (sy, su, sv) = coded_source(cfg, frame);
    // Great Gate P1: ONE lazy signal vector per frame. The lme motion term and
    // the me_wide coherence gate below both read `gmc_residual` — memoization
    // collapses what used to be TWO full global-MC probes into one.
    let sig = FrameSignals::new(&sy, fe.cw, fe.mb_w, fe.mb_h, refs.first().map(|r| &r.y[..]));
    apply_screen_t8_veto(&mut fe, &sig);
    // CONTENT GATE for P-slice trellis. Flat-on is refuted (loses on 4 of 6 clips at
    // strength 32); grain and screen are the two classes where it wins hugely
    // (-30.12% / -12.11% BD-SSIM). The discriminator is exactly the reference-structure
    // argument in `cabac_rdoq_p`'s doc: a P frame is a reference, so trading its
    // distortion for rate propagates -- UNLESS what is being traded away is noise
    // (grain) or flat runs (screen), which propagate nothing.
    fe.rdoq_strength = if sig.grain_signature() || sig.is_screen() {
        cfg.cabac_rdoq_p
    } else {
        0.0
    };
    let lambda = 0.85 * fe.tune_lambda_scale * crate::fastmath::lambda_qp(qp);
    // Hoisted to SLICE level: the texture median is O(pixels) and the site below
    // sits inside the macroblock loop, where recomputing it would be quadratic.
    // Great Gate P1 (opt-in, BD-gate pending — great-gate.md §6 P2): `tune_lme_q` /
    // `RFF_LME_Q` converts the lme TEXTURE veto from an absolute frame-median test
    // (which cannot separate bus 454 from football 583 — they want opposite values)
    // to the population-shaped per-MB form: THIS frame's top-q highest-variance MBs
    // take the conservative scale individually. None/unset = frame form, byte-identical.
    // SHAPE-RD TEXTURE GUARD (Great Gate). shape-rd wins on 12 of 13 clips on
    // BOTH metrics and on 13 of 13 on PSNR; the lone loser is mobile (+1.99
    // BD-SSIM while WINNING -0.44 PSNR), the natural-corpus texture extreme.
    //
    // THIS IS A CONSERVATIVE GUARD, NOT A CAUSAL MODEL -- read before touching.
    // Four candidate mechanisms were tested and ALL FOUR REFUTED (gate-ledger):
    // texture-causes-the-loss is refuted by `maxtex_plaid`, a SYNTHESIZED clip
    // at median_var 2583 -- above mobile's 1494 -- which WINS -1.87 unvetoed.
    // dcfrac separated only under an unrelated lambda config. AQ accounts for
    // at most a fifth (aq=0 still loses +1.64). Chroma-weighting the RD SSD
    // moves it MONOTONICALLY THE WRONG WAY (+2.11 at weight 0).
    //
    // So median_var does not explain the loss; it merely BOUNDS it. Within the
    // 24-clip natural truth table exactly one clip exceeds this threshold and
    // that clip regresses, so the guard can only ever forgo a win, never create
    // a loss. It costs `maxtex_plaid` its -1.87 -- accepted, because synthetic
    // single-frequency texture is not content anyone ships. Threshold sits in
    // the open natural gap (highest winner foreman_qcif 793 -> mobile 1494).
    // Delete this guard the moment a real mechanism is found.
    let shape_rd_tex_veto = sig.median_var() > shape_rd_tex_max();
    let lme_q = lme_q_env().or(cfg.tune_lme_q).filter(|&q| q > 0.0);
    let lme_scale = me_lambda_scale(cfg, &sig, lme_q.is_some());
    let lme_mb_thresh: Option<i64> = match lme_q {
        Some(q) if lme_scale != cfg.cabac_lambda_scale => Some(sig.var_percentile_thresh(q)),
        _ => None,
    };
    let num_refs = refs.len();
    // me_wide content gate (pure-pan → global-MC residual ≈ 0 → off; see encode_slice_data).
    if fe.me_wide && !refs.is_empty() {
        let coh = sig.gmc_residual();
        if rusty_h264_common::knob("RFF_ME_COH_DBG").is_some() {
            eprintln!("ME_COH qp{qp} residual={coh:.2}");
        }
        if coh < fe.me_wide_coh {
            fe.me_wide = false;
        }
    }
    // me_wide HEAD-ROOM GATE (the dispatcher the truth table asked for). The rescue
    // only pays where a wide search actually beats a predictor-local one; measure
    // that directly per frame and route the frame. `RFF_ME_HR` sets the threshold
    // (percent); 0 disables the gate and restores the always-on behaviour.
    // Skip the probe entirely when the gate is disabled: it must not tax the
    // default path (`RFF_ME_HR=0`), which stays byte-identical to pre-gate output.
    if fe.me_wide && !refs.is_empty() && (me_wide_hr_thresh() > 0.0 || me_wide_hr_dbg()) {
        let hr = sig.headroom();
        if me_wide_hr_dbg() {
            eprintln!("ME_HR qp{qp} headroom={hr:.2}");
        }
        if me_wide_hr_thresh() > 0.0 && hr < me_wide_hr_thresh() {
            fe.me_wide = false;
        }
    }
    // Track-B B2 DISPATCH — same probe/route as the CAVLC driver above (the two
    // drivers must stay in lockstep; the U5-struct bug came from patching one).
    if me_sadfp_mode() == 1 && !fe.fast && !refs.is_empty() {
        let (mg, dc) = sig.mgain_dc();
        if me_sadt_dbg() {
            eprintln!("B2_MG qp{qp} mgain={mg:.3} dcfrac={dc:.3}");
        }
        fe.sadfp = mg >= me_sadt() && dc <= me_sad_dcmax();
        // H-24: the mv-cost SHAPE rides the same probe (its BD sign-flip tracks
        // motion for the same physical reason B2's does).
        if mv_smooth_mode() == 1 {
            // dcfrac veto mirrors B2's: crew-class FLASH frames satisfy the mgain
            // test but SAD/mvd statistics mislead there (H-13/H-26).
            fe.mv_smooth = mg >= mv_smooth_t() && dc <= me_sad_dcmax();
        }
        // H-13: near-static frames skip the split searches entirely.
        let smg = split_mg();
        if smg > 0.0 {
            fe.do_splits = mg >= smg;
        }
    }
    if fe.satd_q > 0.0 {
        fe.satd_var_thresh = sig.var_percentile_thresh(fe.satd_q);
    }
    let mut aq_qp = aq_qp_map(&sig, qp, fe.aq_strength);
    apply_mbtree_qpo(&mut aq_qp, qpo); // mb-tree temporal AQ (empty = byte-identical)
    signals::harvest(
        &sig,
        'P',
        qp,
        &signals::GateDecisions {
            me_wide: fe.me_wide,
            sadfp: fe.sadfp,
            mv_smooth: fe.mv_smooth,
            do_splits: fe.do_splits,
            lme_scale,
            satd_thresh: fe.satd_var_thresh,
        },
    );
    let mut mb_qpy = enc_scratch::take_qpy();
    mb_qpy.clear();
    mb_qpy.resize(fe.mb_w * fe.mb_h, qp);

    // Same online free-skip dispatch as the CAVLC path gates the greedy P_Skip on
    // (see `encode_slice_data`): measured over the frame so far, within-frame so it
    // stays deterministic under GOP-parallel encode.
    // Online split-payoff census (see `sub8_pay_cfg`): `seen` = macroblocks that
    // ran the split search, `paid` = those whose split survived the RD trial.
    let (sub8_minpay, sub8_learn) = sub8_pay_cfg();
    let (mut sub8_seen, mut sub8_gain) = (0usize, 0f64);
    let mut sub8_paying = true;
    let mut greedy_free = 0usize;
    let mut greedy_seen = 0usize;
    let mut greedy_on = fe.greedy_min_free == 0;
    let greedy_learn = (fe.mb_w * fe.mb_h / 8).max(64);
    let mut cab = CabacEncoder::new_with_out(
        qp as i32,
        cfg.cabac_init_idc,
        false,
        enc_scratch::take_payload(),
    ); // P-slice
    let mut cs = CabacState::pooled(fe.mb_w * fe.mb_h);
    let total = fe.mb_w * fe.mb_h;

    // E15 hammer (inline-execution.md 11.8): the asm read showed this driver's
    // "1,730 un-vectorised xmm ops" are 24% SEH unwind DIRECTIVES and ~1,200
    // scalar f64 moves/compares of RD currency spilled around 461 calls - there
    // is no pixel loop to vectorise. The lever is Law 5: every line below was
    // re-read per MACROBLOCK (several per CODED macroblock) though each is
    // slice-constant. All are cached atomics or pure values - hoisting is
    // byte-identical by the same process-constant contract the pre-loop gates
    // above already assume.
    let lme_base = rusty_h264_common::fmath::sqrt(lambda);
    let grain = sig.grain_signature();
    let split_t_v = split_t();
    let shape_rd = shape_rd_on().unwrap_or(cfg.tune_shape_rd);
    let want_split_knob = sub8x8_split_on() || cfg.tune_sub8x8_split;
    let sub8_gveto = sub8_grain_veto_on();
    let sub8_rd = sub8_rd_on() || cfg.tune_sub8_rd;
    let use_rd_frame = (intra_rd_on() || cfg.tune_intra_rd) && (!intra_rd_grain_gate() || grain);
    let acct = crate::bitacct::enabled();
    let greedy_zero = fe.greedy_min_free == 0;
    // lme tex veto operands, resolved once (mb_vars is built whenever the
    // threshold exists; the map keeps the never-built path allocation-free).
    let lme_vars = lme_mb_thresh.map(|t| (sig.mb_vars(), t));
    // REFUTED, do not retry (E15 r2): binding `&aq_qp[..total]` removed ZERO
    // of the loop's bounds checks (43 -> 43) - the `mb_idx = mb_y*w + mb_x`
    // shape defeats the slice-length hint here just as runtime-extent row
    // slices did in the decoder campaign.

    // ② residue naming: the CABAC driver's MB loop was untapped (the CAVLC twin
    // has this scope) — `EncMbLoop − Σ(per-MB stages)` is the per-MB glue.
    let _g_loop = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncMbLoop);
    for mb_y in 0..fe.mb_h {
        for mb_x in 0..fe.mb_w {
            let mb_idx = mb_y * fe.mb_w + mb_x;
            let addr = mb_idx;
            let top = if mb_y > 0 { Some(addr - fe.mb_w) } else { None };
            let left = if mb_x > 0 { Some(addr - 1) } else { None };
            let mb_qp = aq_qp[mb_idx];
            fe.qp = mb_qp;
            fe.qpc = chroma_qp(mb_qp);
            // LAMBDA MUST MATCH THE QP THIS MB IS ACTUALLY QUANTIZED AT. The
            // slice `lambda` is built ONCE from the FRAME qp, but AQ (default
            // strength 1.0) and mb-tree rewrite `fe.qp` per macroblock on the
            // line above. Every RD site below compares SSD_recon against
            // lambda*bits, so using the frame lambda misprices rate by
            // 2^((qp_frame-qp_mb)/3) -- and AQ moves QP FURTHEST on the
            // highest-variance macroblocks, so the error is largest exactly
            // where the shape/split decision is hardest. Same family as the
            // SATD-vs-recon-SSE wrong-proxy bug that this campaign already
            // fixed twice: the currency has to match the decision.
            let lam_mb = if cfg.tune_rd_lambda_mb {
                0.85 * fe.tune_lambda_scale * crate::fastmath::lambda_qp(fe.qp)
            } else {
                lambda
            };

            // ---- P_Skip check (identical logic to encode_slice_data) ----
            let mut inter: Option<InterChoice> = None;
            // P3.3: sub_mb_type per 8x8 quad when `inter` is mode 3 ([0;4] = all
            // 8x8). A companion local rather than an InterChoice field so every
            // other constructor site stays untouched.
            let mut inter_subs = [0u8; 4];
            let mut did_skip = false;
            if num_refs > 0 {
                let mv_skip = fe.skip_mv(mb_x, mb_y);
                let skip_y = fe.skip_predict_luma(refs, mb_x, mb_y, mv_skip);
                let luma_free = fe.skip_luma_is_free(&sy, mb_x, mb_y, &skip_y);
                // (fast && !luma_free) never READS the chroma prediction — the
                // old zero-array arm was a dead 128-byte fill per MB (E15 r2 W18).
                let skip_c: Option<[[u8; 64]; 2]> = if luma_free || !fe.fast {
                    Some(fe.skip_predict_chroma(refs, mb_x, mb_y, mv_skip))
                } else {
                    None
                };
                let is_free = luma_free
                    && skip_c
                        .as_ref()
                        .is_some_and(|c| fe.skip_chroma_is_free(&su, &sv, mb_x, mb_y, c));
                let skip_sad = if fe.fast {
                    0
                } else {
                    let (lx, ly) = (mb_x * 16, mb_y * 16);
                    let mut s = 0u32;
                    for dy in 0..16 {
                        let src = &sy[(ly + dy) * fe.cw + lx..][..16];
                        let p = &skip_y[dy * 16..][..16];
                        s += src
                            .iter()
                            .zip(p)
                            .map(|(&a, &b)| a.abs_diff(b) as u32)
                            .sum::<u32>();
                    }
                    s
                };
                greedy_seen += 1;
                if greedy_seen >= greedy_learn {
                    greedy_on = greedy_zero
                        || greedy_free * 100 >= greedy_seen * fe.greedy_min_free as usize;
                }
                if is_free {
                    // `is_free` implies the Some arm above.
                    fe.commit_skip(mb_x, mb_y, mv_skip, &skip_y, skip_c.as_ref().unwrap());
                    if !fe.fast {
                        fe.mb_was_skip[mb_idx] = true;
                        fe.mb_skip_sad[mb_idx] = skip_sad;
                    }
                    greedy_free += 1;
                    did_skip = true;
                } else {
                    signals::census::work(signals::census::W_MB_CODED);
                    let (lx, ly) = (mb_x * 16, mb_y * 16);
                    let nb = {
                        let _g = rusty_h264_common::prof::scope(
                            rusty_h264_common::prof::Stage::EncMvPred,
                        );
                        fe.mv_neighbors_block(mb_x as isize * 4, mb_y as isize * 4, 4)
                    };
                    // Per-MB tex veto (`tune_lme_q`): top-q variance MBs take the
                    // conservative scale; None = the frame-level `lme_scale` exactly.
                    let lme = lme_base
                        * match lme_vars {
                            Some((v, t)) if v[mb_idx] >= t => cfg.cabac_lambda_scale,
                            _ => lme_scale,
                        };
                    if fe.fast {
                        // Dedup (E15 W9): `sig.mb_vars()` IS memoized `mb_variance`
                        // over the same planes; every path here with `satd_q > 0`
                        // already built it via the percentile threshold.
                        fe.mb_use_satd =
                            fe.satd_q > 0.0 && sig.mb_vars()[mb_idx] >= fe.satd_var_thresh;
                        let (r16, mv16, cost_inter) =
                            fe.best_part(refs, &sy, &nb, num_refs, lx, ly, 16, 16, &[], lme);
                        let cost_intra = if fe.mb_use_satd {
                            fe.best_i16_satd(&sy, mb_x, mb_y)
                        } else {
                            fe.best_i16_sad(&sy, mb_x, mb_y)
                        } + (lme * fe.tune_intra_penalty) as i64;
                        inter = if cost_intra < cost_inter {
                            None
                        } else {
                            Some((0, vec![(r16, mv16)]))
                        };
                    } else {
                        // Quality preset: greedy P_Skip, then 16x16 baseline + sub-partitions + intra.
                        if fe.greedy_skip && greedy_on && skip_sad < fe.pred_skip_sad(mb_x, mb_y) {
                            // Quality preset: the Some arm always ran above.
                            fe.commit_skip(mb_x, mb_y, mv_skip, &skip_y, skip_c.as_ref().unwrap());
                            fe.mb_was_skip[mb_idx] = true;
                            fe.mb_skip_sad[mb_idx] = skip_sad;
                            did_skip = true;
                        } else {
                            let (r16, mv16, c16) =
                                fe.best_part(refs, &sy, &nb, num_refs, lx, ly, 16, 16, &[], lme);
                            let mut best_c = c16;
                            let mut pick: Option<InterChoice> = Some((0, vec![(r16, mv16)]));
                            // P3.3: the winning P_8x8 candidate's sub_mb_types
                            // ([0;4] unless a split arm won).
                            let mut pick_subs = [0u8; 4];
                            // Probe #3: every shape the SATD search evaluated, so
                            // the RD re-rank can score them all. Cheap to collect
                            // (the parts vectors already exist); empty unless the
                            // probe is on, so the default path allocates nothing.
                            let mut shape_cands: Vec<(u8, Vec<(i32, (i32, i32))>, [u8; 4])> =
                                Vec::new();
                            // Some(true) = the RD re-rank says INTRA; Some(false) =
                            // it says the chosen shape; None = probe off, use SATD.
                            let mut shape_rd_intra: Option<bool> = None;
                            if shape_rd {
                                shape_cands.push((0u8, vec![(r16, mv16)], [0u8; 4]));
                            }
                            const QSTEP16: [i64; 6] = [10, 11, 13, 14, 16, 18];
                            let qstep16 = QSTEP16[(fe.qp % 6) as usize] << (fe.qp / 6);
                            let split_gate = ((30 * (qstep16 + 160)) >> 3) * 2;
                            if fe.do_splits
                                && c16 > split_gate
                                && (split_t_v <= 0.0 || (c16 as f64) >= split_t_v * lme)
                            {
                                let (rt, mvt, ct) = fe.best_part(
                                    refs,
                                    &sy,
                                    &nb,
                                    num_refs,
                                    lx,
                                    ly,
                                    16,
                                    8,
                                    &[mv16],
                                    lme,
                                );
                                let (rb, mvb, cb) = fe.best_part(
                                    refs,
                                    &sy,
                                    &nb,
                                    num_refs,
                                    lx,
                                    ly + 8,
                                    16,
                                    8,
                                    &[mv16],
                                    lme,
                                );
                                let (rl, mvl, cl) = fe.best_part(
                                    refs,
                                    &sy,
                                    &nb,
                                    num_refs,
                                    lx,
                                    ly,
                                    8,
                                    16,
                                    &[mv16],
                                    lme,
                                );
                                let (rr, mvr, cr) = fe.best_part(
                                    refs,
                                    &sy,
                                    &nb,
                                    num_refs,
                                    lx + 8,
                                    ly,
                                    8,
                                    16,
                                    &[mv16],
                                    lme,
                                );
                                if shape_rd {
                                    shape_cands.push((1u8, vec![(rt, mvt), (rb, mvb)], [0u8; 4]));
                                    shape_cands.push((2u8, vec![(rl, mvl), (rr, mvr)], [0u8; 4]));
                                }
                                if ct + cb < best_c {
                                    best_c = ct + cb;
                                    pick = Some((1u8, vec![(rt, mvt), (rb, mvb)]));
                                }
                                if cl + cr < best_c {
                                    best_c = cl + cr;
                                    pick = Some((2u8, vec![(rl, mvl), (rr, mvr)]));
                                }
                                // P_8x8: four 8×8 sub-partitions (see the CAVLC path).
                                // P3.3: with RFF_SUB8X8_SPLIT=1 each quad also
                                // trials 8x4/4x8/4x4 (single-ref only: ref_idx is
                                // per-QUAD syntax, and best_part searches refs per
                                // part -- mixed sub-part refs are unrepresentable).
                                // Per-quad arm cost = sub-part J sum + REAL
                                // sub_mb_type bins (0->1, 1->2, 2/3->3) priced at
                                // lme -- the uncharged-syntax lesson. The all-8x8
                                // total is arithmetically IDENTICAL to the old
                                // `lme*4 + sum(c)` form.
                                // GRAIN VETO, third consumer (P3.3 gate). With the
                                // decision re-priced in the RD currency the corpus
                                // has exactly two remaining losers and both are
                                // grain: splitting noise buys prediction error the
                                // quantizer discards, and no amount of correct
                                // pricing makes fitting noise worthwhile. Frame
                                // grain -> no split arm (byte-identical to the
                                // all-8x8 P_8x8 the encoder shipped before P3.3).
                                let want_split = want_split_knob && num_refs == 1;
                                let grain_veto = sub8_gveto && grain;
                                if want_split {
                                    signals::census::bump(signals::census::SUB8_GRAIN, grain_veto);
                                }
                                let split_arm = want_split && !grain_veto && sub8_paying;
                                if fe.sub8x8 && !split_arm {
                                    // Knob OFF: the EXACT legacy pricing, including
                                    // its single `(lme*4.0) as i64` truncation --
                                    // per-quad `lme as i64` truncates 4x and picks
                                    // DIFFERENT candidates (byte-identity bug found
                                    // at review, not by the gate).
                                    let mut c8 = (lme * 4.0) as i64;
                                    let mut p8 = Vec::with_capacity(4);
                                    for &(qx, qy) in &[(0usize, 0usize), (8, 0), (0, 8), (8, 8)] {
                                        let (r, mv, c) = fe.best_part(
                                            refs,
                                            &sy,
                                            &nb,
                                            num_refs,
                                            lx + qx,
                                            ly + qy,
                                            8,
                                            8,
                                            &[mv16],
                                            lme,
                                        );
                                        c8 += c;
                                        p8.push((r, mv));
                                    }
                                    if shape_rd {
                                        shape_cands.push((3u8, p8.clone(), [0u8; 4]));
                                    }
                                    if c8 < best_c {
                                        best_c = c8;
                                        pick = Some((3u8, p8));
                                    }
                                } else if fe.sub8x8 {
                                    debug_assert!(split_arm);
                                    let mut c8 = 0i64;
                                    let mut p8: Vec<(i32, (i32, i32))> = Vec::with_capacity(4);
                                    let mut subs = [0u8; 4];
                                    // The all-8x8 arm, kept whatever the SATD search
                                    // picks -- it is the RD probe's other candidate.
                                    let mut p8_flat: Vec<(i32, (i32, i32))> = Vec::with_capacity(4);
                                    let mut c8_flat = 0i64;
                                    let mut hrows: Vec<sub8_harvest::Row> = Vec::new();
                                    for (q, &(qx, qy)) in [(0usize, 0usize), (8, 0), (0, 8), (8, 8)]
                                        .iter()
                                        .enumerate()
                                    {
                                        let (r, mv, c) = fe.best_part(
                                            refs,
                                            &sy,
                                            &nb,
                                            num_refs,
                                            lx + qx,
                                            ly + qy,
                                            8,
                                            8,
                                            &[mv16],
                                            lme,
                                        );
                                        let j8 = c + lme as i64; // + 1 sub_mb_type bin
                                        let mut q_best = j8;
                                        let mut q_parts: Vec<(i32, (i32, i32))> = vec![(r, mv)];
                                        let mut q_st = 0u8;
                                        let mut j_split = i64::MAX; // best split arm, win or lose
                                        {
                                            for st in 1u8..=3 {
                                                let bins = if st == 1 { 2.0 } else { 3.0 };
                                                let mut cs = (lme * bins) as i64;
                                                let mut ps: Vec<(i32, (i32, i32))> =
                                                    Vec::with_capacity(4);
                                                for &(srx, sry, srw, srh) in sub_mb_partitions_p(st)
                                                {
                                                    let (rr, mm, cc) = fe.best_part(
                                                        refs,
                                                        &sy,
                                                        &nb,
                                                        num_refs,
                                                        lx + qx + srx,
                                                        ly + qy + sry,
                                                        srw,
                                                        srh,
                                                        &[mv16, mv],
                                                        lme,
                                                    );
                                                    cs += cc;
                                                    ps.push((rr, mm));
                                                }
                                                j_split = j_split.min(cs);
                                                if cs < q_best {
                                                    q_best = cs;
                                                    q_parts = ps;
                                                    q_st = st;
                                                }
                                            }
                                        }
                                        if sub8_harvest::enabled() {
                                            hrows.push(sub8_harvest::Row {
                                                j8,
                                                jsplit: j_split,
                                                st: q_st,
                                                lme,
                                                mbvar: sig.mb_vars()[mb_idx],
                                                mvdiv: (mv.0 - mv16.0).abs()
                                                    + (mv.1 - mv16.1).abs(),
                                            });
                                        }
                                        signals::census::bump(
                                            signals::census::SUB8_SPLIT,
                                            q_st != 0,
                                        );
                                        c8 += q_best;
                                        c8_flat += j8;
                                        subs[q] = q_st;
                                        p8.extend(q_parts);
                                        p8_flat.push((r, mv));
                                    }
                                    // RD RE-PRICE (probe): the SATD search has picked
                                    // `subs`; ask the CODED macroblock which arm it
                                    // actually prefers. Both arms are planned for real
                                    // (transform+quantize+reconstruct) and scored
                                    // J = SSD_recon + lambda*bits, with the macroblock
                                    // state snapshotted and restored around each trial.
                                    // RD J the surviving split SAVED over all-8x8,
                                    // in lambda units — the census's value unit.
                                    let mut split_gain = 0f64;
                                    if sub8_rd && subs != [0u8; 4] {
                                        let mut snap = enc_scratch::take_snap_a();
                                        fe.save_mb_into(mb_x, mb_y, &mut snap);
                                        let pa = fe.plan_inter_mb(
                                            refs, &sy, &su, &sv, mb_x, mb_y, 3, &p8, None, subs,
                                        );
                                        let ja = fe.mb_ssd(&sy, &su, &sv, mb_x, mb_y) as f64
                                            + lam_mb * plan_rate_bits(&pa, subs);
                                        // E11 (11.16a): the restore BETWEEN trials is
                                        // redundant — `plan_inter_mb` fully overwrites
                                        // every MbState field it writes (recon + the
                                        // four P grids), and the rest (nnz, modes,
                                        // cur_qp) move only at EMIT. One final restore
                                        // suffices; byte-identity gated with the RD
                                        // knobs FORCED ON.
                                        let pb = fe.plan_inter_mb(
                                            refs, &sy, &su, &sv, mb_x, mb_y, 3, &p8_flat, None,
                                            [0u8; 4],
                                        );
                                        let jb = fe.mb_ssd(&sy, &su, &sv, mb_x, mb_y) as f64
                                            + lam_mb * plan_rate_bits(&pb, [0u8; 4]);
                                        fe.load_mb(mb_x, mb_y, &snap);
                                        enc_scratch::put_snap_a(snap);
                                        signals::census::bump(
                                            signals::census::SUB8_RD_REVERT,
                                            jb <= ja,
                                        );
                                        if sub8_regret::enabled() {
                                            // R1 pre-check: keep the MAGNITUDE, which
                                            // the `split_gain` path below discards on
                                            // the revert branch.
                                            sub8_regret::record(
                                                ja,
                                                jb,
                                                lam_mb,
                                                subs.iter().filter(|&&x| x != 0).count(),
                                            );
                                        }
                                        if jb <= ja {
                                            // The split was a SATD mirage on this MB.
                                            subs = [0u8; 4];
                                            p8 = core::mem::take(&mut p8_flat);
                                            c8 = c8_flat;
                                        } else {
                                            split_gain = (jb - ja).max(0.0) / lam_mb.max(1e-9);
                                        }
                                        sub8_harvest::flush(&hrows, subs != [0u8; 4]);
                                    } else {
                                        sub8_harvest::flush(&hrows, subs != [0u8; 4]);
                                    }
                                    if sub8_minpay > 0 {
                                        sub8_seen += 1;
                                        // Value, not a tally: how much RD J this
                                        // macroblock's surviving split actually saved
                                        // over the all-8x8 arm. Zero when the split
                                        // lost — a searched-and-rejected MB is cost
                                        // with no payoff, which is what we want the
                                        // mean to reflect.
                                        sub8_gain += split_gain;
                                        if sub8_seen >= sub8_learn {
                                            sub8_paying =
                                                sub8_gain >= sub8_seen as f64 * sub8_minpay as f64;
                                        }
                                    }
                                    if shape_rd {
                                        shape_cands.push((3u8, p8.clone(), subs));
                                    }
                                    if c8 < best_c {
                                        best_c = c8;
                                        pick = Some((3u8, p8));
                                        pick_subs = subs;
                                    }
                                }
                            }
                            // SHAPE RD RE-RANK (probe #3). The SATD search above
                            // has produced `pick`; ask the CODED macroblock to
                            // re-rank the shapes it actually considered. Each
                            // candidate is planned for real and scored
                            // J = SSD_recon + lambda*bits, state restored between
                            // trials. Candidates are collected as (mode, parts,
                            // subs) so the sub-8x8 winner competes on equal terms.
                            if shape_rd && shape_cands.len() > 1 && !shape_rd_tex_veto {
                                let mut snap0 = enc_scratch::take_snap_a();
                                fe.save_mb_into(mb_x, mb_y, &mut snap0);
                                let mut best_j = f64::INFINITY;
                                let mut best_i = 0usize;
                                for (i, (m, parts, subs)) in shape_cands.iter().enumerate() {
                                    let pl = fe.plan_inter_mb(
                                        refs, &sy, &su, &sv, mb_x, mb_y, *m, parts, None, *subs,
                                    );
                                    let j = fe.mb_ssd(&sy, &su, &sv, mb_x, mb_y) as f64
                                        + lam_mb * plan_rate_bits(&pl, *subs);
                                    if j < best_j {
                                        best_j = j;
                                        best_i = i;
                                    }
                                }
                                // E11 (11.16a): ONE restore after the loop — each
                                // candidate's plan fully overwrites the last (see the
                                // sub8-RD note); the intra trial below then starts
                                // from the same pre-trial state as before.
                                fe.load_mb(mb_x, mb_y, &snap0);
                                enc_scratch::put_snap_a(snap0);
                                let (m, parts, subs) = shape_cands[best_i].clone();
                                signals::census::bump(
                                    signals::census::SHAPE_RD_FLIP,
                                    pick.as_ref().map(|p| p.0) != Some(m),
                                );
                                pick = Some((m, parts));
                                pick_subs = subs;
                                // INTRA COMPETES IN THE SAME CURRENCY. Writing the
                                // RD J back into `best_c` and letting the SATD
                                // intra test below read it mixes two scales —
                                // an SSD+lambda*bits value dwarfs a SATD one, so
                                // intra won essentially every macroblock and the
                                // probe measured +17..+56% BD. Caught because a
                                // better cost function CANNOT lose 50%
                                // (codec-measurement §7: an impossible number is
                                // the instrument asking for help). One decision,
                                // one currency: score intra by RD here and skip
                                // the SATD comparison entirely.
                                let (ssd_i, bits_i) =
                                    fe.trial_intra(&sy, &su, &sv, mb_x, mb_y, true);
                                let j_intra = ssd_i as f64 + lam_mb * bits_i as f64;
                                shape_rd_intra = Some(j_intra < best_j);
                            }
                            // U5-struct: refine ONLY the winning shape (see the twin
                            // block in the CAVLC driver). This site is the CABAC path —
                            // which is now the DEFAULT, so omitting it here left sub-pel
                            // deferred but never refined on every default encode.
                            if fe.sp_defer.get() {
                                if let Some((mode, parts)) = pick.as_mut() {
                                    // P3.3: a sub-split winner's regions are its
                                    // SUB-partitions. Skipping the refine for them
                                    // instead would leave exactly those macroblocks
                                    // on INTEGER-pel motion while every other shape
                                    // got sub-pel — the same "deferring DELETES the
                                    // refinement" failure (+91..+145% BD) the
                                    // preset guard above exists to prevent. Inert
                                    // today (sp_defer is off unless set explicitly)
                                    // but a landmine under that knob.
                                    let split_regions: Vec<(usize, usize, usize, usize)> =
                                        if *mode == 3 && pick_subs != [0u8; 4] {
                                            (0..4)
                                                .flat_map(|p8: usize| {
                                                    let (bx, by) = ((p8 % 2) * 8, (p8 / 2) * 8);
                                                    sub_mb_partitions_p(pick_subs[p8]).iter().map(
                                                        move |&(sx, sy, sw, sh)| {
                                                            (bx + sx, by + sy, sw, sh)
                                                        },
                                                    )
                                                })
                                                .collect()
                                        } else {
                                            Vec::new()
                                        };
                                    let regions: &[(usize, usize, usize, usize)] =
                                        if !split_regions.is_empty() {
                                            &split_regions
                                        } else {
                                            match mode {
                                                1 => &[(0, 0, 16, 8), (0, 8, 16, 8)],
                                                2 => &[(0, 0, 8, 16), (8, 0, 8, 16)],
                                                3 => &[
                                                    (0, 0, 8, 8),
                                                    (8, 0, 8, 8),
                                                    (0, 8, 8, 8),
                                                    (8, 8, 8, 8),
                                                ],
                                                _ => &[(0, 0, 16, 16)],
                                            }
                                        };
                                    // sub_mb_type bins, charged exactly as the search did.
                                    let mut tot = if *mode == 3 {
                                        if pick_subs == [0u8; 4] {
                                            (lme * 4.0) as i64
                                        } else {
                                            pick_subs
                                                .iter()
                                                .map(|&st| {
                                                    if st == 0 {
                                                        lme as i64
                                                    } else {
                                                        (lme * if st == 1 { 2.0 } else { 3.0 })
                                                            as i64
                                                    }
                                                })
                                                .sum()
                                        }
                                    } else {
                                        0
                                    };
                                    for (i, &(qx, qy, pw, ph)) in regions.iter().enumerate() {
                                        let (r, mv) = parts[i];
                                        let (m2, c2) = fe.refine_part(
                                            refs,
                                            &sy,
                                            &nb,
                                            num_refs,
                                            lx + qx,
                                            ly + qy,
                                            pw,
                                            ph,
                                            lme,
                                            r,
                                            mv,
                                        );
                                        parts[i] = (r, m2);
                                        tot += c2;
                                    }
                                    best_c = tot;
                                }
                            }
                            let c_intra = fe.best_i16_satd(&sy, mb_x, mb_y)
                                + (lme * fe.tune_intra_penalty) as i64;
                            let satd_says_intra = c_intra < best_c;
                            // GRAIN-GATED (4th consumer of `grain_signature`).
                            // Measured flip rates: grain 18.4%, screen 12.6%,
                            // foreman 1.1%, harbour 0.3%, akiyo 0.0% — the SATD
                            // proxy is essentially RIGHT on natural content and
                            // badly wrong on noise, so paying 1.71x CPU
                            // everywhere buys ~0 off grain. Gated: grain wins
                            // -4.73 PSNR / -5.22 SSIM, everything else is
                            // byte-identical. screen_text is deliberately NOT
                            // included: its metrics disagree (+0.53 PSNR /
                            // -1.48 SSIM), and a split verdict is not a win.
                            let use_rd = use_rd_frame;
                            let take_intra = if let Some(rd_says) = shape_rd_intra {
                                // The shape re-rank already priced intra against the
                                // winning shape in the RD currency — reuse it rather
                                // than paying a second trial or mixing scales.
                                rd_says
                            } else if use_rd {
                                // RD arm: plan BOTH candidates for real and compare
                                // J = SSD_recon + lambda*bits. `trial_intra` restores
                                // the macroblock, so the loser leaves no trace.
                                let (ssd_i, bits_i) =
                                    fe.trial_intra(&sy, &su, &sv, mb_x, mb_y, true);
                                let j_intra = ssd_i as f64 + lambda * bits_i as f64;
                                let j_inter = match pick.as_ref() {
                                    Some((m, parts)) => {
                                        let mut snap = enc_scratch::take_snap_a();
                                        fe.save_mb_into(mb_x, mb_y, &mut snap);
                                        let pl = fe.plan_inter_mb(
                                            refs, &sy, &su, &sv, mb_x, mb_y, *m, parts, None,
                                            pick_subs,
                                        );
                                        let j = fe.mb_ssd(&sy, &su, &sv, mb_x, mb_y) as f64
                                            + lambda * plan_rate_bits(&pl, pick_subs);
                                        fe.load_mb(mb_x, mb_y, &snap);
                                        enc_scratch::put_snap_a(snap);
                                        j
                                    }
                                    None => f64::INFINITY,
                                };
                                j_intra < j_inter
                            } else {
                                satd_says_intra
                            };
                            if use_rd {
                                signals::census::bump(
                                    signals::census::INTRA_RD_FLIP,
                                    take_intra != satd_says_intra,
                                );
                            }
                            inter = if take_intra { None } else { pick };
                            if matches!(inter, Some((3, _))) {
                                inter_subs = pick_subs; // P3.3: ride with the winner
                            }
                            fe.mb_was_skip[mb_idx] = false;
                            fe.mb_skip_sad[mb_idx] = skip_sad;
                        }
                    }
                }
                // ---- RD P_Skip, CABAC port (Great Gate P3 item 2 —
                // docs/gate-ledger.md rdskip-preset-gate) ------------------------
                // THRESHOLD form only: the CAVLC driver's trial-encode-and-splice
                // arm does not transfer to an arithmetic coder, but its fast gate
                // (`SSD(skip) ≤ T·λ` — take the null arm without pricing the coded
                // one) is exactly the λ-priced-distortion form the RD B_Skip gate
                // already proved under CABAC. Distortion is the skip's
                // RECONSTRUCTION SSD (a P_Skip's recon IS its prediction), never
                // SAD — the wrong-sign-proxy lesson. Gated on the ONLINE free-skip
                // census (engage where free skips are COMMON = temporally
                // redundant content, the CAVLC fit's separating signal; the same
                // `greedy_*` counters already track it here). `tune_rd_skip` off
                // (default) = byte-identical; `fast_t ≤ 0` is inert on this path
                // (no full-compare arm exists under CABAC — recorded limitation).
                if !did_skip
                    && fe.rd_skip
                    && fe.rd_skip_fast_t > 0.0
                    && inter.is_some()
                    && greedy_seen >= greedy_learn
                    && greedy_free * 100 >= greedy_seen * fe.rd_skip_min_free as usize
                {
                    // Fast preset only built the chroma prediction on the free
                    // path; the RD decision needs the real one.
                    let skip_cp = if fe.fast {
                        fe.skip_predict_chroma(refs, mb_x, mb_y, mv_skip)
                    } else {
                        skip_c.unwrap() // quality preset: always Some
                    };
                    let ssd_s = fe.pred_ssd(&sy, &su, &sv, mb_x, mb_y, &skip_y, &skip_cp);
                    if (ssd_s as f64) <= lambda * fe.rd_skip_fast_t {
                        fe.commit_skip(mb_x, mb_y, mv_skip, &skip_y, &skip_cp);
                        if !fe.fast {
                            fe.mb_was_skip[mb_idx] = true;
                            fe.mb_skip_sad[mb_idx] = skip_sad;
                        }
                        did_skip = true;
                    }
                }
            }

            // ---- emit ----
            if did_skip {
                emit_p_skip_cabac(&mut cab, &mut cs, addr, top, left);
                mb_qpy[mb_idx] = fe.cur_qp;
                cb_term_acct(&mut cab, mb_idx + 1 == total, acct);
                continue;
            }
            // mb_skip_flag = 0
            let sctx = 11
                + left.map_or(0, |a| (!cs.mb_skip[a]) as usize)
                + top.map_or(0, |a| (!cs.mb_skip[a]) as usize);
            let tskip = if acct { cab.pos() } else { 0 };
            cb_mb_skip(&mut cab, sctx, false);
            if acct {
                crate::bitacct::add(crate::bitacct::B::SkipFlag, cab.pos() - tskip);
            }
            cs.mb_skip[addr] = false;
            match inter {
                Some((mode, parts)) => {
                    let plan = fe.plan_inter_mb(
                        refs, &sy, &su, &sv, mb_x, mb_y, mode, &parts, None, inter_subs,
                    );
                    // ② residue naming: the CABAC entropy EMIT was untapped on the
                    // (default) CABAC driver — the whole encoder-side arithmetic
                    // coder was landing in `mgmt/other`.
                    let _ge =
                        rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncEmit);
                    emit_mb_cabac_p_inter(
                        &mut fe, &mut cab, &mut cs, mode, &plan, mb_x, mb_y, num_refs,
                    );
                    signals::census::commit_mb(plan.t8x8);
                }
                None => {
                    fe.intra_in_p = true;
                    let plan = plan_mb(&mut fe, mb_x, mb_y, &sy, &su, &sv);
                    let _ge =
                        rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncEmit);
                    let t8 = plan.i8.is_some() && plan.use_i4;
                    emit_mb_cabac_p_intra(&mut fe, &mut cab, &mut cs, &plan, mb_x, mb_y);
                    signals::census::commit_mb(t8);
                }
            }
            mb_qpy[mb_idx] = fe.cur_qp;
            cb_term_acct(&mut cab, mb_idx + 1 == total, acct);
        }
    }
    enc_scratch::put_cs(cs);

    while !w.is_byte_aligned() {
        w.write_bit(true);
    }
    let payload = cab.into_bytes();
    w.write_aligned_bytes(&payload);
    enc_scratch::put_payload(payload);

    // Deblock -> inter reference (same as encode_slice_data).
    let mut ref_id = enc_scratch::take_refid();
    ref_id.clear();
    ref_id.extend(
        fe.ref_idx_y
            .iter()
            .map(|&r| if r >= 0 { r } else { i8::MIN }),
    );
    let info = rusty_h264_common::deblock::BlockInfo {
        inter: &fe.inter_y,
        nnz: &fe.nnz_y,
        mv: &fe.mv_y,
        ref_id: &ref_id,
        mv1: &[],
        ref_id1: &[],
        w4: fe.mb_w * 4,
        t8x8: &[],
        poc0: &[],
        poc1: &[],
        bs: &[],
        kind: &[],
    };
    rusty_h264_common::deblock::filter_frame(
        &mut fe.rec_y,
        &mut fe.rec_u,
        &mut fe.rec_v,
        fe.mb_w,
        fe.mb_h,
        &mb_qpy,
        0,
        0,
        0,
        &info,
    );
    enc_scratch::put_qpy(mb_qpy);
    enc_scratch::put_refid(ref_id);
    let w4 = fe.mb_w * 4;
    crate::RefFrame {
        y: fe.rec_y,
        u: fe.rec_u,
        v: fe.rec_v,
        poc: 0,
        frame_num: 0,
        mv: fe.mv_y,
        ref_idx: fe.ref_idx_y,
        mv1: Vec::new(),
        ref_idx1: Vec::new(),
        w4,
        // Filtered lazily on first sub-pel search use (see `RefFrame::hpel`).
        hpel: crate::HpelOnce::new(),
    }
}

// ============================================================================
// CABAC B-slice entropy coding — inverse of the decoder's decode_slice_data_cabac
// B-slice path. Scope: the modes the encoder's B decision produces — B_Skip,
// B_Direct_16x16 (0), B_L0/L1/Bi_16x16 (1/2/3) — no sub_mb_type, no intra-in-B.
// The new piece vs P is the dual-list (L0 + L1) mvd/ref neighbour cache.
// ============================================================================

/// Per-MB slice terminate + bit-accountant tap — the block was a byte-for-byte
/// twin at SIX sites across the P and B CABAC drivers (E15 rounds 1-2).
#[inline]
fn cb_term_acct(cab: &mut CabacEncoder, last: bool, acct: bool) {
    let tt = if acct { cab.pos() } else { 0 };
    cab.encode_terminate(last);
    if acct {
        crate::bitacct::add(crate::bitacct::B::Terminate, cab.pos() - tt);
    }
}

/// Fill one list's 30-entry mvd/ref neighbour cache from the per-MB export grids
/// (openh264 WelsFillCacheInterCabac). Shared by P (List-0) and B (both lists).
fn cb_fill_inter_cache(
    mb_ref: &[[i8; 16]],
    mb_mvd: &[[[u8; 2]; 16]],
    refc: &mut [i8; 30],
    mvdc: &mut [[i16; 2]; 30],
    top: Option<usize>,
    left: Option<usize>,
    addr: usize,
    mb_w: usize,
) {
    if let Some(l) = left {
        // Row-bind once: the outer runtime index carried a bounds check per
        // access (8 in the emitted body); one per array per side suffices.
        let (rr, rm) = (&mb_ref[l], &mb_mvd[l]);
        for (ci, bi) in [(6usize, 3usize), (12, 7), (18, 11), (24, 15)] {
            refc[ci] = rr[bi];
            mvdc[ci] = [rm[bi][0] as i16, rm[bi][1] as i16];
        }
    }
    if let Some(t) = top {
        for (ci, bi) in [(1usize, 12usize), (2, 13), (3, 14), (4, 15)] {
            refc[ci] = mb_ref[t][bi];
            mvdc[ci] = [mb_mvd[t][bi][0] as i16, mb_mvd[t][bi][1] as i16];
        }
    }
    let mb_x = addr % mb_w;
    let mb_y = addr / mb_w;
    if mb_x > 0 && mb_y > 0 {
        let a = addr - mb_w - 1;
        (refc[0], mvdc[0]) =
            (mb_ref[a][15], [mb_mvd[a][15][0] as i16, mb_mvd[a][15][1] as i16]);
    }
    if mb_y > 0 && mb_x + 1 < mb_w {
        let a = addr - mb_w + 1;
        (refc[5], mvdc[5]) =
            (mb_ref[a][12], [mb_mvd[a][12][0] as i16, mb_mvd[a][12][1] as i16]);
    }
}

/// B-slice `mb_type` for the encoder's B modes (0 = B_Direct_16x16, 1 = B_L0_16x16,
/// 2 = B_L1_16x16, 3 = B_Bi_16x16) — inverse of `parse_mb_type_b_cabac` (ctx 27).
/// B `mb_type` (Table 7-14) for a two-partition macroblock. `p0`/`p1` are 1=L0,
/// 2=L1, 3=Bi; `mvmode` 1 = 16x8, 2 = 8x16 (the odd types).
pub fn b_part_mb_type(p0: u8, p1: u8, mvmode: u8) -> u32 {
    let base = match (p0, p1) {
        (1, 1) => 4,
        (2, 2) => 6,
        (1, 2) => 8,
        (2, 1) => 10,
        (1, 3) => 12,
        (2, 3) => 14,
        (3, 1) => 16,
        (3, 2) => 18,
        _ => 20, // (Bi, Bi)
    };
    base + if mvmode == 2 { 1 } else { 0 }
}

/// The two partition rects `(x, y, w, h)` and their z-order block lists for a B
/// 16x8 / 8x16 macroblock — the same split the decoder's `b_inter_layout` uses.
fn b_part_layout(
    mvmode: u8,
) -> (
    [(usize, usize, usize, usize); 2],
    [(usize, &'static [usize]); 2],
) {
    if mvmode == 1 {
        (
            [(0, 0, 16, 8), (0, 8, 16, 8)],
            [
                (0, &[0, 1, 2, 3, 4, 5, 6, 7][..]),
                (8, &[8, 9, 10, 11, 12, 13, 14, 15][..]),
            ],
        )
    } else {
        (
            [(0, 0, 8, 16), (8, 0, 8, 16)],
            [
                (0, &[0, 1, 2, 3, 8, 9, 10, 11][..]),
                (4, &[4, 5, 6, 7, 12, 13, 14, 15][..]),
            ],
        )
    }
}

/// B `mb_type` CABAC — the exact inverse of the decoder's `parse_mb_type_b_cabac`
/// (ctx base 27). Accepts the FULL spec range 0..=22, not just the four 16x16
/// modes, so the B 16x8 / 8x16 / 8x8 partitions become emittable.
///
/// Binarization, derived from the decoder branch-for-branch:
/// ```text
///   prefix (non-direct):  B+ctx_inc = 1 ; B+3 = 1
///   4-bit m4:             B+4 = bit3 ; B+5 = bit2 ; B+5 = bit1 ; B+5 = bit0
///   type 3..10  -> m4 = type - 3        (m4 < 8 -> bit3 = 0)      4 bins
///   type 11     -> m4 = 14  (B_Bi_8x16)                           4 bins
///   type 22     -> m4 = 15  (B_8x8)                               4 bins
///   type 12..21 -> v = type + 4 ; m4 = v >> 1 ; B+5 = v & 1       5 bins
/// ```
/// The decoder returns `m + 3` for `m < 8`, escapes at 13 (intra) / 14 / 15, and
/// otherwise reads a 5th bin and returns `m - 4`; the mapping above reproduces
/// every one of those branches. For 0..=3 the value equals the old `dir`, so
/// existing call sites are unchanged.
pub fn cb_mb_type_b(cab: &mut CabacEncoder, ctx_inc: usize, mb_type: u32) {
    const B: usize = 27;
    if mb_type == 0 {
        cab.encode_decision(B + ctx_inc, 0); // B_Direct_16x16
        return;
    }
    cab.encode_decision(B + ctx_inc, 1);
    if mb_type <= 2 {
        cab.encode_decision(B + 3, 0);
        cab.encode_decision(B + 5, mb_type - 1); // 16x16 L0 / L1
        return;
    }
    cab.encode_decision(B + 3, 1);
    let (m4, extra) = if mb_type <= 10 {
        (mb_type - 3, None)
    } else if mb_type == 11 {
        (14, None) // B_Bi_8x16
    } else if mb_type == 22 {
        (15, None) // B_8x8
    } else {
        let v = mb_type + 4;
        (v >> 1, Some(v & 1))
    };
    cab.encode_decision(B + 4, (m4 >> 3) & 1);
    cab.encode_decision(B + 5, (m4 >> 2) & 1);
    cab.encode_decision(B + 5, (m4 >> 1) & 1);
    cab.encode_decision(B + 5, m4 & 1);
    if let Some(e) = extra {
        cab.encode_decision(B + 5, e);
    }
}

const CB_ALL16: [usize; 16] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15];

/// Emit one planned INTER B macroblock (mb_skip_flag already coded 0). `dir` is the
/// B direction 0/1/2/3; `plan.mvds` holds mvd_l0 then mvd_l1 (per used list).
/// B-slice INTRA `mb_type` — the inverse of the decoder's `parse_mb_type_b`
/// `m == 13` escape + `parse_intra_mb_type_cabac(cab, 32)`. Prefix bins walk to
/// m4 = 0b1101 = 13, then the intra suffix at ctx base 32 (NOT the I-slice
/// layout: bin0 has no ctxInc, cbp/chroma/mode bins share base+1..base+3).
fn cb_mb_type_b_intra(cab: &mut CabacEncoder, ctx_inc: usize, plan: &MbPlan) {
    const B: usize = 27;
    cab.encode_decision(B + ctx_inc, 1); // not B_Direct_16x16
    cab.encode_decision(B + 3, 1); // not 16x16 L0/L1
    cab.encode_decision(B + 4, 1); // m = 1101 = 13 -> intra escape
    cab.encode_decision(B + 5, 1);
    cab.encode_decision(B + 5, 0);
    cab.encode_decision(B + 5, 1);
    const O: usize = 32; // intra suffix ctx base (Table 9-37 suffix)
    if plan.use_i4 {
        cab.encode_decision(O, 0); // I_NxN
        return;
    }
    cab.encode_decision(O, 1);
    cab.encode_terminate(false); // not I_PCM
    cab.encode_decision(O + 1, plan.i16_cbp15 as u32);
    if plan.cbp_chroma != 0 {
        cab.encode_decision(O + 2, 1);
        cab.encode_decision(O + 2, (plan.cbp_chroma == 2) as u32);
    } else {
        cab.encode_decision(O + 2, 0);
    }
    cab.encode_decision(O + 3, (plan.i16_mode as u32 >> 1) & 1);
    cab.encode_decision(O + 3, plan.i16_mode as u32 & 1);
}

fn emit_mb_cabac_b(
    fe: &mut FrameEncoder,
    cab: &mut CabacEncoder,
    cs: &mut CabacState,
    dir: u8,
    bsplit: Option<(u8, [(u8, (i32, i32), (i32, i32)); 2])>,
    plan: &InterPlan,
    mb_x: usize,
    mb_y: usize,
) {
    let mb_w = fe.mb_w;
    let addr = mb_y * mb_w + mb_x;
    let top = if mb_y > 0 { Some(addr - mb_w) } else { None };
    let left = if mb_x > 0 { Some(addr - 1) } else { None };

    let bci = left.map_or(0, |a| (!cs.mb_direct[a]) as usize)
        + top.map_or(0, |a| (!cs.mb_direct[a]) as usize);
    let bmt = match bsplit {
        Some((mvmode, parts2)) => b_part_mb_type(parts2[0].0, parts2[1].0, mvmode),
        None => dir as u32,
    };
    cb_mb_type_b(cab, bci, bmt);

    // Dual-list mvd/ref caches (L0 = mb_ref/mb_mvd, L1 = mb_ref1/mb_mvd1).
    let mut mvdc0 = [[0i16; 2]; 30];
    let mut refc0 = [-1i8; 30];
    let mut mvdc1 = [[0i16; 2]; 30];
    let mut refc1 = [-1i8; 30];
    cb_fill_inter_cache(
        &cs.mb_ref, &cs.mb_mvd, &mut refc0, &mut mvdc0, top, left, addr, mb_w,
    );
    cb_fill_inter_cache(
        &cs.mb_ref1,
        &cs.mb_mvd1,
        &mut refc1,
        &mut mvdc1,
        top,
        left,
        addr,
        mb_w,
    );
    let mut mmvd0 = [[0i16; 2]; 16];
    let mut mref0 = [-1i8; 16];
    let mut mmvd1 = [[0i16; 2]; 16];
    let mut mref1 = [-1i8; 16];
    let (use0, use1) = (dir == 1 || dir == 3, dir == 2 || dir == 3);
    if let Some((mvmode, parts2)) = bsplit {
        // Two partitions: mvds arrive LIST-major from the plan (spec 7.3.5.1), and
        // `cb_emit_mvd_partition` needs each partition's z-order block list so a
        // later macroblock's mvd ctxInc sees the right neighbours. B here runs a
        // single L0 and single L1, so num_ref_idx_active is 1 and NO ref_idx is
        // coded — only the mvds.
        let (_, zb) = b_part_layout(mvmode);
        let mut k = 0;
        for list in 0..2 {
            for part in 0..2 {
                let pred = parts2[part].0;
                let used = if list == 0 {
                    pred == 1 || pred == 3
                } else {
                    pred == 2 || pred == 3
                };
                if !used {
                    continue;
                }
                let (pidx, blocks) = zb[part];
                if list == 0 {
                    cb_emit_mvd_partition(
                        cab,
                        pidx,
                        blocks,
                        &mut mvdc0,
                        &mut refc0,
                        &mut mmvd0,
                        &mut mref0,
                        plan.mvds[k],
                        0,
                    );
                } else {
                    cb_emit_mvd_partition(
                        cab,
                        pidx,
                        blocks,
                        &mut mvdc1,
                        &mut refc1,
                        &mut mmvd1,
                        &mut mref1,
                        plan.mvds[k],
                        0,
                    );
                }
                k += 1;
            }
        }
    } else if dir == 0 {
        // B_Direct_16x16: no coded motion; ref 0 in both lists (mvd stays 0) so a
        // later MB's mvd ctxInc sums |0|.
        mref0 = [0i8; 16];
        mref1 = [0i8; 16];
    } else {
        // mvd parse order: list-major (L0 then L1); a single 16x16 partition (idx 0).
        let mut k = 0;
        if use0 {
            cb_emit_mvd_partition(
                cab,
                0,
                &CB_ALL16,
                &mut mvdc0,
                &mut refc0,
                &mut mmvd0,
                &mut mref0,
                plan.mvds[k],
                0,
            );
            k += 1;
        }
        if use1 {
            cb_emit_mvd_partition(
                cab,
                0,
                &CB_ALL16,
                &mut mvdc1,
                &mut refc1,
                &mut mmvd1,
                &mut mref1,
                plan.mvds[k],
                0,
            );
        }
    }
    cs.mb_mvd[addr] = cb_sat_mvd(mmvd0);
    cs.mb_ref[addr] = mref0;
    cs.mb_mvd1[addr] = cb_sat_mvd(mmvd1);
    cs.mb_ref1[addr] = mref1;
    cs.mb_direct[addr] = dir == 0 && bsplit.is_none();
    cs.cat[addr] = 100;
    // The B rule (R6-5), mirroring the decoder's `allow8` and `plan_inter_mb`'s
    // `allow_t8`: direct_8x8_inference_flag = 1 in our SPS, so B_Direct_16x16
    // partitions count as 8×8 and every shape we emit may carry the flag
    // (`bsplit` only produces 16x8 / 8x16, both >= 8x8; no B_8x8 sub-types).
    let allow8 = true;
    cb_emit_inter_residual(fe, cab, cs, plan, mb_x, mb_y, addr, top, left, allow8);
}

/// Emit a B_Skip macroblock's mb_skip_flag = 1 (ctx 24 base) + neighbour state. The
/// direct motion was committed by `commit_direct_motion`; ref 0 in both lists, mvd 0
/// (matching the decoder's decode_b_skip handling).
fn emit_b_skip_cabac(
    cab: &mut CabacEncoder,
    cs: &mut CabacState,
    addr: usize,
    top: Option<usize>,
    left: Option<usize>,
) {
    let sctx = 24
        + left.map_or(0, |a| (!cs.mb_skip[a]) as usize)
        + top.map_or(0, |a| (!cs.mb_skip[a]) as usize);
    let acct = crate::bitacct::enabled();
    let t0 = if acct { cab.pos() } else { 0 };
    cb_mb_skip(cab, sctx, true);
    if acct {
        crate::bitacct::add(crate::bitacct::B::SkipFlag, cab.pos() - t0);
    }
    cs.mb_skip[addr] = true;
    cs.cat[addr] = 100;
    cs.mb_direct[addr] = true;
    cs.mb_ref[addr] = [0i8; 16];
    cs.mb_ref1[addr] = [0i8; 16];
    cs.last_delta_qp = 0;
}

/// CABAC B-slice data coder. Mirrors `encode_slice_data_b`'s B_Skip-free check +
/// L0/L1/Bi/Direct RD decision verbatim; only the emit differs (per-MB
/// mb_skip_flag + CABAC + per-MB terminate). B is non-reference → no deblock/return.
/// B-slice mode census (env `RFF_BSTATS=1`), so our B_Skip / B_Direct / coded
/// split can be compared directly with x264's `mb B ... direct:N% skip:N%` line.
/// Counts only; no effect on the bitstream.
/// SEQUENCE-SCOPED sub-pel override, set once by `encode_all`.
///
/// REINSTATED after D5i. It was reverted on the finding that it "did not land", which
/// was WRONG: the acceptance test demanded byte-identity with `--preset fast`, and that
/// is unreachable because `Preset::Fast` ALSO sets `rd_skip_min_free` to 60 (mb16.rs
/// ~1777) which this flag cannot touch. The veto only ever controlled sub-pel, so it
/// must be judged against the same preset with sub-pel off -- which is what it does.
pub struct SeqFastPath;

// The flag is per thread wherever there are threads: `encode_all`'s GOP
// workers each set their own copy, and a test binary's other tests cannot
// see it. Without threads (`no_std`, not a test build) a static is the same
// thing and costs nothing.
#[cfg(any(feature = "std", test))]
::std::thread_local! {
    static SEQ_FAST: core::cell::Cell<bool> = const { core::cell::Cell::new(false) };
}
#[cfg(any(feature = "std", test))]
fn seq_fast_store(on: bool) {
    SEQ_FAST.with(|f| f.set(on));
}
#[cfg(any(feature = "std", test))]
fn seq_fast_load() -> bool {
    SEQ_FAST.with(|f| f.get())
}
#[cfg(not(any(feature = "std", test)))]
static SEQ_FAST: core::sync::atomic::AtomicBool = core::sync::atomic::AtomicBool::new(false);
#[cfg(not(any(feature = "std", test)))]
fn seq_fast_store(on: bool) {
    SEQ_FAST.store(on, core::sync::atomic::Ordering::Relaxed);
}
#[cfg(not(any(feature = "std", test)))]
fn seq_fast_load() -> bool {
    SEQ_FAST.load(core::sync::atomic::Ordering::Relaxed)
}

impl SeqFastPath {
    pub(crate) fn set(on: bool) -> Self {
        seq_fast_store(on);
        SeqFastPath
    }
    pub(crate) fn get() -> bool {
        seq_fast_load()
    }
}
impl Drop for SeqFastPath {
    fn drop(&mut self) {
        seq_fast_store(false);
    }
}

pub mod bstats {
    #[allow(unused_imports)]
    use alloc::{
        boxed::Box,
        format,
        string::{String, ToString},
        vec,
        vec::Vec,
    };
    use core::sync::atomic::Ordering::Relaxed;
    use rusty_h264_common::atomic::AtomicU64;
    #[allow(unused_imports)]
    use rusty_h264_common::once::OnceLock;
    pub static SKIP: AtomicU64 = AtomicU64::new(0);
    pub static CODED: AtomicU64 = AtomicU64::new(0);
    /// Of the NOT-free macroblocks, how often direct still won the mode decision.
    /// B_Skip rides direct-mode motion, so this is the physical quality of the
    /// thing the skip is betting on -- the candidate dispatch signal for how hard
    /// to push the skip.
    pub static DIRWIN: AtomicU64 = AtomicU64::new(0);
    /// Of the NOT-free macroblocks, how often a 16×8 / 8×16 partition beat every
    /// 16×16 mode. x264 puts 13.5% of its B macroblocks here; this is the column
    /// that makes ours comparable.
    pub static SPLIT: AtomicU64 = AtomicU64::new(0);
    /// Of the NOT-free macroblocks, how often intra beat every inter candidate
    /// (the m4 == 13 escape) — scene cuts / occlusion inside a GOP.
    pub static INTRA: AtomicU64 = AtomicU64::new(0);
    pub fn on() -> bool {
        static E: rusty_h264_common::once::OnceLock<bool> =
            rusty_h264_common::once::OnceLock::new();
        *E.get_or_init(|| rusty_h264_common::knob("RFF_BSTATS").is_some())
    }
    pub fn bump(c: &AtomicU64) {
        if on() {
            c.fetch_add(1, Relaxed);
        }
    }
    /// NOTE: this splits B_Skip from everything-else only. It does NOT separate
    /// B_Direct_16x16 (chosen inside the coded path) from genuinely coded modes, so
    /// it is comparable to x264's `skip:` column but NOT to its `direct:` column.
    pub fn dump() {
        let (s, c) = (SKIP.load(Relaxed), CODED.load(Relaxed));
        let t = (s + c).max(1) as f64;
        eprintln!(
            "B-slice census: B_Skip {:.1}%  not-skipped {:.1}%  direct-wins-of-coded {:.1}%  16x8/8x16-of-coded {:.1}%  intra-of-coded {:.1}%   (n={})",
            s as f64 * 100.0 / t, c as f64 * 100.0 / t,
            DIRWIN.load(Relaxed) as f64 * 100.0 / c.max(1) as f64,
            SPLIT.load(Relaxed) as f64 * 100.0 / c.max(1) as f64,
            INTRA.load(Relaxed) as f64 * 100.0 / c.max(1) as f64, s + c
        );
    }
}

pub(crate) fn encode_slice_data_cabac_b(
    w: &mut BitWriter,
    cfg: &EncoderConfig,
    frame: &YuvPlanes<'_>,
    qp: u8,
    poc: i32,
    l0: &crate::RefFrame,
    l1: &crate::RefFrame,
    qpo: &[i32],
    // b-pyramid: a REFERENCE B must produce a deblocked reconstruction for
    // the DPB, exactly like a P anchor; a leaf B returns `None` and skips the
    // filter (the decoder deblocks for display).
    is_ref: bool,
) -> Option<crate::RefFrame> {
    let mut fe = FrameEncoder::new(cfg);
    fe.qp = qp;
    fe.qpc = chroma_qp(qp);
    fe.cur_qp = qp;
    // Recycled per-thread scratch (the P/I coders already pool this exact
    // buffer); `clear` + `resize` reproduces the fresh `vec![qp; n]` contents.
    let mut mb_qpy = enc_scratch::take_qpy();
    mb_qpy.clear();
    mb_qpy.resize(cfg.mb_width() * cfg.mb_height(), qp);
    if cfg.cabac_dz_div > 0 {
        fe.idz = cfg.cabac_dz_div; // CABAC-specific dead-zone override
    }
    // Inter trellis (opt-in, Great Gate P2): B is NON-REFERENCE — the clean
    // arm per the structure-adaptive law. 0 = off, byte-identical.
    fe.rdoq_strength = cfg.cabac_rdoq_b;
    fe.bi_w = implicit_bi_weights(poc, l0.poc, l1.poc);
    let (sy, su, sv) = coded_source(cfg, frame);
    // Great Gate P1: the shared per-frame signal vector (List-0 anchor as ref).
    let sig = FrameSignals::new(&sy, fe.cw, fe.mb_w, fe.mb_h, Some(&l0.y[..]));
    apply_screen_t8_veto(&mut fe, &sig);
    let lambda = 0.85 * fe.tune_lambda_scale * crate::fastmath::lambda_qp(qp);
    // B path keeps the frame-median tex veto even under `tune_lme_q` (its `lme` is
    // hoisted, not per-MB) — recorded limitation until the knob clears its BD gate.
    let lme_scale = me_lambda_scale(cfg, &sig, false);
    let lme = rusty_h264_common::fmath::sqrt(lambda) * lme_scale;
    let refs = core::slice::from_ref(l0);
    if fe.satd_q > 0.0 {
        fe.satd_var_thresh = sig.var_percentile_thresh(fe.satd_q);
    }
    let mut aq_qp = aq_qp_map(&sig, qp, fe.aq_strength);
    apply_mbtree_qpo(&mut aq_qp, qpo); // mb-tree temporal AQ (empty = byte-identical)
    signals::harvest(
        &sig,
        'B',
        qp,
        &signals::GateDecisions {
            lme_scale,
            satd_thresh: fe.satd_var_thresh,
            ..Default::default()
        },
    );

    let mut cab = CabacEncoder::new_with_out(
        qp as i32,
        cfg.cabac_init_idc,
        false,
        enc_scratch::take_payload(),
    );
    let mut cs = CabacState::pooled(fe.mb_w * fe.mb_h);
    let total = fe.mb_w * fe.mb_h;
    // E15 round 2: slice-constant accountant knob, was re-read up to 10x per MB.
    let acct = crate::bitacct::enabled();
    // RD B_Skip knobs + the online free-skip census that dispatches it.
    let bskip_t = rusty_h264_common::knob("RFF_BSKIP_T")
        .and_then(|v| v.parse::<f64>().ok())
        .or(cfg.tune_bskip_rd)
        .unwrap_or(0.0);
    let bskip_busy_pct = rusty_h264_common::knob("RFF_BSKIP_BUSY")
        .and_then(|v| v.parse::<usize>().ok())
        .or(cfg.tune_bskip_busy_pct)
        .unwrap_or(60);
    let (mut b_seen, mut b_free) = (0usize, 0usize);
    // B 16x8/8x16 partition search. Opt-in until the 4-QP per-clip table clears.
    let bsplit_env = rusty_h264_common::knob("RFF_BSPLIT").and_then(|v| v.parse::<u32>().ok());
    let bsplit_on = bsplit_env.map(|v| v == 1).unwrap_or(cfg.tune_b_split);
    let bsplit_probe = bsplit_env.filter(|&v| v >= 2).unwrap_or(0);
    // Online DIRECT-WIN rate: of the macroblocks that were not exactly-free, how
    // often direct still won the mode decision. B_Skip rides direct-mode motion,
    // so this measures the quality of the thing the skip bets on.
    let (mut b_coded, mut b_dirwin) = (0usize, 0usize);
    let bskip_dirwin_pct = rusty_h264_common::knob("RFF_BSKIP_DIRWIN")
        .and_then(|v| v.parse::<usize>().ok())
        .or(cfg.tune_bskip_dirwin_pct)
        .unwrap_or(10);

    for mb_y in 0..fe.mb_h {
        for mb_x in 0..fe.mb_w {
            let mb_idx = mb_y * fe.mb_w + mb_x;
            let addr = mb_idx;
            let top = if mb_y > 0 { Some(addr - fe.mb_w) } else { None };
            let left = if mb_x > 0 { Some(addr - 1) } else { None };
            let mb_qp = aq_qp[mb_idx];
            fe.qp = mb_qp;
            fe.qpc = chroma_qp(mb_qp);
            let (lx, ly) = (mb_x * 16, mb_y * 16);
            let (pbx, pby) = (mb_x as isize * 4, mb_y as isize * 4);
            // Dedup (E15 W9): memoized twin, already built when satd_q > 0.
            fe.mb_use_satd =
                fe.satd_q > 0.0 && sig.mb_vars()[mb_y * fe.mb_w + mb_x] >= fe.satd_var_thresh;
            let n0 = fe.mv_neighbors_block_list(pbx, pby, 4, 0);
            let n1 = fe.mv_neighbors_block_list(pbx, pby, 4, 1);
            let pmv0 = predict_partition_mv(0, 0, n0[0], n0[1], n0[2], 0);
            let pmv1 = predict_partition_mv(0, 0, n1[0], n1[1], n1[2], 0);
            let (dp, dc, dmotion) = fe.b_direct(l0, l1, mb_x, mb_y);
            // B_Skip: free direct prediction → mb_skip_flag = 1.
            let free_skip = fe.skip_luma_is_free(&sy, mb_x, mb_y, &dp)
                && fe.skip_chroma_is_free(&su, &sv, mb_x, mb_y, &dc);
            b_seen += 1;
            if free_skip {
                b_free += 1;
            }
            if free_skip {
                bstats::bump(&bstats::SKIP);
                fe.commit_direct_motion(mb_x, mb_y, &dmotion);
                fe.commit_pred_pixels(mb_x, mb_y, &dp, &dc);
                emit_b_skip_cabac(&mut cab, &mut cs, addr, top, left);
                mb_qpy[mb_idx] = fe.cur_qp;
                cb_term_acct(&mut cab, mb_idx + 1 == total, acct);
                continue;
            }
            bstats::bump(&bstats::CODED);
            let d_direct = fe.pred_dist(&sy, lx, ly, &dp);
            let (mv0, j0) = fe.motion_search(l0, &sy, lx, ly, 16, 16, &[pmv0], lme, None);
            let (mv1, j1) = fe.motion_search(l1, &sy, lx, ly, 16, 16, &[pmv1], lme, None);
            let d_bi = fe.bi_dist(l0, l1, &sy, lx, ly, mv0, mv1);
            let r_bi = mvd_bits(mv0.0 - pmv0.0)
                + mvd_bits(mv0.1 - pmv0.1)
                + mvd_bits(mv1.0 - pmv1.0)
                + mvd_bits(mv1.1 - pmv1.1);
            let j_bi = d_bi + (lme * r_bi as f64) as i64;
            let (mut dir, mut best) = (0u8, d_direct);
            if j0 < best {
                dir = 1;
                best = j0;
            }
            if j1 < best {
                dir = 2;
                best = j1;
            }
            if j_bi < best {
                dir = 3;
                best = j_bi;
            }
            if dir == 0 {
                bstats::bump(&bstats::DIRWIN);
            }
            b_coded += 1;
            if dir == 0 {
                b_dirwin += 1;
            }
            // ---- RD B_Skip (env RFF_BSKIP_T; unset = byte-identical) ----------
            // Our B_Skip previously required the direct residual to quantize to
            // EXACTLY zero. Measured against x264 at qp27: that reaches 93.5% of B
            // macroblocks on akiyo and 34.5% on foreman -- at or ABOVE x264 -- but
            // collapses to 7.8% on mobile where x264 still finds 27.4%. The deficit
            // is BUSY-CONTENT-ONLY, a sign flip, so this is a DISPATCH, not a new
            // constant: engage only where the free-skip rate is low.
            //
            // Two terms, both required:
            //   * `dir == 0` -- direct actually WON the mode decision. `best` starts
            //     at `d_direct` and only falls, so without this the test would fire
            //     on macroblocks the search proved are better coded.
            //   * distortion under T*lambda -- the residual is not worth its bits.
            // Gated on the ONLINE free-skip rate of this frame so far (the same
            // signal shape the P path's rd_skip uses, inverted: engage where free
            // skips are RARE, which is exactly where we under-skip).
            if bskip_t > 0.0
                && dir == 0
                && b_seen >= 32
                && b_free * 100 < b_seen * bskip_busy_pct
                // DIRECT-WIN FLOOR. Measured truth table at T=48 (BD-PSNR):
                //   football 7.0% direct-win  -> +0.08  LOSS   <- the only loser
                //   foreman 14.1%             -> -0.17  win
                //   bus     21.4%             -> -0.05  win
                //   akiyo   24.7%             -> inert
                //   tempete 30.9%             -> -0.16  win
                //   mobile  43.1%             -> -0.38  win
                // The one regressing clip has by far the lowest direct-win rate, and
                // the term is justified by exactly that clip: where direct almost
                // never wins the mode decision, its prediction is unreliable and
                // skipping on it costs more than the bits it saves.
                && b_coded >= 32
                && b_dirwin * 100 >= b_coded * bskip_dirwin_pct
                && (d_direct as f64) <= bskip_t * lambda
            {
                bstats::bump(&bstats::SKIP);
                fe.commit_direct_motion(mb_x, mb_y, &dmotion);
                fe.commit_pred_pixels(mb_x, mb_y, &dp, &dc);
                emit_b_skip_cabac(&mut cab, &mut cs, addr, top, left);
                mb_qpy[mb_idx] = fe.cur_qp;
                cb_term_acct(&mut cab, mb_idx + 1 == total, acct);
                continue;
            }
            // mb_skip_flag = 0, then the coded B MB.
            let sctx = 24
                + left.map_or(0, |a| (!cs.mb_skip[a]) as usize)
                + top.map_or(0, |a| (!cs.mb_skip[a]) as usize);
            let tskip = if acct { cab.pos() } else { 0 };
            cb_mb_skip(&mut cab, sctx, false);
            if acct {
                crate::bitacct::add(crate::bitacct::B::SkipFlag, cab.pos() - tskip);
            }
            cs.mb_skip[addr] = false;
            // ---- intra-in-B (the m4 == 13 escape) ------------------------------
            // Same SATD heuristic as the P path: a B macroblock the inter search
            // cannot predict (scene cut inside the GOP, occlusion) is coded intra
            // instead of as a bad inter guess. Compared against the 16x16-level
            // winner with the same signalling penalty the P decision charges.
            let c_intra = fe.best_i16_satd(&sy, mb_x, mb_y) + (lme * fe.tune_intra_penalty) as i64;
            if c_intra < best {
                bstats::bump(&bstats::INTRA);
                fe.intra_in_p = true;
                let plan = plan_mb(&mut fe, mb_x, mb_y, &sy, &su, &sv);
                let bci = left.map_or(0, |a| (!cs.mb_direct[a]) as usize)
                    + top.map_or(0, |a| (!cs.mb_direct[a]) as usize);
                cb_mb_type_b_intra(&mut cab, bci, &plan);
                emit_intra_body_cabac(
                    &mut fe, &mut cab, &mut cs, &plan, mb_x, mb_y, addr, top, left,
                );
                mb_qpy[mb_idx] = fe.cur_qp;
                cb_term_acct(&mut cab, mb_idx + 1 == total, acct);
                continue;
            }
            // ---- B 16x8 / 8x16 partition search --------------------------------
            // x264 puts 13.5% of its B macroblocks here (`B16..8: 31.1 13.5 8.2`);
            // we had none, which is why the B bucket kept reading as a CODING gap
            // after every constant in it had been swept flat. Each half runs the
            // SAME 16x16 motion search that already exists, then the 9 (p0,p1)
            // pairings are priced against the 16x16 winner.
            let mut bsplit: Option<(u8, [(u8, (i32, i32), (i32, i32)); 2])> = None;
            // ORACLE PROBE (RFF_BSPLIT=2/3): force a 16x8 (2) or 8x16 (3) whose two
            // halves carry the SAME pred and the SAME motion as the 16x16 winner.
            // That is semantically identical to the 16x16 macroblock, so the
            // reconstruction MUST match it bit for bit -- any quality loss under this
            // probe is emit/predict PLUMBING drift and nothing to do with the mode
            // decision. (Separating those two is otherwise guesswork: both present as
            // "quality fell at the same rate".)
            if bsplit_probe > 0 && dir != 0 {
                let m = if bsplit_probe == 2 { 1u8 } else { 2u8 };
                bsplit = Some((m, [(dir, mv0, mv1); 2]));
            } else if bsplit_on {
                for mvmode in 1u8..=2 {
                    let (rects, _) = b_part_layout(mvmode);
                    let mut cand = [(0u8, (0i32, 0i32), (0i32, 0i32)); 2];
                    let mut jsum = 0i64;
                    for (part, &(rx, ry, rw, rh)) in rects.iter().enumerate() {
                        let (px, py) = (lx + rx, ly + ry);
                        let (m0, c0) =
                            fe.motion_search(l0, &sy, px, py, rw, rh, &[pmv0], lme, None);
                        let (m1, c1) =
                            fe.motion_search(l1, &sy, px, py, rw, rh, &[pmv1], lme, None);
                        // Bi for this rect: blend distortion + both mvd rates.
                        let dbi = fe.bi_dist_rect(l0, l1, &sy, px, py, rw, rh, m0, m1);
                        let rbi = mvd_bits(m0.0 - pmv0.0)
                            + mvd_bits(m0.1 - pmv0.1)
                            + mvd_bits(m1.0 - pmv1.0)
                            + mvd_bits(m1.1 - pmv1.1);
                        let jbi = dbi + (lme * rbi as f64) as i64;
                        let (mut bp, mut bj) = (1u8, c0);
                        if c1 < bj {
                            bp = 2;
                            bj = c1;
                        }
                        if jbi < bj {
                            bp = 3;
                            bj = jbi;
                        }
                        cand[part] = (bp, m0, m1);
                        jsum += bj;
                    }
                    // ~4 extra bins for the longer mb_type binarization.
                    let jsplit = jsum + (lme * 4.0) as i64;
                    if jsplit < best {
                        best = jsplit;
                        bsplit = Some((mvmode, cand));
                    }
                }
                if bsplit.is_some() {
                    bstats::bump(&bstats::SPLIT);
                }
            }
            let bspec = if let Some((mvmode, parts2)) = bsplit {
                BInter {
                    dir,
                    l1,
                    mv0,
                    mv1,
                    mvmode,
                    parts2,
                }
            } else {
                BInter {
                    dir,
                    l1,
                    mv0,
                    mv1,
                    mvmode: 0,
                    parts2: [(0, (0, 0), (0, 0)); 2],
                }
            };
            let plan = fe.plan_inter_mb(
                refs,
                &sy,
                &su,
                &sv,
                mb_x,
                mb_y,
                0,
                &[],
                Some(bspec),
                [0u8; 4],
            );
            emit_mb_cabac_b(&mut fe, &mut cab, &mut cs, dir, bsplit, &plan, mb_x, mb_y);
            mb_qpy[mb_idx] = fe.cur_qp;
            cb_term_acct(&mut cab, mb_idx + 1 == total, acct);
        }
    }
    enc_scratch::put_cs(cs);

    while !w.is_byte_aligned() {
        w.write_bit(true);
    }
    let payload = cab.into_bytes();
    w.write_aligned_bytes(&payload);
    enc_scratch::put_payload(payload);
    if !is_ref {
        // Leaf B: non-reference — no deblock, no RefFrame (the decoder
        // deblocks for display).
        enc_scratch::put_qpy(mb_qpy);
        return None;
    }
    // REFERENCE B (b-pyramid): deblock and package the recon exactly like a P
    // anchor, but with BOTH lists' motion in the boundary-strength derivation
    // (raw per-block indices + single-entry index→POC maps — the two-list bS
    // rule compares PICTURES, and an index alone cannot cross lists).
    let mut ref_id = enc_scratch::take_refid();
    ref_id.clear();
    ref_id.extend(
        fe.ref_idx_y
            .iter()
            .map(|&r| if r >= 0 { r } else { i8::MIN }),
    );
    // List-1 grid from the recycled slot — was a fresh Vec per ref-B slice.
    let mut ref_id1 = enc_scratch::take_refid1();
    ref_id1.clear();
    ref_id1.extend(
        fe.ref_idx1_y
            .iter()
            .map(|&r| if r >= 0 { r } else { i8::MIN }),
    );
    let poc0 = [l0.poc];
    let poc1 = [l1.poc];
    let info = rusty_h264_common::deblock::BlockInfo {
        inter: &fe.inter_y,
        nnz: &fe.nnz_y,
        mv: &fe.mv_y,
        ref_id: &ref_id,
        mv1: &fe.mv1_y,
        ref_id1: &ref_id1,
        w4: fe.mb_w * 4,
        t8x8: &[],
        poc0: &poc0,
        poc1: &poc1,
        bs: &[],
        kind: &[],
    };
    rusty_h264_common::deblock::filter_frame(
        &mut fe.rec_y,
        &mut fe.rec_u,
        &mut fe.rec_v,
        fe.mb_w,
        fe.mb_h,
        &mb_qpy,
        0,
        0,
        0,
        &info,
    );
    enc_scratch::put_refid(ref_id);
    enc_scratch::put_refid1(ref_id1);
    enc_scratch::put_qpy(mb_qpy);
    let w4 = fe.mb_w * 4;
    Some(crate::RefFrame {
        y: fe.rec_y,
        u: fe.rec_u,
        v: fe.rec_v,
        poc: 0,
        frame_num: 0,
        mv: fe.mv_y,
        ref_idx: fe.ref_idx_y,
        mv1: fe.mv1_y,
        ref_idx1: fe.ref_idx1_y,
        w4,
        hpel: crate::HpelOnce::new(),
    })
}

/// Minimal all-B_Skip CABAC B-slice (the rare no-bracketing-anchor fallback in
/// `code_picture`): every MB is mb_skip_flag = 1. B is non-reference so the recon
/// is irrelevant; this only needs to be a legal CABAC slice.
pub fn encode_all_skip_b_cabac(w: &mut BitWriter, cfg: &EncoderConfig, qp: u8, n: usize) {
    let mut cab = CabacEncoder::new_with_out(
        qp as i32,
        cfg.cabac_init_idc,
        false,
        enc_scratch::take_payload(),
    );
    for i in 0..n {
        // ctxInc = 24 + (left avail & not-skip) + (top avail & not-skip). Every
        // neighbour is either a skip (contributes 0) or unavailable (0) → always 24.
        cab.encode_decision(24, 1);
        cab.encode_terminate(i + 1 == n);
    }
    while !w.is_byte_aligned() {
        w.write_bit(true);
    }
    let payload = cab.into_bytes();
    w.write_aligned_bytes(&payload);
    enc_scratch::put_payload(payload);
}