rusty_h264_encoder/

mb16.rs

//! I_16x16 macroblock encoding (DC prediction) — the compressing intra path.
//!
//! Index-based loops below drive pixel/block-position arithmetic and read
//! clearer than iterator adapters for this raster math.
#![allow(clippy::needless_range_loop)]
//!
//! Each macroblock is DC-predicted from already-reconstructed neighbors, the
//! residual is transformed/quantized (luma DC via the secondary Hadamard), the
//! coefficients are CAVLC-coded, and the macroblock is reconstructed so the next
//! one can predict from it. `nnz` grids feed the CAVLC `nC` context exactly as a
//! conforming decoder derives it.

use crate::cabac::CabacEncoder;
use crate::config::EncoderConfig;
use rusty_h264_common::cavlc::{
    encode_residual_block, scan_4x4_ac, scan_4x4_dcac, write_cbp_inter, write_cbp_intra,
};
use rusty_h264_common::inter::{
    inter_partitions, mc_chroma, mc_luma, predict_mv, predict_partition_mv, MvNeighbor,
};
use rusty_h264_common::predict::{
    add_residual_4x4, add_residual_8x8, chroma8x8_pred, chroma_mode_available, chroma_qp,
    intra4x4_pred, intra8x8_pred, luma16x16_pred, reconstruct_4x4, I16Mode, CHROMA_4X4_SCAN_XY,
    LUMA_4X4_SCAN_XY,
};
use rusty_h264_common::transform::{
    dequantize, forward_core, forward_core_8x8, forward_dct_blocks, forward_quant_chroma_dc,
    forward_quant_luma_dc, inverse_dct_blocks, inverse_quant_8x8, inverse_quant_chroma_dc,
    inverse_quant_luma_dc, quantize, quantize_8x8, satd_4x4_sum,
};
use rusty_h264_common::aligned::AlignedBytes;
use rusty_h264_common::{BitWriter, YuvFrame};

/// A/B switch for the batched full-pel rescue grid (`RFF_ME_BATCH=0` disables).
///
/// Read ONCE per process, not per call: this sits inside the motion-search rescue
/// path, and `std::env::var` allocates a `String` and takes the process-wide
/// environment lock every time. A runtime switch inside a hot loop is its own
/// measurable tax — cache it.


/// λ-normalised threshold for the partition-split search gate (U2).
///
/// The existing `split_gate` is a function of qstep ALONE, so it does not scale with
/// the rate/distortion trade the search is actually making. Normalising the null arm
/// by λ — the king feature for any search-skip gate — makes one constant transfer
/// across content AND the whole QP ladder, and in the SAFE direction: the feature is
/// small exactly where the 16×16 null arm is already good, so easy content skips more.
///
/// Harvested over 36 k gated macroblocks (4 clips): at T = 400 the split search is
/// skipped on 2.9–22.5% of them while keeping **100.00%** of the achievable cost gain
/// on every clip; T = 600 skips 11–79% for 93–99% kept. `RFF_SPLIT_T=0` disables.
pub(crate) static DEFER_SUBPEL: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);

pub(crate) static SPLIT_T: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(u32::MAX);

fn split_t() -> f64 {
    let v = SPLIT_T.load(std::sync::atomic::Ordering::Relaxed);
    if v != u32::MAX {
        return v as f64;
    }
    let d: u32 = std::env::var("RFF_SPLIT_T").ok().and_then(|s| s.parse().ok()).unwrap_or(0);
    SPLIT_T.store(d, std::sync::atomic::Ordering::Relaxed);
    d as f64
}

/// Observe-only HARVEST for the PARTITION-split gate (U2/U5).
///
/// The 16×16 search is the null arm and runs first; the 2-way splits and P_8x8 are
/// the expensive arm (7 further `best_part` calls, each with its own full-pel search
/// AND sub-pel refinement). Today they are gated by a fixed `split_gate` formula.
/// Records the null-arm cost, the best split cost, and which won, so the
/// skip-rate-vs-gain-kept ceiling can be swept before any threshold is touched.
mod split_harvest {
    use std::fs::File;
    use std::io::Write;
    use std::sync::{Mutex, OnceLock};

    fn sink() -> &'static Option<Mutex<File>> {
        static S: OnceLock<Option<Mutex<File>>> = OnceLock::new();
        S.get_or_init(|| {
            std::env::var("RFF_SPLIT_HARVEST").ok().and_then(|p| {
                let mut f = File::create(p).ok()?;
                let _ = writeln!(f, "c16,best,lambda,gate,won");
                Some(Mutex::new(f))
            })
        })
    }

    #[inline]
    pub fn enabled() -> bool {
        sink().is_some()
    }

    pub fn record(c16: i64, best: i64, lambda: f64, gate: i64, won: u8) {
        if let Some(m) = sink() {
            if let Ok(mut f) = m.lock() {
                let _ = writeln!(f, "{c16},{best},{lambda:.4},{gate},{won}");
            }
        }
    }
}

/// Descent C escape hatch: `RFF_HPEL_REF=0` restores the copy-then-SATD half-pel path
/// (byte-identical to it either way — this exists as a bisection anchor).
fn hpel_ref_enabled() -> bool {
    static E: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
    *E.get_or_init(|| std::env::var("RFF_HPEL_REF").map(|v| v != "0").unwrap_or(true))
}

/// H-23: smooth (x264-shape) mvd cost table, in quarter-bit units scaled to the
/// same magnitude as the Exp-Golomb model so λ stays calibrated. `RFF_MVCOST=1`.
static MV_COST_TAB: std::sync::OnceLock<Vec<u16>> = std::sync::OnceLock::new();
fn build_mv_cost() -> Vec<u16> {
    (0..4096u32)
        .map(|a| {
            let c = 2.0 * ((a + 1) as f64).log2() + 0.718 + if a != 0 { 1.0 } else { 0.0 };
            // Round to quarter-bits then express in the caller's integer "bits"
            // domain by keeping 4× resolution — λ is rescaled to match below.
            (c * 4.0).round() as u16
        })
        .collect()
}
/// H-24: 0 = off (Exp-Golomb step, byte-identical), 1 = DISPATCHED per frame by
/// the `b2_mgain` motion probe, 2 = force-on. The BD sign-flip (bus −1.31 /
/// football −0.24 vs foreman +0.23 / akiyo +0.11) tracks MOTION: the smooth
/// curve pays where |mvd| is large enough to leave the first bracket, and only
/// adds noise where every vector already sits inside it.
static MV_SMOOTH: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(u32::MAX);
pub fn set_mv_smooth(on: bool) {
    MV_SMOOTH.store(if on { 2 } else { 0 }, core::sync::atomic::Ordering::Relaxed)
}
pub fn set_mv_smooth_mode(m: u32) {
    MV_SMOOTH.store(m.min(3), core::sync::atomic::Ordering::Relaxed)
}
/// Dispatch threshold on the per-frame mgain probe (`RFF_MVCOST_T`, default 0.10).
/// Calibrated on the DEPLOYED probe: bus min-frame 0.185 and football med 0.208
/// route ON; foreman med 0.164 is the boundary case, akiyo ~0.00 routes OFF.
/// H-26: the measured TRUE table plus a COHERENCE BIAS on every d≠0 entry —
/// the cheap scalar form of the MV-field externality H-25 root-caused (a chosen
/// vector that leaves the predictor degrades the neighbours' medians; truth
/// per-vector under-prices that shared damage). `RFF_MVCOST_BIAS` in bits
/// (default 0 = pure truth), read once at first use.
static MV_TRUE_BIASED: std::sync::OnceLock<Vec<u16>> = std::sync::OnceLock::new();
fn build_true_biased() -> Vec<u16> {
    let bias_q4 = (std::env::var("RFF_MVCOST_BIAS")
        .ok()
        .and_then(|v| v.parse::<f64>().ok())
        .unwrap_or(1.0)
        * 4.0)
        .round() as u16;
    crate::mvd_cost_tab::MVD_TRUE_COST4
        .iter()
        .enumerate()
        .map(|(d, &c)| if d == 0 { c } else { c.saturating_add(bias_q4) })
        .collect()
}

fn mv_smooth_t() -> f64 {
    static T: std::sync::OnceLock<f64> = std::sync::OnceLock::new();
    *T.get_or_init(|| std::env::var("RFF_MVCOST_T").ok().and_then(|v| v.parse().ok()).unwrap_or(0.10))
}
/// 0 = step model, 1 = smooth (this frame routed on), 2 = smooth (forced),
/// 3 = the MEASURED true-cost table (H-25) — no dispatch needed if it wins
/// everywhere, since it is the truth both analytic models approximate.
/// `frame_smooth` is the per-frame probe decision, carried on the frame state
/// (like `sadfp`) — a process-global here races under the GOP-parallel encode.
#[inline]
fn mv_cost_kind(frame_smooth: bool) -> u32 {
    match mv_smooth_mode() {
        0 => 0,
        // H-26 verdict: smooth/truth/truth+bias shuffle within ±0.2 BD fit-noise
        // of each other at dispatch (bus prefers truth, football prefers smooth,
        // none dominates), so the dispatch keeps its ORIGINALLY-GATED smooth
        // ON-model; the measured/biased tables remain as modes 2/3 for research.
        1 => frame_smooth as u32,
        2 => 1, // archaeology: the x264 smooth curve, forced
        _ => 2, // the biased-truth table, forced
    }
}
#[inline]
fn mv_smooth_mode() -> u32 {
    match MV_SMOOTH.load(core::sync::atomic::Ordering::Relaxed) {
        m @ 0..=3 => m,
        _ => {
            static E: std::sync::OnceLock<u32> = std::sync::OnceLock::new();
            // DEFAULT 1 = DISPATCHED (H-24). Owner's call: mean −0.27% BD is
            // taken over minimax, accepting a known, bounded +0.16-0.18% on
            // foreman-class content. `RFF_MVCOST=0` restores the pre-H-23 bytes.
            *E.get_or_init(|| {
                std::env::var("RFF_MVCOST").ok().and_then(|v| v.parse().ok()).unwrap_or(1)
            })
        }
    }
}

/// Challenge-1 A3 escape hatch: `RFF_SATD_AVG=0` restores the materialize-then-SATD
/// quarter-pel cost path (byte-identical either way — a bisection anchor, like
/// `RFF_HPEL_REF`).
/// H-14 R3 escape hatch: `RFF_MECTX=0` restores the per-eval safe dispatch
/// (byte-identical either way — MeCtx returns exactly the safe path's values).
fn mectx_enabled() -> bool {
    static E: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
    *E.get_or_init(|| std::env::var("RFF_MECTX").map(|v| v != "0").unwrap_or(true))
}

fn satd_avg_enabled() -> bool {
    static E: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
    *E.get_or_init(|| std::env::var("RFF_SATD_AVG").map(|v| v != "0").unwrap_or(true))
}

/// Track-B B2 (docs/lets-win-optimize.md): run the FULL-PEL phase of the non-fast
/// motion search in the SAD domain (`psadbw`-class, ~3-4× cheaper per candidate) and
/// reprice the winner in SATD before the rescue/sub-pel phases — the cost split
/// every x264 preset uses (SAD fpel, SATD from subme≥2). ⚠ BITSTREAM-CHANGING (a
/// different full-pel winner can emerge), so it ships opt-in until the per-clip
/// 4-QP BD gate clears it. `set_me_sadfp` overrides; unset → `RFF_ME_SADFP` env,
/// default OFF (off = byte-identical to the pre-B2 encoder).
/// Modes: 0 = off (byte-identical), 1 = DISPATCHED per frame by the `b2_mgain`
/// probe (the shipping shape), 2 = force-on everywhere (the truth-table A/B arm).
static ME_SADFP: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(u32::MAX);
pub fn set_me_sadfp(on: bool) {
    // Harness semantics preserved: `true` = the force-on arm truth tables measure.
    ME_SADFP.store(if on { 2 } else { 0 }, core::sync::atomic::Ordering::Relaxed)
}
pub fn set_me_sadfp_mode(m: u32) {
    ME_SADFP.store(m.min(2), core::sync::atomic::Ordering::Relaxed)
}
fn me_sadfp_mode() -> u32 {
    match ME_SADFP.load(core::sync::atomic::Ordering::Relaxed) {
        u32::MAX => {
            static INIT: std::sync::OnceLock<u32> = std::sync::OnceLock::new();
            // DEFAULT = 1 (dispatched) since the H-3 gate: 16-clip corpus mean
            // −0.26% BD, wins bus −1.71 / football −1.84 / foreman −0.44 /
            // shields −0.22, every former loss 0.00; residual tail (soccer +0.09,
            // harbour +0.06) is BD-fit noise — it responds NON-monotonically to
            // threshold changes (less B2 made soccer read WORSE, +0.18).
            // `RFF_ME_SADFP=0` is the escape hatch reproducing the pre-B2 bytes.
            *INIT.get_or_init(|| {
                std::env::var("RFF_ME_SADFP").ok().and_then(|v| v.parse().ok()).unwrap_or(1)
            })
        }
        m => m,
    }
}

/// B2 dispatch threshold on the per-frame `b2_mgain` probe (`RFF_ME_SADT`).
/// Calibrated on the DEPLOYED estimator (recon reference, sampled MBs), not the
/// offline source-frame probe — the recurring R6 law.
fn me_sadt() -> f64 {
    static T: std::sync::OnceLock<f64> = std::sync::OnceLock::new();
    *T.get_or_init(|| std::env::var("RFF_ME_SADT").ok().and_then(|s| s.parse().ok()).unwrap_or(0.13))
}
fn me_sadt_dbg() -> bool {
    static D: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
    *D.get_or_init(|| std::env::var_os("RFF_ME_SADT_DBG").is_some())
}

/// Fixed-centre batched diamond passes on SAD-routed frames (`RFF_ME_FC=0` falls
/// back to the cascading scalar walk — the bisection anchor). Fixed-centre differs
/// from the cascade only when 2+ points improve in one pass, so it rides B2's BD
/// gate; dispatched-OFF frames never take this path and stay byte-identical.
/// ③ Sub-pel ring FC: fixed-centre argmin passes for the HALF-PEL step, batched
/// through `satd_16x16_x4p` (two calls cover the 8-ring; candidates resolve to
/// h/h/v/v and c/c/c/c plane reads from an integer centre). Quarter-step and any
/// declined pass keep the cascading walk. Bitstream-changing → own gate
/// (`AB_SPFC`), `RFF_SP_FC=0` anchor.
static SP_FC: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(u32::MAX);
pub fn set_sp_fc(on: bool) {
    SP_FC.store(on as u32, core::sync::atomic::Ordering::Relaxed)
}
fn sp_fc_enabled() -> bool {
    match SP_FC.load(core::sync::atomic::Ordering::Relaxed) {
        0 => false,
        1 => true,
        _ => {
            static E: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
            *E.get_or_init(|| std::env::var("RFF_SP_FC").map(|v| v != "0").unwrap_or(false))
        }
    }
}

static ME_FC: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(u32::MAX);
pub fn set_me_fc(on: bool) {
    ME_FC.store(on as u32, core::sync::atomic::Ordering::Relaxed)
}
fn me_fc_enabled() -> bool {
    match ME_FC.load(core::sync::atomic::Ordering::Relaxed) {
        0 => false,
        1 => true,
        _ => {
            static E: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
            *E.get_or_init(|| std::env::var("RFF_ME_FC").map(|v| v != "0").unwrap_or(true))
        }
    }
}

/// H-13 SPLIT DISPATCH — measured and REFUTED as a free dispatch, shipped as an
/// OPT-IN rung (default 0 = off = byte-identical). The premise "splits buy
/// ~nothing on near-static frames" is FALSE: at T=0.03 akiyo read +2.45% BD,
/// akiyo_qcif +2.02%, FourPeople +2.00% for only 1.10-1.15× — partition splits
/// EARN BD on every measured content class (the third death of the split-gate
/// idea: U2 T=400, the sum-weighted ceiling, now the mgain axis). foreman/bus
/// route ON at any sane T (min frame mgain 0.061/0.185) and stay byte-identical.
/// `RFF_SPLIT_MG` (fraction) / `set_split_mg` (milli): a priced speed rung, not
/// a free lunch.
static SPLIT_MG: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(u32::MAX);
pub fn set_split_mg(milli: u32) {
    SPLIT_MG.store(milli, core::sync::atomic::Ordering::Relaxed)
}
fn split_mg() -> f64 {
    match SPLIT_MG.load(core::sync::atomic::Ordering::Relaxed) {
        u32::MAX => {
            static E: std::sync::OnceLock<f64> = std::sync::OnceLock::new();
            *E.get_or_init(|| {
                std::env::var("RFF_SPLIT_MG").ok().and_then(|v| v.parse().ok()).unwrap_or(0.0)
            })
        }
        m => m as f64 / 1000.0,
    }
}

/// The flash veto: frames whose zero-MV residual is DC-shift-dominated beyond this
/// fraction route OFF even at high mgain (`RFF_ME_SADDC`). Calibrated on the
/// DEPLOYED per-frame values: crew's harmful ON-frames read dc 0.843–0.859 (the
/// camera flashes) while every good ON-frame on bus/football/foreman reads ≤ 0.478
/// — a 1.76× natural gap; 0.6 sits mid-gap with margin both ways.
fn me_sad_dcmax() -> f64 {
    static T: std::sync::OnceLock<f64> = std::sync::OnceLock::new();
    *T.get_or_init(|| std::env::var("RFF_ME_SADDC").ok().and_then(|s| s.parse().ok()).unwrap_or(0.6))
}

/// The B2 dispatch signal: mean over ~24 sampled interior MBs of
/// `(SAD@zeroMV − bestSAD over a ±8 step-4 full-pel grid) / SAD@zeroMV` — how much
/// a plain TRANSLATIONAL full-pel search improves on zero motion, i.e. exactly the
/// surface B2's SAD diamond exploits. Offline (b2_signals, 16-clip truth table) it
/// separates every B2 loss (crew flash 0.070, city 0.110, tempete 0.008) from
/// every meaningful win (bus 0.323, football/foreman 0.164/0.165, shields 0.361);
/// notably `me_wide_headroom` CANNOT be reused here — crew's headroom is high (20)
/// but B2 loses there, because SAD overprices the DC shifts of its camera flashes.
/// Returns `(mgain, dcfrac)`. `dcfrac` — mean `|Σcur − Σref| / SAD0` per sampled
/// block — is the FLASH detector: under an illumination change the zero-MV residual
/// is mostly a DC shift, which SAD prices fully but the Hadamard largely discounts,
/// so SAD misranks candidates exactly there. Justified by the one clip the
/// single-term gate got wrong (crew: high mgain on its motion frames, +0.54 BD).
fn b2_mgain(sy: &[u8], cw: usize, ch: usize, ref_y: &[u8]) -> (f64, f64) {
    const WIDE: isize = 8;
    const STEP: isize = 4;
    const TARGET: usize = 24;
    let sad16 = |bx: usize, by: usize, rx: isize, ry: isize| -> Option<u32> {
        if rx < 0 || ry < 0 || rx as usize + 16 > cw || ry as usize + 16 > ch {
            return None;
        }
        let (rx, ry) = (rx as usize, ry as usize);
        let mut s = 0u32;
        for dy in 0..16 {
            let a = &sy[(by + dy) * cw + bx..][..16];
            let b = &ref_y[(ry + dy) * cw + rx..][..16];
            s += a.iter().zip(b).map(|(&p, &q)| p.abs_diff(q) as u32).sum::<u32>();
        }
        Some(s)
    };
    let (mbw, mbh) = (cw / 16, ch / 16);
    if mbw < 6 || mbh < 6 {
        return (0.0, 0.0);
    }
    let inner = (mbw - 4) * (mbh - 4);
    let stride = (inner / TARGET).max(1);
    let (mut acc, mut dc, mut n) = (0.0f64, 0.0f64, 0u32);
    let mut i = 0usize;
    while i < inner {
        let (mx, my) = (2 + i % (mbw - 4), 2 + i / (mbw - 4));
        let (bx, by) = (mx * 16, my * 16);
        if let Some(s0) = sad16(bx, by, bx as isize, by as isize) {
            let (mut ms, mut mr) = (0u32, 0u32);
            for dy in 0..16 {
                ms += sy[(by + dy) * cw + bx..][..16].iter().map(|&v| v as u32).sum::<u32>();
                mr += ref_y[(by + dy) * cw + bx..][..16].iter().map(|&v| v as u32).sum::<u32>();
            }
            dc += ms.abs_diff(mr) as f64 / (s0 + 1) as f64;
            let mut best = s0;
            let mut dy = -WIDE;
            while dy <= WIDE {
                let mut dx = -WIDE;
                while dx <= WIDE {
                    if let Some(s) = sad16(bx, by, bx as isize + dx, by as isize + dy) {
                        best = best.min(s);
                    }
                    dx += STEP;
                }
                dy += STEP;
            }
            acc += (s0 - best) as f64 / (s0 + 1) as f64;
            n += 1;
        }
        i += stride;
    }
    if n == 0 { (0.0, 0.0) } else { (acc / n as f64, dc / n as f64) }
}

/// Track-B B3: cap on sub-pel ring ITERATIONS per step (`RFF_SP_MAXIT` /
/// `set_sp_maxit`). 0 = unlimited (the default — byte-identical to the walk-to-
/// convergence encoder); N caps each step's walk at N passes, the bounded budget
/// x264's subme levels have always had. Bitstream-changing when set → BD-gated.
static SP_MAXIT: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(u32::MAX);
pub fn set_sp_maxit(n: u32) {
    SP_MAXIT.store(n, core::sync::atomic::Ordering::Relaxed)
}
fn sp_maxit() -> u32 {
    match SP_MAXIT.load(core::sync::atomic::Ordering::Relaxed) {
        u32::MAX => {
            static INIT: std::sync::OnceLock<u32> = std::sync::OnceLock::new();
            *INIT.get_or_init(|| {
                std::env::var("RFF_SP_MAXIT").ok().and_then(|v| v.parse().ok()).unwrap_or(0)
            })
        }
        n => n,
    }
}

/// B2 calibration: λ multiplier for the SAD-domain full-pel phase (`RFF_ME_SADL`,
/// default 1.0). SATD distortion runs ~2× SAD's scale, so λ tuned for SATD weighs
/// the rate term ~2× heavier in the SAD domain — 0.5 restores the SATD-era
/// rate/distortion balance. Read once per process (hoisted per search).
fn me_sadfp_lambda() -> f64 {
    static E: std::sync::OnceLock<f64> = std::sync::OnceLock::new();
    // 0.5 = the calibrated default (SATD ≈ 2× SAD's scale; at 1.0 the rate term
    // weighs double and foreman flips to a BD loss). Rides with the mode-1 default.
    *E.get_or_init(|| {
        std::env::var("RFF_ME_SADL").ok().and_then(|v| v.parse().ok()).unwrap_or(0.5)
    })
}

/// Descent D: sub-pel ring census — evals/improvements by (step, ring position) and
/// by loop ITERATION, so a position or an iteration that never pays is visible rather
/// than assumed.
#[cfg(feature = "profile")]
pub mod spstats {
    use core::sync::atomic::{AtomicU64, Ordering};
    /// [step 0=half,1=quarter][position 0..8][0=evals,1=improvements]
    pub static POS: [AtomicU64; 2 * 8 * 2] = [const { AtomicU64::new(0) }; 32];
    /// [step][iteration 1..=6 clamped][0=evals,1=improvements]
    pub static IT: [AtomicU64; 2 * 6 * 2] = [const { AtomicU64::new(0) }; 24];
    #[inline]
    pub fn ev(st: usize, pos: usize, it: u32) {
        POS[(st * 8 + pos.min(7)) * 2].fetch_add(1, Ordering::Relaxed);
        IT[(st * 6 + (it.max(1) as usize - 1).min(5)) * 2].fetch_add(1, Ordering::Relaxed);
    }
    #[inline]
    pub fn imp(st: usize, pos: usize, it: u32) {
        POS[(st * 8 + pos.min(7)) * 2 + 1].fetch_add(1, Ordering::Relaxed);
        IT[(st * 6 + (it.max(1) as usize - 1).min(5)) * 2 + 1].fetch_add(1, Ordering::Relaxed);
    }
    /// Sub-pel evaluations that re-price an MV already evaluated in the SAME refinement.
    pub static REDUNDANT: AtomicU64 = AtomicU64::new(0);
    #[inline]
    pub fn redundant() { REDUNDANT.fetch_add(1, Ordering::Relaxed); }
    pub fn reset() {
        for c in POS.iter() { c.store(0, Ordering::Relaxed); }
        for c in IT.iter() { c.store(0, Ordering::Relaxed); }
        REDUNDANT.store(0, Ordering::Relaxed);
    }
    pub fn snapshot() -> (Vec<u64>, Vec<u64>) {
        (POS.iter().map(|c| c.load(Ordering::Relaxed)).collect(),
         IT.iter().map(|c| c.load(Ordering::Relaxed)).collect())
    }
    pub fn redundant_count() -> u64 { REDUNDANT.load(Ordering::Relaxed) }
}

/// Descent B: which path does each ME cost evaluation actually take?
#[cfg(feature = "profile")]
pub mod satdpath {
    use core::sync::atomic::{AtomicU64, Ordering};
    pub static C: [AtomicU64; 3] = [const { AtomicU64::new(0) }; 3];
    #[inline]
    pub fn bump(i: usize) { C[i].fetch_add(1, Ordering::Relaxed); }
    pub fn reset() { for c in C.iter() { c.store(0, Ordering::Relaxed); } }
    pub fn snapshot() -> Vec<u64> { C.iter().map(|c| c.load(Ordering::Relaxed)).collect() }
}

/// The coarse-to-fine step ladder. DEFAULT `[16,8,4]` — the 64 and 32 rungs were
/// REMOVED after the per-rung census showed they are ~39% of full-pel evaluations at a
/// 0.05-0.84% hit rate, and the 20-clip 4-QP BD curve showed those rare hits are actively
/// HARMFUL: a coarse jump finds a distant MV with marginally lower SATD, but it costs
/// more mvd bits AND breaks the spatial coherence of the MV field, degrading every
/// downstream neighbour's predictor. `lambda*mvbits` prices the first effect and is blind
/// to the second. Dropping them is mean -0.93% BD-PSNR / -1.09% BD-SSIM with a WORST clip
/// of +0.00%/+0.00% over 20 clips, and 1.15-1.57x fewer ME cost evaluations.
///
/// The 8 rung is load-bearing: `[16,4]` reads marginally better BD but makes football_cif
/// do 1.55x MORE work, because the step-4 walk then has to crawl the distance the 8 rung
/// covered in one hop. Reach and stride both matter; only the useless TOP is removed.
///
/// Bit i of the mask enables rung i of [64,32,16,8,4]. `RFF_DIA_LADDER=64,32,16,8,4`
/// restores the pre-change ladder byte-for-byte; `set_dia_mask` overrides at runtime so a
/// single process can measure several ladders.
pub const DIA_RUNGS: [i32; 5] = [64, 32, 16, 8, 4];
/// Rungs walked by default: `[16,8,4]`.
pub const DIA_DEFAULT: u32 = 0b11100;
pub static DIA_MASK: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(u32::MAX);
pub fn set_dia_mask(m: u32) { DIA_MASK.store(m, core::sync::atomic::Ordering::Relaxed) }
fn dia_mask() -> u32 {
    let m = DIA_MASK.load(core::sync::atomic::Ordering::Relaxed);
    if m != u32::MAX { return m; }
    static INIT: std::sync::OnceLock<u32> = std::sync::OnceLock::new();
    *INIT.get_or_init(|| match std::env::var("RFF_DIA_LADDER") {
        Ok(v) => {
            let want: Vec<i32> = v.split(',').filter_map(|t| t.trim().parse().ok()).collect();
            let mut m = 0u32;
            for (i, r) in DIA_RUNGS.iter().enumerate() {
                if want.contains(r) { m |= 1 << i; }
            }
            if m == 0 { DIA_DEFAULT } else { m }
        }
        Err(_) => DIA_DEFAULT,
    })
}

/// Descent A: per-STEP-SIZE census of the coarse-to-fine diamond. The ladder is
/// [64,32,16,8,4] quarter-pel (i.e. 16,8,4,2,1 full-pel) and each step walks until it
/// stops improving. Counts evaluations AND improvements per step so a step that never
/// pays can be identified rather than assumed.
#[cfg(feature = "profile")]
pub mod diastats {
    use core::sync::atomic::{AtomicU64, Ordering};
    /// [step_index][0]=evals, [1]=improvements
    pub static C: [AtomicU64; 12] = [const { AtomicU64::new(0) }; 12];
    #[inline]
    pub fn ev(i: usize) { C[i * 2].fetch_add(1, Ordering::Relaxed); }
    #[inline]
    pub fn imp(i: usize) { C[i * 2 + 1].fetch_add(1, Ordering::Relaxed); }
    pub fn reset() { for c in C.iter() { c.store(0, Ordering::Relaxed); } }
    pub fn snapshot() -> Vec<(u64, u64)> {
        (0..6).map(|i| (C[i * 2].load(Ordering::Relaxed), C[i * 2 + 1].load(Ordering::Relaxed))).collect()
    }
}

/// Sub-pel refinement PATTERN (U1). Bit 0 = 4-point diamond ring instead of the
/// 8-point square; bit 1 = single pass instead of walking to convergence.
///
/// Harvested from 280 k real refinements: ~29 evaluations each, but the LAST
/// improvement lands at eval ~14–15 — **half of every refinement is spent confirming
/// an answer already found** — and the first ring alone captures 64–72% of the total
/// gain. An 8-point ring pays 8 evaluations for that confirmation; a 4-point diamond
/// (what x264's subme uses) pays 4.
///
/// `RFF_SUBPEL_PAT`: 0 = 8-point + iterate (the pre-U1 default), 1 = 4-point +
/// iterate, 2 = 8-point single pass, 3 = 4-point single pass.
pub(crate) static SUBPEL_PAT: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(u32::MAX);

/// Learning-window size and ring-1 threshold (percent) for the U1 online dispatcher.
/// `RFF_SUBPEL_DISPATCH=0` disables it (pure `RFF_SUBPEL_PAT` behaviour).
pub(crate) static SP_DISPATCH: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(u32::MAX);

fn sp_dispatch_cfg() -> (u32, i64) {
    use std::sync::OnceLock;
    let forced = SP_DISPATCH.load(std::sync::atomic::Ordering::Relaxed);
    if forced == 0 {
        return (0, 0);
    }
    static C: OnceLock<(u32, i64)> = OnceLock::new();
    *C.get_or_init(|| {
        // DEFAULT OFF — measured and refuted (see the U1 entry in
        // docs/WHYS-speed-gap.md). It only delivers speed where a blanket pattern
        // change already would (bus 1.47x) while costing BD where it delivers none
        // (foreman +0.97% for 1.04x, mobile +0.33% for 0.98x), and mixing refinement
        // quality across frames measured WORSE than a uniform cut (bus +0.81%
        // dispatched vs +0.30% pat2-always) — the refinement feeds the reference
        // chain, so per-frame inconsistency propagates.
        let on = std::env::var("RFF_SUBPEL_DISPATCH").map(|s| s != "0").unwrap_or(false);
        if !on {
            return (0, 0);
        }
        let k = std::env::var("RFF_SUBPEL_LEARN").ok().and_then(|s| s.parse().ok()).unwrap_or(200);
        let t = std::env::var("RFF_SUBPEL_T").ok().and_then(|s| s.parse().ok()).unwrap_or(67);
        (k, t)
    })
}

/// Explicit override only; `None` means "use the preset's default".
fn subpel_pattern_override() -> Option<u32> {
    let v = SUBPEL_PAT.load(std::sync::atomic::Ordering::Relaxed);
    if v != u32::MAX {
        return Some(v);
    }
    if let Some(e) = std::env::var("RFF_SUBPEL_PAT").ok().and_then(|s| s.parse::<u32>().ok()) {
        SUBPEL_PAT.store(e, std::sync::atomic::Ordering::Relaxed);
        return Some(e);
    }
    None
}

fn subpel_pattern() -> u32 {
    let v = SUBPEL_PAT.load(std::sync::atomic::Ordering::Relaxed);
    if v != u32::MAX {
        return v;
    }
    // Unset -> take the env default once and latch it, so the hot path stays a
    // relaxed load rather than an env lookup.
    let d = std::env::var("RFF_SUBPEL_PAT").ok().and_then(|s| s.parse().ok()).unwrap_or(0);
    SUBPEL_PAT.store(d, std::sync::atomic::Ordering::Relaxed);
    d
}

/// Observe-only HARVEST for the sub-pel refinement skip-gate (U1).
///
/// `me-subpel` is 141 ms of a 320 ms quality encode — 44% — at 241 candidate
/// evaluations per macroblock. This tap records, per refinement, the NULL-ARM cost
/// (the full-pel winner, i.e. what we would keep if we skipped) against the cost the
/// refinement actually reached, so the skip-rate-vs-gain-kept ceiling can be swept
/// offline before any gate is written. Writes nothing unless `RFF_SUBPEL_HARVEST`
/// names a file.
mod subpel_harvest {
    use std::fs::File;
    use std::io::Write;
    use std::sync::{Mutex, OnceLock};

    fn sink() -> &'static Option<Mutex<File>> {
        static S: OnceLock<Option<Mutex<File>>> = OnceLock::new();
        S.get_or_init(|| {
            std::env::var("RFF_SUBPEL_HARVEST").ok().and_then(|p| {
                let mut f = File::create(p).ok()?;
                let _ = writeln!(f, "pre,post,lambda,w,h,evals,to_best,ring1");
                Some(Mutex::new(f))
            })
        })
    }

    #[inline]
    pub fn enabled() -> bool {
        sink().is_some()
    }

    #[allow(clippy::too_many_arguments)]
    pub fn record(pre: i64, post: i64, lambda: f64, w: usize, h: usize, evals: u32, to_best: u32, ring1: i64) {
        if let Some(m) = sink() {
            if let Ok(mut f) = m.lock() {
                let _ = writeln!(f, "{pre},{post},{lambda:.4},{w},{h},{evals},{to_best},{ring1}");
            }
        }
    }
}

/// A/B switch for serving B-direct 4×4 MC from the cached half-pel planes
/// (`RFF_BDIRECT_PLANES=0` restores the direct `mc_luma` 6-tap). Byte-identical
/// either way; the knob exists so the arm can be measured in one binary.
fn bdirect_planes_enabled() -> bool {
    use std::sync::OnceLock;
    static ON: OnceLock<bool> = OnceLock::new();
    *ON.get_or_init(|| std::env::var("RFF_BDIRECT_PLANES").map(|s| s != "0").unwrap_or(true))
}

fn me_batch_enabled() -> bool {
    use std::sync::OnceLock;
    static ON: OnceLock<bool> = OnceLock::new();
    *ON.get_or_init(|| std::env::var("RFF_ME_BATCH").map(|s| s != "0").unwrap_or(true))
}

/// A 16-byte-aligned 16×16 luma block — the aligned `op1` openh264's SSE2 SAD/SATD
/// kernels require (`movdqa`). Safe to construct (`forbid(unsafe)` holds); the asm
/// FFI that consumes it lives in `rusty_h264-accel`. Only used on the `asm` feature.
#[cfg(accel)]
#[repr(align(16))]
struct AlignedMb([u8; 256]);

/// A B-slice 16×16 inter-coding spec: the prediction direction and the motion it
/// uses. `dir` 1 = `B_L0_16x16`, 2 = `B_L1_16x16`, 3 = `B_Bi_16x16` (spec Table
/// 7-14). List-0 is `refs[0]` (nearest past anchor); `l1` is List-1 (nearest
/// future anchor). `mv0`/`mv1` are the List-0/List-1 motion vectors (quarter-pel).
#[derive(Clone, Copy)]
struct BInter<'a> {
    dir: u8,
    l1: &'a crate::RefFrame,
    mv0: (i32, i32),
    mv1: (i32, i32),
}

/// 16-byte-aligned 256-`i16` DCT/coefficient buffer — the in-place `movdqa` quant
/// kernel (`WelsQuantFour4x4_sse2`) requires aligned coefficients. `asm`-feature only.
#[cfg(accel)]
#[repr(align(16))]
struct AlignedDct([i16; 256]);

/// Luma variance of the 16×16 source MB at (mb_x, mb_y) — the content signal for
/// the adaptive SAD↔SATD cost dispatch (high variance = detail = SAD misprices).
/// `256·variance` scale (the /256 of the mean-square is kept integer); only the
/// RELATIVE ordering matters for the per-frame percentile, so the constant drops.
fn mb_variance(sy: &[u8], cw: usize, mb_x: usize, mb_y: usize) -> i64 {
    let base = mb_y * 16 * cw + mb_x * 16;
    // Accumulate in u32, not i64: the sum of 256 bytes maxes at 65280 and the sum
    // of squares at 16.6M, so 64-bit accumulators (and a 64-bit multiply per
    // pixel) were pure width — and they stop LLVM vectorising what is otherwise a
    // textbook pair of reductions over 16 contiguous bytes.
    let (mut s, mut ss) = (0u32, 0u32);
    for r in 0..16 {
        let row = &sy[base + r * cw..base + r * cw + 16];
        for &p in row {
            let v = p as u32;
            s += v;
            ss += v * v;
        }
    }
    // Widen once at the end: s*s reaches 4.26e9, which only just fits u32.
    ss as i64 - (s as i64) * (s as i64) / 256 // 256·variance, monotone in variance
}

/// Adaptive-Quantization per-MB QP map: flat (low-variance) macroblocks get a FINER
/// QP (where blocking/banding is visible), busy ones a COARSER QP (where the eye
/// masks error) — moving bits to where they're seen. The shift is `strength ·
/// (log2 var − frame mean log2 var)`, so it's relative to THIS frame's texture
/// distribution (content-invariant), rounded to an integer QP step and clamped.
/// `strength == 0` → uniform base QP (byte-identical: every `mb_qp_delta` is 0).
fn aq_qp_map(sy: &[u8], cw: usize, mb_w: usize, mb_h: usize, base_qp: u8, strength: f64) -> Vec<u8> {
    let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncAq);
    const AQ_DQP_MAX: i32 = 4;
    let n = mb_w * mb_h;
    if strength == 0.0 || n == 0 {
        return vec![base_qp; n];
    }
    // Per-MB variance (the bit-cost weight) and its log2 (+1 avoids log2(0) on a flat
    // MB → reads as maximally flat → finest QP).
    let mut var = Vec::with_capacity(n);
    let mut lv = Vec::with_capacity(n);
    for my in 0..mb_h {
        for mx in 0..mb_w {
            let v = (mb_variance(sy, cw, mx, my) + 1) as f64;
            var.push(v);
            lv.push(v.log2());
        }
    }
    let mean_lv = lv.iter().sum::<f64>() / n as f64;
    // CONTENT-ADAPTIVE STRENGTH: back off where the log-variance SPREAD is high. A
    // wide/bimodal spread means synthetic-ish content (flat regions beside detailed
    // patterns) where "busy = maskable" FAILS and the patterns are salient — full AQ
    // there costs PSNR. Natural content's spread is ~1 (keeps full strength); a
    // synthetic pan's is ~6 (heavily reduced). Ramp 1.0→`AQ_SPREAD_MIN` over
    // [`AQ_SPREAD_LO`, `AQ_SPREAD_HI`].
    const AQ_SPREAD_LO: f64 = 1.5;
    const AQ_SPREAD_HI: f64 = 5.0;
    const AQ_SPREAD_MIN: f64 = 0.0; // extreme spread (pathological synthetic) → AQ OFF
    let std_lv = (lv.iter().map(|&l| (l - mean_lv).powi(2)).sum::<f64>() / n as f64).sqrt();
    let factor = (1.0 - (std_lv - AQ_SPREAD_LO) / (AQ_SPREAD_HI - AQ_SPREAD_LO)).clamp(AQ_SPREAD_MIN, 1.0);
    let eff_strength = strength * factor;
    // Per-MB QP shift (clamped): busy (log-var above mean) coarser, flat finer.
    let dqp: Vec<i32> = lv
        .iter()
        .map(|&l| (eff_strength * (l - mean_lv)).round() as i32)
        .map(|d| d.clamp(-AQ_DQP_MAX, AQ_DQP_MAX))
        .collect();
    // RATE COMPENSATION: AQ nets a rate change (coarsening a busy MB saves more bits
    // than fining a flat one adds), so shift the whole frame's QP by `c` to restore
    // the un-AQ rate — keeping `qp` meaningful. Bit model `bits_i ∝ var_i·2^(−qp_i/6)`
    // (variance as the per-MB cost proxy): `c = 6·log2(Σ var·2^(−dqp/6) / Σ var)`.
    let sum_v: f64 = var.iter().sum();
    // `dqp` is clamped to [-AQ_DQP_MAX, AQ_DQP_MAX], so 2^(-d/6) has only nine
    // possible values — but it was being recomputed with a `powf` for every
    // macroblock of every frame. Same expression, evaluated once per offset:
    // bit-identical, and it retires a transcendental from a per-macroblock loop.
    let qstep: [f64; (2 * AQ_DQP_MAX + 1) as usize] =
        std::array::from_fn(|i| 2f64.powf(-((i as i32 - AQ_DQP_MAX) as f64) / 6.0));
    let sum_vs: f64 = var
        .iter()
        .zip(&dqp)
        .map(|(&v, &d)| v * qstep[(d + AQ_DQP_MAX) as usize])
        .sum();
    let c = (6.0 * (sum_vs / sum_v).log2()).round() as i32;
    dqp.iter()
        .map(|&d| (base_qp as i32 + c + d).clamp(0, 51) as u8)
        .collect()
}

/// Mean per-sampled-pixel residual after GLOBAL-motion compensation of `sy` from
/// `ref_y` (coarse ±12 global ME + ±3 refine, subsampled interior). ~0 on a PURE pan
/// (a single MV predicts the whole frame) — precisely the content where the local ME
/// diamond never genuinely STALLS (its seed = the median = the pan MV is already
/// right), so the `me_wide` rescue can only find SPURIOUS MVs that wreck the B-frame
/// spatial-direct predictors. Gates `me_wide` off there — non-uniform content
/// (real stalls, where me_wide wins) reads well above 0.
/// Per-frame HEAD-ROOM probe for the `me_wide` rescue: on a small subsample of
/// blocks, how much does a WIDE full-pel search beat a PREDICTOR-LOCAL one?
///
/// This measures what the rescue actually buys, before the macroblock loop and
/// without committing any vector — unlike the online payoff gate, which scores its
/// own SATD cost-cut *after* committing MVs and so only ever separated static
/// content. Returns the mean relative SAD improvement, in percent.
///
/// Calibrated against the 20-clip per-clip BD truth table (docs/WHYS-speed-gap.md
/// R5): me_wide earns its 1.4–5.1× on high-head-room content (bus +4.57, blue_sky
/// +4.70, football +1.51, park_joy +0.91) and REGRESSES on low-head-room content
/// (foreman_qcif −1.08, foreman_cif −0.16, tempete −0.12, mobile −0.03).
///
/// Deliberately PER-FRAME, not per-clip: cross-frame adaptive state is
/// nondeterministic under the GOP-parallel encode path (a lesson already paid for
/// by the rescue's own learning window).
/// Head-room threshold (percent) for the `me_wide` frame gate. DEFAULT-ON at 16.
///
/// Calibrated on the DEPLOYED estimator (not the offline probe — they differ) and
/// gated on the full 20-clip `video-tests` corpus plus four synthesized boundary
/// clips, 4-QP BD-rate on PSNR and SSIM:
///
/// | | me_wide always-on | gated at 16 |
/// |---|---|---|
/// | real-corpus mean | +0.62% | +0.547% (88% retained) |
/// | **worst clip** | **−1.08%** (foreman_qcif) | **0.00%** |
/// | clips paying 1.1–3.6× for ~nothing | 13 | 0 |
///
/// Wins preserved: blue_sky +4.70, bus +4.37, park_joy +0.94, football +0.64,
/// shields +0.20; synthesized fast-pan +6.73, rotation +1.72, zoom +1.11.
/// Monotone non-regression — no clip is negative — which is what promotes this from
/// a speed trade to a default.
///
/// `RFF_ME_HR=0` disables the gate and reproduces the pre-gate bytes exactly (the
/// escape hatch / bisection anchor). Thresholds 13 and 16 both clear the boundary
/// clip (foreman_cif +0.07 / +0.03); 10 does NOT (−0.23) — the threshold is
/// calibrated on a narrow boundary pair, so treat it as re-tunable, not settled.
fn me_wide_hr_thresh() -> f64 {
    use std::sync::OnceLock;
    static T: OnceLock<f64> = OnceLock::new();
    *T.get_or_init(|| std::env::var("RFF_ME_HR").ok().and_then(|s| s.parse().ok()).unwrap_or(16.0))
}

/// Cached, because it is read per frame — an `env::var` there is its own tax.
fn me_wide_hr_dbg() -> bool {
    use std::sync::OnceLock;
    static D: OnceLock<bool> = OnceLock::new();
    *D.get_or_init(|| std::env::var_os("RFF_ME_HR_DBG").is_some())
}

fn me_wide_headroom(sy: &[u8], cw: usize, ch: usize, ref_y: &[u8]) -> f64 {
    const LOCAL: isize = 2; // a well-seeded diamond's effective reach
    const WIDE: isize = 24; // the rescue grid's half-extent
    const STEP: isize = 4; // coarse: this is a frame-level statistic, not a search
    const TARGET: usize = 24; // samples per frame — keep the probe ~0.5% of a frame
    let sad16 = |bx: usize, by: usize, rx: isize, ry: isize| -> Option<u32> {
        if rx < 0 || ry < 0 || rx as usize + 16 > cw || ry as usize + 16 > ch {
            return None;
        }
        let (rx, ry) = (rx as usize, ry as usize);
        let mut s = 0u32;
        for dy in 0..16 {
            let a = &sy[(by + dy) * cw + bx..][..16];
            let b = &ref_y[(ry + dy) * cw + rx..][..16];
            s += a.iter().zip(b).map(|(&p, &q)| p.abs_diff(q) as u32).sum::<u32>();
        }
        Some(s)
    };
    // Interior blocks only (the probe must not measure edge clamping), spread over
    // the frame so one moving object cannot dominate.
    let (mbw, mbh) = (cw / 16, ch / 16);
    if mbw < 6 || mbh < 6 {
        return 0.0;
    }
    let inner = (mbw - 4) * (mbh - 4);
    let stride = (inner / TARGET).max(1);
    let (mut acc, mut n) = (0.0f64, 0u32);
    let mut i = 0usize;
    while i < inner {
        let (mx, my) = (2 + i % (mbw - 4), 2 + i / (mbw - 4));
        let (bx, by) = (mx * 16, my * 16);
        let mut best_local = u32::MAX;
        for dy in -LOCAL..=LOCAL {
            for dx in -LOCAL..=LOCAL {
                if let Some(s) = sad16(bx, by, bx as isize + dx, by as isize + dy) {
                    best_local = best_local.min(s);
                }
            }
        }
        let mut best_wide = best_local;
        let mut dy = -WIDE;
        while dy <= WIDE {
            let mut dx = -WIDE;
            while dx <= WIDE {
                if let Some(s) = sad16(bx, by, bx as isize + dx, by as isize + dy) {
                    best_wide = best_wide.min(s);
                }
                dx += STEP;
            }
            dy += STEP;
        }
        if best_local > 0 {
            acc += (best_local - best_wide) as f64 / best_local as f64;
            n += 1;
        }
        i += stride;
    }
    if n == 0 {
        0.0
    } else {
        100.0 * acc / n as f64
    }
}

fn global_mc_residual(sy: &[u8], cw: usize, ch: usize, ref_y: &[u8]) -> f64 {
    if cw < 48 || ch < 48 {
        return f64::INFINITY;
    }
    let sad = |dx: isize, dy: isize| -> u64 {
        let mut s = 0u64;
        let mut y = 16;
        while y < ch - 16 {
            let cbase = (y * cw) as isize;
            let rbase = (y as isize + dy) * cw as isize + dx;
            let mut x = 16isize;
            while x < (cw - 16) as isize {
                let c = sy[(cbase + x) as usize] as i32;
                let r = ref_y[(rbase + x) as usize] as i32;
                s += (c - r).unsigned_abs() as u64;
                x += 8;
            }
            y += 8;
        }
        s
    };
    let (mut best, mut bc) = ((0isize, 0isize), u64::MAX);
    let mut dy = -12;
    while dy <= 12 {
        let mut dx = -12;
        while dx <= 12 {
            let c = sad(dx, dy);
            if c < bc {
                bc = c;
                best = (dx, dy);
            }
            dx += 4;
        }
        dy += 4;
    }
    for dy in best.1 - 3..=best.1 + 3 {
        for dx in best.0 - 3..=best.0 + 3 {
            let c = sad(dx, dy);
            if c < bc {
                bc = c;
            }
        }
    }
    let nx = (16..cw - 16).step_by(8).count();
    let ny = (16..ch - 16).step_by(8).count();
    bc as f64 / (nx * ny).max(1) as f64
}

/// Adds the mb-tree per-MB QP offset (TEMPORAL AQ — [`crate::mbtree`]) to the
/// spatial-AQ `aq_qp` map in place. An empty `qpo` (mb-tree off) or a length
/// mismatch is a no-op → byte-identical. Shared by the CAVLC and CABAC slice paths.
fn apply_mbtree_qpo(aq_qp: &mut [u8], qpo: &[i32]) {
    if qpo.len() == aq_qp.len() {
        for (q, &o) in aq_qp.iter_mut().zip(qpo) {
            *q = (*q as i32 + o).clamp(0, 51) as u8;
        }
    }
}

/// IMPLICIT bi-prediction weights `(w0, w1)` from POC distances (spec §8.4.2.3.2,
/// `weighted_bipred_idc == 2`), IDENTICAL to the decoder's `implicit_weights`. The
/// closer anchor gets more weight; an equidistant B (`bframes == 1`) yields 32:32,
/// i.e. the plain average. `(32, 32)` fallback for the degenerate/out-of-range cases
/// the decoder also averages (no long-term refs here).
fn implicit_bi_weights(cur_poc: i32, l0_poc: i32, l1_poc: i32) -> (i32, i32) {
    let td = (l1_poc - l0_poc).clamp(-128, 127);
    let tb = (cur_poc - l0_poc).clamp(-128, 127);
    if td == 0 {
        return (32, 32);
    }
    let tx = (16384 + td.abs() / 2) / td;
    let dsf = ((tb * tx + 32) >> 6).clamp(-1024, 1023);
    let w1 = dsf >> 2;
    if !(-64..=128).contains(&w1) {
        return (32, 32);
    }
    (64 - w1, w1)
}

/// Bi-prediction blend of two motion-compensated samples `p` (List-0) and `q`
/// (List-1) under weights `(w0, w1)` — the decoder's `b_mc` blend. `(32, 32)` is the
/// plain `(p+q+1)>>1` average.
#[inline(always)]
fn bi_blend(p: i32, q: i32, w: (i32, i32)) -> u8 {
    ((p * w.0 + q * w.1 + 32) >> 6).clamp(0, 255) as u8
}

/// Zig-zag scan of a raster i16 4×4 block into scan-order i32 — the fused-path
/// twin of `scan_4x4_dcac(&q_blocks[..])`, reading quantized levels straight from
/// the hot i16 DCT buffer. Byte-identical: the i16→i32 widening of a quant level
/// is exact (levels always fit i16, being the input to the i16 idct kernel).
#[cfg(accel)]
#[inline]
fn scan_4x4_dcac_i16(d: &[i16]) -> [i32; 16] {
    [
        d[0] as i32, d[1] as i32, d[4] as i32, d[8] as i32, d[5] as i32, d[2] as i32,
        d[3] as i32, d[6] as i32, d[9] as i32, d[12] as i32, d[13] as i32, d[10] as i32,
        d[7] as i32, d[11] as i32, d[14] as i32, d[15] as i32,
    ]
}


/// Per-frame intra encoder state: reconstructed planes (coded size) and the
/// per-4×4-block non-zero-coefficient counts used for CAVLC context.
pub struct FrameEncoder {
    mb_w: usize,
    mb_h: usize,
    qp: u8,  // the CURRENT macroblock's target QPy (AQ varies it per MB)
    qpc: u8, // chroma QP for `qp`
    /// Running QPy of the last macroblock that coded an `mb_qp_delta` (spec QPY_PREV).
    /// `mb_qp_delta = qp − cur_qp`; a skip / cbp==0 MB codes no delta and inherits it.
    cur_qp: u8,
    /// Implicit bi-prediction weights `(w0, w1)` for the current B-frame (from its
    /// L0/L1 anchor POC distances). `(32, 32)` = plain average (P/I frames, `bframes
    /// == 1`); unequal for `bframes > 1`.
    bi_w: (i32, i32),
    cw: usize, // coded luma width
    ccw: usize, // coded chroma width
    // 16-byte aligned (the openh264 deblock/MC/intra asm load aligned row chunks).
    rec_y: AlignedBytes,
    rec_u: AlignedBytes,
    rec_v: AlignedBytes,
    nnz_y: Vec<u8>,    // (mb_w*4) x (mb_h*4)
    nnz_c: [Vec<u8>; 2], // each (mb_w*2) x (mb_h*2)
    modes_y: Vec<u8>,  // intra4x4 mode per 4×4 block (2=DC for I_16x16 blocks)
    coded_y: Vec<bool>, // whether each 4×4 block is reconstructed (top-right avail)
    mv_y: Vec<(i32, i32)>, // motion vector per 4×4 block (quarter-pel) — List-0
    inter_y: Vec<bool>, // whether each 4×4 block is inter-coded
    ref_idx_y: Vec<i32>, // reference index per 4×4 block (-1 = intra/uncoded) — List-0
    // B-slice List-1 motion field (empty for P/I). B_L1/B_Bi commit here so a later
    // partition's List-1 median predictor sees it, mirroring the decoder's
    // `mv_neighbors_list(.., 1)` over `mv1`/`ref_idx1`.
    mv1_y: Vec<(i32, i32)>,
    ref_idx1_y: Vec<i32>,
    idz: i64, // intra dead-zone divisor: 2 for all-intra, 3 when frames reference each other
    rdoq_strength: f64, // CABAC trellis (RDOQ) strength; 0 = off (hard quantize, CAVLC path)
    transform_8x8: bool, // High-profile 8x8 transform enabled (transform_8x8_mode_flag)
    sub8x8: bool, // P_8x8 sub-partition motion (four 8x8 MVs per MB)
    me_wide: bool, // adaptive wide ME grid search rescue (diamond stalls on flat surfaces)
    /// Track-B B2 for THIS frame: SAD-domain full-pel phase. Set at construction
    /// (force mode), or per frame by the `b2_mgain` dispatcher (mode 1).
    sadfp: bool,
    /// H-24 mv-cost SHAPE routing for THIS frame (mv_smooth mode 1), set by the
    /// same `b2_mgain` probe. Per-frame state, NOT a global: the GOP-parallel
    /// encode runs frames concurrently and a global store races across workers.
    mv_smooth: bool,
    /// H-13: search partition splits this frame (routed off on near-static frames).
    do_splits: bool,
    me_wide_var: u64, // per-pixel source variance below which a block is "flat"
    me_rescue: i64, // per-pixel residual SATD (on a flat block) that flags a diamond stall
    me_wide_coh: f64, // gate me_wide off when the frame's global-MC residual is below this (pure pan)
    me_range: i32, // rescue grid half-range in px (16 = ±16; wider reaches FAST motion the diamond misses)
    me_fast: bool, // also fire the rescue on HIGH-VARIANCE high-residual blocks (fast-motion stalls, not just flat)
    // ONLINE per-frame rescue-payoff gate (adaptive; WITHIN-frame so it stays
    // deterministic under the frame-parallel encode). Run the real rescue on the
    // first `me_learn` stalls of a frame, count how many the fine grid improves by
    // ≥6.25%, and if that fraction is below `me_payoff_pct`% disable the rescue for
    // the rest of the frame. This separates genuine diamond stalls (tsrc/zoom fine
    // grid improves ~33% of fires) from IRREDUCIBLE residual (rotation/fractal ~5-8%)
    // using the ACTUAL neighbour-seeded diamond — the only faithful signal (a cheap
    // SAD proxy from (0,0) inverts it: rot reads as highest-payoff). Frame-level, so
    // no per-block selection concentrates the B-direct-poisoning spurious MVs.
    me_learn: u32,
    me_payoff_pct: u32,
    /// U1 online sub-pel dispatcher (within-frame, so it stays deterministic under
    /// GOP-parallel encode). For the first `SP_LEARN` refinements of a frame we run
    /// the full 8-point+iterate pattern and accumulate how much of the total gain the
    /// FIRST ring captured; once the window fills, a frame whose gain is concentrated
    /// in ring 1 switches to the single-pass pattern for the rest of the frame.
    ///
    /// Harvested justification: ring-1 captures 63.7% of the gain on foreman (which
    /// loses +2.34% BD to a blanket single-pass) against 69.9–71.9% on bus/mobile
    /// (which lose only +0.30/+0.74% and gain 1.08–1.31×). The fraction separates the
    /// content that can afford the cut from the content that cannot.
    sp_single_pass: bool,
    /// U5-struct: when set, `motion_search` returns its FULL-PEL winner and skips
    /// sub-pel refinement entirely. The partition driver uses this to search all
    /// candidate shapes cheaply, pick one, and refine ONLY the winner's sub-blocks.
    /// Measured ceiling: 3.4–6.4× less sub-pel work (the losing shapes' refinements
    /// are pure waste), i.e. ~1.42× whole-encode at 44% sub-pel share.
    sp_defer: std::cell::Cell<bool>,
    sp_learn_n: std::cell::Cell<u32>,
    sp_ring1: std::cell::Cell<i64>,
    sp_total: std::cell::Cell<i64>,
    sp_1pass: std::cell::Cell<bool>,
    resc_n: std::cell::Cell<u32>,   // stalls the fine grid ran on this frame (learning phase)
    resc_big: std::cell::Cell<u32>, // of those, how many it improved ≥6.25%
    resc_off: std::cell::Cell<bool>, // rescue disabled for the rest of this frame
    inter8x8: u8, // inter 8x8-transform dispatch: 0=off, 1=always-RD, 2=content-adaptive
    inter8_pen: i64, // extra rate charge (nonzero-equiv) on the inter 8x8 candidate
    fast: bool, // Preset::Fast — SATD mode decision (no RDO), 16×16/I_16x16 only
    skip_accel_check: bool, // A/B knob: whole-MB psadbw gate in the P_Skip free-check
    coded_path_v2: bool,    // A/B knob: route inter coding through encode_inter_mb_v2
    tune_lambda_scale: f64, // tuning knob: scale on the RD λ (1.0 = standard)
    tune_intra_penalty: f64,
    satd_q: f64,               // adaptive: fraction of high-variance MBs routed to SATD cost
    subpel_force: bool,        // force sub-pel refinement even in the fast preset
    me_snap: bool,             // snap the diamond centre to integer-pel (see config)
    me_subpel_iter: bool,      // walk the sub-pel refine to convergence
    greedy_skip: bool,         // quality preset's SAD-thresholded P_Skip (PredictSadSkip)
    greedy_min_free: u32,      // online free-skip % gating greedy_skip on this frame
    rd_skip: bool,             // decide P_Skip by J = SSD + lambda*bits, not exact-zero residual
    rd_skip_min_free: u32,     // online free-skip % gating rd_skip on this frame
    rd_skip_fast_t: f64,       // skip-gate on SSD(skip)/lambda; <= 0 prices every candidate
    satd_var_thresh: i64,      // per-frame variance threshold for the routing (set in a pre-pass)
    aq_strength: f64,          // adaptive quantization: per-MB QP modulation strength (0 = off)
    mb_use_satd: bool,         // per-MB: this MB uses the SATD cost this decision
    // Per-MB luma nnz prediction cache (openh264 scan8 style): a padded 5×5 grid,
    // block (lbx,lby) at (lby+1)*5+(lbx+1); row 0 = top neighbours, col 0 = left.
    // Unavailable edges hold the sentinel 0x80, so the nnz predict is branchless.
    nnz_l_cache: [u8; 25],
    // Same, per chroma plane: a padded 3×3 grid for the 2×2 chroma blocks.
    nnz_c_cache: [[u8; 9]; 2],
    // openh264 predicted-SAD skip apparatus (per MB, mb_w×mb_h): the P_Skip
    // prediction's luma SAD, and whether the MB was actually skipped. The greedy
    // skip threshold for an MB is the median of its skip *neighbours'* skip SADs
    // (`PredictSadSkip`) — so skip propagates only from already-skip regions
    // (seeded by free skips) and self-limits, instead of a fixed bound that drifts.
    mb_skip_sad: Vec<u32>,
    mb_was_skip: Vec<bool>,
}

/// A chosen inter coding for a macroblock: `mb_type` and, per partition, the
/// reference index and motion vector.
type InterChoice = (u8, Vec<(i32, (i32, i32))>);

/// Approximate marginal rate (bits) of one `P_Skip` — it only lengthens the
/// surrounding `mb_skip_run` Exp-Golomb code slightly.
const SKIP_RATE_BITS: f64 = 1.0;



/// EXTERNAL MV SCORING (`RFF_MV_CMP=1`). Holds another encoder's motion field
/// (per frame, 4x4-block raster) so our own coder can price ITS vectors against
/// ours under REAL coded bits instead of SATD — the only way to tell a bad search
/// from a bad cost function.
pub static EXT_MV: std::sync::Mutex<Vec<Vec<(i32, i32)>>> = std::sync::Mutex::new(Vec::new());
/// [n, our bits, ext bits, our SSD, ext SSD, ext won on J, MVs differing]
pub static MVCMP: [std::sync::atomic::AtomicU64; 7] = {
    const Z: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
    [Z; 7]
};
pub static MVCMP_FRAME: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
/// Replace our chosen vector with the external field's, for EVERY macroblock where
/// that field used a single 16x16 partition. Transplanting one vector in isolation
/// is meaningless — `mvd` is coded against the NEIGHBOURS' vectors, so a lone
/// foreign vector prices against the wrong predictor. Only a whole coherent field
/// can be compared fairly.
fn mv_force_on() -> bool {
    static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
    *ON.get_or_init(|| std::env::var("RFF_MV_FORCE").map_or(false, |v| v != "0"))
}
fn mv_cmp_on() -> bool {
    static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
    *ON.get_or_init(|| std::env::var("RFF_MV_CMP").map_or(false, |v| v != "0"))
}

/// [full-pel SATD evals, INTERPOLATED SATD evals] — `RFF_MC_COUNT=1`.
/// x264 precomputes half-pel planes once per frame; we run the 6-tap filter per
/// candidate, so this ratio prices that difference.
pub static MC_COUNT: [std::sync::atomic::AtomicU64; 2] = [
    std::sync::atomic::AtomicU64::new(0),
    std::sync::atomic::AtomicU64::new(0),
];
fn mc_count_on() -> bool {
    static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
    *ON.get_or_init(|| std::env::var("RFF_MC_COUNT").map_or(false, |v| v != "0"))
}

/// [n, sum our cost, sum oracle cost, blocks the oracle beat us on, cost() evals]
pub static ME_PROBE: [std::sync::atomic::AtomicU64; 7] = {
    const Z: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
    [Z; 7]
};

/// Cached — an `env::var` inside the ME loop inflated it 4x when probed naively.
fn me_oracle_on() -> bool {
    static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
    *ON.get_or_init(|| std::env::var("RFF_ME_ORACLE").map_or(false, |v| v != "0"))
}

/// RDO early-termination gate. Sub-partitions (16×8 / 8×16) only help at motion
/// boundaries, which show up as a heavy 16×16 residual; below this many coded bits
/// the 16×16 already fits, so skip their motion search and trials. (Intra is *not*
/// gated — it can win even against a cheap inter prediction, so gating it on inter
/// cost regresses compression badly on textured content.)
const SPLIT_GATE_BITS: f64 = 60.0;

/// Fast preset: signalling-cost penalty (in bits, SATD-weighted by √λ) charged to
/// the intra candidate so it only wins a P-macroblock when its prediction is
/// clearly better than inter — intra's `mb_type` + modes cost more to signal.
const FAST_INTRA_PENALTY_BITS: f64 = 24.0;

/// A snapshot of one macroblock's per-block grids and reconstruction region,
/// used to roll back a trial encode during RD mode decision.
///
/// Every field is a `Vec`, so building one from scratch is ten heap allocations.
/// The RD skip decision snapshots on EVERY candidate macroblock, which made that
/// allocation traffic the decision's dominant cost — hence
/// [`save_mb_into`](FrameEncoder::save_mb_into), which refills a reused buffer.
#[derive(Default)]
struct MbState {
    rec_y: Vec<u8>,
    rec_u: Vec<u8>,
    rec_v: Vec<u8>,
    nnz_y: Vec<u8>,
    nnz_c: [Vec<u8>; 2],
    mv_y: Vec<(i32, i32)>,
    inter_y: Vec<bool>,
    ref_idx_y: Vec<i32>,
    coded_y: Vec<bool>,
    modes_y: Vec<u8>,
    /// QPY_PREV. `qp_delta()` MUTATES this as a side effect of coding
    /// `mb_qp_delta`, so a trial encode advances it; without restoring it the
    /// real encode then codes its delta against the wrong predecessor and the
    /// decoder's QP diverges from the encoder's — a silent stream corruption,
    /// not a quality tweak.
    cur_qp: u8,
}

/// Edge-clamped, coded-size source planes (luma, Cb, Cr).
/// Fast-preset pruned I4x4 mode search ({MPM, DC, V, H} instead of all 9 — the
/// x264-ultrafast-style candidate set). DEFAULT ON for the fast preset (gated:
/// +0.5% size at +0.02 dB on all-intra, +17% all-intra speed); RUSTY_FAST_INTRA=0
/// restores the exhaustive 9-mode search (the pre-flip bitstream).
fn fast_intra_enabled() -> bool {
    static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
    *ON.get_or_init(|| std::env::var("RUSTY_FAST_INTRA").map_or(true, |v| v != "0"))
}

fn coded_source(cfg: &EncoderConfig, frame: &YuvFrame) -> (Vec<u8>, Vec<u8>, Vec<u8>) {
    let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncSource);
    let cw = cfg.mb_width() * 16;
    let ch = cfg.mb_height() * 16;
    // MB-aligned frame: the clamp is the identity — a plane memcpy (clone) replaces
    // the per-pixel clamp loop (bit-exact: same bytes).
    if frame.width == cw && frame.height == ch {
        return (frame.y.clone(), frame.u.clone(), frame.v.clone());
    }
    let y = clamp_plane(&frame.y, frame.width, frame.height, cw, ch);
    let u = clamp_plane(&frame.u, frame.chroma_width(), frame.chroma_height(), cw / 2, ch / 2);
    let v = clamp_plane(&frame.v, frame.chroma_width(), frame.chroma_height(), cw / 2, ch / 2);
    (y, u, v)
}

/// Edge-extends `plane` from `w`×`h` to the coded `ow`×`oh`, replicating the last
/// row/column — the source form the MB grid needs.
///
/// Row-wise, because the per-pixel form is O(pixels) of scalar `min`+multiply and is
/// the DOMINANT cost of `enc-source-copy`: every frame whose height is not a multiple
/// of 16 takes this path, which includes all 1080p content (1080/16 = 67.5 → coded
/// height 1088). The stage measured 579 ms over the corpus while the three plane
/// clones on the MB-aligned fast path account for only ~135 ms of it.
///
/// Byte-identical to the per-pixel form (`clamp_plane_per_pixel`, kept as the test
/// oracle): `x.min(w-1)` is the identity below `w` and pins to the last column above
/// it, so a row is a `copy_from_slice` plus a `fill`; `y.min(h-1)` makes the
/// overhanging rows copies of the final row. Both lower to memcpy/memset.
fn clamp_plane(plane: &[u8], w: usize, h: usize, ow: usize, oh: usize) -> Vec<u8> {
    let mut out = vec![0u8; ow * oh];
    for y in 0..oh {
        let sy = y.min(h - 1);
        let src = &plane[sy * w..sy * w + w];
        let dst = &mut out[y * ow..y * ow + ow];
        if ow <= w {
            dst.copy_from_slice(&src[..ow]);
        } else {
            dst[..w].copy_from_slice(src);
            dst[w..].fill(src[w - 1]);
        }
    }
    out
}

/// The original per-pixel edge extension — kept as the correctness oracle for
/// [`clamp_plane`], per the scalar-twin discipline.
#[cfg(test)]
fn clamp_plane_per_pixel(plane: &[u8], w: usize, h: usize, ow: usize, oh: usize) -> Vec<u8> {
    let mut out = vec![0u8; ow * oh];
    for y in 0..oh {
        for x in 0..ow {
            out[y * ow + x] = plane[y.min(h - 1) * w + x.min(w - 1)];
        }
    }
    out
}

#[cfg(test)]
mod source_tests {
    use super::*;

    #[test]
    fn clamp_plane_matches_per_pixel_oracle() {
        let mut s: u32 = 0xDEAD_BEEF;
        let mut rnd = || {
            s = s.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
            (s >> 24) as u8
        };
        // Real coded geometries plus adversarial ones: width-only overhang,
        // height-only overhang (the 1080p case), both, and neither.
        let cases = [
            (1920usize, 1080usize, 1920usize, 1088usize), // 1080p luma
            (960, 540, 960, 544),                         // 1080p chroma
            (352, 288, 352, 288),                         // exactly aligned
            (100, 100, 112, 112),                         // both axes overhang
            (37, 5, 48, 16),                              // tiny + ragged
            (16, 1, 16, 16),                              // single source row
            (1, 1, 16, 16),                               // single sample
        ];
        for (w, h, ow, oh) in cases {
            let plane: Vec<u8> = (0..w * h).map(|_| rnd()).collect();
            assert_eq!(
                clamp_plane(&plane, w, h, ow, oh),
                clamp_plane_per_pixel(&plane, w, h, ow, oh),
                "clamp mismatch for {w}x{h} -> {ow}x{oh}"
            );
        }
    }
}

impl FrameEncoder {
    fn new(cfg: &EncoderConfig) -> Self {
        let (mb_w, mb_h) = (cfg.mb_width(), cfg.mb_height());
        let (cw, ch) = (mb_w * 16, mb_h * 16);
        let (ccw, cch) = (cw / 2, ch / 2);
        Self {
            mb_w,
            mb_h,
            qp: cfg.qp,
            qpc: chroma_qp(cfg.qp),
            cur_qp: cfg.qp,
            bi_w: (32, 32),
            cw,
            ccw,
            rec_y: AlignedBytes::zeroed(cw * ch),
            rec_u: AlignedBytes::zeroed(ccw * cch),
            rec_v: AlignedBytes::zeroed(ccw * cch),
            nnz_y: vec![0; (mb_w * 4) * (mb_h * 4)],
            nnz_c: [vec![0; (mb_w * 2) * (mb_h * 2)], vec![0; (mb_w * 2) * (mb_h * 2)]],
            modes_y: vec![2; (mb_w * 4) * (mb_h * 4)],
            coded_y: vec![false; (mb_w * 4) * (mb_h * 4)],
            mv_y: vec![(0, 0); (mb_w * 4) * (mb_h * 4)],
            inter_y: vec![false; (mb_w * 4) * (mb_h * 4)],
            ref_idx_y: vec![-1; (mb_w * 4) * (mb_h * 4)],
            mv1_y: vec![(0, 0); (mb_w * 4) * (mb_h * 4)],
            ref_idx1_y: vec![-1; (mb_w * 4) * (mb_h * 4)],
            // All-intra (no inter references) tolerates the larger dead-zone; in
            // an I+P stream the IDR is a reference, so keep the standard offset.
            idz: if cfg.gop_size <= 1 { 2 } else { 3 },
            rdoq_strength: 0.0, // set >0 only in the CABAC slice coders
            transform_8x8: cfg.transform_8x8,
            // sub8x8 stays OPT-IN: the four P_8x8 sub-MVs feed the B-frames'
            // spatial-direct predictor, so on DIVERGENT motion (rotation/zoom/mixed)
            // it regresses with B-frames (mixed +0.24%, rot +0.42%, zoom +0.40%) —
            // a global effect its local RD gate can't see, and no clean dispatch
            // signal separates it yet (unlike me_wide's pure-pan coherence gate).
            // DEFAULT-ON for Quality (net real-content win; a 6-channel discovery
            // harvest proved no cheap gate beats always-on). Quality-only (Fast never
            // runs it). env RFF_SUB8X8 (0/1) > cfg.sub_8x8 (Some) > preset default.
            sub8x8: std::env::var("RFF_SUB8X8").ok().map(|s| s == "1")
                .or(cfg.sub_8x8)
                .unwrap_or(cfg.preset == crate::config::Preset::Quality),
            // me_wide is DEFAULT-ON for the Quality preset. VALIDATED 2026-07-27 on the
            // full 20-clip `video-tests` Derf corpus (4-QP BD-rate, PSNR+SSIM, anchor =
            // me_wide ON): **mean +0.62% BD-PSNR / +0.69% BD-SSIM**, i.e. turning it off
            // costs that much. Biggest wins blue_sky +4.70, bus +4.57, football +1.51,
            // park_joy +0.91; synthesized boundary content (smooth fast-pan / rotation /
            // zoom) reaches +2.6..+6.7%. The static clips (akiyo, FourPeople) sit at
            // exactly 0.00 at ~1.0x — the online payoff gate correctly disables it there.
            //
            // ⚠ UNFINISHED DISPATCH — the per-clip BD SIGN-FLIPS (+4.70 blue_sky ..
            // -1.08 foreman_qcif), and the cost when it fires is 1.0-5.1x. Worst value:
            // soccer_4cif 1.70x for +0.00, park_joy 5.08x for +0.91. `me_range` is NOT
            // the separating axis — it is a compromise dial (foreman_qcif loses at EVERY
            // range 24/16/8/4 = -1.08/-0.55/-0.50/-0.19 while blue_sky wins at every one
            // = +4.70/+3.10/+0.73), so shrinking it just trades the win away. The real
            // fix is a content signal that predicts the sign; the truth table for it is
            // in docs/WHYS-speed-gap.md.
            //
            // Quality-only (Fast never runs it). Precedence:
            // env RFF_ME_WIDE (0/1, for A/B) > cfg.me_wide (Some) > preset default.
            sadfp: me_sadfp_mode() == 2,
            mv_smooth: false,
            do_splits: true,
            me_wide: std::env::var("RFF_ME_WIDE").ok().map(|s| s == "1")
                .or(cfg.me_wide)
                .unwrap_or(cfg.preset == crate::config::Preset::Quality),
            me_wide_var: std::env::var("RFF_ME_WIDE_VAR").ok().and_then(|s| s.parse().ok()).unwrap_or(800),
            me_rescue: std::env::var("RFF_ME_RESCUE").ok().and_then(|s| s.parse().ok()).unwrap_or(3),
            me_wide_coh: std::env::var("RFF_ME_COH").ok().and_then(|s| s.parse().ok()).unwrap_or(4.0),
            me_range: std::env::var("RFF_ME_RANGE").ok().and_then(|s| s.parse().ok()).unwrap_or(24),
            me_fast: std::env::var("RFF_ME_FASTMO").map(|s| s != "0").unwrap_or(true),
            me_learn: std::env::var("RFF_ME_LEARN").ok().and_then(|s| s.parse().ok()).unwrap_or(40),
            me_payoff_pct: std::env::var("RFF_ME_PAYOFF").ok().and_then(|s| s.parse().ok()).unwrap_or(15),
            // U3: `balanced` runs SINGLE-PASS sub-pel. Measured on the 4-QP corpus,
            // a single pass captures 95.5–99.4% of the full refinement's BD benefit
            // (foreman −38.14 vs −39.94, mobile −49.38 vs −49.66, akiyo −26.10 vs
            // −26.43) for 1.03–1.31× less time — a straight Pareto improvement on the
            // preset. `RFF_SUBPEL_PAT=0` restores the full walk-to-convergence.
            sp_single_pass: cfg.preset == crate::config::Preset::Balanced,
            sp_defer: std::cell::Cell::new({
                let a = DEFER_SUBPEL.load(std::sync::atomic::Ordering::Relaxed) != 0
                    || std::env::var("RFF_DEFER_SUBPEL").map(|v| v != "0").unwrap_or(false);
                // ONLY the Quality preset runs the multi-shape partition driver. On the
                // fast/balanced path there is a single 16×16 candidate, so there is no
                // losing shape to skip — deferring there does not save the refinement,
                // it DELETES it (measured +91..+145% BD before this guard).
                a && cfg.preset == crate::config::Preset::Quality
            }),
            sp_learn_n: std::cell::Cell::new(0),
            sp_ring1: std::cell::Cell::new(0),
            sp_total: std::cell::Cell::new(0),
            sp_1pass: std::cell::Cell::new(false),
            resc_n: std::cell::Cell::new(0),
            resc_big: std::cell::Cell::new(0),
            resc_off: std::cell::Cell::new(false),
            inter8x8: std::env::var("RFF_INTER8")
                .ok()
                .and_then(|s| s.parse().ok())
                .unwrap_or(1),
            // ~2 bits per 8x8 luma block (×4) of CAVLC-8x8 overhead the level-aware
            // rate still under-charges (no native 8x8 entropy model in CAVLC). Keeps
            // the per-MB transform RD from over-picking 8x8 on fine-texture MBs where
            // it doesn't compact — content-adaptive: only decisively-favorable MBs win.
            inter8_pen: std::env::var("RFF_INTER8_PEN")
                .ok()
                .and_then(|s| s.parse().ok())
                .unwrap_or(8),
            // Balanced shares Fast's decision path; only sub-pel differs.
            fast: cfg.preset != crate::config::Preset::Quality,
            skip_accel_check: cfg.tune_skip_accel_check,
            coded_path_v2: cfg.coded_path_v2,
            aq_strength: cfg.aq_strength,
            tune_lambda_scale: cfg.tune_lambda_scale,
            tune_intra_penalty: cfg.tune_intra_penalty,
            satd_q: cfg.tune_satd_q,
            subpel_force: cfg.tune_subpel || cfg.preset == crate::config::Preset::Balanced,
            me_snap: cfg.tune_me_snap,
            me_subpel_iter: cfg.tune_me_subpel_iter,
            greedy_skip: cfg.tune_greedy_skip,
            greedy_min_free: cfg.tune_greedy_skip_min_free.unwrap_or(85),
            rd_skip: cfg.tune_rd_skip,
            rd_skip_fast_t: cfg.tune_rd_skip_fast_t.unwrap_or(0.0),
            rd_skip_min_free: cfg.tune_rd_skip_min_free.unwrap_or(
                if cfg.preset == crate::config::Preset::Fast { 60 } else { 90 },
            ),
            satd_var_thresh: i64::MAX,
            mb_use_satd: false,
            nnz_l_cache: [0x80; 25],
            nnz_c_cache: [[0x80; 9]; 2],
            mb_skip_sad: vec![0; mb_w * mb_h],
            mb_was_skip: vec![false; mb_w * mb_h],
        }
    }

    /// openh264 `PredictSadSkip`: the greedy P_Skip threshold = the median of the
    /// skip SADs of the *skip* neighbours (left A, top B, top-right C, top-left
    /// fallback for C). Non-skip neighbours contribute 0, so with no skip neighbour
    /// the threshold is 0 (no greedy skip). This makes the skip self-calibrating —
    /// it only spreads where a neighbour already skipped at a comparable SAD.
    fn pred_skip_sad(&self, mb_x: usize, mb_y: usize) -> u32 {
        let mbw = self.mb_w;
        let at = |x: isize, y: isize| -> Option<(bool, u32)> {
            if x < 0 || y < 0 || x >= mbw as isize {
                return None;
            }
            let i = y as usize * mbw + x as usize;
            Some((self.mb_was_skip[i], self.mb_skip_sad[i]))
        };
        let a = at(mb_x as isize - 1, mb_y as isize); // left
        let b = at(mb_x as isize, mb_y as isize - 1); // top
        let c = at(mb_x as isize + 1, mb_y as isize - 1) // top-right
            .or_else(|| at(mb_x as isize - 1, mb_y as isize - 1)); // top-left fallback
        let sad = |n: Option<(bool, u32)>| n.filter(|&(s, _)| s).map_or(0, |(_, v)| v);
        let (sa, sb, sc) = (sad(a), sad(b), sad(c));
        // B and C unavailable but A available → A only.
        if b.is_none() && c.is_none() && a.is_some() {
            return sa;
        }
        match (
            a.is_some_and(|(s, _)| s),
            b.is_some_and(|(s, _)| s),
            c.is_some_and(|(s, _)| s),
        ) {
            (true, false, false) => sa,
            (false, true, false) => sb,
            (false, false, true) => sc,
            _ => sb.max(sa.min(sc)).min(sa.max(sc)), // median(sa, sb, sc)
        }
    }

    /// The `mb_qp_delta` for the current macroblock (`qp − cur_qp`) and commits the
    /// running QPy — called ONLY where the syntax actually codes a delta (I_16x16
    /// always; inter / I_4x4 when `cbp != 0`), so a skip / cbp==0 MB leaves `cur_qp`
    /// unchanged and inherits it, exactly as the decoder's `step_qp` does.
    fn qp_delta(&mut self) -> i32 {
        let d = self.qp as i32 - self.cur_qp as i32;
        self.cur_qp = self.qp;
        d
    }

    /// MV-predictor neighbors (left, above, above-right) for the 16×16 partition
    /// of macroblock `(mb_x, mb_y)`, read from the per-4×4-block grids.
    fn mv_neighbors(&self, mb_x: usize, mb_y: usize) -> [MvNeighbor; 3] {
        let w4 = self.mb_w * 4;
        let get = |avail: bool, bx: isize, by: isize| {
            if avail {
                let idx = by as usize * w4 + bx as usize;
                MvNeighbor {
                    available: true,
                    mv: self.mv_y[idx],
                    ref_idx: self.ref_idx_y[idx],
                }
            } else {
                MvNeighbor::NONE
            }
        };
        let (bx, by) = (mb_x as isize * 4, mb_y as isize * 4);
        let a = get(mb_x > 0, bx - 1, by);
        let b = get(mb_y > 0, bx, by - 1);
        // C = above-right; if unavailable, fall back to D = above-left.
        let c = if mb_y > 0 && mb_x + 1 < self.mb_w {
            get(true, bx + 4, by - 1)
        } else {
            get(mb_x > 0 && mb_y > 0, bx - 1, by - 1)
        };
        [a, b, c]
    }

    /// The `P_Skip` motion vector (spec §8.4.1.1). P_Skip always references
    /// index 0 (the most recent picture).
    fn skip_mv(&self, mb_x: usize, mb_y: usize) -> (i32, i32) {
        let [a, b, c] = self.mv_neighbors(mb_x, mb_y);
        if !a.available
            || !b.available
            || (a.ref_idx == 0 && a.mv == (0, 0))
            || (b.ref_idx == 0 && b.mv == (0, 0))
        {
            (0, 0)
        } else {
            predict_mv(a, b, c, 0)
        }
    }

    /// Records a macroblock's per-4×4-block motion state (`ref` = reference index
    /// for inter, ignored for intra where `inter` is false).
    fn set_mb_mv(&mut self, mb_x: usize, mb_y: usize, mv: (i32, i32), inter: bool, refi: i32) {
        let w4 = self.mb_w * 4;
        for dy in 0..4 {
            for dx in 0..4 {
                let idx = (mb_y * 4 + dy) * w4 + (mb_x * 4 + dx);
                self.mv_y[idx] = mv;
                self.inter_y[idx] = inter;
                self.ref_idx_y[idx] = if inter { refi } else { -1 };
            }
        }
    }

    /// Block-level MV-predictor neighbors for a partition whose top-left 4×4
    /// block is `(pbx, pby)` and which is `pwb` blocks wide. Availability uses
    /// the decoded-block grid, so in-macroblock partitions see earlier ones.
    fn mv_neighbors_block(&self, pbx: isize, pby: isize, pwb: isize) -> [MvNeighbor; 3] {
        let (w4, h4) = ((self.mb_w * 4) as isize, (self.mb_h * 4) as isize);
        let get = |bx: isize, by: isize| -> MvNeighbor {
            if bx < 0 || by < 0 || bx >= w4 || by >= h4 || !self.coded_y[(by * w4 + bx) as usize] {
                MvNeighbor::NONE
            } else {
                let idx = (by * w4 + bx) as usize;
                MvNeighbor { available: true, mv: self.mv_y[idx], ref_idx: self.ref_idx_y[idx] }
            }
        };
        let a = get(pbx - 1, pby);
        let b = get(pbx, pby - 1);
        let mut c = get(pbx + pwb, pby - 1);
        if !c.available {
            c = get(pbx - 1, pby - 1); // D fallback
        }
        [a, b, c]
    }

    /// List-aware block MV-predictor neighbors (`list` 0 or 1), for the B-slice
    /// per-list `mvd` predictor. Identical geometry to [`Self::mv_neighbors_block`]
    /// but reads the List-1 motion grid when `list == 1`, matching the decoder's
    /// `mv_neighbors_list`. A neighbor not coded in this list reads `ref_idx = -1`
    /// (so `predict_partition_mv` treats it as non-matching, exactly as the decoder).
    fn mv_neighbors_block_list(&self, pbx: isize, pby: isize, pwb: isize, list: usize) -> [MvNeighbor; 3] {
        let (w4, h4) = ((self.mb_w * 4) as isize, (self.mb_h * 4) as isize);
        let (mvg, refg): (&[(i32, i32)], &[i32]) = if list == 0 {
            (&self.mv_y, &self.ref_idx_y)
        } else {
            (&self.mv1_y, &self.ref_idx1_y)
        };
        let get = |bx: isize, by: isize| -> MvNeighbor {
            if bx < 0 || by < 0 || bx >= w4 || by >= h4 || !self.coded_y[(by * w4 + bx) as usize] {
                MvNeighbor::NONE
            } else {
                let idx = (by * w4 + bx) as usize;
                MvNeighbor { available: true, mv: mvg[idx], ref_idx: refg[idx] }
            }
        };
        let a = get(pbx - 1, pby);
        let b = get(pbx, pby - 1);
        let mut c = get(pbx + pwb, pby - 1);
        if !c.available {
            c = get(pbx - 1, pby - 1); // D fallback
        }
        [a, b, c]
    }

    /// SATD cost of a motion-compensated `rw`×`rh` luma region against the source —
    /// THE per-candidate ME cost function (Challenge-1 A2 shape: the per-search
    /// invariants arrive as parameters instead of being re-derived per candidate).
    /// `hp` is the already-resolved plane cache (`None` ⇔ the fast preset, whose
    /// SATD path never reads planes), `hr_on` the hoisted `RFF_HPEL_REF` knob,
    /// `src_row` the hoisted source slice base. Dispatch order (interior full-pel →
    /// in-place plane read → fused avg+SATD → materialize → `mc_luma` fallback) is
    /// the historical `mc_satd` order, so the accepted candidate set — and the
    /// bitstream — are byte-identical to it.
    #[allow(clippy::too_many_arguments)]
    #[inline]
    fn mc_satd_hp(
        &self,
        reference: &crate::RefFrame,
        hp: Option<&rusty_h264_common::inter::HpelPlanes>,
        hr_on: bool,
        // `hr_on && RFF_SATD_AVG` (and accel compiled in) — hoisted per search like
        // `hr_on`, so the fused-kernel gate costs zero OnceLock loads per candidate.
        // Unused (and always false) on non-accel builds.
        sa_on: bool,
        src_row: &[u8],
        lx: usize,
        ly: usize,
        rw: usize,
        rh: usize,
        mv: (i32, i32),
    ) -> i64 {
        #[cfg(not(accel))]
        let _ = sa_on;
        #[cfg(feature = "profile")]
        let _site = rusty_h264_common::inter::mcstats::SiteTag::new(2);
        let ch = self.mb_h * 16;
        let cw = self.cw;
        let (ix0, iy0) = (lx as isize + (mv.0 >> 2) as isize, ly as isize + (mv.1 >> 2) as isize);
        let interior_fullpel = mv.0 & 3 == 0
            && mv.1 & 3 == 0
            && ix0 >= 0
            && iy0 >= 0
            && ix0 + rw as isize <= cw as isize
            && iy0 + rh as isize <= ch as isize;
        #[cfg(feature = "profile")]
        {
            let fullpel = mv.0 & 3 == 0 && mv.1 & 3 == 0;
            satdpath::bump(if interior_fullpel { 0 } else if fullpel { 1 } else { 2 });
        }
        if interior_fullpel {
            let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::MeCost);
            let (rx0, ry0) = (ix0 as usize, iy0 as usize);
            return satd_px(src_row, cw, &reference.y[ry0 * cw + rx0..], cw, rw, rh);
        }
        if let Some(hp) = hp {
            if hr_on {
                if let Some((plane, base, stride)) =
                    rusty_h264_common::inter::hpel_ref(hp, lx, ly, rw, rh, mv.0, mv.1)
                {
                    let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::MeCost);
                    return satd_px(src_row, cw, &plane[base..], stride, rw, rh);
                }
            }
            // A3: QUARTER-pel — fuse the two-plane (a+b+1)>>1 average into the
            // SATD kernel itself (no 256-byte materialize + reload, no FFI hop).
            // `satd_avg` returns the exact `Σ|H·d|` that `satd_px` computes on
            // the materialized average, so the cost value — and the bitstream —
            // are byte-identical; on non-AVX2 (or a declined size) it returns
            // `None` and the old materialize path below runs unchanged.
            #[cfg(accel)]
            if sa_on {
                if let Some((pa, ba, pb, bb, stride)) =
                    rusty_h264_common::inter::hpel_qpel_refs(hp, lx, ly, rw, rh, mv.0, mv.1)
                {
                    let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::MeCost);
                    if let Some(v) = rusty_h264_accel::satd_avg(
                        src_row, cw, &pa[ba..], &pb[bb..], stride, rw, rh,
                    ) {
                        return v as i64;
                    }
                }
            }
            let mut pred = [0u8; 256];
            if rusty_h264_common::inter::hpel_block(hp, lx, ly, rw, rh, mv.0, mv.1, &mut pred) {
                return satd_px(src_row, cw, &pred, rw, rw, rh);
            }
            mc_luma(&reference.y, cw, ch, lx, ly, rw, rh, mv.0, mv.1, &mut pred);
            return satd_px(src_row, cw, &pred, rw, rw, rh);
        }
        let mut pred = [0u8; 256];
        mc_luma(&reference.y, cw, ch, lx, ly, rw, rh, mv.0, mv.1, &mut pred);
        satd_px(src_row, cw, &pred, rw, rw, rh)
    }

    /// Track-B B2.1: the SAD twin of `mc_satd_hp` — the SAME dispatch ladder
    /// (interior full-pel → in-place plane read → fused avg → materialize →
    /// `mc_luma`), with SAD (`psadbw`-class) distortion. `mc_sad` (the fast
    /// preset's function) had NONE of the SATD path's accumulated wins, so the
    /// first B2 cut measured 61% MORE `mc_luma` fallbacks; this is the parity fix.
    /// Every arm reads the same samples the materializing path would, so the SAD
    /// value — and therefore the B2-on bitstream — is unchanged by this function.
    #[allow(clippy::too_many_arguments)]
    #[inline]
    fn mc_sad_hp(
        &self,
        reference: &crate::RefFrame,
        hp: Option<&rusty_h264_common::inter::HpelPlanes>,
        hr_on: bool,
        src_row: &[u8],
        lx: usize,
        ly: usize,
        rw: usize,
        rh: usize,
        mv: (i32, i32),
        _asrc: Option<&[u8; 256]>,
    ) -> i64 {
        #[cfg(feature = "profile")]
        let _site = rusty_h264_common::inter::mcstats::SiteTag::new(2);
        let ch = self.mb_h * 16;
        let cw = self.cw;
        let (ix0, iy0) = (lx as isize + (mv.0 >> 2) as isize, ly as isize + (mv.1 >> 2) as isize);
        let interior_fullpel = mv.0 & 3 == 0
            && mv.1 & 3 == 0
            && ix0 >= 0
            && iy0 >= 0
            && ix0 + rw as isize <= cw as isize
            && iy0 + rh as isize <= ch as isize;
        if interior_fullpel {
            let (rx0, ry0) = (ix0 as usize, iy0 as usize);
            #[cfg(accel)]
            if rw == 16 && rh == 16 {
                if let Some(src) = _asrc {
                    return rusty_h264_accel::sad_16x16(src, 16, &reference.y[ry0 * cw + rx0..], cw)
                        as i64;
                }
            }
            return sad_strided(src_row, cw, &reference.y[ry0 * cw + rx0..], cw, rw, rh);
        }
        if let Some(hp) = hp {
            if hr_on {
                // Single-plane phases (h/v/c half-pel AND edge full-pel via the
                // padded `f` plane — the E-3 move, which `mc_sad` never had).
                if let Some((plane, base, stride)) =
                    rusty_h264_common::inter::hpel_ref(hp, lx, ly, rw, rh, mv.0, mv.1)
                {
                    return sad_strided(src_row, cw, &plane[base..], stride, rw, rh);
                }
                // Quarter-pel: fused (a+b+1)>>1 + SAD, no materialize.
                if let Some((pa, ba, pb, bb, stride)) =
                    rusty_h264_common::inter::hpel_qpel_refs(hp, lx, ly, rw, rh, mv.0, mv.1)
                {
                    return sad_avg_strided(src_row, cw, &pa[ba..], &pb[bb..], stride, rw, rh);
                }
            }
            let mut pred = [0u8; 256];
            if rusty_h264_common::inter::hpel_block(hp, lx, ly, rw, rh, mv.0, mv.1, &mut pred) {
                return sad_strided(src_row, cw, &pred, rw, rw, rh);
            }
            mc_luma(&reference.y, cw, ch, lx, ly, rw, rh, mv.0, mv.1, &mut pred);
            return sad_strided(src_row, cw, &pred, rw, rw, rh);
        }
        let mut pred = [0u8; 256];
        mc_luma(&reference.y, cw, ch, lx, ly, rw, rh, mv.0, mv.1, &mut pred);
        sad_strided(src_row, cw, &pred, rw, rw, rh)
    }

    /// SAD (sum of absolute differences) of a motion-compensated `rw`×`rh` luma
    /// region against the source — the **fast** preset's motion-search cost.
    ///
    /// SAD is far cheaper than SATD (no Hadamard transform), and the inner loop is
    /// written as `Σ a.abs_diff(b)` over `u8` slices, the exact pattern LLVM
    /// auto-vectorizes to the `psadbw` SAD instruction — the same instruction
    /// x264's hand-written assembly uses, but reached without any `unsafe`. (x264's
    /// fast presets use SAD for the full-pel search for precisely this reason.)
    #[allow(clippy::too_many_arguments)]
    fn mc_sad(
        &self,
        reference: &crate::RefFrame,
        sy: &[u8],
        lx: usize,
        ly: usize,
        rw: usize,
        rh: usize,
        mv: (i32, i32),
        // 16-aligned source MB (built once per search) for the asm SAD; `None`
        // (and unused) on the scalar build.
        _asrc: Option<&[u8; 256]>,
    ) -> i64 {
        // Descent E depth-6: tag WHO is calling mc_luma. The search's edge fallback and
        // reconstruction land in the same `inter-mc` bucket; pricing a recon-side lever
        // against the merged total is pricing the wrong population.
        #[cfg(feature = "profile")]
        let _site = rusty_h264_common::inter::mcstats::SiteTag::new(2);
        let ch = self.mb_h * 16;
        let cw = self.cw;
        let (ix0, iy0) = (lx as isize + (mv.0 >> 2) as isize, ly as isize + (mv.1 >> 2) as isize);
        let interior_fullpel = mv.0 & 3 == 0
            && mv.1 & 3 == 0
            && ix0 >= 0
            && iy0 >= 0
            && ix0 + rw as isize <= cw as isize
            && iy0 + rh as isize <= ch as isize;
        // Full-pel interior 16×16: openh264's `psadbw` SAD of the aligned source vs
        // the (movdqu) reference block. SAD is exact, so this is byte-identical to the
        // scalar path — a pure ME speedup (~2.4× the kernel).
        #[cfg(accel)]
        if interior_fullpel && rw == 16 && rh == 16 {
            if let Some(src) = _asrc {
                let (rx0, ry0) = (ix0 as usize, iy0 as usize);
                return rusty_h264_accel::sad_16x16(src, 16, &reference.y[ry0 * cw + rx0..], cw)
                    as i64;
            }
        }
        let mut sad = 0u32;
        if interior_fullpel {
            // Direct from the reference (a copy at full-pel) — no interpolation.
            let (rx0, ry0) = (ix0 as usize, iy0 as usize);
            let refy = &reference.y;
            for dy in 0..rh {
                let s = &sy[(ly + dy) * cw + lx..][..rw];
                let r = &refy[(ry0 + dy) * cw + rx0..][..rw];
                sad += s.iter().zip(r).map(|(&a, &b)| a.abs_diff(b) as u32).sum::<u32>();
            }
        } else {
            let mut pred = [0u8; 256];
            // Same plane-cache read (and same preset gate) as `mc_satd`.
            let from_planes = !self.fast
                && rusty_h264_common::inter::hpel_block(
                    reference.hpel(cw, ch),
                    lx,
                    ly,
                    rw,
                    rh,
                    mv.0,
                    mv.1,
                    &mut pred,
                );
            if !from_planes {
                mc_luma(&reference.y, cw, ch, lx, ly, rw, rh, mv.0, mv.1, &mut pred);
            }
            for dy in 0..rh {
                let s = &sy[(ly + dy) * cw + lx..][..rw];
                let p = &pred[dy * rw..][..rw];
                sad += s.iter().zip(p).map(|(&a, &b)| a.abs_diff(b) as u32).sum::<u32>();
            }
        }
        sad as i64
    }

    /// Luma distortion of a `B_Bi` 16×16 prediction: motion-compensate `l0`/`l1`,
    /// average `(p+q+1)>>1` (the decoder's `b_mc` blend at `weighted_bipred_idc=0`),
    /// and score vs the source with the SAME metric the per-list searches used —
    /// SAD on the fast path, SATD when this MB is SATD-routed — so `J_bi` compares
    /// directly against `J0`/`J1`.
    fn bi_dist(
        &self,
        l0: &crate::RefFrame,
        l1: &crate::RefFrame,
        sy: &[u8],
        lx: usize,
        ly: usize,
        mv0: (i32, i32),
        mv1: (i32, i32),
    ) -> i64 {
        // Descent E/F: identify this mc_luma population by call site.
        #[cfg(feature = "profile")]
        let _site = rusty_h264_common::inter::mcstats::SiteTag::new(4);
        let ch = self.mb_h * 16;
        let (mut a, mut b) = ([0u8; 256], [0u8; 256]);
        mc_luma(&l0.y, self.cw, ch, lx, ly, 16, 16, mv0.0, mv0.1, &mut a);
        mc_luma(&l1.y, self.cw, ch, lx, ly, 16, 16, mv1.0, mv1.1, &mut b);
        let mut avg = [0u8; 256];
        for i in 0..256 {
            avg[i] = bi_blend(a[i] as i32, b[i] as i32, self.bi_w);
        }
        if self.fast && !self.mb_use_satd {
            let mut sad = 0u32;
            for dy in 0..16 {
                let s = &sy[(ly + dy) * self.cw + lx..][..16];
                let p = &avg[dy * 16..][..16];
                sad += s.iter().zip(p).map(|(&x, &y)| x.abs_diff(y) as u32).sum::<u32>();
            }
            sad as i64
        } else {
            satd_px(&sy[ly * self.cw + lx..], self.cw, &avg, 16, 16, 16)
        }
    }

    /// Distortion of a pre-formed 16×16 luma prediction vs the source (SAD on the
    /// fast path, SATD when SATD-routed) — the mode-decision cost for `B_Direct`,
    /// on the same scale as the per-list search J so they compare directly.
    fn pred_dist(&self, sy: &[u8], lx: usize, ly: usize, pred: &[u8; 256]) -> i64 {
        if self.fast && !self.mb_use_satd {
            let mut sad = 0u32;
            for dy in 0..16 {
                let s = &sy[(ly + dy) * self.cw + lx..][..16];
                let p = &pred[dy * 16..][..16];
                sad += s.iter().zip(p).map(|(&a, &b)| a.abs_diff(b) as u32).sum::<u32>();
            }
            sad as i64
        } else {
            satd_px(&sy[ly * self.cw + lx..], self.cw, pred, 16, 16, 16)
        }
    }

    /// `colZeroFlag` for absolute 4×4 block `(bx, by)` (spec §8.4.1.2.2): true when
    /// the co-located picture `RefPicList1[0]` (`l1`) is short-term (always, here —
    /// we use no long-term refs) and its co-located block uses List-0 reference 0
    /// with a near-zero (|·| ≤ 1) motion vector. Must match the decoder's `col_zero`.
    fn col_zero(&self, l1: &crate::RefFrame, bx: usize, by: usize) -> bool {
        if l1.w4 == 0 {
            return false;
        }
        let idx = by * l1.w4 + bx;
        if idx >= l1.ref_idx.len() {
            return false;
        }
        l1.ref_idx[idx] == 0 && l1.mv[idx].0.abs() <= 1 && l1.mv[idx].1.abs() <= 1
    }

    /// Bi-predictive MC of one small region into `pred_y`/`c_pred` at MB-relative
    /// offset `(dx, dy)` — the per-4×4 primitive the spatial-direct derivation uses.
    /// Mirrors the decoder's `b_mc` (average `(p+q+1)>>1` for bi, copy for uni).
    #[allow(clippy::too_many_arguments)]
    fn b_mc_block(
        &self,
        l0: &crate::RefFrame,
        l1: &crate::RefFrame,
        mb_x: usize,
        mb_y: usize,
        dx: usize,
        dy: usize,
        refi0: i32,
        m0: (i32, i32),
        refi1: i32,
        m1: (i32, i32),
        pred_y: &mut [u8; 256],
        c_pred: &mut [[u8; 64]; 2],
    ) {
        // Descent E/F: identify this mc_luma population by call site.
        #[cfg(feature = "profile")]
        let _site = rusty_h264_common::inter::mcstats::SiteTag::new(4);
        let (ch, cch) = (self.mb_h * 16, self.mb_h * 8);
        let (px, py) = (mb_x * 16 + dx, mb_y * 16 + dy);
        let (mut a, mut b) = ([0u8; 16], [0u8; 16]);
        // 4-wide luma exists ONLY here: P partitions bottom out at 8×8, so B-frame
        // spatial-direct is the encoder's only 4×4 MC. With B-frames on it is ~8% of
        // all MC calls and HALF of all sub-pel ones — and `luma_h`/`luma_v` dispatch
        // to asm only at width 16/8, so 4-wide otherwise runs the scalar 6-tap.
        // Serving it from the cached half-pel planes (bit-identical, and they are
        // already built for this reference by the motion search) is strictly better
        // than adding a 4-wide asm kernel.
        let mc4 = |r: &crate::RefFrame, mv: (i32, i32), out: &mut [u8; 16]| {
            if !self.fast
                && bdirect_planes_enabled()
                && rusty_h264_common::inter::hpel_block(
                    r.hpel(self.cw, ch), px, py, 4, 4, mv.0, mv.1, out,
                )
            {
                return;
            }
            mc_luma(&r.y, self.cw, ch, px, py, 4, 4, mv.0, mv.1, out);
        };
        if refi0 >= 0 {
            mc4(l0, m0, &mut a);
        }
        if refi1 >= 0 {
            mc4(l1, m1, &mut b);
        }
        for yy in 0..4 {
            for xx in 0..4 {
                let i = yy * 4 + xx;
                let v = match (refi0 >= 0, refi1 >= 0) {
                    (true, true) => bi_blend(a[i] as i32, b[i] as i32, self.bi_w),
                    (true, false) => a[i],
                    _ => b[i],
                };
                pred_y[(dy + yy) * 16 + (dx + xx)] = v;
            }
        }
        // Chroma: the co-located 2×2 block at half resolution.
        let (cpx, cpy) = (mb_x * 8 + dx / 2, mb_y * 8 + dy / 2);
        for c in 0..2 {
            let (r0, r1) = if c == 0 { (&l0.u, &l1.u) } else { (&l0.v, &l1.v) };
            let (mut ca, mut cb) = ([0u8; 4], [0u8; 4]);
            if refi0 >= 0 {
                mc_chroma(r0, self.ccw, cch, cpx, cpy, 2, 2, m0.0, m0.1, &mut ca);
            }
            if refi1 >= 0 {
                mc_chroma(r1, self.ccw, cch, cpx, cpy, 2, 2, m1.0, m1.1, &mut cb);
            }
            for yy in 0..2 {
                for xx in 0..2 {
                    let i = yy * 2 + xx;
                    let v = match (refi0 >= 0, refi1 >= 0) {
                        (true, true) => bi_blend(ca[i] as i32, cb[i] as i32, self.bi_w),
                        (true, false) => ca[i],
                        _ => cb[i],
                    };
                    c_pred[c][(dy / 2 + yy) * 8 + (dx / 2 + xx)] = v;
                }
            }
        }
    }

    /// Spatial-direct (`direct_spatial_mv_pred_flag == 1`) prediction for a 16×16 B
    /// macroblock — the shared basis of `B_Skip` and `B_Direct_16x16`. Returns the
    /// prediction and the per-4×4 `(refIdxL0, mvL0, refIdxL1, mvL1)` motion the
    /// decoder's `decode_b_direct` derives (so the caller commits identical motion).
    fn b_direct(
        &self,
        l0: &crate::RefFrame,
        l1: &crate::RefFrame,
        mb_x: usize,
        mb_y: usize,
    ) -> ([u8; 256], [[u8; 64]; 2], [(i32, (i32, i32), i32, (i32, i32)); 16]) {
        let (nbx, nby) = ((mb_x * 4) as isize, (mb_y * 4) as isize);
        let n0 = self.mv_neighbors_block_list(nbx, nby, 4, 0);
        let n1 = self.mv_neighbors_block_list(nbx, nby, 4, 1);
        let min_pos = |a: i32, b: i32| if a < 0 { b } else if b < 0 { a } else { a.min(b) };
        let rid = |n: &[MvNeighbor; 3]| min_pos(min_pos(n[0].ref_idx, n[1].ref_idx), n[2].ref_idx);
        let (mut refi0, mut refi1) = (rid(&n0), rid(&n1));
        let direct_zero = refi0 < 0 && refi1 < 0;
        if direct_zero {
            refi0 = 0;
            refi1 = 0;
        }
        let mv0 = if refi0 >= 0 && !direct_zero { predict_mv(n0[0], n0[1], n0[2], refi0) } else { (0, 0) };
        let mv1 = if refi1 >= 0 && !direct_zero { predict_mv(n1[0], n1[1], n1[2], refi1) } else { (0, 0) };
        let mut pred_y = [0u8; 256];
        let mut c_pred = [[0u8; 64]; 2];
        let mut motion = [(0i32, (0i32, 0i32), 0i32, (0i32, 0i32)); 16];
        for sby in 0..4 {
            for sbx in 0..4 {
                let cz = !direct_zero && self.col_zero(l1, mb_x * 4 + sbx, mb_y * 4 + sby);
                let m0 = if refi0 == 0 && cz { (0, 0) } else { mv0 };
                let m1 = if refi1 == 0 && cz { (0, 0) } else { mv1 };
                motion[sby * 4 + sbx] = (refi0, m0, refi1, m1);
                self.b_mc_block(l0, l1, mb_x, mb_y, sbx * 4, sby * 4, refi0, m0, refi1, m1, &mut pred_y, &mut c_pred);
            }
        }
        (pred_y, c_pred, motion)
    }

    /// Commits a spatial-direct MB's per-4×4 motion into the List-0/List-1 grids so
    /// later MBs' neighbor predictors see it (mirrors the decoder's `b_set_motion`).
    fn commit_direct_motion(&mut self, mb_x: usize, mb_y: usize, motion: &[(i32, (i32, i32), i32, (i32, i32)); 16]) {
        let w4 = self.mb_w * 4;
        for sby in 0..4 {
            for sbx in 0..4 {
                let (refi0, m0, refi1, m1) = motion[sby * 4 + sbx];
                let idx = (mb_y * 4 + sby) * w4 + (mb_x * 4 + sbx);
                self.inter_y[idx] = true;
                self.coded_y[idx] = true;
                self.mv_y[idx] = m0;
                self.ref_idx_y[idx] = refi0;
                self.mv1_y[idx] = m1;
                self.ref_idx1_y[idx] = refi1;
            }
        }
    }

    /// Rate-aware motion search for a luma region: full-pel diamond + half/
    /// quarter-pel refinement minimizing `J = SATD + λ·bits(mvd)`, where the
    /// motion cost is measured against `predictors[0]` (the MV predictor the
    /// `mvd` will actually be coded against). The search is seeded from every
    /// entry in `predictors` plus `(0,0)`. Returns the best MV and its `J`.
    ///
    /// The rate term is only a *search heuristic* — whatever MV it picks is still
    /// coded as a correct `mvd`, so this never affects decodability.
    #[allow(clippy::too_many_arguments)]
    /// ME ORACLE PROBE (`RFF_ME_ORACLE=1`): does our search actually FIND the best
    /// motion vector available to it? Accumulates our chosen cost against an
    /// exhaustive +-24 full-pel search refined by the identical sub-pel pass, so a
    /// gap is attributable to the SEARCH, not to the cost function or precision.
    /// [n, sum(our cost), sum(oracle cost), blocks where oracle won, cost() evals]
    fn motion_search(
        &self,
        reference: &crate::RefFrame,
        sy: &[u8],
        lx: usize,
        ly: usize,
        rw: usize,
        rh: usize,
        predictors: &[(i32, i32)],
        lambda_me: f64,
        // Some(mv) => skip the full-pel search entirely and refine THIS vector. The
        // starting COST is recomputed here rather than passed in, so the baseline the
        // refinement must beat is priced by the same closure as every candidate.
        start: Option<(i32, i32)>,
    ) -> ((i32, i32), i64) {
        // Bit length of `se(d)` (Exp-Golomb), i.e. what an `mvd` component costs.
        // Branchless closed form of the old `while n > 1 { n >>= 1; len += 2 }` loop:
        // that loop yields `len = 1 + 2·floor(log2(codenum+1))`, and for x ≥ 1
        // `floor(log2(x)) == 31 - x.leading_zeros()`. Removes a data-dependent branch
        // from the innermost ME cost — bit-identical (verified over the d range).
        let mvk = mv_cost_kind(self.mv_smooth);
        let mvbits = |d: i32| -> u32 {
            // H-23: the ME rate model. `RFF_MVCOST=1` swaps the Exp-Golomb STEP
            // function for x264's smooth curve `2·log2(|d|+1) + 0.718 + (d!=0)`.
            // The step function is FLAT inside a power-of-two bracket — it prices
            // d=4 and d=7 identically, so the search takes the far end of a
            // bracket for free, inflating |mvd| (and with it the sign+prefix bits
            // the accountant found are ~14% of the payload). λ cannot fix this:
            // scaling a flat region leaves it flat. Table is in WHOLE bits to keep
            // the caller's integer arithmetic; ×4 internally then rounded, so the
            // curve's ordering survives quantization.
            match mvk {
                1 => {
                    let a = d.unsigned_abs().min(4095) as usize;
                    MV_COST_TAB.get_or_init(build_mv_cost)[a] as u32
                }
                2 => {
                    let a = d.unsigned_abs().min(4095) as usize;
                    MV_TRUE_BIASED.get_or_init(build_true_biased)[a] as u32
                }
                _ => {
                    let codenum = if d > 0 { (2 * d - 1) as u32 } else { (-2 * d) as u32 };
                    1 + 2 * (31 - (codenum + 1).leading_zeros())
                }
            }
        };
        let center = predictors[0];
        let probe = me_oracle_on();
        // Track-B B2: the full-pel phase (seeds/snap/diamond) prices candidates in
        // the SAD domain; the winner is repriced in SATD before rescue/sub-pel.
        // Refine-only searches have no full-pel phase, so B2 does not apply there.
        // `self.sadfp` is force-mode at construction or the per-frame `b2_mgain`
        // dispatcher's routing (mode 1).
        let sadfp = !self.fast && start.is_none() && self.sadfp;
        // Build the 16-aligned source MB ONCE per search for the asm SAD path (fast
        // preset — and B2's SAD full-pel phase — full 16×16). Amortized over every
        // candidate's SAD; the reference block stays unaligned (movdqu). Scalar
        // build does no copy.
        #[cfg(accel)]
        let asrc_buf = if (self.fast || sadfp) && rw == 16 && rh == 16 {
            let mut a = AlignedMb([0u8; 256]);
            for dy in 0..16 {
                a.0[dy * 16..dy * 16 + 16].copy_from_slice(&sy[(ly + dy) * self.cw + lx..][..16]);
            }
            Some(a)
        } else {
            None
        };
        #[cfg(accel)]
        let asrc: Option<&[u8; 256]> = asrc_buf.as_ref().map(|a| &a.0);
        #[cfg(not(accel))]
        let asrc: Option<&[u8; 256]> = None;
        // Challenge-1 A2: hoist the SATD path's per-search invariants OUT of the
        // per-candidate closure. `mc_satd` re-derived, for EVERY candidate: the
        // plane-cache OnceLock (an acquire load + branch, twice on the quarter-pel
        // arm), the `RFF_HPEL_REF` OnceLock, and the source-row slice base (a bounds
        // check). All are constant across the ~20-50 evaluations of one search.
        // `mc_satd_hp` is the same dispatch with those values passed in — the same
        // arms in the same order, so the accepted candidate set is byte-identical.
        let use_sad = self.fast && !self.mb_use_satd;
        let cw = self.cw;
        // Every non-fast search sub-pel-refines at the end, so the planes are built
        // for any reference a search touches — hoisting the get_or_init here does not
        // build planes a lazy path would have avoided.
        let hp: Option<&rusty_h264_common::inter::HpelPlanes> =
            if !self.fast { Some(reference.hpel(cw, self.mb_h * 16)) } else { None };
        let hr_on = hpel_ref_enabled();
        // A3 gate, hoisted with the rest (`RFF_HPEL_REF=0` restores the FULL pre-C/A3
        // copy path, so the fused kernel rides the same master anchor).
        let sa_on = cfg!(accel) && hr_on && satd_avg_enabled();
        let src_row = &sy[ly * cw + lx..];
        // H-14 R3: the MeCtx fast evaluator — ONE geometry validation per search,
        // then per-eval integer bounds + direct kernel (collects the measured
        // ~23 ns/eval dispatch chain). Values are exactly the safe path's, so a
        // candidate served here cannot change the bitstream; out-of-window
        // candidates fall back to `mc_satd_hp` (equal values there too).
        #[cfg(accel)]
        let mectx = if !use_sad && mectx_enabled() {
            hp.and_then(|p| {
                rusty_h264_accel::MeCtx::new(
                    src_row, cw, &p.f, &p.h, &p.v, &p.c, p.stride, p.pad, p.pw, p.ph,
                    lx, ly, rw, rh,
                )
            })
        } else {
            None
        };
        let cost = |mv: (i32, i32)| -> i64 {
            let rate = mvbits(mv.0 - center.0) + mvbits(mv.1 - center.1);
            // The smooth table carries 4× resolution; fold that into λ so the
            // rate/distortion balance is unchanged and only the SHAPE differs.
            let lam_r = if mvk != 0 { lambda_me * 0.25 } else { lambda_me };
            // Fast preset: SAD (psadbw — asm kernel on `--features asm`, else auto-vec)
            // — far cheaper than SATD, the single biggest reason x264 fast out-runs us.
            let dist = if use_sad {
                self.mc_sad(reference, sy, lx, ly, rw, rh, mv, asrc)
            } else {
                #[cfg(accel)]
                {
                    match mectx.as_ref().and_then(|c| c.eval(mv.0, mv.1)) {
                        Some(d) => d as i64,
                        None => {
                            self.mc_satd_hp(reference, hp, hr_on, sa_on, src_row, lx, ly, rw, rh, mv)
                        }
                    }
                }
                #[cfg(not(accel))]
                {
                    self.mc_satd_hp(reference, hp, hr_on, sa_on, src_row, lx, ly, rw, rh, mv)
                }
            };
            dist + (lam_r * rate as f64) as i64
        };
        // B2's full-pel-phase cost: SAD distortion, λ scaled to the SAD domain
        // (`RFF_ME_SADL`, hoisted). Falls through to `cost` (SATD) whenever B2 is
        // off, so every pre-B2 path is untouched.
        let lam_fp = lambda_me * if sadfp { me_sadfp_lambda() } else { 1.0 };
        let cost_fp = |mv: (i32, i32)| -> i64 {
            if !sadfp {
                return cost(mv);
            }
            let rate = mvbits(mv.0 - center.0) + mvbits(mv.1 - center.1);
            self.mc_sad_hp(reference, hp, hr_on, src_row, lx, ly, rw, rh, mv, asrc)
                + (lam_fp * rate as f64) as i64
        };
        // Seed from (0,0) and each predictor; keep the cheapest.
        let refine_only = start.is_some();
        let (mut best, mut best_c) = match start {
            Some(mv) => (mv, cost(mv)),
            None => {
                let mut b = (0, 0);
                let mut bc = cost_fp(b);
                for &p in predictors {
                    let pc = cost_fp(p);
                    if pc < bc {
                        bc = pc;
                        b = p;
                    }
                }
                (b, bc)
            }
        };
        // SNAP THE DIAMOND CENTRE TO INTEGER-PEL. The diamond below steps by whole
        // pels, so a fractional centre makes EVERY candidate fractional and forces
        // all of them through `mc_luma`'s 6-tap filter — measured at 84-90% of all
        // SATD evaluations. Snapping puts the whole full-pel phase on the direct
        // (no-interpolation) SATD path. The pre-snap seed is kept and re-compared
        // after refinement, so this can only change WHERE we search, never make the
        // returned vector worse than the seed we started from.
        let (seed_mv, mut seed_c) = (best, best_c);
        if !refine_only && self.me_snap && (best.0 & 3 != 0 || best.1 & 3 != 0) {
            let snapped = ((best.0 + 2).div_euclid(4) * 4, (best.1 + 2).div_euclid(4) * 4);
            best_c = cost_fp(snapped);
            best = snapped;
        }
        // Coarse-to-fine full-pel search: a 4-point diamond walked at each step
        // size from 16 px down to 1 px (steps in quarter-pel units: 64,32,…,4).
        // The larger initial steps reach fast motion the predictor missed; the
        // diamond stays orthogonal (no diagonals) — diagonal probes were found to
        // chase equally-good far matches on ambiguous motion, wrecking MV-field
        // coherence and the neighbor predictors.
        // The fast preset trusts the neighbour MV predictor and refines locally
        // (one coarse reach + fine), like x264's `me=dia`; quality sweeps the full
        // coarse-to-fine range. Each step's diamond still walks until no
        // improvement, so even fast reaches far motion — just in smaller hops.
        // Descent A: the coarse rungs are ~76-80% of full-pel evals at a 0.05-1.0% hit
        // rate (near-equal eval counts per rung = the walk almost never walks, so each
        // rung is a flat ~4-eval toll). RFF_DIA_LADDER selects which rungs to pay for.
        let mut ladder = [0i32; 5];
        let mut nladder = 0usize;
        let steps: &[i32] = if self.fast {
            &[16, 4]
        } else {
            let m = dia_mask();
            for (i, r) in DIA_RUNGS.iter().enumerate() {
                if m & (1 << i) != 0 {
                    ladder[nladder] = *r;
                    nladder += 1;
                }
            }
            &ladder[..nladder]
        };
        let _gd = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::MeDiamond);
        // FC: batch a fixed-centre diamond pass through the x4 kernels when every
        // candidate is an interior full-pel 16×16 read — one source band covers all
        // four candidates. Applies to BOTH cost domains (`sad_16x16_x4` on
        // SAD-routed frames, `satd_16x16_x4` otherwise); the fast preset keeps its
        // own untouched path. Argmin-of-4 replaces the first-improver cascade —
        // measured BD-POSITIVE on the SAD domain (bus −1.71→−2.61) and gated on the
        // corpus for the SATD domain the same way. `RFF_ME_FC=0` restores cascade.
        let fc = !self.fast && cfg!(accel) && me_fc_enabled()
            && matches!((rw, rh), (16, 16) | (16, 8) | (8, 16) | (8, 8));
        let ch_px = self.mb_h as isize * 16;
        for (_si, &step) in steps.iter().enumerate() {
            if refine_only {
                break;
            }
            loop {
                #[cfg(accel)]
                if fc && best.0 & 3 == 0 && best.1 & 3 == 0 {
                    // All four candidates full-pel; interior iff the ±step box is.
                    let s = (step >> 2) as isize;
                    let (bx, by) = (lx as isize + (best.0 >> 2) as isize, ly as isize + (best.1 >> 2) as isize);
                    if bx - s >= 0 && by - s >= 0 && bx + s + rw as isize <= cw as isize && by + s + rh as isize <= ch_px {
                        let offs = [
                            (by * cw as isize + bx + s) as usize,
                            (by * cw as isize + bx - s) as usize,
                            ((by + s) * cw as isize + bx) as usize,
                            ((by - s) * cw as isize + bx) as usize,
                        ];
                        // 16-wide shapes go through the batch kernel; 8-wide ones
                        // measured SLOWER batched than the per-candidate Wels asm
                        // (H-8 speed gate), so they evaluate individually inside the
                        // SAME argmin — identical values, identical comparisons,
                        // identical bitstream.
                        let batch = if rw != 16 {
                            None
                        } else if sadfp {
                            rusty_h264_accel::sad_x4(src_row, cw, &reference.y, offs, cw, rw, rh)
                        } else {
                            rusty_h264_accel::satd_x4(src_row, cw, &reference.y, offs, cw, rw, rh)
                        };
                        {
                            let ring = [(step, 0), (-step, 0), (0, step), (0, -step)];
                            let (mut bi, mut bc) = (usize::MAX, best_c);
                            for (i, &(dx, dy)) in ring.iter().enumerate() {
                                let mv = (best.0 + dx, best.1 + dy);
                                let cc = match batch {
                                    Some(sads) => {
                                        let rate = mvbits(mv.0 - center.0) + mvbits(mv.1 - center.1);
                                        sads[i] as i64 + (lam_fp * rate as f64) as i64
                                    }
                                    None => cost_fp(mv),
                                };
                                #[cfg(feature = "profile")]
                                diastats::ev(_si);
                                if cc < bc {
                                    bc = cc;
                                    bi = i;
                                }
                            }
                            if bi == usize::MAX {
                                break;
                            }
                            best_c = bc;
                            best = (best.0 + ring[bi].0, best.1 + ring[bi].1);
                            #[cfg(feature = "profile")]
                            diastats::imp(_si);
                            continue;
                        }
                    }
                }
                let mut improved = false;
                for &(dx, dy) in &[(step, 0), (-step, 0), (0, step), (0, -step)] {
                    let c = (best.0 + dx, best.1 + dy);
                    let cc = cost_fp(c);
                    #[cfg(feature = "profile")]
                    diastats::ev(_si);
                    if cc < best_c {
                        best_c = cc;
                        best = c;
                        improved = true;
                        #[cfg(feature = "profile")]
                        diastats::imp(_si);
                    }
                }
                if !improved {
                    break;
                }
            }
        }
        // DIAMOND-STALLED RESCUE (content-adaptive: fires on the FAILURE, not a proxy).
        // The gradient-descent diamond stalls at a plateau on FLAT cost surfaces and
        // never reaches the far-but-better MV that exists within ±16 (measured: ~+22%
        // BD-rate vs x264's simple dia on smooth content). The precise stall signal is
        // the CONJUNCTION: a FLAT source block (low variance) whose diamond match STILL
        // has a high residual — because on a flat surface the RIGHT MV predicts near-
        // perfectly, so a high residual there means the diamond missed it (a stall).
        // (Residual alone fires on busy blocks where a high residual is inherent — that
        // was 3.3× slower on mand for nothing; variance alone fires on flat-but-well-
        // predicted blocks. The AND targets exactly the stalls.) Then a FINE ±16 step-2
        // grid reaches the true minimum. Fires on a fraction of blocks → affordable.
        // Quality preset only.
        drop(_gd);
        // B2: the full-pel phase priced in the SAD domain — reprice the winner AND
        // the pre-snap seed into the SATD domain the rescue + sub-pel phases (and the
        // final seed-vs-refined comparison) trade in. Two SATD evaluations per
        // search, against the ~20-50 candidate evaluations the SAD domain cheapened.
        if sadfp {
            best_c = cost(best);
            seed_c = cost(seed_mv);
        }
        let _gr = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::MeRescue);
        // H-14 R1 brick 1: `me_fast` defaults TRUE, which makes `flat`'s value
        // IRRELEVANT to the gate below on every default search — yet the full
        // rw×rh sum+sum-of-squares walk (256 pixel loads + muls) ran EAGERLY per
        // search. Lazy-evaluate it: same boolean outcome in every case (me_fast
        // short-circuits first), the dead variance pass simply never runs.
        let flat = |sself: &Self| {
            !refine_only && {
                let (mut s, mut ss) = (0u64, 0u64);
                for dy in 0..rh {
                    for dx in 0..rw {
                        let v = sy[(ly + dy) * sself.cw + lx + dx] as u64;
                        s += v;
                        ss += v * v;
                    }
                }
                let n = (rw * rh) as u64;
                (ss - s * s / n) / n < sself.me_wide_var
            }
        };
        // The online payoff gate may have disabled the rescue for the rest of this
        // frame (irreducible-residual content — rotation/fractal — where the fine grid
        // fixes almost nothing; measured 2.25× on rot for a ~0% BD gain). A gated-off
        // frame runs exactly the diamond → identical to me_wide-off → never worse.
        // FAST-MOTION extension: the flat gate targets smooth-surface stalls, but the
        // diamond ALSO stalls on FAST motion (bus/football: an exhaustive ±24 search
        // recovers 6-15% BD) — those blocks are high-VARIANCE (detail) so `flat` misses
        // them. `me_fast` also fires on any high-residual block; the online payoff gate
        // then keeps it only where a wider search actually pays off (fast motion), and
        // disables it on irreducible-residual detail — the same self-tuning as flat.
        if self.me_wide && !self.fast && (self.me_fast || flat(self)) && !self.resc_off.get() {
            // H-14 R1 brick 2: `best` was priced by the SAME `dist + (λ·rate) as
            // i64` formula on every path that can reach here (cost, cost_fp after
            // the B2 reprice, the FC batch with lam_fp == λ off SAD frames), so
            // its distortion is recoverable EXACTLY by subtraction — the extra
            // full SATD kernel call per search was pure recompute (the
            // codec-eliminate-redundancy "return the already-computed value").
            let rate_b = mvbits(best.0 - center.0) + mvbits(best.1 - center.1);
            let dist = best_c - (lambda_me * rate_b as f64) as i64;
            if dist / (rw * rh).max(1) as i64 > self.me_rescue {
                // FINE ±16 step-2 grid + ±1 refine — recover the true minimum the
                // diamond missed. Fires only on flat-block stalls, so it is affordable.
                // SNAP THE GRID CENTRE TO INTEGER-PEL: the diamond seed can be sub-pel
                // (sub-pel neighbour predictors), and since every grid point shares
                // cx&3, a sub-pel centre forces the WHOLE ±16 grid through mc_luma
                // interpolation — measured 89% of zoom's rescue cost. The rescue only
                // needs the right REGION (a far MV the diamond missed); the sub-pel
                // refine that follows recovers the fraction. Integer centre → the grid
                // hits the fast full-pel SATD path (no interpolation).
                let pre_c = best_c;
                let (cx, cy) = ((best.0 + 2).div_euclid(4) * 4, (best.1 + 2).div_euclid(4) * 4);
                let mut gb = best;
                // BATCHED FULL-PEL GRID (accel): now that the grid centre is integer-pel
                // (all points interior full-pel), hoist the interior/bounds check out of
                // the loop and call the AVX2 SATD directly — skipping mc_satd's per-point
                // interior test + satd_px dispatch. BYTE-IDENTICAL to the cost() path
                // (same 2·satd_16x16 + rate), so it is default-on (RFF_ME_BATCH=0 to A/B
                // it off). ~+7% zoom / +4% tsrc on top of the snap; the SATD kernel itself
                // is already AVX2 and its transform can't amortise across the grid, so
                // this per-call-overhead trim is the ceiling for an "asm grid kernel".
                let cw = self.cw;
                let r = self.me_range;
                let batched = rw == 16 && rh == 16 && cfg!(accel) && {
                    let (icdx, icdy) = (cx >> 2, cy >> 2);
                    lx as i32 + icdx >= r
                        && lx as i32 + icdx + r + 16 <= cw as i32
                        && ly as i32 + icdy >= r
                        && ly as i32 + icdy + r + 16 <= (self.mb_h * 16) as i32
                        && me_batch_enabled()
                };
                #[cfg(accel)]
                if batched {
                    let (icdx, icdy) = ((cx >> 2), (cy >> 2));
                    let src = &sy[ly * cw + lx..];
                    let mut dy = -r;
                    while dy <= r {
                        let rby = (ly as i32 + icdy + dy) as usize;
                        let mut dx = -r;
                        while dx <= r {
                            let rbx = (lx as i32 + icdx + dx) as usize;
                            let satd =
                                2 * rusty_h264_accel::satd_16x16(src, cw, &reference.y[rby * cw + rbx..], cw) as i64;
                            let mv = (cx + dx * 4, cy + dy * 4);
                            let rate = mvbits(mv.0 - center.0) + mvbits(mv.1 - center.1);
                            let cc = satd + (lambda_me * rate as f64) as i64;
                            if cc < best_c {
                                best_c = cc;
                                gb = mv;
                            }
                            dx += 2;
                        }
                        dy += 2;
                    }
                }
                if !batched {
                    let mut dy = -r;
                    while dy <= r {
                        let mut dx = -r;
                        while dx <= r {
                            let cc = cost((cx + dx * 4, cy + dy * 4));
                            if cc < best_c {
                                best_c = cc;
                                gb = (cx + dx * 4, cy + dy * 4);
                            }
                            dx += 2;
                        }
                        dy += 2;
                    }
                }
                best = gb;
                for dy in -1..=1 {
                    for dx in -1..=1 {
                        let c = (best.0 + dx * 4, best.1 + dy * 4);
                        let cc = cost(c);
                        if cc < best_c {
                            best_c = cc;
                            best = c;
                        }
                    }
                }
                // LEARNING PHASE: for the first `me_learn` stalls of the frame, tally
                // whether the grid actually paid off (≥6.25% cost cut). Once the window
                // fills, if too few paid off the residual is irreducible on this content
                // → disable the rescue for the rest of the frame. The window's own MVs
                // are committed, but they're a small spatially-clustered set (not
                // improvement-selected), so on net-neutral content (rot) they can't
                // regress — only frame-level on/off avoids the per-block B-direct
                // selection effect.
                let n = self.resc_n.get();
                if n < self.me_learn {
                    self.resc_n.set(n + 1);
                    if best_c * 16 <= pre_c * 15 {
                        self.resc_big.set(self.resc_big.get() + 1);
                    }
                    if n + 1 == self.me_learn
                        && self.resc_big.get() * 100 < self.me_learn * self.me_payoff_pct
                    {
                        self.resc_off.set(true);
                    }
                }
            }
        }
        drop(_gr);
        let _gs = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::MeSubpel);
        // Sub-pel refinement uses the 6-tap/bilinear interpolation — the expensive
        // per-pixel `mc_luma` path that profiling pinned at ~55% of the entire
        // encode. The fast preset skips it (integer-pel only, like x264's fastest
        // presets `subme=0`): ~3× faster, trading a little quality on sub-pixel
        // motion. The quality preset does the full half-pel + quarter-pel rings.
        if probe {
            // Exhaustive +-24 full-pel around the same centre, then the SAME sub-pel
            // pass, so only the full-pel search strategy differs.
            let mut ob = center;
            let mut oc = i64::MAX;
            for gy in -24i32..=24 {
                for gx in -24i32..=24 {
                    let c = (center.0 + gx * 4, center.1 + gy * 4);
                    let cc = cost(c);
                    if cc < oc {
                        oc = cc;
                        ob = c;
                    }
                }
            }
            let fullpel_best = ob;
            for &st in &[2i32, 1] {
                for &(dx, dy) in &[(st, 0), (-st, 0), (0, st), (0, -st)] {
                    let c = (ob.0 + dx, ob.1 + dy);
                    let cc = cost(c);
                    if cc < oc {
                        oc = cc;
                        ob = c;
                    }
                }
            }
            // EXHAUSTIVE sub-pel: every quarter-pel offset in +-3 around the full-pel
            // winner. Our own pass is a single 4-point probe at half then quarter, so
            // this is what separates a sub-pel deficiency from a full-pel one.
            let mut oc_sp = oc;
            for dy in -3i32..=3 {
                for dx in -3i32..=3 {
                    let c = (fullpel_best.0 + dx, fullpel_best.1 + dy);
                    let cc = cost(c);
                    if cc < oc_sp {
                        oc_sp = cc;
                    }
                }
            }
            // our own sub-pel pass has not run yet; replicate it for a fair compare
            let (mut mb_, mut mc_) = (best, best_c);
            for &st in &[2i32, 1] {
                for &(dx, dy) in &[(st, 0), (-st, 0), (0, st), (0, -st)] {
                    let c = (mb_.0 + dx, mb_.1 + dy);
                    let cc = cost(c);
                    if cc < mc_ {
                        mc_ = cc;
                        mb_ = c;
                    }
                }
            }
            use std::sync::atomic::Ordering::Relaxed;
            ME_PROBE[0].fetch_add(1, Relaxed);
            ME_PROBE[1].fetch_add(mc_.max(0) as u64, Relaxed);
            ME_PROBE[2].fetch_add(oc.max(0) as u64, Relaxed);
            ME_PROBE[3].fetch_add((mc_ > oc) as u64, Relaxed);
            ME_PROBE[5].fetch_add(oc_sp.max(0) as u64, Relaxed);
            ME_PROBE[6].fetch_add((mc_ > oc_sp) as u64, Relaxed);
        }
        let subpel: &[i32] = if (self.fast && !self.subpel_force) || (self.sp_defer.get() && !refine_only) {
            &[]
        } else {
            &[2, 1]
        };
        // U1 harvest: the null arm is the full-pel winner we would keep on a skip.
        let (hv_pre, mut hv_evals) = (best_c, 0u32);
        // `to_best` = eval index of the LAST improvement; `ring1` = the cost after the
        // first 8-point half-pel ring. Together they answer "how many of these 29
        // evaluations actually matter", which is the ceiling for any cheaper pattern.
        let (mut hv_to_best, mut hv_ring1) = (0u32, i64::MIN);
        let mut pat = subpel_pattern_override()
            .unwrap_or(if self.sp_single_pass { 2 } else { 0 });
        let (sp_learn, sp_t) = sp_dispatch_cfg();
        // Only dispatch when the caller has not pinned a pattern (pat 0 = default).
        let sp_dispatching = sp_learn > 0 && pat == 0 && !subpel.is_empty();
        if sp_dispatching && self.sp_learn_n.get() >= sp_learn && self.sp_1pass.get() {
            pat = 2;
        }
        // Descent D-2 MEMO. The ring walks around a MOVING centre, so iteration N+1's
        // ring necessarily re-contains the previous centre and several previous ring
        // points: 27-44% of sub-pel evaluations re-price an MV this refinement already
        // priced. `cost()` is PURE in `mv` (rate from mv-centre; distortion from the
        // fixed reference/source/block captures), so memoizing is EXACT -- identical
        // costs, identical comparisons, identical chosen MV, byte-identical output.
        // A miss simply recomputes, so the table's hit rate is a SPEED property only.
        //
        // 64-entry direct-mapped on the low bits of the MV, tagged with the full MV so
        // a collision is a miss rather than a wrong answer. Stack-resident (1 KiB) and
        // re-initialized per refinement: measured cheaper than a thread-local + RefCell
        // borrow on every evaluation, since ~60% of lookups miss.
        const SP_MEMO_N: usize = 64;
        #[inline(always)]
        fn sp_slot(mv: (i32, i32)) -> usize {
            ((mv.0 & 7) as usize) | (((mv.1 & 7) as usize) << 3)
        }
        let mut memo_mv = [(i32::MIN, i32::MIN); SP_MEMO_N];
        let mut memo_c = [0i64; SP_MEMO_N];
        if !subpel.is_empty() {
            let s0 = sp_slot(best);
            memo_mv[s0] = best;
            memo_c[s0] = best_c;
        }
        // Descent D-2 census: the ring walks around a MOVING centre, so iteration N+1's ring
        // necessarily re-contains the previous centre and several previous ring points.
        // Count how many sub-pel evaluations price an MV this refinement ALREADY priced
        // -- redundant recompute is byte-identically removable, unlike dropping work.
        #[cfg(feature = "profile")]
        let mut seen: Vec<(i32, i32)> = Vec::with_capacity(64);
        #[cfg(feature = "profile")]
        {
            seen.push(best);
        }
        // Track-B B3: the sub-pel iteration BUDGET. The ring walks until no
        // improvement; Descent D's census says iteration 1 carries 55% of evals at
        // an 11-13% hit rate, iteration 2 another 35-40% at 1.5-2.5%, and the tail
        // past that almost never pays — but under B2's SAD-chosen starts the tail
        // GROWS (+27% ns/search), eating the SAD savings. A cap bounds the walk the
        // way x264's fixed subme budget does. 0 (default) = unlimited =
        // byte-identical; bitstream-changing otherwise → BD-gated, opt-in.
        let sp_cap = sp_maxit();
        // ③: batched fixed-centre half-pel ring (see `sp_fc_enabled`).
        let sp_fc = sp_fc_enabled() && !self.fast && cfg!(accel)
            && matches!((rw, rh), (16, 16) | (16, 8) | (8, 16) | (8, 8));
        for &step in subpel {
            // Snapping starts this refine from an integer centre instead of the
            // seed's own fractional lattice, so a single 8-point pass can leave
            // precision behind. Walk it until it stops improving to compensate —
            // the snap is what pays for the extra probes.
            let ring8 = [
                (step, 0), (-step, 0), (0, step), (0, -step),
                (step, step), (-step, -step), (step, -step), (-step, step),
            ];
            let ring4 = [(step, 0), (-step, 0), (0, step), (0, -step)];
            let ring: &[(i32, i32)] = if pat & 1 != 0 { &ring4 } else { &ring8 };
            let mut _iter = 0u32;
            loop {
                // ③: from an INTEGER centre at step 2, all 8 ring candidates are
                // single-plane reads (h/h/v/v axes, c/c/c/c diagonals) — batch them
                // as two x4 kernel calls and take the argmin (first-wins in ring
                // order). Any decline (edge, half-pel centre, ring4 pattern) falls
                // through to the cascading walk for this pass.
                // ③b: the QUARTER step — every ±1 offset makes a component odd, so
                // all 8 candidates are two-plane average pairs regardless of the
                // centre's phase; two `satd_avg_x4` calls cover the ring.
                #[cfg(accel)]
                if sp_fc && step == 1 && pat & 1 == 0 {
                    _iter += 1;
                    let hp8 = hp.expect("sp_fc implies non-fast, which resolves hp");
                    let ring8 = [
                        (1, 0), (-1, 0), (0, 1), (0, -1),
                        (1, 1), (-1, -1), (1, -1), (-1, 1),
                    ];
                    let mut prs: [Option<(&[u8], usize, &[u8], usize, usize)>; 8] = [None; 8];
                    let mut all = true;
                    for (i, &(dx, dy)) in ring8.iter().enumerate() {
                        prs[i] = rusty_h264_common::inter::hpel_qpel_refs(
                            hp8, lx, ly, rw, rh, best.0 + dx, best.1 + dy,
                        );
                        all &= prs[i].is_some();
                    }
                    if all {
                        let stride = prs[0].unwrap().4;
                        // Batch kernel for 16-wide only (8-wide measured slower
                        // batched than the per-candidate fused path — H-8 gate);
                        // either way the SAME argmin over the SAME values.
                        let pack = |a: usize, b: usize, c2: usize, d: usize| {
                            if rw != 16 {
                                return None;
                            }
                            let g = |i: usize| {
                                let (pa, oa, pb, ob, _) = prs[i].unwrap();
                                (pa, oa, pb, ob)
                            };
                            rusty_h264_accel::satd_avg_x4(
                                src_row, cw, [g(a), g(b), g(c2), g(d)], stride, rw, rh,
                            )
                        };
                        {
                            let (ax, di) = (pack(0, 1, 2, 3), pack(4, 5, 6, 7));
                            let (mut bi, mut bc) = (usize::MAX, best_c);
                            for i in 0..8 {
                                let (dx, dy) = ring8[i];
                                let mv = (best.0 + dx, best.1 + dy);
                                let cc = match (i < 4, &ax, &di) {
                                    (true, Some(ax), _) => {
                                        let rate = mvbits(mv.0 - center.0) + mvbits(mv.1 - center.1);
                                        ax[i] as i64 + (lambda_me * rate as f64) as i64
                                    }
                                    (false, _, Some(di)) => {
                                        let rate = mvbits(mv.0 - center.0) + mvbits(mv.1 - center.1);
                                        di[i - 4] as i64 + (lambda_me * rate as f64) as i64
                                    }
                                    _ => cost(mv),
                                };
                                hv_evals += 1;
                                if cc < bc {
                                    bc = cc;
                                    bi = i;
                                }
                            }
                            if hv_ring1 == i64::MIN {
                                hv_ring1 = if bi == usize::MAX { best_c } else { bc };
                            }
                            if bi == usize::MAX
                                || !self.me_subpel_iter
                                || pat & 2 != 0
                                || (sp_cap != 0 && _iter >= sp_cap)
                            {
                                if bi != usize::MAX {
                                    best_c = bc;
                                    best = (best.0 + ring8[bi].0, best.1 + ring8[bi].1);
                                    hv_to_best = hv_evals;
                                }
                                break;
                            }
                            best_c = bc;
                            best = (best.0 + ring8[bi].0, best.1 + ring8[bi].1);
                            hv_to_best = hv_evals;
                            continue;
                        }
                    }
                    _iter -= 1;
                }
                #[cfg(accel)]
                if sp_fc && step == 2 && best.0 & 3 == 0 && best.1 & 3 == 0 && pat & 1 == 0 {
                    _iter += 1;
                    let hp8 = hp.expect("sp_fc implies non-fast, which resolves hp");
                    let ring8 = [
                        (step, 0), (-step, 0), (0, step), (0, -step),
                        (step, step), (-step, -step), (step, -step), (-step, step),
                    ];
                    let mut refs8: [Option<(&[u8], usize, usize)>; 8] = [None; 8];
                    let mut all = true;
                    for (i, &(dx, dy)) in ring8.iter().enumerate() {
                        refs8[i] = rusty_h264_common::inter::hpel_ref(
                            hp8, lx, ly, rw, rh, best.0 + dx, best.1 + dy,
                        );
                        all &= refs8[i].is_some();
                    }
                    if all {
                        let stride = refs8[0].unwrap().2;
                        // 16-wide batches; 8-wide evaluates per candidate (H-8 gate)
                        // — identical values, identical argmin, identical bitstream.
                        let pack = |a: usize, b: usize, c2: usize, d: usize| {
                            if rw != 16 {
                                return None;
                            }
                            let g = |i: usize| {
                                let (p, o, _) = refs8[i].unwrap();
                                (p, o)
                            };
                            rusty_h264_accel::satd_x4p(
                                src_row, cw, [g(a), g(b), g(c2), g(d)], stride, rw, rh,
                            )
                        };
                        {
                            let (ax, di) = (pack(0, 1, 2, 3), pack(4, 5, 6, 7));
                            let (mut bi, mut bc) = (usize::MAX, best_c);
                            for i in 0..8 {
                                let (dx, dy) = ring8[i];
                                let mv = (best.0 + dx, best.1 + dy);
                                let cc = match (i < 4, &ax, &di) {
                                    (true, Some(ax), _) => {
                                        let rate = mvbits(mv.0 - center.0) + mvbits(mv.1 - center.1);
                                        ax[i] as i64 + (lambda_me * rate as f64) as i64
                                    }
                                    (false, _, Some(di)) => {
                                        let rate = mvbits(mv.0 - center.0) + mvbits(mv.1 - center.1);
                                        di[i - 4] as i64 + (lambda_me * rate as f64) as i64
                                    }
                                    _ => cost(mv),
                                };
                                hv_evals += 1;
                                if cc < bc {
                                    bc = cc;
                                    bi = i;
                                }
                            }
                            if hv_ring1 == i64::MIN {
                                hv_ring1 = if bi == usize::MAX { best_c } else { bc };
                            }
                            if bi == usize::MAX
                                || !self.me_subpel_iter
                                || pat & 2 != 0
                                || (sp_cap != 0 && _iter >= sp_cap)
                            {
                                if bi != usize::MAX {
                                    best_c = bc;
                                    best = (best.0 + ring8[bi].0, best.1 + ring8[bi].1);
                                    hv_to_best = hv_evals;
                                }
                                break;
                            }
                            best_c = bc;
                            best = (best.0 + ring8[bi].0, best.1 + ring8[bi].1);
                            hv_to_best = hv_evals;
                            continue;
                        }
                    }
                    _iter -= 1; // declined — the cascade pass below re-counts it
                }
                let mut improved = false;
                _iter += 1;
                for (_pi, &(dx, dy)) in ring.iter().enumerate() {
                    let c = (best.0 + dx, best.1 + dy);
                    let slot = sp_slot(c);
                    let cc = if memo_mv[slot] == c {
                        memo_c[slot]
                    } else {
                        let v = cost(c);
                        memo_mv[slot] = c;
                        memo_c[slot] = v;
                        v
                    };
                    hv_evals += 1;
                    // Descent D: which ring POSITION and which ITERATION actually pay?
                    // Same census that showed the diamond's coarse rungs were noise,
                    // aimed at the stage that is now 41% of encode.
                    #[cfg(feature = "profile")]
                    {
                        spstats::ev(if step == 2 { 0 } else { 1 }, _pi, _iter);
                        if seen.contains(&c) {
                            spstats::redundant();
                        } else {
                            seen.push(c);
                        }
                    }
                    if cc < best_c {
                        best_c = cc;
                        best = c;
                        improved = true;
                        hv_to_best = hv_evals;
                        #[cfg(feature = "profile")]
                        spstats::imp(if step == 2 { 0 } else { 1 }, _pi, _iter);
                    }
                }
                if hv_ring1 == i64::MIN {
                    hv_ring1 = best_c;
                }
                if !improved
                    || !self.me_subpel_iter
                    || pat & 2 != 0
                    || (sp_cap != 0 && _iter >= sp_cap)
                {
                    break;
                }
            }
        }
        if sp_dispatching {
            let n = self.sp_learn_n.get();
            if n < sp_learn {
                self.sp_learn_n.set(n + 1);
                if hv_ring1 != i64::MIN {
                    self.sp_ring1.set(self.sp_ring1.get() + (hv_pre - hv_ring1).max(0));
                    self.sp_total.set(self.sp_total.get() + (hv_pre - best_c).max(0));
                }
                if n + 1 == sp_learn {
                    let tot = self.sp_total.get();
                    // Concentrated in ring 1 -> the later rings are affordable to drop.
                    self.sp_1pass.set(tot > 0 && self.sp_ring1.get() * 100 >= tot * sp_t);
                }
            }
        }
        if !subpel.is_empty() && subpel_harvest::enabled() {
            subpel_harvest::record(hv_pre, best_c, lambda_me, rw, rh, hv_evals, hv_to_best, hv_ring1);
        }
        // The snap moved the search off the seed; if the seed was better after all,
        // keep it. This is what makes the snap safe by construction.
        if self.me_snap && seed_c < best_c {
            best = seed_mv;
            best_c = seed_c;
        }
        (best, best_c)
    }

    /// Encodes macroblock `(mb_x, mb_y)` as an inter macroblock of the given
    /// `mode` (0 = P_L0_16x16, 1 = P_16x8, 2 = P_8x16) with one motion vector
    /// per partition: motion-compensate each partition, code the macroblock
    /// residual, and reconstruct.
    #[allow(clippy::too_many_arguments)]
    /// Dispatch to the current coded path (`_v1`) or the isolated fused path
    /// (`_v2`), selected by the hidden `coded_path_v2` A/B knob. Both produce
    /// byte-identical bitstreams (gated by the `coded_path_ab` test); the split
    /// exists so the two run side-by-side in one binary for honest timing.
    #[allow(clippy::too_many_arguments)]
    fn encode_inter_mb(
        &mut self,
        w: &mut BitWriter,
        refs: &[crate::RefFrame],
        sy: &[u8],
        su: &[u8],
        sv: &[u8],
        mb_x: usize,
        mb_y: usize,
        mode: u8,
        parts: &[(i32, (i32, i32))],
    ) {
        if self.coded_path_v2 {
            self.encode_inter_mb_v2(w, refs, sy, su, sv, mb_x, mb_y, mode, parts);
        } else {
            self.encode_inter_mb_v1(w, refs, sy, su, sv, mb_x, mb_y, mode, parts);
        }
    }

    /// Isolated, coefficient-fused inter coding path (A/B twin of `_v1`). The
    /// quantized luma levels stay in the hot 16-byte-aligned i16 DCT buffer for the
    /// whole MB; the i32 form is materialized on demand only for *coded* blocks
    /// (CAVLC scan + recon dequant), so uncoded quads never pay the conversion and
    /// there is no 256-word i32 `q_blocks` round-trip. Byte-identical to `_v1`
    /// (gated by `coded_path_ab`). Accel-only optimization; the scalar build reuses
    /// `_v1` unchanged.
    #[allow(clippy::too_many_arguments)]
    fn encode_inter_mb_v2(
        &mut self,
        w: &mut BitWriter,
        refs: &[crate::RefFrame],
        sy: &[u8],
        su: &[u8],
        sv: &[u8],
        mb_x: usize,
        mb_y: usize,
        mode: u8,
        parts: &[(i32, (i32, i32))],
    ) {
        // Descent E/F: identify this mc_luma population by call site.
        #[cfg(feature = "profile")]
        let _site = rusty_h264_common::inter::mcstats::SiteTag::new(1);
        #[cfg(not(accel))]
        {
            self.encode_inter_mb_v1(w, refs, sy, su, sv, mb_x, mb_y, mode, parts);
        }
        #[cfg(accel)]
        {
            let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncInterCode);
            let (qp, qpc) = (self.qp, self.qpc);
            let w4 = self.mb_w * 4;
            let (ch, cch) = (self.mb_h * 16, self.mb_h * 8);

            // ---- per-partition motion compensation + MV prediction (== v1) ----
            let mut pred_y = [0u8; 256];
            let mut c_pred = [[0u8; 64]; 2];
            let mut mvds = [(0i32, 0i32); 4];
            let mut n_mvd = 0;
            let _g_mc = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::PredBuf);
            for (part, &(rx, ry, rw, rh)) in inter_partitions(mode).iter().enumerate() {
                let (refi, mv) = parts[part];
                let reference = &refs[refi as usize];
                let (pbx, pby) = ((mb_x * 4 + rx / 4) as isize, (mb_y * 4 + ry / 4) as isize);
                let [a, b, c] = self.mv_neighbors_block(pbx, pby, (rw / 4) as isize);
                let pmv = predict_partition_mv(mode, part, a, b, c, refi);
                mvds[n_mvd] = (mv.0 - pmv.0, mv.1 - pmv.1);
                n_mvd += 1;
                for by in ry / 4..ry / 4 + rh / 4 {
                    for bx in rx / 4..rx / 4 + rw / 4 {
                        let idx = (mb_y * 4 + by) * w4 + (mb_x * 4 + bx);
                        self.mv_y[idx] = mv;
                        self.inter_y[idx] = true;
                        self.ref_idx_y[idx] = refi;
                        self.coded_y[idx] = true;
                    }
                }
                if rw == 16 && rh == 16 {
                    self.mc_luma_cached(reference, mb_x * 16, mb_y * 16, 16, 16, mv.0, mv.1, &mut pred_y);
                } else {
                    let mut tmp = [0u8; 256];
                    self.mc_luma_cached(reference, mb_x * 16 + rx, mb_y * 16 + ry, rw, rh, mv.0, mv.1, &mut tmp);
                    // H-17: the per-pixel re-stride was the runtime-width copy trap
                    // (a bounds-checked store per pixel); const-width row copies are
                    // byte-identical and lower to inline moves.
                    if rw == 8 {
                        for dy in 0..rh {
                            pred_y[(ry + dy) * 16 + rx..][..8].copy_from_slice(&tmp[dy * 8..][..8]);
                        }
                    } else {
                        for dy in 0..rh {
                            pred_y[(ry + dy) * 16 + rx..][..16].copy_from_slice(&tmp[dy * 16..][..16]);
                        }
                    }
                }
                let (crx, cry, crw, crh) = (rx / 2, ry / 2, rw / 2, rh / 2);
                for cc in 0..2 {
                    let rc = if cc == 0 { &reference.u } else { &reference.v };
                    if crw == 8 && crh == 8 {
                        mc_chroma(rc, self.ccw, cch, mb_x * 8, mb_y * 8, 8, 8, mv.0, mv.1, &mut c_pred[cc]);
                    } else {
                        let mut tc = [0u8; 64];
                        mc_chroma(rc, self.ccw, cch, mb_x * 8 + crx, mb_y * 8 + cry, crw, crh, mv.0, mv.1, &mut tc);
                        // H-17: same const-width row-copy fix as luma.
                        if crw == 4 {
                            for dy in 0..crh {
                                c_pred[cc][(cry + dy) * 8 + crx..][..4].copy_from_slice(&tc[dy * 4..][..4]);
                            }
                        } else {
                            for dy in 0..crh {
                                c_pred[cc][(cry + dy) * 8 + crx..][..8].copy_from_slice(&tc[dy * 8..][..8]);
                            }
                        }
                    }
                }
            }

            // ---- luma residual + quantization: keep levels in the i16 buffer ----
            let mut dctw = AlignedDct([0i16; 256]);
            let dct = &mut dctw.0;
            let mut cbp_luma = 0u32;
            drop(_g_mc);
            let _g_tq = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncTq);
            let base = mb_y * 16 * self.cw + mb_x * 16;
            for (qi, &(qx, qy)) in [(0usize, 0usize), (8, 0), (0, 8), (8, 8)].iter().enumerate() {
                rusty_h264_accel::dct_four_t4(
                    &mut dct[qi * 64..qi * 64 + 64],
                    &sy[base + qy * self.cw + qx..],
                    self.cw,
                    &pred_y[qy * 16 + qx..],
                    16,
                );
            }
            let ff = rusty_h264_common::transform::quant_dz_ff(qp, 6);
            let mf = &rusty_h264_common::transform::QUANT_MF_OH[qp as usize];
            for qi in 0..4 {
                rusty_h264_accel::quant_four_4x4(&mut dct[qi * 64..qi * 64 + 64], &ff, mf);
            }
            // cbp per quad straight from the i16 levels (no i32 q_blocks copy).
            for blk in 0..16 {
                if dct[blk * 16..blk * 16 + 16].iter().any(|&v| v != 0) {
                    cbp_luma |= 1 << (blk / 4);
                }
            }

            // ---- chroma residual (identical to v1: c_q stays i32) ----
            let mut c_dc_levels = [[0i32; 4]; 2];
            let mut c_recon_dc = [[0i32; 4]; 2];
            let mut c_q = [[[0i32; 16]; 4]; 2];
            let (mut any_ac, mut any_dc) = (false, false);
            for c in 0..2 {
                let src = if c == 0 { su } else { sv };
                let dc2x2 = {
                    #[repr(align(16))]
                    struct A([i16; 64]);
                    let mut cdct = A([0i16; 64]);
                    rusty_h264_accel::dct_four_t4(
                        &mut cdct.0,
                        &src[(mb_y * 8) * self.ccw + mb_x * 8..],
                        self.ccw,
                        &c_pred[c],
                        8,
                    );
                    let dc = [cdct.0[0] as i32, cdct.0[16] as i32, cdct.0[32] as i32, cdct.0[48] as i32];
                    let ffc = rusty_h264_common::transform::quant_dz_ff(qpc, 6);
                    let mfc = &rusty_h264_common::transform::QUANT_MF_OH[qpc as usize];
                    rusty_h264_accel::quant_four_4x4(&mut cdct.0, &ffc, mfc);
                    for i in 0..4 {
                        let q = &mut c_q[c][i];
                        q[0] = 0;
                        for j in 1..16 {
                            let v = cdct.0[i * 16 + j] as i32;
                            q[j] = v;
                            if v != 0 {
                                any_ac = true;
                            }
                        }
                    }
                    dc
                };
                let dl = forward_quant_chroma_dc(&dc2x2, qpc, false);
                if dl.iter().any(|&v| v != 0) {
                    any_dc = true;
                }
                c_recon_dc[c] = inverse_quant_chroma_dc(&dl, qpc);
                c_dc_levels[c] = dl;
            }
            let cbp_chroma: u32 = if any_ac { 2 } else if any_dc { 1 } else { 0 };
            let cbp = cbp_luma | (cbp_chroma << 4);

            // ---- emit syntax (== v1) ----
            drop(_g_tq);
            let _g_syn = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::Syntax);
            w.write_ue(mode as u32);
            let num_refs = refs.len();
            if num_refs > 1 {
                for &(refi, _) in parts {
                    write_ref_idx(w, refi, num_refs);
                }
            }
            for &(mvdx, mvdy) in &mvds[..n_mvd] {
                w.write_se(mvdx);
                w.write_se(mvdy);
            }
            write_cbp_inter(w, cbp);
            if cbp != 0 {
                w.write_se(self.qp_delta()); // mb_qp_delta (AQ per-MB QPy)
            }
            self.nnz_cache_load(mb_x, mb_y);
            drop(_g_syn);

            // ---- CAVLC: scan straight from the i16 levels for coded blocks ----
            let _g_scan = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::Scatter);
            for (blk, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
                let (bx, by) = (mb_x * 4 + lbx, mb_y * 4 + lby);
                let total = if cbp_luma & (1 << (blk / 4)) != 0 {
                    let nc = self.nc_pred(lbx, lby);
                    let scan16 = scan_4x4_dcac_i16(&dct[blk * 16..blk * 16 + 16]);
                    encode_residual_block(w, &scan16, 16, nc) as u8
                } else {
                    0
                };
                self.nnz_cache_set(lbx, lby, total);
                self.nnz_y[by * w4 + bx] = total;
            }
            if cbp_chroma != 0 {
                for c in 0..2 {
                    encode_residual_block(w, &c_dc_levels[c], 4, -1);
                }
            }
            if cbp_chroma == 2 {
                self.chroma_cache_load(mb_x, mb_y);
                let w2 = self.mb_w * 2;
                for c in 0..2 {
                    for &(bx, by) in &CHROMA_4X4_SCAN_XY {
                        let nc = self.chroma_nc_pred(c, bx, by);
                        let ac = scan_4x4_ac(&c_q[c][by * 2 + bx]);
                        let total = encode_residual_block(w, &ac, 15, nc) as u8;
                        self.chroma_nnz_cache_set(c, bx, by, total);
                        self.nnz_c[c][(mb_y * 2 + by) * w2 + (mb_x * 2 + bx)] = total;
                    }
                }
            }
            drop(_g_scan);

            // ---- reconstruction: dequantize luma straight from the i16 levels ----
            let _g_rec = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::SkipRecon);
            #[repr(align(16))]
            struct Align16([i16; 64]);
            let mut dct_in = Align16([0i16; 64]);
            for (qi, &(qx, qy)) in [(0usize, 0usize), (8, 0), (0, 8), (8, 8)].iter().enumerate() {
                let rec_off = base + qy * self.cw + qx;
                if cbp_luma & (1 << qi) == 0 {
                    for r in 0..8 {
                        let (dsti, srci) = (rec_off + r * self.cw, (qy + r) * 16 + qx);
                        self.rec_y[dsti..dsti + 8].copy_from_slice(&pred_y[srci..srci + 8]);
                    }
                    continue;
                }
                for k in 0..4 {
                    let blk = qi * 4 + k;
                    let mut lvl = [0i32; 16];
                    for i in 0..16 {
                        lvl[i] = dct[blk * 16 + i] as i32;
                    }
                    let deq = dequantize(&lvl, qp);
                    for i in 0..16 {
                        dct_in.0[k * 16 + i] = deq[i] as i16;
                    }
                }
                rusty_h264_accel::idct_four_t4_rec(
                    &mut self.rec_y[rec_off..],
                    self.cw,
                    &pred_y[qy * 16 + qx..],
                    16,
                    &dct_in.0,
                );
            }
            // chroma recon (identical to v1)
            for c in 0..2 {
                let base_c = (mb_y * 8) * self.ccw + mb_x * 8;
                let plane = if c == 0 { &mut self.rec_u } else { &mut self.rec_v };
                if cbp_chroma == 0 {
                    for r in 0..8 {
                        let dsti = base_c + r * self.ccw;
                        plane[dsti..dsti + 8].copy_from_slice(&c_pred[c][r * 8..r * 8 + 8]);
                    }
                } else {
                    #[repr(align(16))]
                    struct A([i16; 64]);
                    let mut d = A([0i16; 64]);
                    for i in 0..4 {
                        let deq = dequantize(&c_q[c][i], qpc);
                        for j in 0..16 {
                            d.0[i * 16 + j] = deq[j] as i16;
                        }
                        d.0[i * 16] = c_recon_dc[c][i] as i16;
                    }
                    rusty_h264_accel::idct_four_t4_rec(&mut plane[base_c..], self.ccw, &c_pred[c], 8, &d.0);
                }
            }
            for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
                self.modes_y[(mb_y * 4 + lby) * w4 + (mb_x * 4 + lbx)] = 2;
            }
        }
    }

    #[allow(clippy::too_many_arguments)]
    fn encode_inter_mb_v1(
        &mut self,
        w: &mut BitWriter,
        refs: &[crate::RefFrame],
        sy: &[u8],
        su: &[u8],
        sv: &[u8],
        mb_x: usize,
        mb_y: usize,
        mode: u8,
        parts: &[(i32, (i32, i32))],
    ) {
        self.encode_inter_mb_v1_b(w, refs, sy, su, sv, mb_x, mb_y, mode, parts, None);
    }

    /// As [`Self::encode_inter_mb_v1`], but `b_mode` selects B-slice framing: the
    /// macroblock is coded as `B_L0_16x16` (`mb_type == 1`) instead of the P-slice
    /// `mb_type == mode`. Everything else — the single List-0 partition, the median
    /// `mvd_l0` predictor, the residual, and the reconstruction — is byte-identical
    /// to `P_L0_16x16`, so the caller passes `mode == 0`, `refs == &[L0_anchor]`
    /// (length 1 ⇒ no `ref_idx` coded), and `parts == &[(0, mv)]`.
    /// Decide + reconstruct one inter macroblock (motion compensation, residual,
    /// quantize, reconstruct, commit motion grids) — everything except entropy
    /// coding. Returns an [`InterPlan`] coded by either backend, so CAVLC and CABAC
    /// share this whole path bit-for-bit (the P/B analogue of [`plan_mb`]).
    #[allow(clippy::too_many_arguments)]
    fn plan_inter_mb(
        &mut self,
        refs: &[crate::RefFrame],
        sy: &[u8],
        su: &[u8],
        sv: &[u8],
        mb_x: usize,
        mb_y: usize,
        mode: u8,
        parts: &[(i32, (i32, i32))],
        bspec: Option<BInter>,
    ) -> InterPlan {
        // Descent E/F: identify this mc_luma population by call site.
        #[cfg(feature = "profile")]
        let _site = rusty_h264_common::inter::mcstats::SiteTag::new(1);
        let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncInterCode);
        let (qp, qpc) = (self.qp, self.qpc);
        let w4 = self.mb_w * 4;
        let (ch, cch) = (self.mb_h * 16, self.mb_h * 8);

        // ---- per-partition motion compensation + MV prediction ----
        let mut pred_y = [0u8; 256];
        let mut c_pred = [[0u8; 64]; 2];
        let mut mvds = [(0i32, 0i32); 4]; // ≤4 partitions; no per-MB Vec alloc
        let mut plan_refs = [0i32; 4]; // per-partition ref_idx_l0 (0 for B / 1-ref)
        let mut n_mvd = 0;
        let _g_mc = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::PredBuf);
        if let Some(b) = bspec.filter(|b| b.dir == 0) {
            // ---- B_Direct_16x16 (mb_type 0): spatial-direct prediction, no mvd ----
            let (dp, dc, motion) = self.b_direct(&refs[0], b.l1, mb_x, mb_y);
            pred_y = dp;
            c_pred = dc;
            self.commit_direct_motion(mb_x, mb_y, &motion);
        } else if let Some(b) = bspec {
            // ---- B 16×16 prediction: List-0 / List-1 / Bi ----
            let use0 = b.dir == 1 || b.dir == 3;
            let use1 = b.dir == 2 || b.dir == 3;
            let (lx, ly) = (mb_x * 16, mb_y * 16);
            let (cx, cy) = (mb_x * 8, mb_y * 8);
            let (pbx, pby) = ((mb_x * 4) as isize, (mb_y * 4) as isize);
            // Per-list `mvd` against the median predictor over that list's neighbors.
            if use0 {
                let [a, c0, c1] = self.mv_neighbors_block_list(pbx, pby, 4, 0);
                let p = predict_partition_mv(0, 0, a, c0, c1, 0);
                mvds[n_mvd] = (b.mv0.0 - p.0, b.mv0.1 - p.1);
                n_mvd += 1;
            }
            if use1 {
                let [a, c0, c1] = self.mv_neighbors_block_list(pbx, pby, 4, 1);
                let p = predict_partition_mv(0, 0, a, c0, c1, 0);
                mvds[n_mvd] = (b.mv1.0 - p.0, b.mv1.1 - p.1);
                n_mvd += 1;
            }
            // Motion compensation. L0/L1 write straight into pred; Bi averages
            // (p+q+1)>>1 — the decoder's `b_mc` blend with weighted_bipred_idc=0.
            let mut a_y = [0u8; 256];
            let mut b_y = [0u8; 256];
            let mut a_c = [[0u8; 64]; 2];
            let mut b_c = [[0u8; 64]; 2];
            if use0 {
                mc_luma(&refs[0].y, self.cw, ch, lx, ly, 16, 16, b.mv0.0, b.mv0.1, &mut a_y);
                mc_chroma(&refs[0].u, self.ccw, cch, cx, cy, 8, 8, b.mv0.0, b.mv0.1, &mut a_c[0]);
                mc_chroma(&refs[0].v, self.ccw, cch, cx, cy, 8, 8, b.mv0.0, b.mv0.1, &mut a_c[1]);
            }
            if use1 {
                mc_luma(&b.l1.y, self.cw, ch, lx, ly, 16, 16, b.mv1.0, b.mv1.1, &mut b_y);
                mc_chroma(&b.l1.u, self.ccw, cch, cx, cy, 8, 8, b.mv1.0, b.mv1.1, &mut b_c[0]);
                mc_chroma(&b.l1.v, self.ccw, cch, cx, cy, 8, 8, b.mv1.0, b.mv1.1, &mut b_c[1]);
            }
            match (use0, use1) {
                (true, true) => {
                    for i in 0..256 {
                        pred_y[i] = bi_blend(a_y[i] as i32, b_y[i] as i32, self.bi_w);
                    }
                    for c in 0..2 {
                        for i in 0..64 {
                            c_pred[c][i] = bi_blend(a_c[c][i] as i32, b_c[c][i] as i32, self.bi_w);
                        }
                    }
                }
                (true, false) => {
                    pred_y = a_y;
                    c_pred = a_c;
                }
                _ => {
                    pred_y = b_y;
                    c_pred = b_c;
                }
            }
            // Commit per-list motion so later MBs' per-list predictors see it.
            for by in 0..4 {
                for bx in 0..4 {
                    let idx = (mb_y * 4 + by) * w4 + (mb_x * 4 + bx);
                    self.inter_y[idx] = true;
                    self.coded_y[idx] = true;
                    self.mv_y[idx] = if use0 { b.mv0 } else { (0, 0) };
                    self.ref_idx_y[idx] = if use0 { 0 } else { -1 };
                    self.mv1_y[idx] = if use1 { b.mv1 } else { (0, 0) };
                    self.ref_idx1_y[idx] = if use1 { 0 } else { -1 };
                }
            }
        } else {
        for (part, &(rx, ry, rw, rh)) in inter_partitions(mode).iter().enumerate() {
            let (refi, mv) = parts[part];
            plan_refs[part] = refi; // per-partition ref_idx_l0 → carried to the CABAC emit
            let reference = &refs[refi as usize];
            let (pbx, pby) = ((mb_x * 4 + rx / 4) as isize, (mb_y * 4 + ry / 4) as isize);
            let [a, b, c] = self.mv_neighbors_block(pbx, pby, (rw / 4) as isize);
            let pmv = predict_partition_mv(mode, part, a, b, c, refi);
            mvds[n_mvd] = (mv.0 - pmv.0, mv.1 - pmv.1);
            n_mvd += 1;
            // Commit this partition's motion so later partitions can predict from it.
            for by in ry / 4..ry / 4 + rh / 4 {
                for bx in rx / 4..rx / 4 + rw / 4 {
                    let idx = (mb_y * 4 + by) * w4 + (mb_x * 4 + bx);
                    self.mv_y[idx] = mv;
                    self.inter_y[idx] = true;
                    self.ref_idx_y[idx] = refi;
                    self.coded_y[idx] = true;
                }
            }
            // Luma MC into the partition's sub-region. A full-MB (16×16) partition is
            // the whole `pred_y`, so MC straight into it — no scratch + repack copy.
            if rw == 16 && rh == 16 {
                self.mc_luma_cached(reference, mb_x * 16, mb_y * 16, 16, 16, mv.0, mv.1, &mut pred_y);
            } else {
                let mut tmp = [0u8; 256];
                self.mc_luma_cached(reference, mb_x * 16 + rx, mb_y * 16 + ry, rw, rh, mv.0, mv.1, &mut tmp);
                // H-17: const-width row copies (see the v2 twin).
                if rw == 8 {
                    for dy in 0..rh {
                        pred_y[(ry + dy) * 16 + rx..][..8].copy_from_slice(&tmp[dy * 8..][..8]);
                    }
                } else {
                    for dy in 0..rh {
                        pred_y[(ry + dy) * 16 + rx..][..16].copy_from_slice(&tmp[dy * 16..][..16]);
                    }
                }
            }
            // Chroma MC (half-resolution region); 8×8 = the whole plane prediction.
            let (crx, cry, crw, crh) = (rx / 2, ry / 2, rw / 2, rh / 2);
            for cc in 0..2 {
                let rc = if cc == 0 { &reference.u } else { &reference.v };
                if crw == 8 && crh == 8 {
                    mc_chroma(rc, self.ccw, cch, mb_x * 8, mb_y * 8, 8, 8, mv.0, mv.1, &mut c_pred[cc]);
                } else {
                    let mut tc = [0u8; 64];
                    mc_chroma(rc, self.ccw, cch, mb_x * 8 + crx, mb_y * 8 + cry, crw, crh, mv.0, mv.1, &mut tc);
                    // H-17: const-width row copies (see the v2 twin).
                    if crw == 4 {
                        for dy in 0..crh {
                            c_pred[cc][(cry + dy) * 8 + crx..][..4].copy_from_slice(&tc[dy * 4..][..4]);
                        }
                    } else {
                        for dy in 0..crh {
                            c_pred[cc][(cry + dy) * 8 + crx..][..8].copy_from_slice(&tc[dy * 8..][..8]);
                        }
                    }
                }
            }
        }
        } // end P per-partition formation (else of the B branch)

        // ---- luma residual + quantization ----
        let mut q_blocks = [[0i32; 16]; 16]; // raster, levels
        let mut cbp_luma = 0u32;
        // Inter 8x8-transform candidate (High profile, scalar path). Filled by the
        // per-MB 4x4-vs-8x8 RD below; false/zero means the 4x4 residual is used.
        #[allow(unused_mut)]
        let mut t8x8 = false;
        #[allow(unused_mut)]
        let mut q8 = [[0i32; 64]; 4];
        drop(_g_mc);
        let _g_tq = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncTq);
        #[cfg(accel)]
        {
            // openh264 `WelsDctFourT4_sse2` (fused residual+DCT) → i16, then
            // `WelsQuantFour4x4_sse2` in place — the whole DCT→quant chain stays in i16,
            // no i32 round-trip. Quant is openh264's structure carrying OUR deadzone
            // (`quant_dz_ff` + `QUANT_MF_OH`), so levels are bit-identical to `quantize`.
            let mut dctw = AlignedDct([0i16; 256]);
            let dct = &mut dctw.0;
            let base = mb_y * 16 * self.cw + mb_x * 16;
            for (qi, &(qx, qy)) in [(0usize, 0usize), (8, 0), (0, 8), (8, 8)].iter().enumerate() {
                rusty_h264_accel::dct_four_t4(
                    &mut dct[qi * 64..qi * 64 + 64],
                    &sy[base + qy * self.cw + qx..],
                    self.cw,
                    &pred_y[qy * 16 + qx..],
                    16,
                );
            }
            let ff = rusty_h264_common::transform::quant_dz_ff(qp, 6);
            let mf = &rusty_h264_common::transform::QUANT_MF_OH[qp as usize];
            for qi in 0..4 {
                rusty_h264_accel::quant_four_4x4(&mut dct[qi * 64..qi * 64 + 64], &ff, mf);
            }
            for (blk, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
                let mut nz = false;
                for i in 0..16 {
                    let v = dct[blk * 16 + i] as i32;
                    q_blocks[lby * 4 + lbx][i] = v;
                    nz |= v != 0;
                }
                if nz {
                    cbp_luma |= 1 << (blk / 4);
                }
            }
        }
        #[cfg(not(accel))]
        {
            // Scalar/`wide`: gather all 16 residual blocks, batched forward-DCT, quantize.
            let mut res_blocks = [[0i32; 16]; 16]; // raster
            for lby in 0..4 {
                for lbx in 0..4 {
                    let b = &mut res_blocks[lby * 4 + lbx];
                    for dy in 0..4 {
                        for dx in 0..4 {
                            let sx = mb_x * 16 + lbx * 4 + dx;
                            let syy = mb_y * 16 + lby * 4 + dy;
                            b[dy * 4 + dx] = sy[syy * self.cw + sx] as i32
                                - pred_y[(lby * 4 + dy) * 16 + (lbx * 4 + dx)] as i32;
                        }
                    }
                }
            }
            let mut coeffs = [[0i32; 16]; 16];
            forward_dct_blocks(&res_blocks, &mut coeffs);
            for (blk, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
                let q = rdoq(&coeffs[lby * 4 + lbx], qp, 6, self.rdoq_strength, 0);
                if q.iter().any(|&v| v != 0) {
                    cbp_luma |= 1 << (blk / 4);
                }
                q_blocks[lby * 4 + lbx] = q;
            }
        }

        // Per-MB transform-size RD (runs in scalar AND accel builds — q_blocks +
        // cbp_luma are filled by whichever quant path ran; the 8x8 candidate + its
        // recon are pure Rust). One 8x8 DCT per 8x8 block vs four 4x4s. Every inter
        // partition here is >= 8x8, so transform_size_8x8_flag is always allowed.
        // Content-adaptive by construction — the winner is chosen per MB.
        {
            if self.transform_8x8 && self.inter8x8 != 0 {
                let lambda =
                    0.85 * self.tune_lambda_scale * 2f64.powf((qp as f64 - 12.0) / 3.0);
                let mut ssd4 = 0i64;
                let mut rate4 = 0f64;
                for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
                    let mut predb = [0i32; 16];
                    for dy in 0..4 {
                        for dx in 0..4 {
                            predb[dy * 4 + dx] =
                                pred_y[(lby * 4 + dy) * 16 + (lbx * 4 + dx)] as i32;
                        }
                    }
                    let deq = dequantize(&q_blocks[lby * 4 + lbx], qp);
                    let s = reconstruct_4x4(&deq, &predb);
                    for dy in 0..4 {
                        for dx in 0..4 {
                            let sx = mb_x * 16 + lbx * 4 + dx;
                            let syy = mb_y * 16 + lby * 4 + dy;
                            let d = s[dy * 4 + dx] as i64 - sy[syy * self.cw + sx] as i64;
                            ssd4 += d * d;
                        }
                    }
                    for &l in &q_blocks[lby * 4 + lbx] {
                        if l != 0 {
                            rate4 += rdoq_rate((l as i64).abs());
                        }
                    }
                }
                let (q8c, cbp8, rate8, _rec8, ssd8) =
                    plan_inter8_luma(sy, self.cw, mb_x, mb_y, &pred_y, qp);
                // Both candidates priced with the SAME level-aware rate (Σ rdoq_rate);
                // `inter8_pen` is an optional extra bias (default 0) on the 8x8 flag.
                let j4 = ssd4 as f64 + lambda * (rate4 + 16.0);
                let j8 = ssd8 as f64 + lambda * (rate8 + 16.0 + self.inter8_pen as f64);
                if cbp8 > 0 && j8 < j4 {
                    t8x8 = true;
                    cbp_luma = cbp8;
                    q8 = q8c;
                }
            }
        }

        // ---- chroma residual (prediction already built per partition) ----
        let mut c_dc_levels = [[0i32; 4]; 2];
        let mut c_recon_dc = [[0i32; 4]; 2];
        let mut c_q = [[[0i32; 16]; 4]; 2];
        let (mut any_ac, mut any_dc) = (false, false);
        for c in 0..2 {
            let src = if c == 0 { su } else { sv };
            // Fast path: one dct_four_t4 covers the whole 8x8 chroma region (all 4
            // blocks, residual+DCT fused straight from the planes); block b's pre-quant
            // DC is dct[b*16] (quad z-scan == 2x2 raster); quant_four_4x4 with our
            // FF/MF is bit-identical to scalar `quantize`. Same pairing the P_Skip
            // free-check proved byte-identical over the corpus.
            #[cfg(accel)]
            let (mut dc2x2, applied) = {
                #[repr(align(16))]
                struct A([i16; 64]);
                let mut dct = A([0i16; 64]);
                rusty_h264_accel::dct_four_t4(
                    &mut dct.0,
                    &src[(mb_y * 8) * self.ccw + mb_x * 8..],
                    self.ccw,
                    &c_pred[c],
                    8,
                );
                let dc = [
                    dct.0[0] as i32,
                    dct.0[16] as i32,
                    dct.0[32] as i32,
                    dct.0[48] as i32,
                ];
                let ffc = rusty_h264_common::transform::quant_dz_ff(qpc, 6);
                let mfc = &rusty_h264_common::transform::QUANT_MF_OH[qpc as usize];
                rusty_h264_accel::quant_four_4x4(&mut dct.0, &ffc, mfc);
                for i in 0..4 {
                    let q = &mut c_q[c][i];
                    q[0] = 0;
                    for j in 1..16 {
                        let v = dct.0[i * 16 + j] as i32;
                        q[j] = v;
                        if v != 0 {
                            any_ac = true;
                        }
                    }
                }
                (dc, true)
            };
            #[cfg(not(accel))]
            let (mut dc2x2, applied) = ([0i32; 4], false);
            if !applied {
                // Scalar/`wide` twin: gather, batch forward DCT, quantize per block.
                let mut res_blocks = [[0i32; 16]; 4];
                for by in 0..2 {
                    for bx in 0..2 {
                        let b = &mut res_blocks[by * 2 + bx];
                        for dy in 0..4 {
                            for dx in 0..4 {
                                let sx = mb_x * 8 + bx * 4 + dx;
                                let syy = mb_y * 8 + by * 4 + dy;
                                b[dy * 4 + dx] = src[syy * self.ccw + sx] as i32
                                    - c_pred[c][(by * 4 + dy) * 8 + (bx * 4 + dx)] as i32;
                            }
                        }
                    }
                }
                let mut coeffs = [[0i32; 16]; 4];
                forward_dct_blocks(&res_blocks, &mut coeffs);
                for i in 0..4 {
                    dc2x2[i] = coeffs[i][0];
                    let mut q = rdoq(&coeffs[i], qpc, 6, self.rdoq_strength, 1);
                    q[0] = 0;
                    if q[1..].iter().any(|&v| v != 0) {
                        any_ac = true;
                    }
                    c_q[c][i] = q;
                }
            }
            let dl = forward_quant_chroma_dc(&dc2x2, qpc, false);
            if dl.iter().any(|&v| v != 0) {
                any_dc = true;
            }
            c_recon_dc[c] = inverse_quant_chroma_dc(&dl, qpc);
            c_dc_levels[c] = dl;
        }
        let cbp_chroma: u32 = if any_ac { 2 } else if any_dc { 1 } else { 0 };
        let cbp = cbp_luma | (cbp_chroma << 4);

        drop(_g_tq);
        let _g_rec = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::SkipRecon);
        // ---- reconstruction (luma) ----
        #[cfg(accel)]
        if t8x8 {
            // 8x8-transform recon is pure Rust (no asm 8x8 kernels yet); inverse of
            // the decoder's t8x8 inter path. Same code as the scalar branch below.
            let weight = [16i32; 64];
            for b8 in 0..4usize {
                let (b8x, b8y) = (b8 % 2, b8 / 2);
                let res_r = inverse_quant_8x8(&q8[b8], qp, &weight);
                let predb: [i32; 64] = std::array::from_fn(|i| {
                    pred_y[(b8y * 8 + i / 8) * 16 + (b8x * 8 + i % 8)] as i32
                });
                let recon = add_residual_8x8(&res_r, &predb);
                for dy in 0..8 {
                    for dx in 0..8 {
                        let px = mb_x * 16 + b8x * 8 + dx;
                        let py = mb_y * 16 + b8y * 8 + dy;
                        self.rec_y[py * self.cw + px] = recon[dy * 8 + dx];
                    }
                }
            }
        } else {
            // Dequantize all 16 blocks into the 4-quadrant int16 layout (16-byte
            // aligned — the kernel uses movdqa coeff loads), then inverse-DCT + add
            // prediction + clip per quadrant via openh264. The inverse butterfly +
            // (x+32)>>6 is bit-identical to reconstruct_4x4 (verified in accel).
            // An 8x8 quad whose cbp bit is clear has ZERO residual: reconstruction
            // IS the prediction (the decoder's own uncoded-region fast path) — a row
            // copy replaces dequant + convert + idct for that quad. Byte-identical:
            // idct of an all-zero block adds (0+32)>>6 = 0 to pred, clip is identity.
            #[repr(align(16))]
            struct Align16([i16; 64]);
            let mut dct_in = Align16([0i16; 64]);
            let base = mb_y * 16 * self.cw + mb_x * 16;
            for (qi, &(qx, qy)) in [(0usize, 0usize), (8, 0), (0, 8), (8, 8)].iter().enumerate() {
                let rec_off = base + qy * self.cw + qx;
                if cbp_luma & (1 << qi) == 0 {
                    for r in 0..8 {
                        let (dsti, srci) = (rec_off + r * self.cw, (qy + r) * 16 + qx);
                        self.rec_y[dsti..dsti + 8].copy_from_slice(&pred_y[srci..srci + 8]);
                    }
                    continue;
                }
                for k in 0..4 {
                    let blk = qi * 4 + k;
                    let (lbx, lby) = LUMA_4X4_SCAN_XY[blk];
                    let deq = dequantize(&q_blocks[lby * 4 + lbx], qp);
                    for i in 0..16 {
                        dct_in.0[k * 16 + i] = deq[i] as i16;
                    }
                }
                rusty_h264_accel::idct_four_t4_rec(
                    &mut self.rec_y[rec_off..],
                    self.cw,
                    &pred_y[qy * 16 + qx..],
                    16,
                    &dct_in.0,
                );
            }
        }
        #[cfg(not(accel))]
        if t8x8 {
            // 8x8-transform reconstruction (inverse of the decoder's t8x8 inter path).
            let weight = [16i32; 64];
            for b8 in 0..4usize {
                let (b8x, b8y) = (b8 % 2, b8 / 2);
                let res_r = inverse_quant_8x8(&q8[b8], qp, &weight);
                let predb: [i32; 64] = std::array::from_fn(|i| {
                    pred_y[(b8y * 8 + i / 8) * 16 + (b8x * 8 + i % 8)] as i32
                });
                let recon = add_residual_8x8(&res_r, &predb);
                for dy in 0..8 {
                    for dx in 0..8 {
                        let px = mb_x * 16 + b8x * 8 + dx;
                        let py = mb_y * 16 + b8y * 8 + dy;
                        self.rec_y[py * self.cw + px] = recon[dy * 8 + dx];
                    }
                }
            }
        } else {
            for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
                let mut predb = [0i32; 16];
                for dy in 0..4 {
                    for dx in 0..4 {
                        predb[dy * 4 + dx] = pred_y[(lby * 4 + dy) * 16 + (lbx * 4 + dx)] as i32;
                    }
                }
                let deq = dequantize(&q_blocks[lby * 4 + lbx], qp);
                let s = reconstruct_4x4(&deq, &predb);
                store(&mut self.rec_y, self.cw, mb_x * 16 + lbx * 4, mb_y * 16 + lby * 4, &s);
            }
        }
        for c in 0..2 {
            // Fast path: dequantize into the quad i16 layout (raster == the kernel's
            // z-order for a 2x2) with the Hadamard DC injected, then ONE
            // idct+add-pred+clip kernel writes the 8x8 straight into the plane —
            // bit-identical to the scalar tail below (verified kernel pairing).
            #[cfg(accel)]
            {
                let base = (mb_y * 8) * self.ccw + mb_x * 8;
                let plane = if c == 0 { &mut self.rec_u } else { &mut self.rec_v };
                if cbp_chroma == 0 {
                    // No chroma residual at all: recon = prediction (row copies).
                    for r in 0..8 {
                        let dsti = base + r * self.ccw;
                        plane[dsti..dsti + 8].copy_from_slice(&c_pred[c][r * 8..r * 8 + 8]);
                    }
                } else {
                    #[repr(align(16))]
                    struct A([i16; 64]);
                    let mut d = A([0i16; 64]);
                    for i in 0..4 {
                        let deq = dequantize(&c_q[c][i], qpc);
                        for j in 0..16 {
                            d.0[i * 16 + j] = deq[j] as i16;
                        }
                        d.0[i * 16] = c_recon_dc[c][i] as i16;
                    }
                    rusty_h264_accel::idct_four_t4_rec(&mut plane[base..], self.ccw, &c_pred[c], 8, &d.0);
                }
            }
            #[cfg(not(accel))]
            {
                // Dequantize the 4 blocks (raster, DC overridden by the 2×2-Hadamard
                // recon), then batch the inverse DCT and share the add+clip tail.
                let mut deq_blocks = [[0i32; 16]; 4];
                for i in 0..4 {
                    deq_blocks[i] = dequantize(&c_q[c][i], qpc);
                    deq_blocks[i][0] = c_recon_dc[c][i];
                }
                let mut res = [[0i32; 16]; 4];
                inverse_dct_blocks(&deq_blocks, &mut res);
                let plane = if c == 0 { &mut self.rec_u } else { &mut self.rec_v };
                for by in 0..2 {
                    for bx in 0..2 {
                        let mut predb = [0i32; 16];
                        for dy in 0..4 {
                            for dx in 0..4 {
                                predb[dy * 4 + dx] = c_pred[c][(by * 4 + dy) * 8 + (bx * 4 + dx)] as i32;
                            }
                        }
                        let s = add_residual_4x4(&res[by * 2 + bx], &predb);
                        store(plane, self.ccw, mb_x * 8 + bx * 4, mb_y * 8 + by * 4, &s);
                    }
                }
            }
        }
        // MV grid + coded flags were set per partition; mark modes as DC.
        for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
            self.modes_y[(mb_y * 4 + lby) * w4 + (mb_x * 4 + lbx)] = 2;
        }
        InterPlan { mvds, plan_refs, n_mvd, cbp, q_blocks, c_dc_levels, c_q, t8x8, q8 }
    }

    /// Code one planned inter macroblock as CAVLC (the original `encode_inter_mb_v1_b`
    /// tail). `plan_inter_mb` already committed the reconstruction + motion grids.
    #[allow(clippy::too_many_arguments)]
    fn encode_inter_mb_v1_b(
        &mut self,
        w: &mut BitWriter,
        refs: &[crate::RefFrame],
        sy: &[u8],
        su: &[u8],
        sv: &[u8],
        mb_x: usize,
        mb_y: usize,
        mode: u8,
        parts: &[(i32, (i32, i32))],
        bspec: Option<BInter>,
    ) {
        let plan = self.plan_inter_mb(refs, sy, su, sv, mb_x, mb_y, mode, parts, bspec);
        let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncEmit);
        self.emit_inter_cavlc(w, refs.len(), mb_x, mb_y, mode, parts, bspec, &plan);
    }

    /// CAVLC entropy coding for a planned inter macroblock.
    #[allow(clippy::too_many_arguments)]
    fn emit_inter_cavlc(
        &mut self,
        w: &mut BitWriter,
        num_refs: usize,
        mb_x: usize,
        mb_y: usize,
        mode: u8,
        parts: &[(i32, (i32, i32))],
        bspec: Option<BInter>,
        plan: &InterPlan,
    ) {
        let w4 = self.mb_w * 4;
        let (cbp, cbp_luma, cbp_chroma) = (plan.cbp, plan.cbp & 15, plan.cbp >> 4);
        // mb_pred order (spec 7.3.5.1): mb_type, then all ref_idx_l0, then all mvd_l0.
        // B-slice mb_type = the B direction 1/2/3; P-slice uses `mode`. ref_idx coded
        // only when >1 reference is active.
        w.write_ue(bspec.map_or(mode as u32, |b| b.dir as u32)); // inter mb_type
        // P_8x8 (mb_type 3): sub_mb_type per 8×8 (spec 7.3.5.2, before ref_idx/mvd).
        // 0 = P_L0_8x8 (one MV) — the only shape emitted for now.
        if mode == 3 {
            for _ in 0..4 {
                w.write_ue(0);
            }
        }
        if num_refs > 1 {
            for &(refi, _) in parts {
                write_ref_idx(w, refi, num_refs);
            }
        }
        for &(mvdx, mvdy) in &plan.mvds[..plan.n_mvd] {
            w.write_se(mvdx);
            w.write_se(mvdy);
        }
        write_cbp_inter(w, cbp);
        // transform_size_8x8_flag: after cbp, before mb_qp_delta, present only when
        // luma has coefficients and the 8x8 transform is enabled. Every inter partition
        // here is >= 8x8, so the spec's allow_8x8 (all partitions >= 8x8) always holds.
        if cbp_luma > 0 && self.transform_8x8 {
            w.write_bit(plan.t8x8);
        }
        if cbp != 0 {
            w.write_se(self.qp_delta()); // mb_qp_delta (AQ per-MB QPy)
        }
        self.nnz_cache_load(mb_x, mb_y);
        if plan.t8x8 {
            // 8x8 residual: four interleaved 4x4 CAVLC sub-blocks per 8x8 block
            // (coeff k of sub s -> 8x8 scan position 4k+s), the inverse of the
            // decoder's t8x8 inter luma read. nnz set per 4x4 sub-block.
            for b8 in 0..4usize {
                let (b8x, b8y) = (b8 % 2, b8 / 2);
                let scan8 = scan_8x8_fwd(&plan.q8[b8]);
                for sub in 0..4usize {
                    let (cx, cy) = (b8x * 2 + sub % 2, b8y * 2 + sub / 2);
                    let (bx, by) = (mb_x * 4 + cx, mb_y * 4 + cy);
                    let total = if cbp_luma & (1 << b8) != 0 {
                        let nc = self.nc_pred(cx, cy);
                        let blk: [i32; 16] = std::array::from_fn(|k| scan8[4 * k + sub]);
                        encode_residual_block(w, &blk, 16, nc) as u8
                    } else {
                        0
                    };
                    self.nnz_cache_set(cx, cy, total);
                    self.nnz_y[by * w4 + bx] = total;
                }
            }
        } else {
            for (blk, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
                let (bx, by) = (mb_x * 4 + lbx, mb_y * 4 + lby);
                let total = if cbp_luma & (1 << (blk / 4)) != 0 {
                    let nc = self.nc_pred(lbx, lby);
                    let scan16 = scan_4x4_dcac(&plan.q_blocks[lby * 4 + lbx]);
                    encode_residual_block(w, &scan16, 16, nc) as u8
                } else {
                    0
                };
                self.nnz_cache_set(lbx, lby, total);
                self.nnz_y[by * w4 + bx] = total;
            }
        }
        if cbp_chroma != 0 {
            for c in 0..2 {
                encode_residual_block(w, &plan.c_dc_levels[c], 4, -1);
            }
        }
        if cbp_chroma == 2 {
            self.chroma_cache_load(mb_x, mb_y);
            let w2 = self.mb_w * 2;
            for c in 0..2 {
                for &(bx, by) in &CHROMA_4X4_SCAN_XY {
                    let nc = self.chroma_nc_pred(c, bx, by);
                    let ac = scan_4x4_ac(&plan.c_q[c][by * 2 + bx]);
                    let total = encode_residual_block(w, &ac, 15, nc) as u8;
                    self.chroma_nnz_cache_set(c, bx, by, total);
                    self.nnz_c[c][(mb_y * 2 + by) * w2 + (mb_x * 2 + bx)] = total;
                }
            }
        }
    }

    /// Descent F: reconstruction / skip-check MC through the cached half-pel planes
    /// instead of the per-pixel 6-tap. `hpel_block` is proven bit-identical to `mc_luma`
    /// (`hpel_block_matches_mc_luma_exactly`) and the `f` plane is the padded,
    /// edge-replicated reference, so both paths are BYTE-IDENTICAL; anything outside the
    /// padded plane still falls back to `mc_luma`.
    ///
    /// Census that motivated it: with the search's edge fallback fixed, `mc_luma` is
    /// 3.8-5.2% of encode and splits recon ~56-67% / skip-check ~24-35%, the latter at a
    /// content-independent one call per macroblock.
    #[inline]
    fn mc_luma_cached(
        &self,
        reference: &crate::RefFrame,
        x0: usize,
        y0: usize,
        bw: usize,
        bh: usize,
        mvx: i32,
        mvy: i32,
        out: &mut [u8],
    ) {
        let ch = self.mb_h * 16;
        let cw = self.cw;
        if !self.fast {
            let p = reference.hpel(cw, ch);
            if rusty_h264_common::inter::hpel_block(p, x0, y0, bw, bh, mvx, mvy, out) {
                return;
            }
            if let Some((plane, base, stride)) =
                rusty_h264_common::inter::hpel_ref(p, x0, y0, bw, bh, mvx, mvy)
            {
                for r in 0..bh {
                    out[r * bw..r * bw + bw].copy_from_slice(&plane[base + r * stride..][..bw]);
                }
                return;
            }
        }
        mc_luma(&reference.y, cw, ch, x0, y0, bw, bh, mvx, mvy, out);
    }

    /// Motion-compensates the `P_Skip` prediction (luma + both chroma) from
    /// reference 0 at the skip MV.
    /// Luma half of the P_Skip prediction. Split out so the fast path can test the
    /// luma residual first and only motion-compensate chroma when luma is free —
    /// for the majority of (non-free) macroblocks the chroma MC is never needed.
    fn skip_predict_luma(
        &self,
        refs: &[crate::RefFrame],
        mb_x: usize,
        mb_y: usize,
        mv: (i32, i32),
    ) -> [u8; 256] {
        // Descent E/F: identify this mc_luma population by call site.
        #[cfg(feature = "profile")]
        let _site = rusty_h264_common::inter::mcstats::SiteTag::new(3);
        let reference = &refs[0]; // P_Skip always references index 0
        let ch = self.mb_h * 16;
        let mut pred_y = [0u8; 256];
        self.mc_luma_cached(reference, mb_x * 16, mb_y * 16, 16, 16, mv.0, mv.1, &mut pred_y);
        pred_y
    }

    /// Chroma half of the P_Skip prediction (see [`Self::skip_predict_luma`]).
    fn skip_predict_chroma(
        &self,
        refs: &[crate::RefFrame],
        mb_x: usize,
        mb_y: usize,
        mv: (i32, i32),
    ) -> [[u8; 64]; 2] {
        let reference = &refs[0];
        let cch = self.mb_h * 8;
        let mut pred_c = [[0u8; 64]; 2];
        for c in 0..2 {
            let rc = if c == 0 { &reference.u } else { &reference.v };
            mc_chroma(rc, self.ccw, cch, mb_x * 8, mb_y * 8, 8, 8, mv.0, mv.1, &mut pred_c[c]);
        }
        pred_c
    }

    /// Whether the luma half of the P_Skip prediction has an all-zero quantized
    /// residual. Tested first and independently so the caller can defer the chroma
    /// MC + test for the common case where luma already disqualifies the skip (a
    /// "free", exact P_Skip costs no bits and is strictly beneficial).
    fn skip_luma_is_free(&self, sy: &[u8], mb_x: usize, mb_y: usize, pred_y: &[u8; 256]) -> bool {
        let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncFree);
        let qp = self.qp;
        // Fast path (deployment): the SAME asm kernels the coding path uses —
        // `dct_four_t4` computes the 4x4 DCTs of (src - pred) for an 8x8 quad
        // STRAIGHT FROM THE PLANES (no scalar gather), `quant_four_4x4` quantizes
        // with the identical FF/MF math as scalar `quantize` (bit-identical), and
        // "free" = all 64 levels zero, which is order-independent. Per-quad early
        // exit. The knob interleaves this against the scalar twin for A/B.
        #[cfg(accel)]
        if self.skip_accel_check {
            #[repr(align(16))]
            struct Align16([i16; 64]);
            let mut dct = Align16([0i16; 64]);
            let ff = rusty_h264_common::transform::quant_dz_ff(qp, 6);
            let mf = &rusty_h264_common::transform::QUANT_MF_OH[qp as usize];
            for &(qx, qy) in &[(0usize, 0usize), (8, 0), (0, 8), (8, 8)] {
                rusty_h264_accel::dct_four_t4(
                    &mut dct.0,
                    &sy[(mb_y * 16 + qy) * self.cw + mb_x * 16 + qx..],
                    self.cw,
                    &pred_y[qy * 16 + qx..],
                    16,
                );
                rusty_h264_accel::quant_four_4x4(&mut dct.0, &ff, mf);
                if dct.0.iter().any(|&v| v != 0) {
                    return false;
                }
            }
            return true;
        }
        // Exact quantize-to-zero bounds (mirrors `quantize`: level != 0 iff
        // (|c| + ff[p])·mf_oh[p] >= 2^16). With |C_ij| <= 4·SAD (max |H| entry = 2)
        // and C_DC = Σres, most blocks are decided by one SAD/sum pass — the full
        // scalar DCT+quant proof only runs for the rare undecided middle band.
        // BIT-EXACT: both shortcuts are sufficient conditions of the exact check.
        let mf = &rusty_h264_common::transform::QUANT_MF_OH[qp as usize];
        let ff = rusty_h264_common::transform::quant_dz_ff(qp, 6);
        let mut t_min = i32::MAX;
        for p in 0..8 {
            let t = (65536 + mf[p] as i32 - 1) / mf[p] as i32 - ff[p] as i32;
            t_min = t_min.min(t);
        }
        let t_dc = (65536 + mf[0] as i32 - 1) / mf[0] as i32 - ff[0] as i32;
        // Whole-MB gate: SAD(any 4x4) <= SAD(MB), so 4*SAD_MB < T_min proves all 16
        // blocks quantize to zero from ONE (psadbw) SAD. On skip-heavy content most
        // free MBs are exact/near-exact copies (SAD_MB ~ 0) - they skip the whole
        // per-block walk. Not-free MBs pay one extra SAD (~2% of their check).
        for by in 0..4 {
            for bx in 0..4 {
                let mut res = [0i32; 16];
                let (mut sad, mut dc) = (0i32, 0i32);
                for dy in 0..4 {
                    for dx in 0..4 {
                        let sx = mb_x * 16 + bx * 4 + dx;
                        let syy = mb_y * 16 + by * 4 + dy;
                        let d = sy[syy * self.cw + sx] as i32
                            - pred_y[(by * 4 + dy) * 16 + (bx * 4 + dx)] as i32;
                        res[dy * 4 + dx] = d;
                        sad += d.abs();
                        dc += d;
                    }
                }
                if 4 * sad < t_min {
                    continue; // every |C| <= 4·SAD < T_min → all levels zero
                }
                if dc.abs() >= t_dc {
                    return false; // DC level provably nonzero
                }
                if quantize(&forward_core(&res), qp, 6).iter().any(|&v| v != 0) {
                    return false;
                }
            }
        }
        true
    }

    /// Chroma half of [`Self::skip_is_free`].
    fn skip_chroma_is_free(
        &self,
        su: &[u8],
        sv: &[u8],
        mb_x: usize,
        mb_y: usize,
        pred_c: &[[u8; 64]; 2],
    ) -> bool {
        let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncFree);
        let qpc = self.qpc;
        // Fast path: one dct_four_t4 covers the whole 8x8 chroma plane region (all 4
        // blocks, residual+DCT fused, no scalar gather). Block order is the quad's
        // z-scan == raster for 2x2, so block b's DC (pre-quant) sits at dct[b*16] —
        // exactly the dc2x2 the Hadamard check needs. quant_four_4x4 with our FF/MF
        // is bit-identical to scalar `quantize`; AC-free = positions 1..16 all zero.
        #[cfg(accel)]
        if self.skip_accel_check {
            #[repr(align(16))]
            struct Align16C([i16; 64]);
            let mut dct = Align16C([0i16; 64]);
            let ff = rusty_h264_common::transform::quant_dz_ff(qpc, 6);
            let mf = &rusty_h264_common::transform::QUANT_MF_OH[qpc as usize];
            for c in 0..2 {
                let src = if c == 0 { su } else { sv };
                rusty_h264_accel::dct_four_t4(
                    &mut dct.0,
                    &src[(mb_y * 8) * self.ccw + mb_x * 8..],
                    self.ccw,
                    &pred_c[c],
                    8,
                );
                let dc2x2 = [
                    dct.0[0] as i32,
                    dct.0[16] as i32,
                    dct.0[32] as i32,
                    dct.0[48] as i32,
                ];
                rusty_h264_accel::quant_four_4x4(&mut dct.0, &ff, mf);
                for b in 0..4 {
                    if dct.0[b * 16 + 1..b * 16 + 16].iter().any(|&v| v != 0) {
                        return false;
                    }
                }
                if forward_quant_chroma_dc(&dc2x2, qpc, false).iter().any(|&v| v != 0) {
                    return false;
                }
            }
            return true;
        }
        for c in 0..2 {
            let src = if c == 0 { su } else { sv };
            let mut dc2x2 = [0i32; 4];
            for &(bx, by) in &CHROMA_4X4_SCAN_XY {
                let mut res = [0i32; 16];
                for dy in 0..4 {
                    for dx in 0..4 {
                        let sx = mb_x * 8 + bx * 4 + dx;
                        let syy = mb_y * 8 + by * 4 + dy;
                        res[dy * 4 + dx] = src[syy * self.ccw + sx] as i32
                            - pred_c[c][(by * 4 + dy) * 8 + (bx * 4 + dx)] as i32;
                    }
                }
                let coeffs = forward_core(&res);
                dc2x2[by * 2 + bx] = coeffs[0];
                if quantize(&coeffs, qpc, 6)[1..].iter().any(|&v| v != 0) {
                    return false;
                }
            }
            if forward_quant_chroma_dc(&dc2x2, qpc, false).iter().any(|&v| v != 0) {
                return false;
            }
        }
        true
    }

    /// SSD between the source and a macroblock prediction (luma + chroma).
    #[allow(clippy::too_many_arguments)]
    fn pred_ssd(
        &self,
        sy: &[u8],
        su: &[u8],
        sv: &[u8],
        mb_x: usize,
        mb_y: usize,
        pred_y: &[u8; 256],
        pred_c: &[[u8; 64]; 2],
    ) -> i64 {
        let mut ssd = 0i64;
        for dy in 0..16 {
            for dx in 0..16 {
                let d = sy[(mb_y * 16 + dy) * self.cw + mb_x * 16 + dx] as i64
                    - pred_y[dy * 16 + dx] as i64;
                ssd += d * d;
            }
        }
        for c in 0..2 {
            let src = if c == 0 { su } else { sv };
            for dy in 0..8 {
                for dx in 0..8 {
                    let d = src[(mb_y * 8 + dy) * self.ccw + mb_x * 8 + dx] as i64
                        - pred_c[c][dy * 8 + dx] as i64;
                    ssd += d * d;
                }
            }
        }
        ssd
    }

    /// SSD between the *reconstructed* macroblock and the source.
    fn mb_ssd(&self, sy: &[u8], su: &[u8], sv: &[u8], mb_x: usize, mb_y: usize) -> i64 {
        let mut ssd = 0i64;
        for dy in 0..16 {
            for dx in 0..16 {
                let i = (mb_y * 16 + dy) * self.cw + mb_x * 16 + dx;
                let d = sy[i] as i64 - self.rec_y[i] as i64;
                ssd += d * d;
            }
        }
        for c in 0..2 {
            let (src, rec) = if c == 0 { (su, &self.rec_u) } else { (sv, &self.rec_v) };
            for dy in 0..8 {
                for dx in 0..8 {
                    let i = (mb_y * 8 + dy) * self.ccw + mb_x * 8 + dx;
                    let d = src[i] as i64 - rec[i] as i64;
                    ssd += d * d;
                }
            }
        }
        ssd
    }

    /// Reconstructs a `P_Skip` macroblock (reconstruction *is* the prediction —
    /// no residual coded) and records its motion state.
    #[allow(clippy::too_many_arguments)]
    fn commit_skip_probe_marker(&self) {}
    fn commit_skip(
        &mut self,
        mb_x: usize,
        mb_y: usize,
        mv: (i32, i32),
        pred_y: &[u8; 256],
        pred_c: &[[u8; 64]; 2],
    ) {
        let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::MvGrid);
        // Skip recon = the prediction verbatim: straight row copies (byte-identical
        // to the old per-4x4 gather + store scatter, ~5x fewer ops).
        let base = mb_y * 16 * self.cw + mb_x * 16;
        for r in 0..16 {
            let d = base + r * self.cw;
            self.rec_y[d..d + 16].copy_from_slice(&pred_y[r * 16..r * 16 + 16]);
        }
        let cbase = mb_y * 8 * self.ccw + mb_x * 8;
        for c in 0..2 {
            let plane = if c == 0 { &mut self.rec_u } else { &mut self.rec_v };
            for r in 0..8 {
                let d = cbase + r * self.ccw;
                plane[d..d + 8].copy_from_slice(&pred_c[c][r * 8..r * 8 + 8]);
            }
        }
        self.set_mb_mv(mb_x, mb_y, mv, true, 0);
        let w4 = self.mb_w * 4;
        for row in 0..4 {
            let st = (mb_y * 4 + row) * w4 + mb_x * 4;
            self.modes_y[st..st + 4].fill(2);
            self.coded_y[st..st + 4].fill(true);
        }
    }

    /// Trial-encodes an inter macroblock to measure its rate-distortion cost
    /// `(SSD, bits)` without committing: snapshot the macroblock's grid + recon
    /// region, run the real `encode_inter_mb` into a scratch writer, read the
    /// bit count and reconstruction SSD, then restore. Neighbor CAVLC context is
    /// read (not mutated), so the bit count is accurate.
    #[allow(clippy::too_many_arguments)]
    fn trial_inter(
        &mut self,
        refs: &[crate::RefFrame],
        sy: &[u8],
        su: &[u8],
        sv: &[u8],
        mb_x: usize,
        mb_y: usize,
        mode: u8,
        parts: &[(i32, (i32, i32))],
    ) -> (i64, usize) {
        let snap = self.save_mb(mb_x, mb_y);
        let mut scratch = BitWriter::new();
        self.encode_inter_mb(&mut scratch, refs, sy, su, sv, mb_x, mb_y, mode, parts);
        let bits = scratch.bit_len();
        let ssd = self.mb_ssd(sy, su, sv, mb_x, mb_y);
        self.load_mb(mb_x, mb_y, &snap);
        (ssd, bits)
    }

    /// Trial-encodes the macroblock as **intra** (`encode_mb` runs its own
    /// I_16x16-vs-I_4x4 decision), measuring `(SSD, bits)` without committing —
    /// the intra candidate for the RD mode decision.
    fn trial_intra(
        &mut self,
        sy: &[u8],
        su: &[u8],
        sv: &[u8],
        mb_x: usize,
        mb_y: usize,
        is_p: bool,
    ) -> (i64, usize) {
        let snap = self.save_mb(mb_x, mb_y);
        let mut scratch = BitWriter::new();
        encode_mb(self, &mut scratch, mb_x, mb_y, sy, su, sv, is_p);
        let bits = scratch.bit_len();
        let ssd = self.mb_ssd(sy, su, sv, mb_x, mb_y);
        self.load_mb(mb_x, mb_y, &snap);
        (ssd, bits)
    }

    /// Best `(ref_idx, mv, cost)` for one partition by `SATD + λ·bits`, searched
    /// across every reference (`cost` is that SATD-domain rate-distortion cost).
    /// `extra` seeds the search with already-found MVs (e.g. the 16×16 result when
    /// refining a sub-partition).
    #[allow(clippy::too_many_arguments)]
    fn best_part(
        &self,
        refs: &[crate::RefFrame],
        sy: &[u8],
        nb: &[MvNeighbor; 3],
        num_refs: usize,
        rx: usize,
        ry: usize,
        rw: usize,
        rh: usize,
        extra: &[(i32, i32)],
        lme: f64,
    ) -> (i32, (i32, i32), i64) {
        let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncMe);
        let [a, b, c] = *nb;
        let (mut br, mut bmv, mut bc) = (0i32, (0, 0), i64::MAX);
        for r in 0..num_refs {
            let mut seeds = vec![predict_mv(a, b, c, r as i32)];
            seeds.extend_from_slice(extra);
            let (mv, cost) = self.motion_search(&refs[r], sy, rx, ry, rw, rh, &seeds, lme, None);
            let cost = cost + (lme * ref_bits(r, num_refs) as f64) as i64;
            if cost < bc {
                bc = cost;
                br = r as i32;
                bmv = mv;
            }
        }
        (br, bmv, bc)
    }

    /// Sub-pel-refines ONE already-chosen partition, reusing `motion_search`'s cost
    /// closure via its `start` hook so the rate term and predictor centre are exactly
    /// the ones the full search used. Companion to `best_part` under `sp_defer`.
    #[allow(clippy::too_many_arguments)]
    fn refine_part(
        &self,
        refs: &[crate::RefFrame],
        sy: &[u8],
        nb: &[MvNeighbor; 3],
        num_refs: usize,
        rx: usize,
        ry: usize,
        rw: usize,
        rh: usize,
        lme: f64,
        r: i32,
        mv: (i32, i32),
    ) -> ((i32, i32), i64) {
        let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncMe);
        let [a, b, c] = *nb;
        let rb = (lme * ref_bits(r as usize, num_refs) as f64) as i64;
        let seeds = [predict_mv(a, b, c, r)];
        let (m, cc) = self.motion_search(&refs[r as usize], sy, rx, ry, rw, rh, &seeds, lme, Some(mv));
        (m, cc + rb)
    }

    /// Cheapest `I_16x16` prediction's SAD over the four whole-block modes, using
    /// the already-reconstructed top/left neighbours — the intra candidate's cost
    /// in the fast (SAD) mode decision, without the full `I_4x4` search.
    fn best_i16_sad(&self, sy: &[u8], mb_x: usize, mb_y: usize) -> i64 {
        let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncIntraCost);
        let (lx, ly) = (mb_x * 16, mb_y * 16);
        let (avail_top, avail_left) = (mb_y > 0, mb_x > 0);
        let mut top = [0u8; 16];
        let mut left = [0u8; 16];
        if avail_top {
            for i in 0..16 {
                top[i] = self.rec_y[(ly - 1) * self.cw + lx + i];
            }
        }
        if avail_left {
            for i in 0..16 {
                left[i] = self.rec_y[(ly + i) * self.cw + lx - 1];
            }
        }
        let corner = if avail_top && avail_left {
            self.rec_y[(ly - 1) * self.cw + lx - 1]
        } else {
            0
        };
        let mut best = i64::MAX;
        for mode in [I16Mode::Dc, I16Mode::Vertical, I16Mode::Horizontal, I16Mode::Plane] {
            if !mode.available(avail_top, avail_left) {
                continue;
            }
            let pred = i16_pred(self, mode, avail_top, avail_left, &top, &left, corner, lx, ly);
            best = best.min(sad_16x16(sy, self.cw, lx, ly, &pred));
        }
        best
    }

    /// SATD sibling of [`Self::best_i16_sad`] — the intra candidate's cost in the
    /// quality preset's SATD mode decision (openh264's `WelsMdI16x16`).
    fn best_i16_satd(&self, sy: &[u8], mb_x: usize, mb_y: usize) -> i64 {
        let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncIntraCost);
        let (lx, ly) = (mb_x * 16, mb_y * 16);
        let (avail_top, avail_left) = (mb_y > 0, mb_x > 0);
        let mut top = [0u8; 16];
        let mut left = [0u8; 16];
        if avail_top {
            for i in 0..16 {
                top[i] = self.rec_y[(ly - 1) * self.cw + lx + i];
            }
        }
        if avail_left {
            for i in 0..16 {
                left[i] = self.rec_y[(ly + i) * self.cw + lx - 1];
            }
        }
        let corner = if avail_top && avail_left {
            self.rec_y[(ly - 1) * self.cw + lx - 1]
        } else {
            0
        };
        let mut best = i64::MAX;
        for mode in [I16Mode::Dc, I16Mode::Vertical, I16Mode::Horizontal, I16Mode::Plane] {
            if !mode.available(avail_top, avail_left) {
                continue;
            }
            let pred = i16_pred(self, mode, avail_top, avail_left, &top, &left, corner, lx, ly);
            best = best.min(satd_16x16(sy, self.cw, lx, ly, &pred));
        }
        best
    }

    /// Snapshots the per-block grids and reconstruction for one macroblock, so a
    /// trial encode can be rolled back.
    fn save_mb(&self, mb_x: usize, mb_y: usize) -> MbState {
        let mut d = MbState::default();
        self.save_mb_into(mb_x, mb_y, &mut d);
        d
    }

    /// [`save_mb`](Self::save_mb) into an existing buffer, reusing its allocations.
    /// The per-macroblock region is a fixed size, so after the first call every
    /// `Vec` already has the capacity it needs and refilling is a pure copy.
    fn save_mb_into(&self, mb_x: usize, mb_y: usize, d: &mut MbState) {
        let w4 = self.mb_w * 4;
        let w2 = self.mb_w * 2;
        macro_rules! reg4 {
            ($v:expr, $o:expr) => {{
                $o.clear();
                for dy in 0..4 {
                    for dx in 0..4 {
                        $o.push($v[(mb_y * 4 + dy) * w4 + mb_x * 4 + dx]);
                    }
                }
            }};
        }
        macro_rules! regn {
            ($v:expr, $o:expr, $n:expr, $ox:expr, $oy:expr, $stride:expr) => {{
                $o.clear();
                for dy in 0..$n {
                    for dx in 0..$n {
                        $o.push($v[($oy + dy) * $stride + $ox + dx]);
                    }
                }
            }};
        }
        regn!(self.rec_y, d.rec_y, 16, mb_x * 16, mb_y * 16, self.cw);
        regn!(self.rec_u, d.rec_u, 8, mb_x * 8, mb_y * 8, self.ccw);
        regn!(self.rec_v, d.rec_v, 8, mb_x * 8, mb_y * 8, self.ccw);
        reg4!(self.nnz_y, d.nnz_y);
        regn!(self.nnz_c[0], d.nnz_c[0], 2, mb_x * 2, mb_y * 2, w2);
        regn!(self.nnz_c[1], d.nnz_c[1], 2, mb_x * 2, mb_y * 2, w2);
        reg4!(self.mv_y, d.mv_y);
        reg4!(self.inter_y, d.inter_y);
        reg4!(self.ref_idx_y, d.ref_idx_y);
        reg4!(self.coded_y, d.coded_y);
        reg4!(self.modes_y, d.modes_y);
        d.cur_qp = self.cur_qp;
    }

    /// Restores a macroblock's grids + reconstruction from a [`save_mb`] snapshot.
    fn load_mb(&mut self, mb_x: usize, mb_y: usize, s: &MbState) {
        let w4 = self.mb_w * 4;
        let w2 = self.mb_w * 2;
        macro_rules! put4 {
            ($v:expr, $src:expr) => {
                for dy in 0..4 {
                    for dx in 0..4 {
                        $v[(mb_y * 4 + dy) * w4 + mb_x * 4 + dx] = $src[dy * 4 + dx];
                    }
                }
            };
        }
        macro_rules! putn {
            ($v:expr, $src:expr, $n:expr, $ox:expr, $oy:expr, $stride:expr) => {
                for dy in 0..$n {
                    for dx in 0..$n {
                        $v[($oy + dy) * $stride + $ox + dx] = $src[dy * $n + dx];
                    }
                }
            };
        }
        putn!(self.rec_y, s.rec_y, 16, mb_x * 16, mb_y * 16, self.cw);
        putn!(self.rec_u, s.rec_u, 8, mb_x * 8, mb_y * 8, self.ccw);
        putn!(self.rec_v, s.rec_v, 8, mb_x * 8, mb_y * 8, self.ccw);
        put4!(self.nnz_y, s.nnz_y);
        putn!(self.nnz_c[0], s.nnz_c[0], 2, mb_x * 2, mb_y * 2, w2);
        putn!(self.nnz_c[1], s.nnz_c[1], 2, mb_x * 2, mb_y * 2, w2);
        put4!(self.mv_y, s.mv_y);
        put4!(self.inter_y, s.inter_y);
        put4!(self.ref_idx_y, s.ref_idx_y);
        put4!(self.coded_y, s.coded_y);
        put4!(self.modes_y, s.modes_y);
        self.cur_qp = s.cur_qp;
    }

    /// Loads the per-MB luma nnz prediction cache (openh264 `scan8` style): the top
    /// row from the macroblock above and the left column from the macroblock to the
    /// left (both already in `nnz_y`), with `0x80` at the picture edges. After this,
    /// neighbour nnz reads are branchless cache indexing — no bounds-checked `Option`.
    fn nnz_cache_load(&mut self, mb_x: usize, mb_y: usize) {
        let w4 = self.mb_w * 4;
        for lbx in 0..4 {
            self.nnz_l_cache[1 + lbx] = if mb_y == 0 {
                0x80
            } else {
                self.nnz_y[(mb_y * 4 - 1) * w4 + (mb_x * 4 + lbx)]
            };
        }
        for lby in 0..4 {
            self.nnz_l_cache[(lby + 1) * 5] = if mb_x == 0 {
                0x80
            } else {
                self.nnz_y[(mb_y * 4 + lby) * w4 + (mb_x * 4 - 1)]
            };
        }
    }

    /// Branchless nnz prediction (`nC`) for luma block `(lbx,lby)` from the cache —
    /// the `0x80` sentinel + `& 0x7f` mask collapse the four availability cases
    /// (matches the scalar nnz predict). Call after the block's left/top are cached.
    #[inline]
    fn nc_pred(&self, lbx: usize, lby: usize) -> i32 {
        let left = self.nnz_l_cache[(lby + 1) * 5 + lbx] as i32; // (lbx-1)+1
        let top = self.nnz_l_cache[lby * 5 + (lbx + 1)] as i32; // (lby-1)+1
        let r = left + top;
        if r < 0x80 {
            (r + 1) >> 1
        } else {
            r & 0x7f
        }
    }

    /// Records a luma block's nnz into the per-MB cache (for later neighbour reads).
    #[inline]
    fn nnz_cache_set(&mut self, lbx: usize, lby: usize, total: u8) {
        self.nnz_l_cache[(lby + 1) * 5 + (lbx + 1)] = total;
    }

    /// Loads the per-MB chroma nnz prediction cache (both planes) from the chroma
    /// blocks above/left, `0x80` at the picture edges — the chroma analogue of
    /// [`Self::nnz_cache_load`] (2×2 blocks → padded 3×3 grid).
    fn chroma_cache_load(&mut self, mb_x: usize, mb_y: usize) {
        let w2 = self.mb_w * 2;
        for c in 0..2 {
            for bx in 0..2 {
                self.nnz_c_cache[c][1 + bx] = if mb_y == 0 {
                    0x80
                } else {
                    self.nnz_c[c][(mb_y * 2 - 1) * w2 + (mb_x * 2 + bx)]
                };
            }
            for by in 0..2 {
                self.nnz_c_cache[c][(by + 1) * 3] = if mb_x == 0 {
                    0x80
                } else {
                    self.nnz_c[c][(mb_y * 2 + by) * w2 + (mb_x * 2 - 1)]
                };
            }
        }
    }

    /// Branchless chroma nnz prediction (`nC`) for plane `c`, block `(bx,by)`.
    #[inline]
    fn chroma_nc_pred(&self, c: usize, bx: usize, by: usize) -> i32 {
        let left = self.nnz_c_cache[c][(by + 1) * 3 + bx] as i32;
        let top = self.nnz_c_cache[c][by * 3 + (bx + 1)] as i32;
        let r = left + top;
        if r < 0x80 {
            (r + 1) >> 1
        } else {
            r & 0x7f
        }
    }

    /// Records a chroma block's nnz into the per-MB cache.
    #[inline]
    fn chroma_nnz_cache_set(&mut self, c: usize, bx: usize, by: usize, total: u8) {
        self.nnz_c_cache[c][(by + 1) * 3 + (bx + 1)] = total;
    }
}

/// Encodes a slice's macroblocks then RBSP trailing bits, returning the
/// **deblocked** reconstruction to serve as the next frame's reference.
///
/// `is_p` selects P-slice framing (`mb_skip_run` prefix + intra `mb_type` +5
/// offset). In phase 4a every macroblock is still coded intra; motion-compensated
/// macroblocks arrive in 4b (using `reference`).
/// Boundary strengths for one macroblock, derived from the encoder's own grids
/// the moment it finishes coding.
///
/// `ref_idx_y` holds raw indices (-1 for intra) rather than the deblocker's
/// `NO_REF` sentinel; safe because reference identity is only compared between
/// two INTER blocks, which always carry a valid index.
// NOT inlined: this sits at three exits of the hottest loop in the encoder, and
// inlining it there costs more in I-cache and register pressure on the
// surrounding code than the call saves (measured: the loop grew ~2x the
// derivation's own cost).
#[inline(never)]
fn derive_mb_bs_from(
    fe: &FrameEncoder,
    mb_x: usize,
    mb_y: usize,
    kind: rusty_h264_common::deblock::MbKind,
) -> rusty_h264_common::deblock::MbBs {
    let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncBs);
    let view = rusty_h264_common::deblock::BlockInfo {
        inter: &fe.inter_y,
        nnz: &fe.nnz_y,
        mv: &fe.mv_y,
        ref_id: &fe.ref_idx_y,
        mv1: &[],
        ref_id1: &[],
        w4: fe.mb_w * 4,
        t8x8: &[],
        bs: &[],
    };
    rusty_h264_common::deblock::derive_mb_kind(&view, mb_x, mb_y, kind)
}

pub fn encode_slice_data(
    w: &mut BitWriter,
    cfg: &EncoderConfig,
    frame: &YuvFrame,
    qp: u8,
    is_p: bool,
    refs: &[crate::RefFrame],
    qpo: &[i32],
) -> crate::RefFrame {
    let _g_prep = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncPrep);
    let mut fe = FrameEncoder::new(cfg);
    let precomp = rusty_h264_common::deblock::precomputed_bs_enabled();
    let mut bs_grid =
        vec![rusty_h264_common::deblock::MbBs::UNSET; if precomp { fe.mb_w * fe.mb_h } else { 0 }];
    fe.qp = qp;
    fe.qpc = chroma_qp(qp);
    fe.cur_qp = qp;
    if cfg.cabac_dz_div > 0 {
        fe.idz = cfg.cabac_dz_div; // CABAC-specific dead-zone override
    } // QPY_PREV starts at the slice QP so the first mb_qp_delta is 0
    let (sy, su, sv) = coded_source(cfg, frame);
    let lambda = 0.85 * fe.tune_lambda_scale * 2f64.powf((qp as f64 - 12.0) / 3.0);
    let num_refs = refs.len();
    // me_wide CONTENT GATE: on a pure PAN the global-MC residual ≈ 0, so the diamond's
    // seed (median = pan MV) is already right and the wide rescue only over-fits
    // (spurious MVs that hurt the B-frames' spatial-direct — the panc regression).
    // Gate it off there; non-uniform content (real stalls) reads well above 0.
    if is_p && fe.me_wide && !refs.is_empty()
        && global_mc_residual(&sy, fe.cw, fe.mb_h * 16, &refs[0].y) < fe.me_wide_coh
    {
        fe.me_wide = false;
    }
    // me_wide HEAD-ROOM GATE (the dispatcher the truth table asked for). The rescue
    // only pays where a wide search actually beats a predictor-local one; measure
    // that directly per frame and route the frame. `RFF_ME_HR` sets the threshold
    // (percent); 0 disables the gate and restores the always-on behaviour.
    // Skip the probe entirely when the gate is disabled: it must not tax the
    // default path (`RFF_ME_HR=0`), which stays byte-identical to pre-gate output.
    if fe.me_wide && !refs.is_empty() && (me_wide_hr_thresh() > 0.0 || me_wide_hr_dbg()) {
        let hr = me_wide_headroom(&sy, fe.cw, fe.mb_h * 16, &refs[0].y);
        if me_wide_hr_dbg() {
            eprintln!("ME_HR qp{qp} headroom={hr:.2}");
        }
        if me_wide_hr_thresh() > 0.0 && hr < me_wide_hr_thresh() {
            fe.me_wide = false;
        }
    }
    // Track-B B2 DISPATCH (WHYS H-2): SAD full-pel wins where a plain full-pel
    // translational search actually improves on zero motion (`b2_mgain`) and loses
    // on flash/fine-detail content. Probe per frame, route the frame — per-frame,
    // not cross-frame, so it stays deterministic under GOP-parallel encode.
    if me_sadfp_mode() == 1 && !fe.fast && !refs.is_empty() {
        let (mg, dc) = b2_mgain(&sy, fe.cw, fe.mb_h * 16, &refs[0].y);
        if me_sadt_dbg() {
            eprintln!("B2_MG qp{qp} mgain={mg:.3} dcfrac={dc:.3}");
        }
        fe.sadfp = mg >= me_sadt() && dc <= me_sad_dcmax();
        // H-24: the mv-cost SHAPE rides the same probe (its BD sign-flip tracks
        // motion for the same physical reason B2's does).
        if mv_smooth_mode() == 1 {
            // dcfrac veto mirrors B2's: crew-class FLASH frames satisfy the mgain
            // test but SAD/mvd statistics mislead there (H-13/H-26).
            fe.mv_smooth = mg >= mv_smooth_t() && dc <= me_sad_dcmax();
        }
        // H-13: near-static frames skip the split searches entirely.
        let smg = split_mg();
        if smg > 0.0 {
            fe.do_splits = mg >= smg;
        }
    }
    // Content-adaptive cost-function dispatch (codec-content-adaptive-dispatch): the
    // fast preset prices modes by cheap SAD, which is rate-blind on detailed MBs;
    // route the top `satd_q` fraction of highest-VARIANCE MBs to the rate-faithful
    // SATD cost. A per-frame PERCENTILE threshold makes the routed fraction — hence
    // the speed/quality split — content-invariant (same q → same fraction on any
    // clip). `satd_q == 0` leaves the threshold at MAX (pure SAD, byte-identical).
    if is_p && fe.satd_q > 0.0 {
        let mut vars: Vec<i64> = (0..fe.mb_h)
            .flat_map(|my| (0..fe.mb_w).map(move |mx| (mx, my)))
            .map(|(mx, my)| mb_variance(&sy, fe.cw, mx, my))
            .collect();
        vars.sort_unstable();
        let idx = (((1.0 - fe.satd_q) * vars.len() as f64) as usize).min(vars.len() - 1);
        fe.satd_var_thresh = vars[idx];
    }
    // Adaptive Quantization: per-MB target QPy from content (finer on flat MBs,
    // coarser on busy ones). `mb_qpy` records each MB's ACTUAL QPy (a skip / cbp==0
    // MB inherits `cur_qp`), for the deblock filter. `strength 0` → uniform → the
    // mb_qp_delta stays 0, byte-identical.
    let mut aq_qp = aq_qp_map(&sy, fe.cw, fe.mb_w, fe.mb_h, qp, fe.aq_strength);
    apply_mbtree_qpo(&mut aq_qp, qpo); // mb-tree temporal AQ (empty = byte-identical)
    fe.cur_qp = qp;
    let mut mb_qpy = vec![qp; fe.mb_w * fe.mb_h];
    let mut skip_run = 0u32;
    // ---- adaptive RD-skip gate -------------------------------------------
    // RD P_Skip is a large win on temporally redundant content and a large LOSS
    // on detailed content (SSIM: akiyo -13.1%, FourPeople -5.6% vs in_to_tree
    // +34.0%, stockholm +95.7%). The separating signal is the content's own
    // FREE-skip rate — how much of it is already exactly redundant — and the gap
    // is wide (winners >=58.7%, losers <=6.4%). Measure it ONLINE over the first
    // slice of the frame and enable RD skip for the remainder only if it clears
    // the bar. Within-frame, so it stays deterministic under GOP-parallel encode.
    if is_p && mv_cmp_on() {
        MVCMP_FRAME.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
    }
    // Reused across every RD-skip candidate — see `MbState`.
    let mut rdskip_snap = MbState::default();
    let mut rdskip_free = 0usize;
    let mut rdskip_seen = 0usize;
    let mut rdskip_on = false;
    let mut greedy_on = fe.greedy_min_free == 0; // 0 = ungated (historic behaviour)
    let rdskip_learn = (fe.mb_w * fe.mb_h / 8).max(64);
    let rdskip_min_free = fe.rd_skip_min_free as usize;

    drop(_g_prep);
    let _g_loop = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncMbLoop);
    for mb_y in 0..fe.mb_h {
        for mb_x in 0..fe.mb_w {
            let mb_idx = mb_y * fe.mb_w + mb_x;
            fe.qp = aq_qp[mb_idx];
            fe.qpc = chroma_qp(aq_qp[mb_idx]);
            // P_Skip: motion-compensate from the most-recent reference; accept if free.
            // Chosen inter coding: (mb_type, per-partition (ref_idx, mv)).
            let mut inter: Option<InterChoice> = None;
            // Bits of an inter macroblock already encoded by the skip decision
            // below. When present the emit path splices them instead of encoding
            // the same macroblock a second time.
            let mut coded: Option<BitWriter> = None;
            if is_p {
                if num_refs > 0 {
                    // P_Skip prediction (reference 0). A free skip (zero residual) is
                    // taken immediately; the quality preset also takes a greedy P_Skip
                    // when its SAD is below the neighbour-predicted bound (below).
                    let _g_skip = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncSkip);
                    let _g_smc = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::Neighbors);
                    rdskip_seen += 1;
                    if rdskip_seen >= rdskip_learn {
                        rdskip_on = rdskip_free * 100 >= rdskip_seen * rdskip_min_free;
                        greedy_on = fe.greedy_min_free == 0
                            || rdskip_free * 100 >= rdskip_seen * fe.greedy_min_free as usize;
                    }
                    let mv_skip = fe.skip_mv(mb_x, mb_y);
                    let skip_y = fe.skip_predict_luma(refs, mb_x, mb_y, mv_skip);
                    drop(_g_smc);
                    let luma_free = fe.skip_luma_is_free(&sy, mb_x, mb_y, &skip_y);
                    // Chroma MC only when it can matter: luma already free (so the
                    // skip might be taken) or the quality path needs it below.
                    let skip_c = if luma_free || !fe.fast {
                        fe.skip_predict_chroma(refs, mb_x, mb_y, mv_skip)
                    } else {
                        [[0u8; 64]; 2]
                    };
                    let is_free =
                        luma_free && fe.skip_chroma_is_free(&su, &sv, mb_x, mb_y, &skip_c);
                    // Skip-prediction luma SAD (the quality preset's predicted-SAD apparatus).
                    let skip_sad = if fe.fast {
                        0
                    } else {
                        let (lx, ly) = (mb_x * 16, mb_y * 16);
                        let mut s = 0u32;
                        for dy in 0..16 {
                            let src = &sy[(ly + dy) * fe.cw + lx..][..16];
                            let p = &skip_y[dy * 16..][..16];
                            s += src.iter().zip(p).map(|(&a, &b)| a.abs_diff(b) as u32).sum::<u32>();
                        }
                        s
                    };
                    if is_free {
                        fe.commit_skip(mb_x, mb_y, mv_skip, &skip_y, &skip_c);
                        if !fe.fast {
                            fe.mb_was_skip[mb_idx] = true;
                            fe.mb_skip_sad[mb_idx] = skip_sad;
                        }
                        mb_qpy[mb_idx] = fe.cur_qp; // skip inherits QPy
                        rdskip_free += 1;
                        if precomp {
                            bs_grid[mb_idx] = derive_mb_bs_from(&fe, mb_x, mb_y, rusty_h264_common::deblock::MbKind::Skip);
                        }
                        skip_run += 1;
                        continue;
                    }
                    drop(_g_skip);
                    let (lx, ly) = (mb_x * 16, mb_y * 16);
                    let nb = {
                        let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncMvPred);
                        fe.mv_neighbors_block(mb_x as isize * 4, mb_y as isize * 4, 4)
                    };
                    let lme = lambda.sqrt();

                    if fe.fast {
                        // Fast preset: pick the cheapest *prediction* by SATD (no
                        // trial-encoding), then always code its residual — P_16x16 vs
                        // I_16x16 only, no sub-partitions. Crucially it does NOT make a
                        // SATD skip-vs-code decision: P_Skip is taken only for a truly
                        // free (zero-residual) macroblock, handled above. Pricing skip
                        // by SATD would drop residual the QP wants coded and tank PSNR;
                        // like x264's fast presets, fast trades *efficiency* (more bits)
                        // for speed, not quality. The faster ME is what makes it fast.
                        // Adaptive dispatch: high-variance MBs price by SATD (both
                        // inter — via `mb_use_satd` in `best_part` — and intra), the
                        // rest by cheap SAD. Set the per-MB flag before best_part.
                        fe.mb_use_satd = fe.satd_q > 0.0
                            && mb_variance(&sy, fe.cw, mb_x, mb_y) >= fe.satd_var_thresh;
                        let (r16, mv16, cost_inter) =
                            fe.best_part(refs, &sy, &nb, num_refs, lx, ly, 16, 16, &[], lme);
                        let cost_intra = if fe.mb_use_satd {
                            fe.best_i16_satd(&sy, mb_x, mb_y)
                        } else {
                            fe.best_i16_sad(&sy, mb_x, mb_y)
                        } + (lme * fe.tune_intra_penalty) as i64;
                        inter = if cost_intra < cost_inter {
                            None // intra wins → encode_mb below
                        } else {
                            Some((0, vec![(r16, mv16)]))
                        };
                    } else {
                        // Quality preset: openh264's mode-decision model — SATD + λ·mvbits
                        // cost ESTIMATE (no per-candidate trial-encode); modes are ranked
                        // by that cost and only the winner is encoded (once) below. This
                        // removes ~the 93%-of-quality re-encode cost.

                        // Greedy P_Skip (openh264 `PredictSadSkip`): take the skip when its
                        // luma SAD is below the neighbour-predicted skip SAD. The threshold
                        // is what skip neighbours achieved, so the skip propagates from the
                        // free skips and self-limits — no fixed bound, no inter-chain drift.
                        if fe.greedy_skip && greedy_on && skip_sad < fe.pred_skip_sad(mb_x, mb_y) {
                            fe.commit_skip(mb_x, mb_y, mv_skip, &skip_y, &skip_c);
                            fe.mb_was_skip[mb_idx] = true;
                            fe.mb_skip_sad[mb_idx] = skip_sad;
                            mb_qpy[mb_idx] = fe.cur_qp; // skip inherits QPy
                            if precomp {
                                bs_grid[mb_idx] = derive_mb_bs_from(&fe, mb_x, mb_y, rusty_h264_common::deblock::MbKind::Skip);
                            }
                            skip_run += 1;
                            continue;
                        }

                        // 16×16 baseline (SATD + λ·bits, with sub-pel refinement).
                        let (r16, mv16, c16) =
                            fe.best_part(refs, &sy, &nb, num_refs, lx, ly, 16, 16, &[], lme);
                        let mut best_c = c16;
                        let mut pick: Option<InterChoice> = Some((0, vec![(r16, mv16)]));

                        // Sub-partitions, ranked by SATD, gated on a heavy 16×16 (a likely
                        // motion boundary — the 4 sub-pel searches are the expensive part).
                        const QSTEP16: [i64; 6] = [10, 11, 13, 14, 16, 18];
                        let qstep16 = QSTEP16[(fe.qp % 6) as usize] << (fe.qp / 6);
                        let split_gate = ((30 * (qstep16 + 160)) >> 3) * 2;
                        let split_t = split_t();
                        if fe.do_splits && c16 > split_gate && (split_t <= 0.0 || (c16 as f64) >= split_t * lme) {
                            let (rt, mvt, ct) = fe.best_part(refs, &sy, &nb, num_refs, lx, ly, 16, 8, &[mv16], lme);
                            let (rb, mvb, cb) = fe.best_part(refs, &sy, &nb, num_refs, lx, ly + 8, 16, 8, &[mv16], lme);
                            let (rl, mvl, cl) = fe.best_part(refs, &sy, &nb, num_refs, lx, ly, 8, 16, &[mv16], lme);
                            let (rr, mvr, cr) = fe.best_part(refs, &sy, &nb, num_refs, lx + 8, ly, 8, 16, &[mv16], lme);
                            if ct + cb < best_c {
                                best_c = ct + cb;
                                pick = Some((1u8, vec![(rt, mvt), (rb, mvb)]));
                            }
                            if cl + cr < best_c {
                                best_c = cl + cr;
                                pick = Some((2u8, vec![(rl, mvl), (rr, mvr)]));
                            }

                            // P_8x8: four independent 8×8 sub-partitions (finer motion
                            // granularity — the win on complex/boundary motion). Each 8×8
                            // seeded by the 16×16 MV; the exact chained MVD is computed in
                            // plan_inter_mb. Same heavy-16×16 gate as the 2-way splits.
                            if fe.sub8x8 {
                                let mut c8 = (lme * 4.0) as i64; // ~4 sub_mb_type bits
                                let mut p8 = Vec::with_capacity(4);
                                for &(qx, qy) in &[(0usize, 0usize), (8, 0), (0, 8), (8, 8)] {
                                    let (r, mv, c) = fe.best_part(
                                        refs, &sy, &nb, num_refs, lx + qx, ly + qy, 8, 8, &[mv16], lme,
                                    );
                                    c8 += c;
                                    p8.push((r, mv));
                                }
                                if c8 < best_c {
                                    best_c = c8;
                                    pick = Some((3u8, p8));
                                }
                            }
                        }

                        // U5-struct: everything above searched FULL-PEL only when
                        // `sp_defer` is set. Now that a shape has won, refine just its
                        // sub-blocks — the losing shapes' refinements were the waste
                        // (measured 3.4–6.4× more refinement than necessary).
                        if fe.sp_defer.get() {
                            if let Some((mode, parts)) = pick.as_mut() {
                                let regions: &[(usize, usize, usize, usize)] = match mode {
                                    1 => &[(0, 0, 16, 8), (0, 8, 16, 8)],
                                    2 => &[(0, 0, 8, 16), (8, 0, 8, 16)],
                                    3 => &[(0, 0, 8, 8), (8, 0, 8, 8), (0, 8, 8, 8), (8, 8, 8, 8)],
                                    _ => &[(0, 0, 16, 16)],
                                };
                                let mut tot = if *mode == 3 { (lme * 4.0) as i64 } else { 0 };
                                for (i, &(qx, qy, pw, ph)) in regions.iter().enumerate() {
                                    let (r, mv) = parts[i];
                                    let (m2, c2) = fe.refine_part(
                                        refs, &sy, &nb, num_refs, lx + qx, ly + qy, pw, ph, lme, r, mv,
                                    );
                                    parts[i] = (r, m2);
                                    tot += c2;
                                }
                                best_c = tot;
                            }
                        }
                        if split_harvest::enabled() {
                            let won = match pick.as_ref().map(|p| p.0) {
                                Some(0) | None => 0u8,
                                Some(m) => m,
                            };
                            split_harvest::record(c16, best_c, lme, split_gate, won);
                        }
                        // Intra is ALWAYS a candidate (textured / occluded content):
                        // I_16x16 SATD + λ·mode bits.
                        let c_intra = fe.best_i16_satd(&sy, mb_x, mb_y)
                            + (lme * fe.tune_intra_penalty) as i64;
                        inter = if c_intra < best_c { None } else { pick };
                        fe.mb_was_skip[mb_idx] = false;
                        fe.mb_skip_sad[mb_idx] = skip_sad;
                    }

                    // ---- RD P_Skip ----------------------------------------
                    // The default criterion skips only when the residual quantizes
                    // to EXACTLY zero. That matches x264 at both extremes (akiyo
                    // 72.5% vs 73.6%, mobile 1.0% vs 1.4%) but falls 17-23 points
                    // short in the middle (foreman 6.4% vs 23.6%), because x264
                    // also skips macroblocks whose residual is small-but-nonzero.
                    // Decide it properly: trial-encode the chosen mode for real
                    // bits + reconstruction SSD, and compare J = SSD + lambda*R
                    // against the skip. Raw-SAD versions of this comparison fail
                    // badly (coding REPAIRS the residual, skipping keeps it), so
                    // the distortion term has to come from the reconstruction.
                    if fe.rd_skip && rdskip_on && inter.is_some() {
                        let skip_cp = fe.skip_predict_chroma(refs, mb_x, mb_y, mv_skip);
                        // A P_Skip carries no residual, so its RECONSTRUCTION *is*
                        // its prediction — the skip SSD needs no state mutation at
                        // all. The commit / mb_ssd / restore round trip this
                        // replaces cost a full macroblock save+restore on every
                        // candidate, including the ones that go on to code.
                        let ssd_s = fe.pred_ssd(&sy, &su, &sv, mb_x, mb_y, &skip_y, &skip_cp);
                        debug_assert_eq!(ssd_s, {
                            let snap = fe.save_mb(mb_x, mb_y);
                            fe.commit_skip(mb_x, mb_y, mv_skip, &skip_y, &skip_cp);
                            let v = fe.mb_ssd(&sy, &su, &sv, mb_x, mb_y);
                            fe.load_mb(mb_x, mb_y, &snap);
                            v
                        }, "skip prediction SSD must equal the committed-skip reconstruction SSD");
                        // A skip inside a run costs ~1 bit of mb_skip_run.
                        let j_skip = ssd_s as f64 + lambda;
                        // Search-skip gate: when the null arm is this cheap it
                        // almost always wins, so take it without pricing the coded
                        // arm at all. This is where the decision's remaining cost
                        // lives — the coded arm is encoded and then discarded on
                        // 55-80% of candidates.
                        let take_skip = if fe.rd_skip_fast_t > 0.0
                            && (ssd_s as f64) <= lambda * fe.rd_skip_fast_t
                        {
                            true
                        } else {
                        // Otherwise encode ONCE, into scratch, and KEEP the state.
                        // If the skip loses, those are the real bits and they splice
                        // straight into the slice. The previous shape trial-encoded,
                        // threw the result away, and then encoded again — paying
                        // twice on the path that actually codes.
                            fe.save_mb_into(mb_x, mb_y, &mut rdskip_snap);
                            let mut scratch = BitWriter::new();
                            {
                                let (m, p) = inter.as_ref().unwrap();
                                fe.encode_inter_mb(
                                    &mut scratch, refs, &sy, &su, &sv, mb_x, mb_y, *m, p,
                                );
                            }
                            let bits_c = scratch.bit_len();
                            let ssd_c = fe.mb_ssd(&sy, &su, &sv, mb_x, mb_y);
                            let won = j_skip <= ssd_c as f64 + lambda * bits_c as f64;
                            if won {
                                fe.load_mb(mb_x, mb_y, &rdskip_snap); // undo it; take the skip
                                true
                            } else {
                                coded = Some(scratch); // keep it — no second encode
                                false
                            }
                        };
                        if take_skip {
                            fe.commit_skip(mb_x, mb_y, mv_skip, &skip_y, &skip_cp);
                            if !fe.fast {
                                fe.mb_was_skip[mb_idx] = true;
                                fe.mb_skip_sad[mb_idx] = skip_sad;
                            }
                            mb_qpy[mb_idx] = fe.cur_qp;
                            if precomp {
                                bs_grid[mb_idx] = derive_mb_bs_from(
                                    &fe, mb_x, mb_y,
                                    rusty_h264_common::deblock::MbKind::Skip,
                                );
                            }
                            skip_run += 1;
                            continue;
                        }
                    }
                }
                w.write_ue(skip_run); // run of skipped macroblocks before this one
                skip_run = 0;
            }
            if mv_force_on() && is_p && inter.is_some() {
                let fi = MVCMP_FRAME.load(std::sync::atomic::Ordering::Relaxed);
                let ext = EXT_MV.lock().unwrap();
                if let Some(field) = ext.get(fi) {
                    let w4 = fe.mb_w * 4;
                    let b0 = (mb_y * 4) * w4 + mb_x * 4;
                    // uniform 16x16 only: a sub-partitioned macroblock has no single
                    // vector to transplant, so leave those to our own decision
                    let uniform = (0..4).all(|r| {
                        (0..4).all(|c| field.get(b0 + r * w4 + c) == field.get(b0))
                    });
                    if uniform {
                        if let Some(&emv) = field.get(b0) {
                            inter = Some((0, vec![(0, emv)]));
                            MVCMP[6].fetch_add(1, std::sync::atomic::Ordering::Relaxed);
                        }
                    }
                }
            }
            if mv_cmp_on() && is_p {
                if let Some((mode, parts)) = inter.as_ref() {
                    let fi = MVCMP_FRAME.load(std::sync::atomic::Ordering::Relaxed);
                    let ext = EXT_MV.lock().unwrap();
                    if let Some(field) = ext.get(fi) {
                        let bidx = (mb_y * 4) * (fe.mb_w * 4) + mb_x * 4;
                        if let Some(&emv) = field.get(bidx) {
                            let (mode, parts) = (*mode, parts.clone());
                            drop(ext);
                            // Both priced through the SAME pipeline: MC, transform,
                            // quantize, CAVLC. Real bits, real reconstruction SSD.
                            let (so, bo) =
                                fe.trial_inter(refs, &sy, &su, &sv, mb_x, mb_y, mode, &parts);
                            let (se, be) = fe.trial_inter(
                                refs, &sy, &su, &sv, mb_x, mb_y, 0, &[(0, emv)],
                            );
                            let jo = so as f64 + lambda * bo as f64;
                            let je = se as f64 + lambda * be as f64;
                            use std::sync::atomic::Ordering::Relaxed;
                            MVCMP[0].fetch_add(1, Relaxed);
                            MVCMP[1].fetch_add(bo as u64, Relaxed);
                            MVCMP[2].fetch_add(be as u64, Relaxed);
                            MVCMP[3].fetch_add(so.max(0) as u64, Relaxed);
                            MVCMP[4].fetch_add(se.max(0) as u64, Relaxed);
                            MVCMP[5].fetch_add((je < jo) as u64, Relaxed);
                            MVCMP[6].fetch_add((parts[0].1 != emv) as u64, Relaxed);
                        }
                    }
                }
            }
            // Capture the kind before `inter` is consumed: the deblocking
            // strengths of an intra macroblock are pure constants.
            let mb_kind = match &inter {
                // A single partition covers the whole macroblock with one
                // (ref, mv), which collapses the internal derivation to nnz.
                Some((_, parts)) if parts.len() == 1 => {
                    rusty_h264_common::deblock::MbKind::InterUniform
                }
                Some(_) => rusty_h264_common::deblock::MbKind::Inter,
                None => rusty_h264_common::deblock::MbKind::Intra,
            };
            match inter {
                Some((mode, parts)) => match coded {
                    // Encoded already, during the skip decision — splice the bits in
                    // rather than encoding this macroblock for a second time.
                    Some(sc) => w.append(&sc),
                    None => {
                        fe.encode_inter_mb(w, refs, &sy, &su, &sv, mb_x, mb_y, mode, &parts)
                    }
                },
                None => encode_mb(&mut fe, w, mb_x, mb_y, &sy, &su, &sv, is_p),
            }
            mb_qpy[mb_idx] = fe.cur_qp; // ACTUAL QPy (updated iff an mb_qp_delta was coded)
            if precomp {
                bs_grid[mb_idx] = derive_mb_bs_from(&fe, mb_x, mb_y, mb_kind);
            }
        }
    }
    debug_assert!(
        !precomp || bs_grid.iter().all(|b| *b != rusty_h264_common::deblock::MbBs::UNSET),
        "a macroblock loop exit failed to store its boundary strengths"
    );
    if is_p && skip_run > 0 {
        w.write_ue(skip_run); // trailing skipped macroblocks
    }
    w.rbsp_trailing_bits();

    // Deblock the reconstruction; the result is the inter reference. Baseline: the
    // intra mask is `!inter_y` (passed directly, no alloc); no B (List-1 empty); no
    // 8×8 transform (t8x8 empty). ref_id is each block's List-0 ref index.
    drop(_g_loop);
    let _g_fin = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncFinal);
    // No NO_REF-mapping collect: it ran over every 4x4 block every frame (~1.9 MB
    // of allocation + map at 1080p) to produce a grid that is only ever read for
    // INTER-vs-INTER comparisons, where the encoder's raw indices are already
    // equivalent. Intra blocks short-circuit before reference identity is touched.
    let info = rusty_h264_common::deblock::BlockInfo {
        inter: &fe.inter_y,
        nnz: &fe.nnz_y,
        mv: &fe.mv_y,
        ref_id: &fe.ref_idx_y,
        mv1: &[],
        ref_id1: &[],
        w4: fe.mb_w * 4,
        t8x8: &[],
        bs: &bs_grid,
    };
    // Per-MB actual QPy (AQ varies it; `mb_qp_delta`-driven). With `aq_strength 0`
    // this is uniform, reproducing the old scalar-QP filtering exactly.
    drop(_g_fin);
    rusty_h264_common::deblock::filter_frame(
        &mut fe.rec_y,
        &mut fe.rec_u,
        &mut fe.rec_v,
        fe.mb_w,
        fe.mb_h,
        &mb_qpy,
        0, // chroma_qp_index_offset — the encoder emits 0
        0, // slice_alpha_c0_offset — the encoder always signals zero offsets
        0, // slice_beta_offset
        &info,
    );
    let w4 = fe.mb_w * 4;
    crate::RefFrame {
        y: fe.rec_y,
        u: fe.rec_u,
        v: fe.rec_v,
        poc: 0,       // set by the caller (it knows the display order)
        frame_num: 0, // set by the caller
        // List-0 motion field, for a later B-frame's spatial-direct colZeroFlag.
        mv: fe.mv_y,
        ref_idx: fe.ref_idx_y,
        w4,
        // Filtered lazily on first sub-pel search use (see `RefFrame::hpel`).
        hpel: std::sync::OnceLock::new(),
    }
}

/// Codes a B-slice's macroblock layer. B-frames are **non-reference**, so the
/// reconstruction is computed (the CAVLC nnz predictor needs it) but discarded.
///
/// This brick: every MB is coded `B_L0_16x16` (`mb_type == 1`) — a real
/// motion-compensated prediction from `l0` (the nearest PAST anchor, List-0 index
/// 0) plus a coded residual. Because every MB is List-0-only with `ref_idx`
/// inferred 0, the per-4×4 List-0 motion field and its median `mvd` predictor are
/// byte-identical to the P-slice `P_L0_16x16` path — so this reuses
/// [`FrameEncoder::encode_inter_mb_v1_b`] verbatim, differing from P only in the
/// `mb_type` value. `l1` (nearest future anchor) is unused until `B_Bi` lands.
#[allow(clippy::too_many_arguments)]
#[allow(clippy::too_many_arguments)]
pub fn encode_slice_data_b(
    w: &mut BitWriter,
    cfg: &EncoderConfig,
    frame: &YuvFrame,
    qp: u8,
    poc: i32,
    l0: &crate::RefFrame,
    l1: &crate::RefFrame,
    qpo: &[i32],
) {
    let mut fe = FrameEncoder::new(cfg);
    fe.qp = qp;
    fe.qpc = chroma_qp(qp);
    fe.cur_qp = qp;
    if cfg.cabac_dz_div > 0 {
        fe.idz = cfg.cabac_dz_div; // CABAC-specific dead-zone override
    } // QPY_PREV starts at the slice QP so the first mb_qp_delta is 0
    // Implicit bi-prediction weights from the anchor POC distances (matches the
    // decoder). Equidistant B (bframes==1) → 32:32 (plain average); unequal → weighted.
    fe.bi_w = implicit_bi_weights(poc, l0.poc, l1.poc);
    let (sy, su, sv) = coded_source(cfg, frame);
    let lambda = 0.85 * fe.tune_lambda_scale * 2f64.powf((qp as f64 - 12.0) / 3.0);
    let lme = lambda.sqrt();
    let refs = std::slice::from_ref(l0); // List-0 = [nearest past anchor]
    // Same content-adaptive SAD→SATD dispatch as the P path (codec-content-adaptive-
    // dispatch): the top `satd_q` fraction of highest-variance MBs price by SATD.
    if fe.satd_q > 0.0 {
        let mut vars: Vec<i64> = (0..fe.mb_h)
            .flat_map(|my| (0..fe.mb_w).map(move |mx| (mx, my)))
            .map(|(mx, my)| mb_variance(&sy, fe.cw, mx, my))
            .collect();
        vars.sort_unstable();
        let idx = (((1.0 - fe.satd_q) * vars.len() as f64) as usize).min(vars.len() - 1);
        fe.satd_var_thresh = vars[idx];
    }
    let mut skip_run = 0u32; // run of consecutive B_Skip MBs pending a coded MB
    for mb_y in 0..fe.mb_h {
        for mb_x in 0..fe.mb_w {
            let (lx, ly) = (mb_x * 16, mb_y * 16);
            let (pbx, pby) = (mb_x as isize * 4, mb_y as isize * 4);
            fe.mb_use_satd =
                fe.satd_q > 0.0 && mb_variance(&sy, fe.cw, mb_x, mb_y) >= fe.satd_var_thresh;
            // Per-list median MV predictors — the search-rate center AND the actual
            // `mvd` predictor (identical to the decoder's `predict_partition_mv`).
            let n0 = fe.mv_neighbors_block_list(pbx, pby, 4, 0);
            let n1 = fe.mv_neighbors_block_list(pbx, pby, 4, 1);
            let pmv0 = predict_partition_mv(0, 0, n0[0], n0[1], n0[2], 0);
            let pmv1 = predict_partition_mv(0, 0, n1[0], n1[1], n1[2], 0);
            // Independent List-0 / List-1 motion searches (their J already includes
            // the mvd rate against the matching predictor, so J0/J1 compare directly).
            // Spatial-direct prediction (basis of B_Skip and B_Direct_16x16).
            let (dp, dc, dmotion) = fe.b_direct(l0, l1, mb_x, mb_y);
            // B_Skip: take the direct prediction with NO coded residual (~1 bit in
            // the mb_skip_run) only when it is truly FREE — its residual quantizes to
            // zero at the B QP, so skipping loses nothing. (A looser SATD-threshold
            // skip was measured strictly WORSE: on B's derived prediction the SATD
            // proxy over-values the skip, dropping residual the quantizer wanted —
            // the same proxy-vs-quantization gap seen on sub-pel. So skip only when
            // provably free; the rest goes through the L0/L1/Bi/Direct RD decision.)
            if fe.skip_luma_is_free(&sy, mb_x, mb_y, &dp)
                && fe.skip_chroma_is_free(&su, &sv, mb_x, mb_y, &dc)
            {
                fe.commit_direct_motion(mb_x, mb_y, &dmotion);
                skip_run += 1;
                continue;
            }
            let d_direct = fe.pred_dist(&sy, lx, ly, &dp);
            let (mv0, j0) = fe.motion_search(l0, &sy, lx, ly, 16, 16, &[pmv0], lme, None);
            let (mv1, j1) = fe.motion_search(l1, &sy, lx, ly, 16, 16, &[pmv1], lme, None);
            // Bi: average the two winners' predictions; rate = both mvds.
            let d_bi = fe.bi_dist(l0, l1, &sy, lx, ly, mv0, mv1);
            let r_bi = mvd_bits(mv0.0 - pmv0.0) + mvd_bits(mv0.1 - pmv0.1)
                + mvd_bits(mv1.0 - pmv1.0) + mvd_bits(mv1.1 - pmv1.1);
            let j_bi = d_bi + (lme * r_bi as f64) as i64;
            // B_Direct (mb_type 0): spatial-direct prediction, NO coded MV — so its
            // J (d_direct, computed above) carries zero mvd rate and it wins wherever
            // the derived motion predicts as well as an explicit vector.
            // Pick the cheapest of {0=Direct, 1=L0, 2=L1, 3=Bi}; Direct wins ties.
            let (mut dir, mut best) = (0u8, d_direct);
            if j0 < best { dir = 1; best = j0; }
            if j1 < best { dir = 2; best = j1; }
            if j_bi < best { dir = 3; best = j_bi; }
            let _ = best;
            w.write_ue(skip_run); // run of B_Skips preceding this coded MB
            skip_run = 0;
            let bspec = BInter { dir, l1, mv0, mv1 };
            fe.encode_inter_mb_v1_b(w, refs, &sy, &su, &sv, mb_x, mb_y, 0, &[], Some(bspec));
        }
    }
    if skip_run > 0 {
        w.write_ue(skip_run); // trailing B_Skip run
    }
    w.rbsp_trailing_bits();
}

/// `se(d)` Exp-Golomb bit length — the `mvd`-component rate for the B mode
/// decision. Same closed form as `motion_search`'s private `mvbits` (kept separate
/// so the P search's heuristic — and thus P output — is untouched).
#[inline(always)]
fn mvd_bits(d: i32) -> u32 {
    let codenum = if d > 0 { (2 * d - 1) as u32 } else { (-2 * d) as u32 };
    1 + 2 * (31 - (codenum + 1).leading_zeros())
}

/// Reads a 4×4 residual block (source minus a raster prediction block).
/// Writes `ref_idx_l0` (spec: `te(v)` when two references are active — a single
/// flag — else `ue(v)`). Only called when more than one reference is active.
fn write_ref_idx(w: &mut BitWriter, refi: i32, num_refs: usize) {
    if num_refs == 2 {
        w.write_bit(refi == 0); // te(v): value = !bit
    } else {
        w.write_ue(refi as u32);
    }
}

/// Approximate bit cost of coding `ref_idx = r` with `num_refs` active, for the
/// motion-estimation rate term. Zero with a single reference (no `ref_idx` coded).
fn ref_bits(r: usize, num_refs: usize) -> u32 {
    if num_refs <= 1 {
        0
    } else if num_refs == 2 {
        1
    } else {
        let mut n = r as u32 + 1;
        let mut len = 1;
        while n > 1 {
            n >>= 1;
            len += 2;
        }
        len
    }
}

fn residual(src: &[u8], stride: usize, x0: usize, y0: usize, pred: &[i32; 16]) -> [i32; 16] {
    let mut r = [0i32; 16];
    for dy in 0..4 {
        for dx in 0..4 {
            r[dy * 4 + dx] = src[(y0 + dy) * stride + (x0 + dx)] as i32 - pred[dy * 4 + dx];
        }
    }
    r
}

/// Writes reconstructed samples back into a plane.
fn store(plane: &mut [u8], stride: usize, x0: usize, y0: usize, s: &[u8; 16]) {
    for dy in 0..4 {
        for dx in 0..4 {
            plane[(y0 + dy) * stride + (x0 + dx)] = s[dy * 4 + dx];
        }
    }
}

/// Extracts the 4×4 raster prediction block at `(bx, by)` from a 16×16 (256-sample)
/// luma prediction.
fn pred_block(pred: &[u8; 256], bx: usize, by: usize) -> [i32; 16] {
    let mut p = [0i32; 16];
    for dy in 0..4 {
        for dx in 0..4 {
            p[dy * 4 + dx] = pred[(by * 4 + dy) * 16 + (bx * 4 + dx)] as i32;
        }
    }
    p
}

/// Sum of absolute transformed differences over a 16×16 luma macroblock — the
/// mode-decision cost (correlates with coded bits better than plain SAD).
/// SATD of a `w`×`h` luma block: `src` (stride `ss`) vs `pred` (stride `ps`).
///
/// With `--features asm` and a supported size this is `2 · WelsSampleSatd_sse2`, which
/// is **byte-identical** to the scalar `Σ|H·d|` Hadamard: the openh264 kernel returns
/// `(Σ+1)>>1`, and `Σ` is always even (every 4×4 Hadamard coefficient shares the block
/// sum's parity, so 16 of them sum even), so `×2` recovers `Σ` exactly — proven over
/// 20 k random blocks at 4×4/8×8/16×16 in `tests/satd_asm_compare.rs`. Without asm (or
/// for an unsupported size) it falls back to the scalar Hadamard — the original path.
#[inline]
pub(crate) fn satd_px(src: &[u8], ss: usize, pred: &[u8], ps: usize, w: usize, h: usize) -> i64 {
    #[cfg(accel)]
    {
        let asm = match (w, h) {
            (16, 16) => Some(rusty_h264_accel::satd_16x16(src, ss, pred, ps)),
            (16, 8) => Some(rusty_h264_accel::satd_16x8(src, ss, pred, ps)),
            (8, 16) => Some(rusty_h264_accel::satd_8x16(src, ss, pred, ps)),
            (8, 8) => Some(rusty_h264_accel::satd_8x8(src, ss, pred, ps)),
            (4, 4) => Some(rusty_h264_accel::satd_4x4(src, ss, pred, ps)),
            _ => None,
        };
        if let Some(v) = asm {
            return 2 * v as i64;
        }
    }
    // Scalar Hadamard (also the no-asm path): Σ over the 4×4 sub-blocks.
    let (nbx, nby) = (w / 4, h / 4);
    let mut blocks = [[0i32; 16]; 16];
    let mut bi = 0;
    for by in 0..nby {
        for bx in 0..nbx {
            let blk = &mut blocks[bi];
            for dy in 0..4 {
                for dx in 0..4 {
                    blk[dy * 4 + dx] =
                        src[(by * 4 + dy) * ss + bx * 4 + dx] as i32 - pred[(by * 4 + dy) * ps + bx * 4 + dx] as i32;
                }
            }
            bi += 1;
        }
    }
    satd_4x4_sum(&blocks[..nbx * nby])
}

/// SAD of a `w`×`h` block: `src` (stride `ss`) vs a strided region `r` (stride `rs`)
/// — the openh264 `psadbw` kernels for the shapes that ship them (they take strides,
/// so in-place plane reads need NO materialize), scalar `Σ abs_diff` rows otherwise
/// (LLVM lowers the idiom to `psadbw` for contiguous rows).
#[inline]
fn sad_strided(src: &[u8], ss: usize, r: &[u8], rs: usize, w: usize, h: usize) -> i64 {
    #[cfg(accel)]
    {
        match (w, h) {
            (16, 16) => return rusty_h264_accel::sad_16x16(src, ss, r, rs) as i64,
            (16, 8) => return rusty_h264_accel::sad_16x8(src, ss, r, rs) as i64,
            (8, 16) => return rusty_h264_accel::sad_8x16(src, ss, r, rs) as i64,
            _ => {}
        }
    }
    let mut sad = 0u32;
    for dy in 0..h {
        let a = &src[dy * ss..][..w];
        let b = &r[dy * rs..][..w];
        sad += a.iter().zip(b).map(|(&x, &y)| x.abs_diff(y) as u32).sum::<u32>();
    }
    sad as i64
}

/// Fused `SAD(src, (a+b+1)>>1)` — the quarter-pel SAD without materializing the
/// average (the B2 sibling of the A3 `satd_avg` kernel, scalar because the avg+SAD
/// idiom auto-vectorizes and quarter-phase SAD evals are seed-frequency only).
#[inline]
fn sad_avg_strided(src: &[u8], ss: usize, a: &[u8], b: &[u8], rs: usize, w: usize, h: usize) -> i64 {
    let mut sad = 0u32;
    for dy in 0..h {
        let s = &src[dy * ss..][..w];
        let pa = &a[dy * rs..][..w];
        let pb = &b[dy * rs..][..w];
        for i in 0..w {
            let p = ((pa[i] as u16 + pb[i] as u16 + 1) >> 1) as u8;
            sad += s[i].abs_diff(p) as u32;
        }
    }
    sad as i64
}

fn satd_16x16(src: &[u8], stride: usize, lx: usize, ly: usize, pred: &[u8; 256]) -> i64 {
    satd_px(&src[ly * stride + lx..], stride, pred, 16, 16, 16)
}

/// SAD over a 16×16 luma macroblock against a prediction — the fast preset's
/// intra cost, kept in the same (SAD) domain as its inter cost. `Σ a.abs_diff(b)`
/// over `u8` slices auto-vectorizes to `psadbw`.
fn sad_16x16(src: &[u8], stride: usize, lx: usize, ly: usize, pred: &[u8; 256]) -> i64 {
    let mut sad = 0u32;
    for dy in 0..16 {
        let s = &src[(ly + dy) * stride + lx..][..16];
        let p = &pred[dy * 16..][..16];
        sad += s.iter().zip(p).map(|(&a, &b)| a.abs_diff(b) as u32).sum::<u32>();
    }
    sad as i64
}

/// SATD over an 8×8 chroma block (four 4×4 sub-blocks) against a prediction.
fn satd_8x8(src: &[u8], stride: usize, x0: usize, y0: usize, pred: &[u8; 64]) -> i64 {
    satd_px(&src[y0 * stride + x0..], stride, pred, 8, 8, 8)
}

/// SATD of one 4×4 luma block against a prediction.
fn satd_4x4(src: &[u8], stride: usize, px: usize, py: usize, pred: &[u8; 16]) -> i64 {
    satd_px(&src[py * stride + px..], stride, pred, 4, 4, 4)
}

/// Whether an `Intra_4x4` mode is usable given top/left neighbor availability.
fn i4_mode_available(mode: u8, top: bool, left: bool) -> bool {
    match mode {
        0 | 3 | 7 => top,        // vertical, diag-down-left, vertical-left
        1 | 8 => left,           // horizontal, horizontal-up
        2 => true,               // DC
        _ => top && left,        // diag-down-right, vertical-right, horizontal-down
    }
}

/// Result of planning an I_4x4 macroblock (luma). Reconstruction has already
/// been written into the frame's `rec_y` and `coded_y` by [`plan_i4x4`].
struct I4Plan {
    modes: [u8; 16],       // per-block intra4x4 mode, raster [lby*4+lbx]
    q: [[i32; 16]; 16],    // per-block quantized coefficients (full, raster)
    cbp_luma: u32,         // 4-bit coded-block-pattern (one bit per 8×8 region)
    nonzero: i64,          // total non-zero coefficients (rate proxy)
}

/// A fully-decided intra macroblock: the mode decision, the quantized coefficients,
/// and the committed reconstruction. Produced by [`plan_mb`] (which reuses the
/// entire mode-decision + transform + reconstruct path), then consumed by an
/// entropy backend — `emit_mb_cavlc` or `emit_mb_cabac` — so the two coders share
/// every non-entropy decision bit-for-bit (the bringup-encoder reuse guarantee).
struct MbPlan {
    use_i4: bool,
    // I_16x16 (when !use_i4): prediction mode, whether any AC is coded (cbp_luma=15),
    // luma DC levels (block order), per-4×4 quantized AC (raster).
    i16_mode: I16Mode,
    i16_cbp15: bool,
    i16_dc_levels: [i32; 16],
    i16_q: [[i32; 16]; 16],
    // I_4x4 (when use_i4 && i8 is None): the sub-plan, already reconstructed.
    i4: Option<I4Plan>,
    // I_8x8 (High profile; when use_i4 && i8 is Some): the sub-plan, already
    // reconstructed. use_i4 means "I_NxN"; i8 present disambiguates 8x8 from 4x4.
    i8: Option<I8Plan>,
    // Chroma (shared by both luma types).
    chroma_mode: u8,
    cbp_chroma: u32,
    c_dc_levels: [[i32; 4]; 2],
    c_q_blocks: [[[i32; 16]; 4]; 2],
}

/// A fully-decided inter macroblock: the per-partition motion residuals, coded
/// block pattern, and quantized residual, with the reconstruction + motion grids
/// already committed. Produced by [`FrameEncoder::plan_inter_mb`] (which reuses the
/// whole MC + residual + reconstruct path), then coded by `emit_inter_cavlc` or
/// `emit_inter_cabac` — so the two entropy backends share every non-entropy
/// decision bit-for-bit (the P/B analogue of [`MbPlan`]).
struct InterPlan {
    mvds: [(i32, i32); 4], // per-partition mvd (P: mvd_l0; B: mvd_l0 then mvd_l1)
    plan_refs: [i32; 4],   // per-partition ref_idx_l0 (multi-ref P; 0 for B / single-ref)
    n_mvd: usize,
    cbp: u32,
    q_blocks: [[i32; 16]; 16], // luma quantized levels (raster) — used when !t8x8
    c_dc_levels: [[i32; 4]; 2],
    c_q: [[[i32; 16]; 4]; 2],
    t8x8: bool,           // transform_size_8x8_flag (High profile, 8x8 luma residual)
    q8: [[i32; 64]; 4],   // per-8x8-block quantized levels (raster) — used when t8x8
}

/// Gathers the 4×4 luma intra neighbors at pixel `(px, py)` from `rec_y`.
fn gather_i4(
    fe: &FrameEncoder,
    px: usize,
    py: usize,
    avail_top: bool,
    avail_left: bool,
    bx: usize,
    by: usize,
) -> ([u8; 8], [u8; 4], u8) {
    let (cw, w4) = (fe.cw, fe.mb_w * 4);
    let mut top = [0u8; 8];
    let mut left = [0u8; 4];
    let mut corner = 0;
    if avail_top {
        for i in 0..4 {
            top[i] = fe.rec_y[(py - 1) * cw + px + i];
        }
        let tr_avail = bx + 1 < w4 && fe.coded_y[(by - 1) * w4 + (bx + 1)];
        for i in 0..4 {
            top[4 + i] = if tr_avail {
                fe.rec_y[(py - 1) * cw + px + 4 + i]
            } else {
                top[3]
            };
        }
    }
    if avail_left {
        for i in 0..4 {
            left[i] = fe.rec_y[(py + i) * cw + px - 1];
        }
    }
    if avail_top && avail_left {
        corner = fe.rec_y[(py - 1) * cw + px - 1];
    }
    (top, left, corner)
}

/// Plans an I_4x4 macroblock: picks a mode per 4×4 block (lowest-SATD available
/// mode), quantizes, and reconstructs serially into `rec_y` so each block can
/// predict from the previous one.
/// Neighbour 4x4 block intra mode for the MPM candidate: in-MB blocks read the
/// in-progress local `modes`; blocks in earlier MBs read `fe.modes_y`. (bx, by)
/// are the current block's absolute 4x4 grid coords.
#[inline]
fn modes_at(fe: &FrameEncoder, modes: &[u8; 16], lbx: usize, lby: usize, dx: isize, dy: isize, bx: usize, by: usize) -> u8 {
    let (nx, ny) = (lbx as isize + dx, lby as isize + dy);
    if (0..4).contains(&nx) && (0..4).contains(&ny) {
        modes[ny as usize * 4 + nx as usize]
    } else {
        let w4 = fe.mb_w * 4;
        let gx = (bx as isize + dx) as usize;
        let gy = (by as isize + dy) as usize;
        fe.modes_y[gy * w4 + gx]
    }
}

fn plan_i4x4(fe: &mut FrameEncoder, sy: &[u8], mb_x: usize, mb_y: usize, qp: u8) -> I4Plan {
    let w4 = fe.mb_w * 4;
    let mut modes = [2u8; 16];
    let mut q = [[0i32; 16]; 16];
    let mut cbp_luma = 0u32;
    let mut nonzero = 0i64;

    for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
        let (bx, by) = (mb_x * 4 + lbx, mb_y * 4 + lby);
        let (px, py) = (bx * 4, by * 4);
        let avail_top = by > 0;
        let avail_left = bx > 0;
        let (top, left, corner) = gather_i4(fe, px, py, avail_top, avail_left, bx, by);

        // Pick the lowest-SATD available mode. RUSTY_FAST_INTRA prunes the
        // candidate set to {MPM, DC, V, H} (x264-ultrafast-style); the H.264
        // predicted mode (min of left/top block modes, DC on the edge) keeps the
        // 1-bit prev_intra4x4_pred_mode signalling cheap for the common winner.
        let mut best_m = 2u8;
        let mut best_cost = i64::MAX;
        if fe.fast && fast_intra_enabled() {
            let lm = if bx > 0 { modes_at(fe, &modes, lbx, lby, -1, 0, bx, by) } else { 2 };
            let tm = if by > 0 { modes_at(fe, &modes, lbx, lby, 0, -1, bx, by) } else { 2 };
            let mpm = lm.min(tm);
            let mut cands = [mpm, 2u8, 0, 1];
            for i in 1..4 {
                for j in 0..i {
                    if cands[i] == cands[j] {
                        cands[i] = 255;
                    }
                }
            }
            for &m in cands.iter() {
                if m == 255 || !i4_mode_available(m, avail_top, avail_left) {
                    continue;
                }
                let pred = intra4x4_pred(m, avail_top, avail_left, &top, &left, corner);
                let cost = satd_4x4(sy, fe.cw, px, py, &pred);
                if cost < best_cost {
                    best_cost = cost;
                    best_m = m;
                }
            }
        } else {
            for m in 0..9u8 {
                if !i4_mode_available(m, avail_top, avail_left) {
                    continue;
                }
                let pred = intra4x4_pred(m, avail_top, avail_left, &top, &left, corner);
                let cost = satd_4x4(sy, fe.cw, px, py, &pred);
                if cost < best_cost {
                    best_cost = cost;
                    best_m = m;
                }
            }
        }

        // Quantize + reconstruct with the chosen mode.
        let pred = intra4x4_pred(best_m, avail_top, avail_left, &top, &left, corner);
        let mut predb = [0i32; 16];
        for i in 0..16 {
            predb[i] = pred[i] as i32;
        }
        let res = residual(sy, fe.cw, px, py, &predb);
        let qb = rdoq(&forward_core(&res), qp, fe.idz, fe.rdoq_strength, 0); // full 16 incl DC
        let s = reconstruct_4x4(&dequantize(&qb, qp), &predb);
        store(&mut fe.rec_y, fe.cw, px, py, &s);
        fe.coded_y[by * w4 + bx] = true;

        let nz = qb.iter().filter(|&&v| v != 0).count();
        if nz > 0 {
            cbp_luma |= 1 << ((lby / 2) * 2 + (lbx / 2));
        }
        nonzero += nz as i64;
        modes[lby * 4 + lbx] = best_m;
        q[lby * 4 + lbx] = qb;
    }
    I4Plan {
        modes,
        q,
        cbp_luma,
        nonzero,
    }
}

/// A planned I_8x8 macroblock (High profile): one intra8x8 mode + one 8x8 DCT per
/// 8x8 block. Reconstructed serially into `rec_y` (each block predicts from the
/// previous), and `modes_y` written per block so later blocks' MPM sees earlier.
struct I8Plan {
    modes: [u8; 4],     // per-8x8-block intra8x8 mode (raster b8 0..3)
    q: [[i32; 64]; 4],  // per-8x8-block quantized levels (raster)
    cbp_luma: u32,      // 4-bit coded-block-pattern (one bit per 8x8 block)
    nonzero: i64,       // rate proxy
}

/// Forward zig-zag scan of a raster 8x8 block: `scan[i] = raster[ZIGZAG_8X8[i]]`
/// (the inverse of the decoder's `un_scan_8x8`).
const ZIGZAG_8X8: [usize; 64] = [
    0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, 20,
    13, 6, 7, 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, 44, 51, 58, 59,
    52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63,
];

#[inline]
fn scan_8x8_fwd(raster: &[i32; 64]) -> [i32; 64] {
    std::array::from_fn(|i| raster[ZIGZAG_8X8[i]])
}

/// Gather the 8x8 intra reference samples (top[16] incl top-right, left[8], corner)
/// from `rec_y` — the encoder counterpart of the decoder's `gather_i8`.
fn gather_i8_enc(
    fe: &FrameEncoder,
    px: usize,
    py: usize,
    avail_top: bool,
    avail_left: bool,
    bx: usize,
    by: usize,
) -> ([u8; 16], [u8; 8], u8, bool) {
    let (cw, w4) = (fe.cw, fe.mb_w * 4);
    let mut top = [0u8; 16];
    let mut left = [0u8; 8];
    let mut corner = 0;
    if avail_top {
        for i in 0..8 {
            top[i] = fe.rec_y[(py - 1) * cw + px + i];
        }
        let tr_avail = bx + 2 < w4 && fe.coded_y[(by - 1) * w4 + (bx + 2)];
        for i in 0..8 {
            top[8 + i] = if tr_avail {
                fe.rec_y[(py - 1) * cw + px + 8 + i]
            } else {
                top[7]
            };
        }
    }
    if avail_left {
        for i in 0..8 {
            left[i] = fe.rec_y[(py + i) * cw + px - 1];
        }
    }
    let avail_corner = avail_top && avail_left;
    if avail_corner {
        corner = fe.rec_y[(py - 1) * cw + px - 1];
    }
    (top, left, corner, avail_corner)
}

/// Plans an I_8x8 macroblock: per 8x8 block, picks the lowest-SATD intra8x8 mode,
/// 8x8-forward-transforms + quantizes, and reconstructs serially into `rec_y`.
fn plan_i8x8(fe: &mut FrameEncoder, sy: &[u8], mb_x: usize, mb_y: usize, qp: u8) -> I8Plan {
    let w4 = fe.mb_w * 4;
    let mut modes = [2u8; 4];
    let mut q = [[0i32; 64]; 4];
    let mut cbp_luma = 0u32;
    let mut nonzero = 0i64;
    let weight = [16i32; 64];

    for b8 in 0..4usize {
        let (b8x, b8y) = (b8 % 2, b8 / 2);
        let (px, py) = (mb_x * 16 + b8x * 8, mb_y * 16 + b8y * 8);
        let (bx, by) = (mb_x * 4 + b8x * 2, mb_y * 4 + b8y * 2); // top-left 4x4 cell
        let avail_top = b8y > 0 || mb_y > 0;
        let avail_left = b8x > 0 || mb_x > 0;
        let (top, left, corner, avail_corner) =
            gather_i8_enc(fe, px, py, avail_top, avail_left, bx, by);

        // Mode decision: lowest-SATD available intra8x8 mode (same 9 modes / avail
        // rules as intra4x4). The MPM (predict_i4_mode on the top-left 4x4) keeps the
        // 1-bit prev-mode signalling cheap; a small penalty biases toward it.
        let predicted = predict_i4_mode(fe, bx, by);
        let mut best_m = 2u8;
        let mut best_cost = i64::MAX;
        for m in 0..9u8 {
            if !i4_mode_available(m, avail_top, avail_left) {
                continue;
            }
            let pred = intra8x8_pred(m, avail_top, avail_left, avail_corner, &top, &left, corner);
            let mut cost = satd_8x8(sy, fe.cw, px, py, &pred);
            if m != predicted {
                cost += 4 * fe.qp as i64; // ~mode-signal penalty (rem vs prev flag)
            }
            if cost < best_cost {
                best_cost = cost;
                best_m = m;
            }
        }
        modes[b8] = best_m;

        // Forward 8x8 transform + quantize + reconstruct (shared decoder primitives).
        let pred = intra8x8_pred(best_m, avail_top, avail_left, avail_corner, &top, &left, corner);
        let mut res = [0i32; 64];
        for dy in 0..8 {
            for dx in 0..8 {
                res[dy * 8 + dx] =
                    sy[(py + dy) * fe.cw + (px + dx)] as i32 - pred[dy * 8 + dx] as i32;
            }
        }
        let levels = quantize_8x8(&forward_core_8x8(&res), qp, &weight, fe.idz);
        let nz = levels.iter().filter(|&&v| v != 0).count();
        if nz > 0 {
            cbp_luma |= 1 << b8;
        }
        nonzero += nz as i64;
        q[b8] = levels;

        let res_r = inverse_quant_8x8(&levels, qp, &weight);
        let predb: [i32; 64] = std::array::from_fn(|i| pred[i] as i32);
        let recon = add_residual_8x8(&res_r, &predb);
        for dy in 0..8 {
            for dx in 0..8 {
                fe.rec_y[(py + dy) * fe.cw + (px + dx)] = recon[dy * 8 + dx];
            }
        }
        // Publish the mode into all four 4x4 cells + mark coded — so the next 8x8
        // block's MPM (and later MBs' neighbours) see it, exactly as the decoder does.
        for sry in 0..2 {
            for srx in 0..2 {
                fe.modes_y[(by + sry) * w4 + (bx + srx)] = best_m;
                fe.coded_y[(by + sry) * w4 + (bx + srx)] = true;
            }
        }
    }
    I8Plan {
        modes,
        q,
        cbp_luma,
        nonzero,
    }
}

/// Inter 8×8-transform luma candidate. Forward-8×8 + quantize + reconstruct each of
/// the four 8×8 blocks of the motion-compensated residual `(source − pred_y)`, the
/// pure inverse of the decoder's t8x8 inter luma path (`inv_quant8` ∘ `un_scan_8x8`
/// ∘ `add_residual_8x8`). Returns the quantized levels, `cbp_luma`, a LEVEL-AWARE rate
/// estimate (Σ `rdoq_rate(|level|)` — charges the 8×8's fewer-but-larger coeffs at
/// their true bit cost, not a blind count), the 256-sample reconstruction, and its
/// SSD vs source. Inter deadzone `dz_div = 6`; scaling list flat (16).
#[allow(clippy::too_many_arguments)]
fn plan_inter8_luma(
    sy: &[u8],
    cw: usize,
    mb_x: usize,
    mb_y: usize,
    pred_y: &[u8; 256],
    qp: u8,
) -> ([[i32; 64]; 4], u32, f64, [u8; 256], i64) {
    let weight = [16i32; 64];
    let mut q8 = [[0i32; 64]; 4];
    let mut cbp = 0u32;
    let mut rate = 0f64;
    let mut rec = [0u8; 256];
    let mut ssd = 0i64;
    for b8 in 0..4usize {
        let (b8x, b8y) = (b8 % 2, b8 / 2);
        let mut res = [0i32; 64];
        for dy in 0..8 {
            for dx in 0..8 {
                let sx = mb_x * 16 + b8x * 8 + dx;
                let syy = mb_y * 16 + b8y * 8 + dy;
                let p = pred_y[(b8y * 8 + dy) * 16 + (b8x * 8 + dx)] as i32;
                res[dy * 8 + dx] = sy[syy * cw + sx] as i32 - p;
            }
        }
        let levels = quantize_8x8(&forward_core_8x8(&res), qp, &weight, 6);
        let mut nz = false;
        for &l in &levels {
            if l != 0 {
                nz = true;
                rate += rdoq_rate((l as i64).abs());
            }
        }
        if nz {
            cbp |= 1 << b8;
        }
        q8[b8] = levels;

        let res_r = inverse_quant_8x8(&levels, qp, &weight);
        let predb: [i32; 64] =
            std::array::from_fn(|i| pred_y[(b8y * 8 + i / 8) * 16 + (b8x * 8 + i % 8)] as i32);
        let recon = add_residual_8x8(&res_r, &predb);
        for dy in 0..8 {
            for dx in 0..8 {
                let ri = (b8y * 8 + dy) * 16 + (b8x * 8 + dx);
                rec[ri] = recon[dy * 8 + dx];
                let sx = mb_x * 16 + b8x * 8 + dx;
                let syy = mb_y * 16 + b8y * 8 + dy;
                let d = recon[dy * 8 + dx] as i64 - sy[syy * cw + sx] as i64;
                ssd += d * d;
            }
        }
    }
    (q8, cbp, rate, rec, ssd)
}

/// 16×16 luma intra prediction. For interior MBs (both neighbors available) this
/// dispatches to openh264's `WelsI16x16LumaPred*_sse2` (bit-identical to the spec
/// predictor); edge MBs (partial availability → C-only DC variants) use the scalar
/// path. The scalar `top`/`left`/`corner` are gathered by the caller regardless.
#[inline]
fn i16_pred(
    fe: &FrameEncoder,
    mode: I16Mode,
    avail_top: bool,
    avail_left: bool,
    top: &[u8; 16],
    left: &[u8; 16],
    corner: u8,
    lx: usize,
    ly: usize,
) -> [u8; 256] {
    #[cfg(accel)]
    if avail_top && avail_left {
        let mode_n = match mode {
            I16Mode::Vertical => 0,
            I16Mode::Horizontal => 1,
            I16Mode::Dc => 2,
            I16Mode::Plane => 3,
        };
        let mut p = AlignedMb([0; 256]);
        rusty_h264_accel::i16x16_luma_pred(mode_n, &mut p.0, &fe.rec_y[..], ly * fe.cw + lx, fe.cw);
        return p.0;
    }
    let _ = (fe, lx, ly);
    luma16x16_pred(mode, avail_top, avail_left, top, left, corner)
}

/// 8×8 chroma intra prediction. Interior MBs use openh264's `WelsIChromaPred{V,Plane}_sse2`
/// for the V/Plane modes (bit-identical); DC/Horizontal (C-only in openh264) and edge MBs
/// use the scalar path.
#[inline]
#[allow(clippy::too_many_arguments)]
fn chroma_pred(
    fe: &FrameEncoder,
    mode: u8,
    avail_top: bool,
    avail_left: bool,
    c: usize,
    top: &[u8; 8],
    left: &[u8; 8],
    corner: u8,
    cx: usize,
    cy: usize,
) -> [u8; 64] {
    #[cfg(accel)]
    if avail_top && avail_left && (mode == 2 || mode == 3) {
        let plane = if c == 0 { &fe.rec_u } else { &fe.rec_v };
        let mut p = AlignedMb([0; 256]);
        rusty_h264_accel::chroma8x8_pred(mode, &mut p.0[..64], &plane[..], cy * fe.ccw + cx, fe.ccw);
        let mut out = [0u8; 64];
        out.copy_from_slice(&p.0[..64]);
        return out;
    }
    let _ = (fe, c, cx, cy);
    chroma8x8_pred(mode, avail_top, avail_left, top, left, corner)
}

/// Predicted `Intra_4x4` mode for the block at absolute coords `(bx, by)` —
/// `min` of the left/top neighbor modes, or DC if either is unavailable.
fn predict_i4_mode(fe: &FrameEncoder, bx: usize, by: usize) -> u8 {
    if bx == 0 || by == 0 {
        return 2;
    }
    let w4 = fe.mb_w * 4;
    fe.modes_y[by * w4 + (bx - 1)].min(fe.modes_y[(by - 1) * w4 + bx])
}

#[allow(clippy::too_many_arguments)]
/// Zig-zag scan: block (raster 4×4) index at scan position i.
const RDOQ_ZZ: [usize; 16] = [0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15];

/// Approximate CABAC bit cost of coding one residual coefficient at magnitude
/// `level`: significant_coeff_flag (~1) + coeff_abs_level_minus1 bins (gt1 + UEG0)
/// + sign (~1); `level == 0` is significant_coeff_flag = 0 (~1). A coarse model —
/// the transform-norm/bin-to-bit scaling is absorbed into the calibrated strength.
#[inline]
fn rdoq_rate(level: i64) -> f64 {
    if level == 0 {
        1.0
    } else if level == 1 {
        3.0 // sig(1) + gt1=0 (1) + sign(1)
    } else {
        // sig(1) + gt1=1 (1) + UEG0(level-2) prefix (~level-1, capped) + sign(1)
        3.0 + (level - 1).min(13) as f64
    }
}

/// Rate-distortion optimized quantization (CABAC trellis, RDOQ) for one 4×4 residual
/// block. Refines the hard-decision levels toward min over {|q|, |q|-1} of
/// `SSD_coef + λ·R_cabac` per coefficient (coefficient-domain distortion
/// `(|coeff| - level·deq_step)²`; `λ = strength·2^((qp-12)/3)`). `strength == 0`
/// returns the hard quantization unchanged (the CAVLC path). `first` = 1 skips the
/// DC (AC-only categories: I_16x16 AC, chroma AC), else 0.
fn rdoq(coeffs: &[i32; 16], qp: u8, dz_div: i64, strength: f64, first: usize) -> [i32; 16] {
    let mut q = quantize(coeffs, qp, dz_div);
    if strength <= 0.0 {
        return q;
    }
    let lambda = strength * 2f64.powf((qp as f64 - 12.0) / 3.0);
    // Distortion is measured in the QUANTIZER-INPUT (forward-transform) domain, where
    // level L reconstructs to L·qstep, qstep = 2^16 / MF (the inverse of the forward
    // quant scale). The transform norm (forward↔pixel) folds into `strength`.
    let mf = &rusty_h264_common::transform::QUANT_MF_OH[qp as usize];
    const POS: [usize; 16] = [0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7];
    let dist = |p: usize, level: i64| -> f64 {
        let e = coeffs[p].unsigned_abs() as f64 - level as f64 * (65536.0 / mf[POS[p]] as f64);
        e * e
    };
    // Pass 1: per-coefficient level lowering (|q| → |q|-1) minimizing D + λ·R.
    for i in first..16 {
        let p = RDOQ_ZZ[i];
        let m = q[p].unsigned_abs() as i64;
        if m == 0 {
            continue;
        }
        let j_keep = dist(p, m) + lambda * rdoq_rate(m);
        let j_down = dist(p, m - 1) + lambda * rdoq_rate(m - 1);
        if j_down < j_keep {
            let nl = (m - 1) as i32;
            q[p] = if q[p] < 0 { -nl } else { nl };
        }
    }
    // Pass 2: last-significant-position trimming. Zeroing the trailing significant
    // coefficient frees its own bits AND the last_significant flag + every sig=0 flag
    // between it and the previous significant coefficient (positions past the new last
    // aren't coded at all) — the dominant RDOQ gain on sparse (inter) residuals.
    loop {
        let Some(li) = (first..16).rev().find(|&i| q[RDOQ_ZZ[i]] != 0) else {
            break;
        };
        let p = RDOQ_ZZ[li];
        let m = q[p].unsigned_abs() as i64;
        let prev = (first..li).rev().find(|&i| q[RDOQ_ZZ[i]] != 0);
        let base = prev.map_or(first, |j| j + 1);
        let bits = rdoq_rate(m) + 1.0 + (li - base) as f64; // coeff + last-flag + freed sig=0
        let d_add = dist(p, 0) - dist(p, m);
        if d_add < lambda * bits {
            q[p] = 0;
        } else {
            break;
        }
    }
    q
}

/// Decide one intra macroblock (I_16x16 vs I_4x4, prediction modes, chroma),
/// forward-transform + quantize, and commit the reconstruction + neighbour mode
/// state — everything except entropy coding. The returned [`MbPlan`] is coded by
/// either entropy backend, so CAVLC and CABAC share this whole path bit-for-bit.
fn plan_mb(
    fe: &mut FrameEncoder,
    mb_x: usize,
    mb_y: usize,
    sy: &[u8],
    su: &[u8],
    sv: &[u8],
) -> MbPlan {
    let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncIntraCode);
    let qp = fe.qp;
    let qpc = fe.qpc;
    // Lagrangian λ for rate-distortion decisions (standard H.264 form).
    let lambda = 0.85 * fe.tune_lambda_scale * 2f64.powf((qp as f64 - 12.0) / 3.0);

    // ---------------- luma ----------------
    let (lx, ly) = (mb_x * 16, mb_y * 16);
    let avail_top = mb_y > 0;
    let avail_left = mb_x > 0;
    let mut top = [0u8; 16];
    let mut left = [0u8; 16];
    if avail_top {
        for i in 0..16 {
            top[i] = fe.rec_y[(ly - 1) * fe.cw + lx + i];
        }
    }
    if avail_left {
        for i in 0..16 {
            left[i] = fe.rec_y[(ly + i) * fe.cw + lx - 1];
        }
    }
    let corner = if avail_top && avail_left {
        fe.rec_y[(ly - 1) * fe.cw + lx - 1]
    } else {
        0
    };

    let w4 = fe.mb_w * 4;

    // ============ I_16x16 plan (reconstruct into a local buffer) ============
    let mut i16_mode = I16Mode::Dc;
    let mut best_pred = i16_pred(fe, I16Mode::Dc, avail_top, avail_left, &top, &left, corner, lx, ly);
    let mut best_cost = satd_16x16(sy, fe.cw, lx, ly, &best_pred);
    for mode in [I16Mode::Vertical, I16Mode::Horizontal, I16Mode::Plane] {
        if !mode.available(avail_top, avail_left) {
            continue;
        }
        let pred = i16_pred(fe, mode, avail_top, avail_left, &top, &left, corner, lx, ly);
        let cost = satd_16x16(sy, fe.cw, lx, ly, &pred);
        if cost < best_cost {
            best_cost = cost;
            i16_mode = mode;
            best_pred = pred;
        }
    }
    // I_16x16 blocks are independent (one fixed whole-MB prediction), so batch the
    // forward DCT (`forward_dct_blocks` → SIMD), bit-identical to `forward_core`.
    let mut dc4x4 = [0i32; 16];
    let mut i16_q = [[0i32; 16]; 16];
    // Fast path: forward DCT of (src - pred) straight from the planes per 8x8 quad,
    // quantize with the identical FF/MF math (deadzone = fe.idz), recon via the
    // bit-identical idct+add+clip kernel — the same pairing encode_inter_mb and the
    // P_Skip free-check already use, byte-identical to the scalar twin below.
    #[cfg(accel)]
    let (i16_dc_levels, _i16_recon_dc, recon16) = {
        #[repr(align(16))]
        struct A([i16; 256]);
        let mut dct = A([0i16; 256]);
        let base = ly * fe.cw + lx;
        for (qi, &(qx, qy)) in [(0usize, 0usize), (8, 0), (0, 8), (8, 8)].iter().enumerate() {
            rusty_h264_accel::dct_four_t4(
                &mut dct.0[qi * 64..qi * 64 + 64],
                &sy[base + qy * fe.cw + qx..],
                fe.cw,
                &best_pred[qy * 16 + qx..],
                16,
            );
        }
        for (blk, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
            dc4x4[lby * 4 + lbx] = dct.0[blk * 16] as i32;
        }
        if fe.rdoq_strength > 0.0 {
            // Trellis (all-intra only): scalar RDOQ from the asm DCT output instead of
            // the asm hard quantizer. dct.0 keeps the raw DCT here; the recon loop below
            // overwrites it with the dequantized RDOQ levels.
            for (blk, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
                let coeffs: [i32; 16] = std::array::from_fn(|i| dct.0[blk * 16 + i] as i32);
                let mut q = rdoq(&coeffs, qp, fe.idz, fe.rdoq_strength, 1);
                q[0] = 0;
                i16_q[lby * 4 + lbx] = q;
            }
        } else {
            let ff = rusty_h264_common::transform::quant_dz_ff(qp, fe.idz);
            let mf = &rusty_h264_common::transform::QUANT_MF_OH[qp as usize];
            for qi in 0..4 {
                rusty_h264_accel::quant_four_4x4(&mut dct.0[qi * 64..qi * 64 + 64], &ff, mf);
            }
            for (blk, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
                let q = &mut i16_q[lby * 4 + lbx];
                q[0] = 0;
                for i in 1..16 {
                    q[i] = dct.0[blk * 16 + i] as i32;
                }
            }
        }
        let i16_dc_levels = forward_quant_luma_dc(&dc4x4, qp, true);
        let i16_recon_dc = inverse_quant_luma_dc(&i16_dc_levels, qp);
        // Recon: dequantize (DC injected from the Hadamard) back into quad layout,
        // then idct+add-pred+clip into the trial buffer.
        let mut recon16 = [0u8; 256];
        for (blk, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
            let mut deq = dequantize(&i16_q[lby * 4 + lbx], qp);
            deq[0] = i16_recon_dc[lby * 4 + lbx];
            for i in 0..16 {
                dct.0[blk * 16 + i] = deq[i] as i16;
            }
        }
        for (qi, &(qx, qy)) in [(0usize, 0usize), (8, 0), (0, 8), (8, 8)].iter().enumerate() {
            rusty_h264_accel::idct_four_t4_rec(
                &mut recon16[qy * 16 + qx..],
                16,
                &best_pred[qy * 16 + qx..],
                16,
                &dct.0[qi * 64..qi * 64 + 64],
            );
        }
        (i16_dc_levels, i16_recon_dc, recon16)
    };
    #[cfg(not(accel))]
    let (i16_dc_levels, _i16_recon_dc, recon16) = {
        let mut res_blocks = [[0i32; 16]; 16];
        for by in 0..4 {
            for bx in 0..4 {
                let predb = pred_block(&best_pred, bx, by);
                res_blocks[by * 4 + bx] = residual(sy, fe.cw, lx + bx * 4, ly + by * 4, &predb);
            }
        }
        let mut coeffs = [[0i32; 16]; 16];
        forward_dct_blocks(&res_blocks, &mut coeffs);
        for i in 0..16 {
            dc4x4[i] = coeffs[i][0];
            let mut q = rdoq(&coeffs[i], qp, fe.idz, fe.rdoq_strength, 1);
            q[0] = 0;
            i16_q[i] = q;
        }
        let i16_dc_levels = forward_quant_luma_dc(&dc4x4, qp, true);
        let i16_recon_dc = inverse_quant_luma_dc(&i16_dc_levels, qp);
        let mut recon16 = [0u8; 256];
        let mut deq_blocks = [[0i32; 16]; 16];
        for i in 0..16 {
            deq_blocks[i] = dequantize(&i16_q[i], qp);
            deq_blocks[i][0] = i16_recon_dc[i];
        }
        let mut idct = [[0i32; 16]; 16];
        inverse_dct_blocks(&deq_blocks, &mut idct);
        for by in 0..4 {
            for bx in 0..4 {
                let s = add_residual_4x4(&idct[by * 4 + bx], &pred_block(&best_pred, bx, by));
                for dy in 0..4 {
                    for dx in 0..4 {
                        recon16[(by * 4 + dy) * 16 + (bx * 4 + dx)] = s[dy * 4 + dx];
                    }
                }
            }
        }
        (i16_dc_levels, i16_recon_dc, recon16)
    };
    let i16_cbp15 = i16_q.iter().any(|b| b[1..].iter().any(|&c| c != 0));
    let i16_dc_nz = i16_dc_levels.iter().filter(|&&v| v != 0).count() as i64;
    let i16_ac_nz: i64 = i16_q
        .iter()
        .map(|b| b[1..].iter().filter(|&&v| v != 0).count() as i64)
        .sum();
    // I_16x16 AC is all-or-nothing: any AC ⇒ all 16 blocks pay a coeff_token.
    let i16_rate = i16_dc_nz + i16_ac_nz + if i16_cbp15 { 16 } else { 0 };
    // Reconstruction distortion (SSD) for the rate-distortion decision.
    let mut ssd16 = 0i64;
    for dy in 0..16 {
        for dx in 0..16 {
            let d = recon16[dy * 16 + dx] as i64 - sy[(ly + dy) * fe.cw + (lx + dx)] as i64;
            ssd16 += d * d;
        }
    }

    // ============ chroma (shared by both luma types; commit immediately) ============
    let (cx, cy) = (mb_x * 8, mb_y * 8);
    // Gather both components' neighbors, then pick a chroma mode by combined SATD.
    let mut ntop = [[0u8; 8]; 2];
    let mut nleft = [[0u8; 8]; 2];
    let mut ncorner = [0u8; 2];
    for c in 0..2 {
        let rec_c = if c == 0 { &fe.rec_u } else { &fe.rec_v };
        if avail_top {
            for i in 0..8 {
                ntop[c][i] = rec_c[(cy - 1) * fe.ccw + cx + i];
            }
        }
        if avail_left {
            for i in 0..8 {
                nleft[c][i] = rec_c[(cy + i) * fe.ccw + cx - 1];
            }
        }
        if avail_top && avail_left {
            ncorner[c] = rec_c[(cy - 1) * fe.ccw + cx - 1];
        }
    }
    let mut chroma_mode = 0u8;
    let mut best_c_cost = i64::MAX;
    for m in 0..4u8 {
        if !chroma_mode_available(m, avail_top, avail_left) {
            continue;
        }
        let mut cost = 0i64;
        for c in 0..2 {
            let src = if c == 0 { su } else { sv };
            let pred8 = chroma_pred(fe, m, avail_top, avail_left, c, &ntop[c], &nleft[c], ncorner[c], cx, cy);
            cost += satd_8x8(src, fe.ccw, cx, cy, &pred8);
        }
        if cost < best_c_cost {
            best_c_cost = cost;
            chroma_mode = m;
        }
    }

    let mut c_dc_levels = [[0i32; 4]; 2];
    let mut c_q_blocks = [[[0i32; 16]; 4]; 2];
    let mut any_chroma_ac = false;
    let mut any_chroma_dc = false;
    for c in 0..2 {
        let src = if c == 0 { su } else { sv };
        let pred8 =
            chroma_pred(fe, chroma_mode, avail_top, avail_left, c, &ntop[c], &nleft[c], ncorner[c], cx, cy);
        let pblk = |bx: usize, by: usize| -> [i32; 16] {
            let mut predb = [0i32; 16];
            for dy in 0..4 {
                for dx in 0..4 {
                    predb[dy * 4 + dx] = pred8[(by * 4 + dy) * 8 + (bx * 4 + dx)] as i32;
                }
            }
            predb
        };
        // Fast path: forward DCT of (src - pred8) straight from the planes, quantize
        // with identical FF/MF (idz deadzone), recon via one idct+add+clip kernel —
        // bit-identical to the scalar twin below (proven kernel pairings).
        let mut dc2x2 = [0i32; 4];
        let mut qbs = [[0i32; 16]; 4];
        #[cfg(accel)]
        let recon_dc = {
            #[repr(align(16))]
            struct A([i16; 64]);
            let mut d = A([0i16; 64]);
            rusty_h264_accel::dct_four_t4(&mut d.0, &src[cy * fe.ccw + cx..], fe.ccw, &pred8, 8);
            for i in 0..4 {
                dc2x2[i] = d.0[i * 16] as i32;
            }
            if fe.rdoq_strength > 0.0 {
                // Trellis (all-intra only): scalar RDOQ from the asm chroma DCT.
                for i in 0..4 {
                    let coeffs: [i32; 16] = std::array::from_fn(|j| d.0[i * 16 + j] as i32);
                    let mut q = rdoq(&coeffs, qpc, fe.idz, fe.rdoq_strength, 1);
                    q[0] = 0;
                    if q[1..].iter().any(|&v| v != 0) {
                        any_chroma_ac = true;
                    }
                    qbs[i] = q;
                }
            } else {
                let ff = rusty_h264_common::transform::quant_dz_ff(qpc, fe.idz);
                let mf = &rusty_h264_common::transform::QUANT_MF_OH[qpc as usize];
                rusty_h264_accel::quant_four_4x4(&mut d.0, &ff, mf);
                for i in 0..4 {
                    let q = &mut qbs[i];
                    q[0] = 0;
                    for j in 1..16 {
                        let v = d.0[i * 16 + j] as i32;
                        q[j] = v;
                        if v != 0 {
                            any_chroma_ac = true;
                        }
                    }
                }
            }
            let dl = forward_quant_chroma_dc(&dc2x2, qpc, true);
            if dl.iter().any(|&v| v != 0) {
                any_chroma_dc = true;
            }
            let recon_dc = inverse_quant_chroma_dc(&dl, qpc);
            for i in 0..4 {
                let deq = dequantize(&qbs[i], qpc);
                for j in 0..16 {
                    d.0[i * 16 + j] = deq[j] as i16;
                }
                d.0[i * 16] = recon_dc[i] as i16;
            }
            let plane = if c == 0 { &mut fe.rec_u } else { &mut fe.rec_v };
            rusty_h264_accel::idct_four_t4_rec(&mut plane[cy * fe.ccw + cx..], fe.ccw, &pred8, 8, &d.0);
            c_dc_levels[c] = dl;
            recon_dc
        };
        #[cfg(not(accel))]
        let recon_dc = {
            let mut res_blocks = [[0i32; 16]; 4];
            for by in 0..2 {
                for bx in 0..2 {
                    res_blocks[by * 2 + bx] =
                        residual(src, fe.ccw, cx + bx * 4, cy + by * 4, &pblk(bx, by));
                }
            }
            let mut coeffs = [[0i32; 16]; 4];
            forward_dct_blocks(&res_blocks, &mut coeffs);
            for i in 0..4 {
                dc2x2[i] = coeffs[i][0];
                let mut q = rdoq(&coeffs[i], qpc, fe.idz, fe.rdoq_strength, 1);
                q[0] = 0;
                qbs[i] = q;
                if q[1..].iter().any(|&v| v != 0) {
                    any_chroma_ac = true;
                }
            }
            let dl = forward_quant_chroma_dc(&dc2x2, qpc, true);
            if dl.iter().any(|&v| v != 0) {
                any_chroma_dc = true;
            }
            let recon_dc = inverse_quant_chroma_dc(&dl, qpc);
            let mut deq_blocks = [[0i32; 16]; 4];
            for i in 0..4 {
                deq_blocks[i] = dequantize(&qbs[i], qpc);
                deq_blocks[i][0] = recon_dc[i];
            }
            let mut idct = [[0i32; 16]; 4];
            inverse_dct_blocks(&deq_blocks, &mut idct);
            let plane = if c == 0 { &mut fe.rec_u } else { &mut fe.rec_v };
            for by in 0..2 {
                for bx in 0..2 {
                    let s = add_residual_4x4(&idct[by * 2 + bx], &pblk(bx, by));
                    store(plane, fe.ccw, cx + bx * 4, cy + by * 4, &s);
                }
            }
            c_dc_levels[c] = dl;
            recon_dc
        };
        let _ = recon_dc;
        c_q_blocks[c] = qbs;
    }
    let cbp_chroma: u32 = if any_chroma_ac {
        2
    } else if any_chroma_dc {
        1
    } else {
        0
    };

    // ============ I_NxN plan + RD: I_16x16 vs I_4x4 vs (High profile) I_8x8 ============
    // I_4x4 and I_8x8 both reconstruct serially into rec_y, but each block predicts
    // only from NEIGHBOURS + earlier blocks it fills itself — never the stale MB
    // content — so running I_8x8 after I_4x4 needs no restore. J = SSD + λ·R picks the
    // per-MB transform (the content-adaptive win: 8x8 on smooth, 4x4 on detail).
    let base = ly * fe.cw + lx;
    let i4 = if i16_rate > 2 {
        Some(plan_i4x4(fe, sy, mb_x, mb_y, qp))
    } else {
        None
    };
    let (j4, i4_recon) = match &i4 {
        Some(p) => {
            let mut ssd = 0i64;
            let mut rec = [0u8; 256];
            for i in 0..256 {
                let v = fe.rec_y[base + (i / 16) * fe.cw + i % 16];
                rec[i] = v;
                let d = v as i64 - sy[base + (i / 16) * fe.cw + i % 16] as i64;
                ssd += d * d;
            }
            (ssd as f64 + lambda * (p.nonzero + 16) as f64, Some(rec))
        }
        None => (f64::INFINITY, None),
    };
    let i8 = if fe.transform_8x8 {
        Some(plan_i8x8(fe, sy, mb_x, mb_y, qp))
    } else {
        None
    };
    let j8 = match &i8 {
        Some(p) => {
            let mut ssd = 0i64;
            for i in 0..256 {
                let d = fe.rec_y[base + (i / 16) * fe.cw + i % 16] as i64
                    - sy[base + (i / 16) * fe.cw + i % 16] as i64;
                ssd += d * d;
            }
            ssd as f64 + lambda * (p.nonzero + 16) as f64
        }
        None => f64::INFINITY,
    };
    let j16 = ssd16 as f64 + lambda * i16_rate as f64;

    // ============ commit the RD winner's reconstruction + neighbour modes ============
    let (use_i4, i4, i8) = if i8.is_some() && j8 <= j4 && j8 <= j16 {
        // I_8x8: plan_i8x8 already committed rec_y AND modes_y (per 8x8 block).
        (true, None, i8)
    } else if i4.is_some() && j4 < j16 {
        // I_4x4: restore its reconstruction (I_8x8 may have overwritten rec_y), publish modes.
        let rec = i4_recon.unwrap();
        for i in 0..256 {
            fe.rec_y[base + (i / 16) * fe.cw + i % 16] = rec[i];
        }
        let modes = i4.as_ref().unwrap().modes;
        for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
            fe.modes_y[(mb_y * 4 + lby) * w4 + (mb_x * 4 + lbx)] = modes[lby * 4 + lbx];
        }
        (true, i4, None)
    } else {
        // I_16x16: commit its reconstruction, mark modes DC.
        for by in 0..4 {
            for bx in 0..4 {
                for dy in 0..4 {
                    for dx in 0..4 {
                        fe.rec_y[(ly + by * 4 + dy) * fe.cw + (lx + bx * 4 + dx)] =
                            recon16[(by * 4 + dy) * 16 + (bx * 4 + dx)];
                    }
                }
            }
        }
        for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
            fe.modes_y[(mb_y * 4 + lby) * w4 + (mb_x * 4 + lbx)] = 2;
        }
        (false, None, None)
    };
    // Mark all luma blocks coded for the next macroblock's top-right availability.
    for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
        fe.coded_y[(mb_y * 4 + lby) * w4 + (mb_x * 4 + lbx)] = true;
    }

    MbPlan {
        use_i4,
        i16_mode,
        i16_cbp15,
        i16_dc_levels,
        i16_q,
        i4,
        i8,
        chroma_mode,
        cbp_chroma,
        c_dc_levels,
        c_q_blocks,
    }
}

/// Emit one planned intra macroblock as CAVLC (the original `encode_mb` tail). Reads
/// only the decided values from `plan`; `plan_mb` already committed recon + modes.
fn encode_mb(
    fe: &mut FrameEncoder,
    w: &mut BitWriter,
    mb_x: usize,
    mb_y: usize,
    sy: &[u8],
    su: &[u8],
    sv: &[u8],
    is_p: bool,
) {
    let plan = plan_mb(fe, mb_x, mb_y, sy, su, sv);
    // In a P-slice, intra macroblock types are offset by 5 (0..4 are inter).
    let mb_type_offset = if is_p { 5 } else { 0 };
    let w4 = fe.mb_w * 4;
    let cbp_chroma = plan.cbp_chroma;

    // ============ emit luma ============
    if let Some(i8) = plan.i8.as_ref().filter(|_| plan.use_i4) {
        // ---- I_8x8 (High profile): mb_type = I_NxN, transform_size_8x8_flag = 1, then
        // one intra8x8 mode per 8x8 block, cbp, mb_qp_delta, and the 8x8 residual as
        // four interleaved 4x4 CAVLC sub-blocks (coeff k of sub s -> 8x8 scan 4k+s). ----
        let cbp = i8.cbp_luma | (cbp_chroma << 4);
        w.write_ue(mb_type_offset); // mb_type = I_NxN
        w.write_bit(true); // transform_size_8x8_flag = 1
        for b8 in 0..4usize {
            let (bx, by) = (mb_x * 4 + (b8 % 2) * 2, mb_y * 4 + (b8 / 2) * 2);
            let predicted = predict_i4_mode(fe, bx, by);
            let actual = i8.modes[b8];
            if actual == predicted {
                w.write_bit(true);
            } else {
                w.write_bit(false);
                let rem = if actual < predicted { actual } else { actual - 1 };
                w.write_bits(rem as u32, 3);
            }
        }
        w.write_ue(plan.chroma_mode as u32); // intra_chroma_pred_mode
        write_cbp_intra(w, cbp);
        if cbp != 0 {
            w.write_se(fe.qp_delta());
        }
        fe.nnz_cache_load(mb_x, mb_y);
        for b8 in 0..4usize {
            let (b8x, b8y) = (b8 % 2, b8 / 2);
            let scan8 = scan_8x8_fwd(&i8.q[b8]);
            for sub in 0..4usize {
                let (cx, cy) = (b8x * 2 + sub % 2, b8y * 2 + sub / 2);
                let (bx, by) = (mb_x * 4 + cx, mb_y * 4 + cy);
                let total = if i8.cbp_luma & (1 << b8) != 0 {
                    let nc = fe.nc_pred(cx, cy);
                    let blk: [i32; 16] = std::array::from_fn(|k| scan8[4 * k + sub]);
                    encode_residual_block(w, &blk, 16, nc) as u8
                } else {
                    0
                };
                fe.nnz_cache_set(cx, cy, total);
                fe.nnz_y[by * w4 + bx] = total;
            }
        }
    } else if plan.use_i4 {
        let i4 = plan.i4.as_ref().unwrap();
        let cbp = i4.cbp_luma | (cbp_chroma << 4);
        w.write_ue(mb_type_offset); // mb_type = I_4x4 (+5 in P-slices)
        if fe.transform_8x8 {
            w.write_bit(false); // transform_size_8x8_flag = 0 (this I_NxN is 4x4)
        }
        for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
            let (bx, by) = (mb_x * 4 + lbx, mb_y * 4 + lby);
            let predicted = predict_i4_mode(fe, bx, by);
            let actual = i4.modes[lby * 4 + lbx];
            if actual == predicted {
                w.write_bit(true);
            } else {
                w.write_bit(false);
                let rem = if actual < predicted { actual } else { actual - 1 };
                w.write_bits(rem as u32, 3);
            }
        }
        w.write_ue(plan.chroma_mode as u32); // intra_chroma_pred_mode
        write_cbp_intra(w, cbp);
        if cbp != 0 {
            w.write_se(fe.qp_delta()); // mb_qp_delta (AQ per-MB QPy)
        }
        fe.nnz_cache_load(mb_x, mb_y);
        for (blk, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
            let (bx, by) = (mb_x * 4 + lbx, mb_y * 4 + lby);
            let total = if i4.cbp_luma & (1 << (blk / 4)) != 0 {
                let nc = fe.nc_pred(lbx, lby);
                let scan16 = scan_4x4_dcac(&i4.q[lby * 4 + lbx]);
                encode_residual_block(w, &scan16, 16, nc) as u8
            } else {
                0
            };
            fe.nnz_cache_set(lbx, lby, total);
            fe.nnz_y[by * w4 + bx] = total;
        }
    } else {
        let mb_type = 1 + plan.i16_mode as u32 + 4 * cbp_chroma + if plan.i16_cbp15 { 12 } else { 0 };
        w.write_ue(mb_type + mb_type_offset);
        w.write_ue(plan.chroma_mode as u32); // intra_chroma_pred_mode
        w.write_se(fe.qp_delta()); // mb_qp_delta (I_16x16 always codes it; AQ per-MB QPy)
        fe.nnz_cache_load(mb_x, mb_y);
        let nc_dc = fe.nc_pred(0, 0);
        let dc_scan = scan_4x4_dcac(&plan.i16_dc_levels);
        encode_residual_block(w, &dc_scan, 16, nc_dc);
        for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
            fe.nnz_cache_set(lbx, lby, 0);
            fe.nnz_y[(mb_y * 4 + lby) * w4 + (mb_x * 4 + lbx)] = 0;
        }
        if plan.i16_cbp15 {
            for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
                let (bx, by) = (mb_x * 4 + lbx, mb_y * 4 + lby);
                let nc = fe.nc_pred(lbx, lby);
                let ac = scan_4x4_ac(&plan.i16_q[lby * 4 + lbx]);
                let total = encode_residual_block(w, &ac, 15, nc) as u8;
                fe.nnz_cache_set(lbx, lby, total);
                fe.nnz_y[by * w4 + bx] = total;
            }
        }
    }

    // ============ emit chroma residual (shared) ============
    if cbp_chroma != 0 {
        for c in 0..2 {
            encode_residual_block(w, &plan.c_dc_levels[c], 4, -1);
        }
    }
    if cbp_chroma == 2 {
        fe.chroma_cache_load(mb_x, mb_y);
        let w2 = fe.mb_w * 2;
        for c in 0..2 {
            for &(bx, by) in &CHROMA_4X4_SCAN_XY {
                let nc = fe.chroma_nc_pred(c, bx, by);
                let ac = scan_4x4_ac(&plan.c_q_blocks[c][by * 2 + bx]);
                let total = encode_residual_block(w, &ac, 15, nc) as u8;
                fe.chroma_nnz_cache_set(c, bx, by, total);
                fe.nnz_c[c][(mb_y * 2 + by) * w2 + (mb_x * 2 + bx)] = total;
            }
        }
    }
}

// ============================================================================
// CABAC I-slice entropy coding — the exact forward inverse of the decoder's
// `decode_slice_data_cabac` I-slice path (rusty_h264-decoder mb16.rs). Every
// binarization + context-selection here mirrors a `parse_*_cabac` there; the
// neighbour state (nzc cache, cbf_dc, cat, cmode, mb_cbp, last_delta_qp) is
// reconstructed identically so the contexts evolve bit-for-bit. Reuses `plan_mb`
// for the entire mode-decision/transform/recon (shared with CAVLC).
// ============================================================================

// --- res-property tables (must match the decoder's mb16.rs g_kBlockCat2CtxOffset*) ---
const CB_NZC_CACHE: [usize; 24] = [
    9, 10, 17, 18, 11, 12, 19, 20, 25, 26, 33, 34, 27, 28, 35, 36, // luma
    14, 15, 22, 23, // Cb
    38, 39, 46, 47, // Cr
];
const CB_RES_MAXPOS: [i32; 11] = [0, 15, 14, 15, 3, 14, 63, 3, 3, 14, 14];
const CB_RES_MAXC2: [i32; 11] = [0, 4, 4, 4, 3, 4, 4, 3, 3, 4, 4];
const CB_RES_CBF: [usize; 11] = [0, 0, 4, 8, 12, 16, 0, 12, 12, 16, 16];
const CB_RES_MAP: [usize; 11] = [0, 0, 15, 29, 44, 47, 0, 44, 44, 47, 47];
const CB_RES_ONE: [usize; 11] = [0, 0, 10, 20, 30, 39, 0, 30, 30, 39, 39];
const CB_RP_I16_DC: usize = 1;
const CB_RP_I16_AC: usize = 2;
const CB_RP_LUMA_4X4: usize = 3;
const CB_RP_CHROMA_DC: usize = 7;
const CB_RP_CHROMA_AC: usize = 9;

/// Inverse of `cabac_unary(ctx, off)`: bin0 at `ctx`; for value >= 1, `value-1` ones
/// then a terminating 0, all at `ctx+off`.
fn cb_unary(cab: &mut CabacEncoder, ctx: usize, off: usize, value: u32) {
    if value == 0 {
        cab.encode_decision(ctx, 0);
        return;
    }
    cab.encode_decision(ctx, 1);
    for _ in 0..value - 1 {
        cab.encode_decision(ctx + off, 1);
    }
    cab.encode_decision(ctx + off, 0);
}

/// Exp-Golomb order-`k` in bypass — inverse of `cabac_exp_bypass(k)`.
fn cb_exp_bypass(cab: &mut CabacEncoder, mut k: i32, mut n: u32) {
    while n >= (1 << k) {
        cab.encode_bypass(1);
        n -= 1 << k;
        k += 1;
    }
    cab.encode_bypass(0);
    while k > 0 {
        k -= 1;
        cab.encode_bypass((n >> k) & 1);
    }
}

/// UEG0 coeff-level suffix — inverse of `cabac_ueg_level(ctx)` (TU prefix <=13 at
/// `ctx`, then an EG0 bypass suffix).
fn cb_ueg_level(cab: &mut CabacEncoder, ctx: usize, value: u32) {
    if value == 0 {
        cab.encode_decision(ctx, 0);
        return;
    }
    let ones = value.min(13);
    for _ in 0..ones {
        cab.encode_decision(ctx, 1);
    }
    if value < 13 {
        cab.encode_decision(ctx, 0);
    } else {
        cb_exp_bypass(cab, 0, value - 13);
    }
}

/// `mb_qp_delta` — inverse of `parse_mb_qp_delta_cabac` (ctxIdxOffset 60).
fn cb_mb_qp_delta(cab: &mut CabacEncoder, last_delta_qp: &mut i32, delta: i32) {
    const O: usize = 60;
    let ctx_inc = (*last_delta_qp != 0) as usize;
    if delta == 0 {
        cab.encode_decision(O + ctx_inc, 0);
    } else {
        cab.encode_decision(O + ctx_inc, 1);
        // code = 2|d| - (d>0); the decode's cabac_unary sees code-1.
        let code = 2 * delta.unsigned_abs() - (delta > 0) as u32;
        cb_unary(cab, O + 2, 1, code - 1);
    }
    *last_delta_qp = delta;
}

/// `intra_chroma_pred_mode` (TU cMax=3) — inverse of `parse_intra_chroma_pred_mode_cabac`.
fn cb_chroma_pred_mode(cab: &mut CabacEncoder, ctx_inc: usize, mode: u8) {
    const C: usize = 64;
    if mode == 0 {
        cab.encode_decision(C + ctx_inc, 0);
        return;
    }
    cab.encode_decision(C + ctx_inc, 1);
    if mode == 1 {
        cab.encode_decision(C + 3, 0);
    } else if mode == 2 {
        cab.encode_decision(C + 3, 1);
        cab.encode_decision(C + 3, 0);
    } else {
        cab.encode_decision(C + 3, 1);
        cab.encode_decision(C + 3, 1);
    }
}

/// I-slice `mb_type` — inverse of `parse_mb_type_i_cabac` (ctxIdxOffset 3).
fn cb_mb_type_i(
    cab: &mut CabacEncoder,
    ctx_inc: usize,
    use_i4: bool,
    i16_mode: u32,
    cbp_chroma: u32,
    cbp_luma15: bool,
) {
    const O: usize = 3;
    if use_i4 {
        cab.encode_decision(O + ctx_inc, 0); // I_NxN
        return;
    }
    cab.encode_decision(O + ctx_inc, 1);
    cab.encode_terminate(false); // not I_PCM
    cab.encode_decision(O + 3, cbp_luma15 as u32);
    if cbp_chroma != 0 {
        cab.encode_decision(O + 4, 1);
        cab.encode_decision(O + 5, (cbp_chroma == 2) as u32);
    } else {
        cab.encode_decision(O + 4, 0);
    }
    cab.encode_decision(O + 6, (i16_mode >> 1) & 1);
    cab.encode_decision(O + 7, i16_mode & 1);
}

/// One `Intra_4x4` pred-mode — inverse of `parse_intra4x4_pred_mode_cabac` (ctx 68).
fn cb_intra4x4_pred_mode(cab: &mut CabacEncoder, predicted: u8, actual: u8) {
    const IPR: usize = 68;
    if actual == predicted {
        cab.encode_decision(IPR, 1);
    } else {
        cab.encode_decision(IPR, 0);
        let rem = if actual < predicted { actual } else { actual - 1 } as u32;
        cab.encode_decision(IPR + 1, rem & 1);
        cab.encode_decision(IPR + 1, (rem >> 1) & 1);
        cab.encode_decision(IPR + 1, (rem >> 2) & 1);
    }
}

/// `coded_block_pattern` — inverse of `parse_cbp_cabac` (ctxIdxOffset 73).
fn cb_cbp(cab: &mut CabacEncoder, top: Option<u8>, left: Option<u8>, cbp: u32) {
    const CBP: usize = 73;
    let t = |m: u32| top.map_or(0u32, |c| ((c as u32 & m) == 0) as u32);
    let l = |m: u32| left.map_or(0u32, |c| ((c as u32 & m) == 0) as u32);
    let nb = |x: u32| (x == 0) as u32;
    let b0 = cbp & 1;
    let b1 = (cbp >> 1) & 1;
    let b2 = (cbp >> 2) & 1;
    let b3 = (cbp >> 3) & 1;
    cab.encode_decision(CBP + (l(1 << 1) + (t(1 << 2) << 1)) as usize, b0);
    cab.encode_decision(CBP + (nb(b0) + (t(1 << 3) << 1)) as usize, b1);
    cab.encode_decision(CBP + (l(1 << 3) + (nb(b0) << 1)) as usize, b2);
    cab.encode_decision(CBP + (nb(b2) + (nb(b1) << 1)) as usize, b3);
    let cbp_chroma = cbp >> 4;
    let ct = top.map_or(0u32, |c| ((c >> 4) != 0) as u32);
    let cl = left.map_or(0u32, |c| ((c >> 4) != 0) as u32);
    cab.encode_decision(CBP + 4 + (cl + (ct << 1)) as usize, (cbp_chroma != 0) as u32);
    if cbp_chroma != 0 {
        let ct2 = top.map_or(0u32, |c| ((c >> 4) == 2) as u32);
        let cl2 = left.map_or(0u32, |c| ((c >> 4) == 2) as u32);
        cab.encode_decision(CBP + 8 + (cl2 + (ct2 << 1)) as usize, (cbp_chroma == 2) as u32);
    }
}

/// One residual block — inverse of `parse_residual_cabac`. `coeffs` is scan-order
/// (len >= maxPos+1). Returns totalCoeffNum (for the nzc cache + deblock nnz).
#[allow(clippy::too_many_arguments)]
fn cb_residual(
    cab: &mut CabacEncoder,
    nzc: &mut [u8; 48],
    cbf_dc: &mut u16,
    iz: usize,
    rp: usize,
    is_intra: bool,
    ndc: (Option<u16>, Option<u16>),
    coeffs: &[i32],
) -> u32 {
    let is_dc = rp == CB_RP_I16_DC || rp == CB_RP_CHROMA_DC || rp == CB_RP_CHROMA_DC + 1;
    let (mut na, mut nb) = (is_intra as u8, is_intra as u8);
    let scan = CB_NZC_CACHE[iz.min(23)];
    if is_dc {
        if let Some(t) = ndc.0 {
            nb = ((t >> rp) & 1) as u8;
        }
        if let Some(l) = ndc.1 {
            na = ((l >> rp) & 1) as u8;
        }
    } else {
        if nzc[scan - 8] != 0xff {
            nb = (nzc[scan - 8] != 0) as u8;
        }
        if nzc[scan - 1] != 0xff {
            na = (nzc[scan - 1] != 0) as u8;
        }
    }
    let maxpos = CB_RES_MAXPOS[rp] as usize;
    let coeff_num = coeffs[..=maxpos].iter().filter(|&&c| c != 0).count() as u32;
    let cbf = coeff_num != 0;
    cab.encode_decision(85 + CB_RES_CBF[rp] + (na + (nb << 1)) as usize, cbf as u32);
    if !cbf {
        if !is_dc {
            nzc[scan] = 0;
        }
        return 0;
    }
    if is_dc {
        *cbf_dc |= 1 << rp;
    }
    // significance map
    let map = 105 + CB_RES_MAP[rp];
    let last = 166 + CB_RES_MAP[rp];
    let lastnz = (0..=maxpos).rev().find(|&i| coeffs[i] != 0).unwrap();
    for i in 0..maxpos {
        let s = coeffs[i] != 0;
        cab.encode_decision(map + i, s as u32);
        if s {
            let is_last = i == lastnz;
            cab.encode_decision(last + i, is_last as u32);
            if is_last {
                break;
            }
        }
    }
    // levels (reverse scan)
    let one = 227 + CB_RES_ONE[rp];
    let abs = 232 + CB_RES_ONE[rp];
    let maxc2 = CB_RES_MAXC2[rp];
    let (mut c1, mut c2) = (1i32, 0i32);
    for i in (0..=maxpos).rev() {
        if coeffs[i] != 0 {
            let av = coeffs[i].unsigned_abs();
            let gt1 = av > 1;
            cab.encode_decision(one + c1 as usize, gt1 as u32);
            if gt1 {
                cb_ueg_level(cab, abs + c2 as usize, av - 2);
                c2 = (c2 + 1).min(maxc2);
                c1 = 0;
            } else if c1 != 0 {
                c1 = (c1 + 1).min(4);
            }
            cab.encode_bypass((coeffs[i] < 0) as u32);
        }
    }
    if !is_dc {
        nzc[scan] = coeff_num as u8;
    }
    coeff_num
}

/// Build the 48-entry padded nzc cache from the top/left neighbour MB exports
/// (openh264 `WelsFillCacheNonZeroCount`) — identical to the decoder.
fn cb_build_nzc(mb_nzc: &[[u8; 24]], top: Option<usize>, left: Option<usize>) -> [u8; 48] {
    let mut nzc = [0xffu8; 48];
    if let Some(t) = top {
        let tn = mb_nzc[t];
        nzc[1..5].copy_from_slice(&tn[12..16]);
        (nzc[0], nzc[5], nzc[29]) = (0, 0, 0);
        (nzc[6], nzc[7]) = (tn[20], tn[21]);
        (nzc[30], nzc[31]) = (tn[22], tn[23]);
    }
    if let Some(l) = left {
        let ln = mb_nzc[l];
        (nzc[8], nzc[16], nzc[24], nzc[32]) = (ln[3], ln[7], ln[11], ln[15]);
        (nzc[13], nzc[21], nzc[37], nzc[45]) = (ln[17], ln[21], ln[19], ln[23]);
    }
    nzc
}

/// Extract the 24-entry per-MB nzc (raster luma + chroma) for future neighbours.
fn cb_export_nzc(nzc: &[u8; 48]) -> [u8; 24] {
    let mut mn = [0u8; 24];
    for k in 0..4 {
        mn[k] = nzc[9 + k];
        mn[4 + k] = nzc[17 + k];
        mn[8 + k] = nzc[25 + k];
        mn[12 + k] = nzc[33 + k];
    }
    (mn[16], mn[17], mn[20], mn[21]) = (nzc[14], nzc[15], nzc[22], nzc[23]);
    (mn[18], mn[19], mn[22], mn[23]) = (nzc[38], nzc[39], nzc[46], nzc[47]);
    for v in mn.iter_mut() {
        if *v == 0xff {
            *v = 0;
        }
    }
    mn
}

/// Per-frame CABAC neighbour state (I-slice): one entry per macroblock, mirroring
/// the arrays the decoder's `decode_slice_data_cabac` maintains.
struct CabacState {
    cat: Vec<u8>,          // 2 = I_16x16, 0 = I_NxN, 100 = inter (mb_type / skip ctxInc)
    cmode: Vec<i32>,       // per-MB chroma mode (chroma-pred ctxInc)
    mb_cbp: Vec<u8>,       // per-MB cbp byte (cbp ctxInc)
    cbf_dc: Vec<u16>,      // per-MB DC coded_block_flag mask (residual DC ctxInc)
    mb_nzc: Vec<[u8; 24]>, // per-MB nzc export (residual AC ctxInc)
    // Inter (P/B) neighbour state — mirrors the decoder's WelsFillCacheInterCabac.
    mb_mvd: Vec<[[i16; 2]; 16]>,  // per-MB per-4x4 List-0 mvd (raster), for the mvd ctxInc cache
    mb_ref: Vec<[i8; 16]>,        // per-MB per-4x4 List-0 ref idx (raster); -1 = unavailable
    mb_mvd1: Vec<[[i16; 2]; 16]>, // B: per-MB per-4x4 List-1 mvd
    mb_ref1: Vec<[i8; 16]>,       // B: per-MB per-4x4 List-1 ref idx
    mb_skip: Vec<bool>,           // per-MB mb_skip_flag (skip ctxInc)
    mb_direct: Vec<bool>,         // B: per-MB B_Direct/B_Skip (B mb_type ctxInc)
    last_delta_qp: i32,
}

impl CabacState {
    fn new(n: usize) -> Self {
        CabacState {
            cat: vec![0; n],
            cmode: vec![0; n],
            mb_cbp: vec![0; n],
            cbf_dc: vec![0; n],
            mb_nzc: vec![[0u8; 24]; n],
            mb_mvd: vec![[[0i16; 2]; 16]; n],
            mb_ref: vec![[-1i8; 16]; n],
            mb_mvd1: vec![[[0i16; 2]; 16]; n],
            mb_ref1: vec![[-1i8; 16]; n],
            mb_skip: vec![false; n],
            mb_direct: vec![false; n],
            last_delta_qp: 0,
        }
    }
}

/// Emit one planned intra macroblock as CABAC (I-slice). Mirrors the decoder's
/// I-slice MB body exactly: `mb_type`, then per luma-type the intra modes / cbp /
/// `mb_qp_delta` / residual in spec order, maintaining `cs` and `fe.nnz_y`.
fn emit_mb_cabac_i(
    fe: &mut FrameEncoder,
    cab: &mut CabacEncoder,
    cs: &mut CabacState,
    plan: &MbPlan,
    mb_x: usize,
    mb_y: usize,
) {
    let mb_w = fe.mb_w;
    let addr = mb_y * mb_w + mb_x;
    let top = if mb_y > 0 { Some(addr - mb_w) } else { None };
    let left = if mb_x > 0 { Some(addr - 1) } else { None };

    // ---- mb_type (I-slice prefix; carries I_16x16 pred-mode/cbp) ----
    let li = left.map_or(0, |a| (cs.cat[a] >= 2) as usize);
    let ti = top.map_or(0, |a| (cs.cat[a] >= 2) as usize);
    let acct = crate::bitacct::enabled();
    let t0 = if acct { cab.pos() } else { 0 };
    if plan.use_i4 {
        cb_mb_type_i(cab, li + ti, true, 0, 0, false);
    } else {
        cb_mb_type_i(cab, li + ti, false, plan.i16_mode as u32, plan.cbp_chroma, plan.i16_cbp15);
    }
    if acct {
        crate::bitacct::add(crate::bitacct::B::MbType, cab.pos() - t0);
    }
    let t1 = if acct { cab.pos() } else { 0 };
    emit_intra_body_cabac(fe, cab, cs, plan, mb_x, mb_y, addr, top, left);
    if acct {
        crate::bitacct::add(crate::bitacct::B::IntraBody, cab.pos() - t1);
    }
}

/// The intra macroblock body (chroma pred mode, intra modes, cbp, mb_qp_delta,
/// residual) shared by I-slice intra and P/B-slice intra — everything AFTER the
/// slice-specific `mb_type` prefix (which already carries the I_16x16 pred-mode/cbp).
#[allow(clippy::too_many_arguments)]
fn emit_intra_body_cabac(
    fe: &mut FrameEncoder,
    cab: &mut CabacEncoder,
    cs: &mut CabacState,
    plan: &MbPlan,
    mb_x: usize,
    mb_y: usize,
    addr: usize,
    top: Option<usize>,
    left: Option<usize>,
) {
    let w4 = fe.mb_w * 4;
    let cbp_chroma = plan.cbp_chroma;
    // chroma-pred-mode ctxInc from neighbour chroma modes (1..=3).
    let cci = left.map_or(0, |a| (1..=3).contains(&cs.cmode[a]) as usize)
        + top.map_or(0, |a| (1..=3).contains(&cs.cmode[a]) as usize);

    let mut nzc;
    let mut cbfdc = 0u16;
    let ndc = (top.map(|a| cs.cbf_dc[a]), left.map(|a| cs.cbf_dc[a]));

    if !plan.use_i4 {
        // ---- I_16x16 ----
        cb_chroma_pred_mode(cab, cci, plan.chroma_mode);
        cs.cmode[addr] = plan.chroma_mode as i32;
        cs.cat[addr] = 2;
        cs.mb_cbp[addr] = ((cbp_chroma as u8) << 4) | if plan.i16_cbp15 { 15 } else { 0 };
        nzc = cb_build_nzc(&cs.mb_nzc, top, left);

        let delta = fe.qp_delta();
        cb_mb_qp_delta(cab, &mut cs.last_delta_qp, delta);

        // luma DC
        let dc_scan = scan_4x4_dcac(&plan.i16_dc_levels);
        cb_residual(cab, &mut nzc, &mut cbfdc, 0, CB_RP_I16_DC, true, ndc, &dc_scan);
        // luma AC
        for (iz, &(lbx, lby)) in LUMA_4X4_SCAN_XY.iter().enumerate() {
            let (bx, by) = (mb_x * 4 + lbx, mb_y * 4 + lby);
            let total = if plan.i16_cbp15 {
                let ac = scan_4x4_ac(&plan.i16_q[lby * 4 + lbx]);
                cb_residual(cab, &mut nzc, &mut cbfdc, iz, CB_RP_I16_AC, true, ndc, &ac)
            } else {
                nzc[CB_NZC_CACHE[iz]] = 0;
                0
            };
            fe.nnz_y[by * w4 + bx] = total as u8;
        }
        cb_emit_chroma_residual(cab, fe, &mut nzc, &mut cbfdc, ndc, true, plan.cbp_chroma, &plan.c_dc_levels, &plan.c_q_blocks, mb_x, mb_y);
    } else {
        // ---- I_NxN (I_4x4) ----
        let i4 = plan.i4.as_ref().unwrap();
        for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
            let (bx, by) = (mb_x * 4 + lbx, mb_y * 4 + lby);
            let predicted = predict_i4_mode(fe, bx, by);
            cb_intra4x4_pred_mode(cab, predicted, i4.modes[lby * 4 + lbx]);
        }
        cb_chroma_pred_mode(cab, cci, plan.chroma_mode);
        cs.cmode[addr] = plan.chroma_mode as i32;
        cs.cat[addr] = 0;
        let cbp = i4.cbp_luma | (cbp_chroma << 4);
        cb_cbp(cab, top.map(|a| cs.mb_cbp[a]), left.map(|a| cs.mb_cbp[a]), cbp);
        cs.mb_cbp[addr] = cbp as u8;
        nzc = cb_build_nzc(&cs.mb_nzc, top, left);

        if cbp == 0 {
            cs.last_delta_qp = 0;
            for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
                fe.nnz_y[(mb_y * 4 + lby) * w4 + (mb_x * 4 + lbx)] = 0;
            }
        } else {
            let delta = fe.qp_delta();
            cb_mb_qp_delta(cab, &mut cs.last_delta_qp, delta);
            for id8 in 0..4usize {
                for id4 in 0..4usize {
                    let iz = id8 * 4 + id4;
                    let (lbx, lby) = LUMA_4X4_SCAN_XY[iz];
                    let (bx, by) = (mb_x * 4 + lbx, mb_y * 4 + lby);
                    let total = if i4.cbp_luma & (1 << id8) != 0 {
                        let sc = scan_4x4_dcac(&i4.q[lby * 4 + lbx]);
                        cb_residual(cab, &mut nzc, &mut cbfdc, iz, CB_RP_LUMA_4X4, true, ndc, &sc)
                    } else {
                        nzc[CB_NZC_CACHE[iz]] = 0;
                        0
                    };
                    fe.nnz_y[by * w4 + bx] = total as u8;
                }
            }
            cb_emit_chroma_residual(cab, fe, &mut nzc, &mut cbfdc, ndc, true, plan.cbp_chroma, &plan.c_dc_levels, &plan.c_q_blocks, mb_x, mb_y);
        }
    }

    cs.cbf_dc[addr] = cbfdc;
    cs.mb_nzc[addr] = cb_export_nzc(&nzc);
}

/// Chroma DC + AC residual (shared by intra I_16x16/I_NxN and inter) — matches the
/// decoder's chroma residual order. `is_intra` selects the coded_block_flag default
/// (nA=nB default to is_intra). Populates the chroma nnz grid for deblock.
#[allow(clippy::too_many_arguments)]
fn cb_emit_chroma_residual(
    cab: &mut CabacEncoder,
    fe: &mut FrameEncoder,
    nzc: &mut [u8; 48],
    cbfdc: &mut u16,
    ndc: (Option<u16>, Option<u16>),
    is_intra: bool,
    cbp_chroma: u32,
    c_dc_levels: &[[i32; 4]; 2],
    c_q: &[[[i32; 16]; 4]; 2],
    mb_x: usize,
    mb_y: usize,
) {
    let w2 = fe.mb_w * 2;
    if cbp_chroma >= 1 {
        for i in 0..2usize {
            cb_residual(cab, nzc, cbfdc, 16 + i * 4, CB_RP_CHROMA_DC + i, is_intra, ndc, &c_dc_levels[i]);
        }
    }
    if cbp_chroma == 2 {
        for i in 0..2usize {
            for (id4, &(bx, by)) in CHROMA_4X4_SCAN_XY.iter().enumerate() {
                let ac = scan_4x4_ac(&c_q[i][by * 2 + bx]);
                let total = cb_residual(
                    cab, nzc, cbfdc, 16 + i * 4 + id4, CB_RP_CHROMA_AC + i, is_intra, ndc, &ac,
                );
                fe.nnz_c[i][(mb_y * 2 + by) * w2 + (mb_x * 2 + bx)] = total as u8;
            }
        }
    }
}

/// CABAC all-intra slice-data coder (IDR / I-slice). Mirrors `encode_slice_data`'s
/// setup + deblock + `RefFrame` construction, but codes every MB via `plan_mb` +
/// `emit_mb_cabac_i` into a CABAC bitstream. `w` already holds the byte-aligned
/// slice header; the CABAC bytes are appended after `cabac_alignment_one_bit`.
pub fn encode_slice_data_cabac_intra(
    w: &mut BitWriter,
    cfg: &EncoderConfig,
    frame: &YuvFrame,
    qp: u8,
    qpo: &[i32],
) -> crate::RefFrame {
    let mut fe = FrameEncoder::new(cfg);
    fe.qp = qp;
    fe.qpc = chroma_qp(qp);
    fe.cur_qp = qp;
    if cfg.cabac_dz_div > 0 {
        fe.idz = cfg.cabac_dz_div; // CABAC-specific dead-zone override
    }
    let (sy, su, sv) = coded_source(cfg, frame);
    let mut aq_qp = aq_qp_map(&sy, fe.cw, fe.mb_w, fe.mb_h, qp, fe.aq_strength);
    apply_mbtree_qpo(&mut aq_qp, qpo); // mb-tree temporal AQ (empty = byte-identical)
    fe.cur_qp = qp;
    let mut mb_qpy = vec![qp; fe.mb_w * fe.mb_h];

    // CABAC trellis (RDOQ): structure-adaptive. ON only for ALL-INTRA streams
    // (gop_size<=1), where each IDR is independent so trading a little distortion for
    // rate is a clean −0.5..−1.3% BD-rate win. OFF inside a GOP: there the I-frame is
    // a REFERENCE, and degrading it costs the dependent P-frames more than the I-frame
    // saves (measured ~+0.1% net) — so the safe end is a true no-op (never regresses).
    fe.rdoq_strength = if cfg.gop_size <= 1 { cfg.cabac_rdoq } else { 0.0 };
    // Contexts init from SliceQPY (the slice qp), init_idc unused for I, is_i = true.
    let mut cab = CabacEncoder::new(qp as i32, 0, true);
    let mut cs = CabacState::new(fe.mb_w * fe.mb_h);
    let total = fe.mb_w * fe.mb_h;

    for mb_y in 0..fe.mb_h {
        for mb_x in 0..fe.mb_w {
            let mb_idx = mb_y * fe.mb_w + mb_x;
            fe.qp = aq_qp[mb_idx];
            fe.qpc = chroma_qp(aq_qp[mb_idx]);
            let plan = plan_mb(&mut fe, mb_x, mb_y, &sy, &su, &sv);
            emit_mb_cabac_i(&mut fe, &mut cab, &mut cs, &plan, mb_x, mb_y);
            mb_qpy[mb_idx] = fe.cur_qp;
            // end_of_slice_flag (EncodeTerminate): 1 on the last MB, else 0.
            {
                let tt = if crate::bitacct::enabled() { cab.pos() } else { 0 };
                cab.encode_terminate(mb_idx + 1 == total);
                if crate::bitacct::enabled() {
                    crate::bitacct::add(crate::bitacct::B::Terminate, cab.pos() - tt);
                }
            }
        }
    }

    // Append CABAC slice data after cabac_alignment_one_bit (pad header with 1-bits).
    while !w.is_byte_aligned() {
        w.write_bit(true);
    }
    for b in cab.into_bytes() {
        w.write_bits(b as u32, 8);
    }

    // Deblock the reconstruction (all-intra: BS derives from intra-ness) -> reference.
    let ref_id: Vec<i32> = fe.ref_idx_y.iter().map(|&r| if r >= 0 { r } else { i32::MIN }).collect();
    let info = rusty_h264_common::deblock::BlockInfo {
        inter: &fe.inter_y,
        nnz: &fe.nnz_y,
        mv: &fe.mv_y,
        ref_id: &ref_id,
        mv1: &[],
        ref_id1: &[],
        w4: fe.mb_w * 4,
        t8x8: &[],
        bs: &[],
        };
    rusty_h264_common::deblock::filter_frame(
        &mut fe.rec_y, &mut fe.rec_u, &mut fe.rec_v, fe.mb_w, fe.mb_h, &mb_qpy, 0, 0, 0, &info,
    );
    let w4 = fe.mb_w * 4;
    crate::RefFrame {
        y: fe.rec_y,
        u: fe.rec_u,
        v: fe.rec_v,
        poc: 0,
        frame_num: 0,
        mv: fe.mv_y,
        ref_idx: fe.ref_idx_y,
        w4,
        // Filtered lazily on first sub-pel search use (see `RefFrame::hpel`).
        hpel: std::sync::OnceLock::new(),
    }
}

// ============================================================================
// CABAC P-slice entropy coding — the forward inverse of the decoder's
// decode_slice_data_cabac P-slice path. mb_skip_flag / mb_type_p / mvd (UEG3) /
// inter residual, plus intra-in-P (the shared intra body under a P mb_type prefix).
// Scope: 1 reference (no ref_idx), P_16x16/16x8/8x16 (no P_8x8/sub_mb_type) — the
// modes the encoder's decision produces.
// ============================================================================

// z-order 4x4 block -> 30-entry (6-stride) mvd/ref cache index (openh264 g_kCache30ScanIdx).
const CB_CACHE30: [usize; 16] = [7, 8, 13, 14, 9, 10, 15, 16, 19, 20, 25, 26, 21, 22, 27, 28];
// z-order 4x4 block -> raster index (openh264 g_kuiScan4): the per-MB mvd/ref grid layout.
const CB_G_SCAN4: [usize; 16] = [0, 1, 4, 5, 2, 3, 6, 7, 8, 9, 12, 13, 10, 11, 14, 15];

/// UEG3 mvd suffix — inverse of `decode_ueg_mv(base)` (TU prefix at base+{0,1,2,3,3..},
/// cMax 7, then EG3 bypass). `v` is the value decode_ueg_mv returns.
fn cb_ueg_mv(cab: &mut CabacEncoder, base: usize, v: u32) {
    const P2C: [usize; 8] = [0, 1, 2, 3, 3, 3, 3, 3];
    if v == 0 {
        cab.encode_decision(base, 0);
        return;
    }
    cab.encode_decision(base, 1);
    if v <= 7 {
        // (v-1) ones then a terminating 0, at base+P2C[count] for count = 1..
        let mut count = 1;
        for _ in 0..v - 1 {
            cab.encode_decision(base + P2C[count], 1);
            count += 1;
        }
        cab.encode_decision(base + P2C[count], 0);
    } else {
        // prefix maxes out: 7 ones (count 1..7) then EG3(v-8).
        let mut count = 1;
        for _ in 0..7 {
            cab.encode_decision(base + P2C[count], 1);
            count += 1;
        }
        let tb = if crate::bitacct::enabled() { cab.pos() } else { 0 };
        cb_exp_bypass(cab, 3, v - 8);
        if crate::bitacct::enabled() {
            crate::bitacct::add(crate::bitacct::B::MvdBypass, cab.pos() - tb);
        }
    }
}

/// One `mvd` component — inverse of `parse_mvd_cabac(comp, ctx_inc)` (ctxIdxOffset
/// 40 for x, 47 for y).
fn cb_mvd(cab: &mut CabacEncoder, comp: usize, ctx_inc: usize, d: i32) {
    let th = if crate::bitacct::enabled() { cab.pos() } else { u64::MAX };
    let base = 40 + comp * 7;
    if d == 0 {
        cab.encode_decision(base + ctx_inc, 0);
        if th != u64::MAX {
            crate::bitacct::add_mvd_sample(0, cab.pos() - th);
        }
        return;
    }
    cab.encode_decision(base + ctx_inc, 1);
    cb_ueg_mv(cab, base + 3, d.unsigned_abs() - 1); // decode adds 1 back
    let ts = if crate::bitacct::enabled() { cab.pos() } else { 0 };
    cab.encode_bypass((d < 0) as u32);
    if crate::bitacct::enabled() {
        crate::bitacct::add(crate::bitacct::B::MvdSign, cab.pos() - ts);
    }
    if th != u64::MAX {
        crate::bitacct::add_mvd_sample(d.unsigned_abs(), cab.pos() - th);
    }
}

/// `mb_skip_flag` — inverse of `parse_mb_skip_cabac` (ctx 11 P + neighbour-not-skip).
fn cb_mb_skip(cab: &mut CabacEncoder, ctx_inc: usize, skip: bool) {
    cab.encode_decision(ctx_inc, skip as u32);
}

/// `ref_idx_l0` (P) — inverse of `parse_ref_idx_cabac`. Unary binarization,
/// ctxIdxOffset 54: binIdx 0 → `ctx0` (condTermFlagA + 2·condTermFlagB, condTermFlagN =
/// neighbour partition's ref_idx > 0), binIdx 1 → 4, binIdx ≥2 → 5 (spec 9.3.3.1.1.6).
fn cb_ref_idx(cab: &mut CabacEncoder, ctx0: usize, r: u32) {
    const B: usize = 54;
    let mut v = r;
    let mut bin_idx = 0u32;
    loop {
        let bin = (v > 0) as u32;
        let ctx = match bin_idx {
            0 => ctx0,
            1 => 4,
            _ => 5,
        };
        cab.encode_decision(B + ctx, bin);
        if bin == 0 {
            break;
        }
        v -= 1;
        bin_idx += 1;
    }
}

/// P-slice inter `mb_type` (0/1/2 = P_L0_16x16 / P_16x8 / P_8x16) — inverse of the
/// inter branch of `parse_mb_type_p_cabac` (ctx base 11).
fn cb_mb_type_p_inter(cab: &mut CabacEncoder, mode: u8) {
    const S: usize = 11;
    cab.encode_decision(S + 3, 0); // inter (prefix bit 0)
    match mode {
        0 => {
            cab.encode_decision(S + 4, 0);
            cab.encode_decision(S + 5, 0);
        }
        3 => {
            // P_8x8 (bins "0 0 1")
            cab.encode_decision(S + 4, 0);
            cab.encode_decision(S + 5, 1);
        }
        1 => {
            cab.encode_decision(S + 4, 1);
            cab.encode_decision(S + 6, 1);
        }
        _ => {
            // mode == 2 (P_8x16)
            cab.encode_decision(S + 4, 1);
            cab.encode_decision(S + 6, 0);
        }
    }
}

/// P `sub_mb_type` CABAC — inverse of `parse_sub_mb_type_p_cabac` (ctx base 21).
/// Only 0 = P_L0_8x8 (bin "1") is emitted (8×8 sub-partitions only).
fn cb_sub_mb_type_p(cab: &mut CabacEncoder, sub_type: u8) {
    const S: usize = 21;
    match sub_type {
        0 => cab.encode_decision(S, 1),
        _ => unreachable!("only 8x8 sub_mb_type (0) emitted"),
    }
}

/// P-slice intra `mb_type` prefix — inverse of the intra branch of
/// `parse_mb_type_p_cabac` (ctx base 11). Carries the I_16x16 pred-mode/cbp exactly
/// like the I-slice mb_type, so the shared intra body re-emits neither.
fn cb_mb_type_p_intra(cab: &mut CabacEncoder, plan: &MbPlan) {
    const S: usize = 11;
    cab.encode_decision(S + 3, 1); // intra (prefix bit 1)
    if plan.use_i4 {
        cab.encode_decision(S + 6, 0); // I_4x4
        return;
    }
    cab.encode_decision(S + 6, 1); // I_16x16
    cab.encode_terminate(false); // not I_PCM
    cab.encode_decision(S + 7, plan.i16_cbp15 as u32);
    if plan.cbp_chroma != 0 {
        cab.encode_decision(S + 8, 1);
        cab.encode_decision(S + 8, (plan.cbp_chroma == 2) as u32);
    } else {
        cab.encode_decision(S + 8, 0);
    }
    cab.encode_decision(S + 9, (plan.i16_mode as u32 >> 1) & 1);
    cab.encode_decision(S + 9, plan.i16_mode as u32 & 1);
}

/// P-slice partition layout: `(part_idx, z-blocks)` per motion partition (matches
/// the decoder's `part!` invocations). part_idx = the partition's top-left z-block
/// (its `CACHE30` slot for the mvd ctxInc); z-blocks = every 4x4 it covers.
fn p_partition_layout(mode: u8) -> &'static [(usize, &'static [usize])] {
    match mode {
        1 => &[(0, &[0, 1, 2, 3, 4, 5, 6, 7]), (8, &[8, 9, 10, 11, 12, 13, 14, 15])],
        2 => &[(0, &[0, 1, 2, 3, 8, 9, 10, 11]), (4, &[4, 5, 6, 7, 12, 13, 14, 15])],
        // P_8x8: four 8×8 quads (z-order 4×4 blocks), part order == inter_partitions(3).
        3 => &[(0, &[0, 1, 2, 3]), (4, &[4, 5, 6, 7]), (8, &[8, 9, 10, 11]), (12, &[12, 13, 14, 15])],
        _ => &[(0, &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15])],
    }
}

/// Emit one motion partition's `mvd` (x,y) and splat it into the 30-entry cache +
/// per-MB raster mvd/ref grids — inverse of the decoder's `parse_mvd_partition`.
#[allow(clippy::too_many_arguments)]
fn cb_emit_mvd_partition(
    cab: &mut CabacEncoder,
    part_idx: usize,
    zblocks: &[usize],
    mvdc: &mut [[i16; 2]; 30],
    refc: &mut [i8; 30],
    mmvd: &mut [[i16; 2]; 16],
    mref: &mut [i8; 16],
    mvd: (i32, i32),
    ref_idx: i8, // the partition's ref_idx_l0 (0 for single-ref) — stored for neighbour context
) {
    let s = CB_CACHE30[part_idx];
    let ctx = |comp: usize| -> usize {
        let mut a = 0i32;
        if refc[s - 6] >= 0 {
            a += mvdc[s - 6][comp].unsigned_abs() as i32;
        }
        if refc[s - 1] >= 0 {
            a += mvdc[s - 1][comp].unsigned_abs() as i32;
        }
        if a >= 3 {
            1 + (a > 32) as usize
        } else {
            0
        }
    };
    cb_mvd(cab, 0, ctx(0), mvd.0);
    cb_mvd(cab, 1, ctx(1), mvd.1);
    let (mx, my) = (mvd.0 as i16, mvd.1 as i16);
    for &zb in zblocks {
        mvdc[CB_CACHE30[zb]] = [mx, my];
        refc[CB_CACHE30[zb]] = ref_idx;
        mmvd[CB_G_SCAN4[zb]] = [mx, my];
        mref[CB_G_SCAN4[zb]] = ref_idx;
    }
}

/// Emit one planned INTER macroblock as CABAC (P-slice, mb_skip_flag already coded
/// as 0). `mode`/`parts` + `plan` from `plan_inter_mb`. 1-ref: no ref_idx.
fn emit_mb_cabac_p_inter(
    fe: &mut FrameEncoder,
    cab: &mut CabacEncoder,
    cs: &mut CabacState,
    mode: u8,
    plan: &InterPlan,
    mb_x: usize,
    mb_y: usize,
    num_refs: usize,
) {
    let mb_w = fe.mb_w;
    let addr = mb_y * mb_w + mb_x;
    let top = if mb_y > 0 { Some(addr - mb_w) } else { None };
    let left = if mb_x > 0 { Some(addr - 1) } else { None };

    // Bit accountant (instrument #6): each tap is a `pos()` delta — exact coded
    // bits for that element — behind an atomic-bool check when disabled.
    let acct = crate::bitacct::enabled();
    let mut t0 = if acct { cab.pos() } else { 0 };
    cb_mb_type_p_inter(cab, mode);
    // P_8x8: four sub_mb_type (all 0 = 8×8), spec order before ref_idx/mvd.
    if mode == 3 {
        for _ in 0..4 {
            cb_sub_mb_type_p(cab, 0);
        }
    }
    if acct {
        crate::bitacct::add(crate::bitacct::B::MbType, cab.pos() - t0);
        t0 = cab.pos();
    }

    // ---- mb_pred (spec 7.3.5.1): all ref_idx_l0 FIRST, then all mvd_l0 ----
    let mut mvdc = [[0i16; 2]; 30];
    let mut refc = [-1i8; 30];
    cb_fill_inter_cache(&cs.mb_ref, &cs.mb_mvd, &mut refc, &mut mvdc, top, left, addr, mb_w);
    let mut mmvd = [[0i16; 2]; 16];
    let mut mref = [0i8; 16];
    let layout = p_partition_layout(mode);
    // Phase 1: ref_idx_l0 per partition, only when the slice has >1 active reference.
    // Update refc after each so a later partition's ref context sees the earlier one.
    if num_refs > 1 {
        for (part, &(part_idx, zblocks)) in layout.iter().enumerate() {
            let r = plan.plan_refs[part];
            let s = CB_CACHE30[part_idx];
            let ctx0 = (refc[s - 1] > 0) as usize + 2 * (refc[s - 6] > 0) as usize;
            cb_ref_idx(cab, ctx0, r as u32);
            for &zb in zblocks {
                refc[CB_CACHE30[zb]] = r as i8;
            }
        }
    }
    if acct {
        crate::bitacct::add(crate::bitacct::B::RefIdx, cab.pos() - t0);
        t0 = cab.pos();
    }
    // Phase 2: mvd per partition (carries the ref into refc/mref for neighbour context).
    for (part, &(part_idx, zblocks)) in layout.iter().enumerate() {
        cb_emit_mvd_partition(
            cab, part_idx, zblocks, &mut mvdc, &mut refc, &mut mmvd, &mut mref, plan.mvds[part],
            plan.plan_refs[part] as i8,
        );
    }
    if acct {
        crate::bitacct::add(crate::bitacct::B::Mvd, cab.pos() - t0);
    }
    cs.mb_mvd[addr] = mmvd;
    cs.mb_ref[addr] = mref;
    cs.cat[addr] = 100;
    cb_emit_inter_residual(fe, cab, cs, plan, mb_x, mb_y, addr, top, left);
}

/// Inter cbp + residual (is_intra = false) — shared by P and B inter MBs. Maintains
/// cs.mb_cbp/cbf_dc/mb_nzc/last_delta_qp + fe.nnz_y.
#[allow(clippy::too_many_arguments)]
fn cb_emit_inter_residual(
    fe: &mut FrameEncoder,
    cab: &mut CabacEncoder,
    cs: &mut CabacState,
    plan: &InterPlan,
    mb_x: usize,
    mb_y: usize,
    addr: usize,
    top: Option<usize>,
    left: Option<usize>,
) {
    let w4 = fe.mb_w * 4;
    let cbp = plan.cbp;
    let (cbp_luma, cbp_chroma) = (cbp & 15, cbp >> 4);
    let acct = crate::bitacct::enabled();
    let mut t0 = if acct { cab.pos() } else { 0 };
    cb_cbp(cab, top.map(|a| cs.mb_cbp[a]), left.map(|a| cs.mb_cbp[a]), cbp);
    if acct {
        crate::bitacct::add(crate::bitacct::B::Cbp, cab.pos() - t0);
    }
    cs.mb_cbp[addr] = cbp as u8;
    let mut nzc = cb_build_nzc(&cs.mb_nzc, top, left);
    let mut cbfdc = 0u16;
    let ndc = (top.map(|a| cs.cbf_dc[a]), left.map(|a| cs.cbf_dc[a]));

    if cbp == 0 {
        cs.last_delta_qp = 0;
        for &(lbx, lby) in &LUMA_4X4_SCAN_XY {
            fe.nnz_y[(mb_y * 4 + lby) * w4 + (mb_x * 4 + lbx)] = 0;
        }
    } else {
        let delta = fe.qp_delta();
        if acct { t0 = cab.pos(); }
        cb_mb_qp_delta(cab, &mut cs.last_delta_qp, delta);
        if acct {
            crate::bitacct::add(crate::bitacct::B::QpDelta, cab.pos() - t0);
            t0 = cab.pos();
        }
        for id8 in 0..4usize {
            for id4 in 0..4usize {
                let iz = id8 * 4 + id4;
                let (lbx, lby) = LUMA_4X4_SCAN_XY[iz];
                let (bx, by) = (mb_x * 4 + lbx, mb_y * 4 + lby);
                let total = if cbp_luma & (1 << id8) != 0 {
                    let sc = scan_4x4_dcac(&plan.q_blocks[lby * 4 + lbx]);
                    cb_residual(cab, &mut nzc, &mut cbfdc, iz, CB_RP_LUMA_4X4, false, ndc, &sc)
                } else {
                    nzc[CB_NZC_CACHE[iz]] = 0;
                    0
                };
                fe.nnz_y[by * w4 + bx] = total as u8;
            }
        }
        if acct {
            crate::bitacct::add(crate::bitacct::B::ResidLuma, cab.pos() - t0);
            t0 = cab.pos();
        }
        cb_emit_chroma_residual(cab, fe, &mut nzc, &mut cbfdc, ndc, false, cbp_chroma, &plan.c_dc_levels, &plan.c_q, mb_x, mb_y);
        if acct {
            crate::bitacct::add(crate::bitacct::B::ResidChroma, cab.pos() - t0);
        }
    }
    cs.cbf_dc[addr] = cbfdc;
    cs.mb_nzc[addr] = cb_export_nzc(&nzc);
}

/// Emit one planned INTRA macroblock inside a P-slice: the P mb_type prefix (which
/// carries the I_16x16 pred-mode/cbp) then the shared intra body.
fn emit_mb_cabac_p_intra(
    fe: &mut FrameEncoder,
    cab: &mut CabacEncoder,
    cs: &mut CabacState,
    plan: &MbPlan,
    mb_x: usize,
    mb_y: usize,
) {
    let mb_w = fe.mb_w;
    let addr = mb_y * mb_w + mb_x;
    let top = if mb_y > 0 { Some(addr - mb_w) } else { None };
    let left = if mb_x > 0 { Some(addr - 1) } else { None };
    let acct = crate::bitacct::enabled();
    let t0 = if acct { cab.pos() } else { 0 };
    cb_mb_type_p_intra(cab, plan);
    emit_intra_body_cabac(fe, cab, cs, plan, mb_x, mb_y, addr, top, left);
    if acct {
        // Whole intra MB (mb_type + modes + its residual) — intra MBs are ~5% of
        // P-frame MBs; splitting them further is a separate tap set.
        crate::bitacct::add(crate::bitacct::B::IntraBody, cab.pos() - t0);
    }
}

/// Emit a P_Skip macroblock's `mb_skip_flag = 1` and update neighbour state. The
/// motion grid was committed by `commit_skip`; the mvd/ref cache is LEFT at its
/// init (-1 ref) — matching the decoder, which does not touch mb_mvd/mb_ref for a
/// P_Skip (so a skip neighbour contributes nothing to a later mvd ctxInc).
fn emit_p_skip_cabac(cab: &mut CabacEncoder, cs: &mut CabacState, addr: usize, top: Option<usize>, left: Option<usize>) {
    let sctx = 11
        + left.map_or(0, |a| (!cs.mb_skip[a]) as usize)
        + top.map_or(0, |a| (!cs.mb_skip[a]) as usize);
    let t0 = if crate::bitacct::enabled() { cab.pos() } else { 0 };
    cb_mb_skip(cab, sctx, true);
    if crate::bitacct::enabled() {
        crate::bitacct::add(crate::bitacct::B::SkipFlag, cab.pos() - t0);
    }
    cs.mb_skip[addr] = true;
    cs.cat[addr] = 100;
    cs.last_delta_qp = 0;
}

/// CABAC P-slice data coder. Mirrors `encode_slice_data`'s decision (P_Skip check +
/// fast/quality inter-vs-intra RD) exactly — only the emit differs (per-MB
/// mb_skip_flag + CABAC syntax + per-MB end_of_slice terminate).
pub fn encode_slice_data_cabac_p(
    w: &mut BitWriter,
    cfg: &EncoderConfig,
    frame: &YuvFrame,
    qp: u8,
    refs: &[crate::RefFrame],
    qpo: &[i32],
) -> crate::RefFrame {
    let mut fe = FrameEncoder::new(cfg);
    fe.qp = qp;
    fe.qpc = chroma_qp(qp);
    fe.cur_qp = qp;
    if cfg.cabac_dz_div > 0 {
        fe.idz = cfg.cabac_dz_div; // CABAC-specific dead-zone override
    }
    let (sy, su, sv) = coded_source(cfg, frame);
    let lambda = 0.85 * fe.tune_lambda_scale * 2f64.powf((qp as f64 - 12.0) / 3.0);
    let num_refs = refs.len();
    // me_wide content gate (pure-pan → global-MC residual ≈ 0 → off; see encode_slice_data).
    if fe.me_wide && !refs.is_empty() {
        let coh = global_mc_residual(&sy, fe.cw, fe.mb_h * 16, &refs[0].y);
        if std::env::var("RFF_ME_COH_DBG").is_ok() {
            eprintln!("ME_COH qp{qp} residual={coh:.2}");
        }
        if coh < fe.me_wide_coh {
            fe.me_wide = false;
        }
    }
    // me_wide HEAD-ROOM GATE (the dispatcher the truth table asked for). The rescue
    // only pays where a wide search actually beats a predictor-local one; measure
    // that directly per frame and route the frame. `RFF_ME_HR` sets the threshold
    // (percent); 0 disables the gate and restores the always-on behaviour.
    // Skip the probe entirely when the gate is disabled: it must not tax the
    // default path (`RFF_ME_HR=0`), which stays byte-identical to pre-gate output.
    if fe.me_wide && !refs.is_empty() && (me_wide_hr_thresh() > 0.0 || me_wide_hr_dbg()) {
        let hr = me_wide_headroom(&sy, fe.cw, fe.mb_h * 16, &refs[0].y);
        if me_wide_hr_dbg() {
            eprintln!("ME_HR qp{qp} headroom={hr:.2}");
        }
        if me_wide_hr_thresh() > 0.0 && hr < me_wide_hr_thresh() {
            fe.me_wide = false;
        }
    }
    // Track-B B2 DISPATCH — same probe/route as the CAVLC driver above (the two
    // drivers must stay in lockstep; the U5-struct bug came from patching one).
    if me_sadfp_mode() == 1 && !fe.fast && !refs.is_empty() {
        let (mg, dc) = b2_mgain(&sy, fe.cw, fe.mb_h * 16, &refs[0].y);
        if me_sadt_dbg() {
            eprintln!("B2_MG qp{qp} mgain={mg:.3} dcfrac={dc:.3}");
        }
        fe.sadfp = mg >= me_sadt() && dc <= me_sad_dcmax();
        // H-24: the mv-cost SHAPE rides the same probe (its BD sign-flip tracks
        // motion for the same physical reason B2's does).
        if mv_smooth_mode() == 1 {
            // dcfrac veto mirrors B2's: crew-class FLASH frames satisfy the mgain
            // test but SAD/mvd statistics mislead there (H-13/H-26).
            fe.mv_smooth = mg >= mv_smooth_t() && dc <= me_sad_dcmax();
        }
        // H-13: near-static frames skip the split searches entirely.
        let smg = split_mg();
        if smg > 0.0 {
            fe.do_splits = mg >= smg;
        }
    }
    if fe.satd_q > 0.0 {
        let mut vars: Vec<i64> = (0..fe.mb_h)
            .flat_map(|my| (0..fe.mb_w).map(move |mx| (mx, my)))
            .map(|(mx, my)| mb_variance(&sy, fe.cw, mx, my))
            .collect();
        vars.sort_unstable();
        let idx = (((1.0 - fe.satd_q) * vars.len() as f64) as usize).min(vars.len() - 1);
        fe.satd_var_thresh = vars[idx];
    }
    let mut aq_qp = aq_qp_map(&sy, fe.cw, fe.mb_w, fe.mb_h, qp, fe.aq_strength);
    apply_mbtree_qpo(&mut aq_qp, qpo); // mb-tree temporal AQ (empty = byte-identical)
    fe.cur_qp = qp;
    let mut mb_qpy = vec![qp; fe.mb_w * fe.mb_h];

    // Same online free-skip dispatch as the CAVLC path gates the greedy P_Skip on
    // (see `encode_slice_data`): measured over the frame so far, within-frame so it
    // stays deterministic under GOP-parallel encode.
    let mut greedy_free = 0usize;
    let mut greedy_seen = 0usize;
    let mut greedy_on = fe.greedy_min_free == 0;
    let greedy_learn = (fe.mb_w * fe.mb_h / 8).max(64);
    let mut cab = CabacEncoder::new(qp as i32, cfg.cabac_init_idc, false); // P-slice
    let mut cs = CabacState::new(fe.mb_w * fe.mb_h);
    let total = fe.mb_w * fe.mb_h;

    // ② residue naming: the CABAC driver's MB loop was untapped (the CAVLC twin
    // has this scope) — `EncMbLoop − Σ(per-MB stages)` is the per-MB glue.
    let _g_loop = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncMbLoop);
    for mb_y in 0..fe.mb_h {
        for mb_x in 0..fe.mb_w {
            let mb_idx = mb_y * fe.mb_w + mb_x;
            let addr = mb_idx;
            let top = if mb_y > 0 { Some(addr - fe.mb_w) } else { None };
            let left = if mb_x > 0 { Some(addr - 1) } else { None };
            fe.qp = aq_qp[mb_idx];
            fe.qpc = chroma_qp(aq_qp[mb_idx]);

            // ---- P_Skip check (identical logic to encode_slice_data) ----
            let mut inter: Option<InterChoice> = None;
            let mut did_skip = false;
            if num_refs > 0 {
                let mv_skip = fe.skip_mv(mb_x, mb_y);
                let skip_y = fe.skip_predict_luma(refs, mb_x, mb_y, mv_skip);
                let luma_free = fe.skip_luma_is_free(&sy, mb_x, mb_y, &skip_y);
                let skip_c = if luma_free || !fe.fast {
                    fe.skip_predict_chroma(refs, mb_x, mb_y, mv_skip)
                } else {
                    [[0u8; 64]; 2]
                };
                let is_free = luma_free && fe.skip_chroma_is_free(&su, &sv, mb_x, mb_y, &skip_c);
                let skip_sad = if fe.fast {
                    0
                } else {
                    let (lx, ly) = (mb_x * 16, mb_y * 16);
                    let mut s = 0u32;
                    for dy in 0..16 {
                        let src = &sy[(ly + dy) * fe.cw + lx..][..16];
                        let p = &skip_y[dy * 16..][..16];
                        s += src.iter().zip(p).map(|(&a, &b)| a.abs_diff(b) as u32).sum::<u32>();
                    }
                    s
                };
                greedy_seen += 1;
                if greedy_seen >= greedy_learn {
                    greedy_on = fe.greedy_min_free == 0
                        || greedy_free * 100 >= greedy_seen * fe.greedy_min_free as usize;
                }
                if is_free {
                    fe.commit_skip(mb_x, mb_y, mv_skip, &skip_y, &skip_c);
                    if !fe.fast {
                        fe.mb_was_skip[mb_idx] = true;
                        fe.mb_skip_sad[mb_idx] = skip_sad;
                    }
                    greedy_free += 1;
                    did_skip = true;
                } else {
                    let (lx, ly) = (mb_x * 16, mb_y * 16);
                    let nb = {
                        let _g = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncMvPred);
                        fe.mv_neighbors_block(mb_x as isize * 4, mb_y as isize * 4, 4)
                    };
                    let lme = lambda.sqrt() * cfg.cabac_lambda_scale;
                    if fe.fast {
                        fe.mb_use_satd = fe.satd_q > 0.0
                            && mb_variance(&sy, fe.cw, mb_x, mb_y) >= fe.satd_var_thresh;
                        let (r16, mv16, cost_inter) =
                            fe.best_part(refs, &sy, &nb, num_refs, lx, ly, 16, 16, &[], lme);
                        let cost_intra = if fe.mb_use_satd {
                            fe.best_i16_satd(&sy, mb_x, mb_y)
                        } else {
                            fe.best_i16_sad(&sy, mb_x, mb_y)
                        } + (lme * fe.tune_intra_penalty) as i64;
                        inter = if cost_intra < cost_inter {
                            None
                        } else {
                            Some((0, vec![(r16, mv16)]))
                        };
                    } else {
                        // Quality preset: greedy P_Skip, then 16x16 baseline + sub-partitions + intra.
                        if fe.greedy_skip && greedy_on && skip_sad < fe.pred_skip_sad(mb_x, mb_y) {
                            fe.commit_skip(mb_x, mb_y, mv_skip, &skip_y, &skip_c);
                            fe.mb_was_skip[mb_idx] = true;
                            fe.mb_skip_sad[mb_idx] = skip_sad;
                            did_skip = true;
                        } else {
                            let (r16, mv16, c16) =
                                fe.best_part(refs, &sy, &nb, num_refs, lx, ly, 16, 16, &[], lme);
                            let mut best_c = c16;
                            let mut pick: Option<InterChoice> = Some((0, vec![(r16, mv16)]));
                            const QSTEP16: [i64; 6] = [10, 11, 13, 14, 16, 18];
                            let qstep16 = QSTEP16[(fe.qp % 6) as usize] << (fe.qp / 6);
                            let split_gate = ((30 * (qstep16 + 160)) >> 3) * 2;
                            let split_t = split_t();
                        if fe.do_splits && c16 > split_gate && (split_t <= 0.0 || (c16 as f64) >= split_t * lme) {
                                let (rt, mvt, ct) = fe.best_part(refs, &sy, &nb, num_refs, lx, ly, 16, 8, &[mv16], lme);
                                let (rb, mvb, cb) = fe.best_part(refs, &sy, &nb, num_refs, lx, ly + 8, 16, 8, &[mv16], lme);
                                let (rl, mvl, cl) = fe.best_part(refs, &sy, &nb, num_refs, lx, ly, 8, 16, &[mv16], lme);
                                let (rr, mvr, cr) = fe.best_part(refs, &sy, &nb, num_refs, lx + 8, ly, 8, 16, &[mv16], lme);
                                if ct + cb < best_c {
                                    best_c = ct + cb;
                                    pick = Some((1u8, vec![(rt, mvt), (rb, mvb)]));
                                }
                                if cl + cr < best_c {
                                    best_c = cl + cr;
                                    pick = Some((2u8, vec![(rl, mvl), (rr, mvr)]));
                                }
                                // P_8x8: four 8×8 sub-partitions (see the CAVLC path).
                                if fe.sub8x8 {
                                    let mut c8 = (lme * 4.0) as i64;
                                    let mut p8 = Vec::with_capacity(4);
                                    for &(qx, qy) in &[(0usize, 0usize), (8, 0), (0, 8), (8, 8)] {
                                        let (r, mv, c) = fe.best_part(
                                            refs, &sy, &nb, num_refs, lx + qx, ly + qy, 8, 8, &[mv16], lme,
                                        );
                                        c8 += c;
                                        p8.push((r, mv));
                                    }
                                    if c8 < best_c {
                                        best_c = c8;
                                        pick = Some((3u8, p8));
                                    }
                                }
                            }
                            // U5-struct: refine ONLY the winning shape (see the twin
                            // block in the CAVLC driver). This site is the CABAC path —
                            // which is now the DEFAULT, so omitting it here left sub-pel
                            // deferred but never refined on every default encode.
                            if fe.sp_defer.get() {
                                if let Some((mode, parts)) = pick.as_mut() {
                                    let regions: &[(usize, usize, usize, usize)] = match mode {
                                        1 => &[(0, 0, 16, 8), (0, 8, 16, 8)],
                                        2 => &[(0, 0, 8, 16), (8, 0, 8, 16)],
                                        3 => &[(0, 0, 8, 8), (8, 0, 8, 8), (0, 8, 8, 8), (8, 8, 8, 8)],
                                        _ => &[(0, 0, 16, 16)],
                                    };
                                    let mut tot = if *mode == 3 { (lme * 4.0) as i64 } else { 0 };
                                    for (i, &(qx, qy, pw, ph)) in regions.iter().enumerate() {
                                        let (r, mv) = parts[i];
                                        let (m2, c2) = fe.refine_part(
                                            refs, &sy, &nb, num_refs, lx + qx, ly + qy, pw, ph, lme, r, mv,
                                        );
                                        parts[i] = (r, m2);
                                        tot += c2;
                                    }
                                    best_c = tot;
                                }
                            }
                            let c_intra = fe.best_i16_satd(&sy, mb_x, mb_y)
                                + (lme * fe.tune_intra_penalty) as i64;
                            inter = if c_intra < best_c { None } else { pick };
                            fe.mb_was_skip[mb_idx] = false;
                            fe.mb_skip_sad[mb_idx] = skip_sad;
                        }
                    }
                }
            }

            // ---- emit ----
            if did_skip {
                emit_p_skip_cabac(&mut cab, &mut cs, addr, top, left);
                mb_qpy[mb_idx] = fe.cur_qp;
                {
                    let tt = if crate::bitacct::enabled() { cab.pos() } else { 0 };
                    {
                let tt = if crate::bitacct::enabled() { cab.pos() } else { 0 };
                cab.encode_terminate(mb_idx + 1 == total);
                if crate::bitacct::enabled() {
                    crate::bitacct::add(crate::bitacct::B::Terminate, cab.pos() - tt);
                }
            }
                    if crate::bitacct::enabled() {
                        crate::bitacct::add(crate::bitacct::B::Terminate, cab.pos() - tt);
                    }
                }
                continue;
            }
            // mb_skip_flag = 0
            let sctx = 11
                + left.map_or(0, |a| (!cs.mb_skip[a]) as usize)
                + top.map_or(0, |a| (!cs.mb_skip[a]) as usize);
            let tskip = if crate::bitacct::enabled() { cab.pos() } else { 0 };
            cb_mb_skip(&mut cab, sctx, false);
            if crate::bitacct::enabled() {
                crate::bitacct::add(crate::bitacct::B::SkipFlag, cab.pos() - tskip);
            }
            cs.mb_skip[addr] = false;
            match inter {
                Some((mode, parts)) => {
                    let plan = fe.plan_inter_mb(refs, &sy, &su, &sv, mb_x, mb_y, mode, &parts, None);
                    // ② residue naming: the CABAC entropy EMIT was untapped on the
                    // (default) CABAC driver — the whole encoder-side arithmetic
                    // coder was landing in `mgmt/other`.
                    let _ge = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncEmit);
                    emit_mb_cabac_p_inter(&mut fe, &mut cab, &mut cs, mode, &plan, mb_x, mb_y, num_refs);
                }
                None => {
                    let plan = plan_mb(&mut fe, mb_x, mb_y, &sy, &su, &sv);
                    let _ge = rusty_h264_common::prof::scope(rusty_h264_common::prof::Stage::EncEmit);
                    emit_mb_cabac_p_intra(&mut fe, &mut cab, &mut cs, &plan, mb_x, mb_y);
                }
            }
            mb_qpy[mb_idx] = fe.cur_qp;
            {
                let tt = if crate::bitacct::enabled() { cab.pos() } else { 0 };
                cab.encode_terminate(mb_idx + 1 == total);
                if crate::bitacct::enabled() {
                    crate::bitacct::add(crate::bitacct::B::Terminate, cab.pos() - tt);
                }
            }
        }
    }

    while !w.is_byte_aligned() {
        w.write_bit(true);
    }
    for b in cab.into_bytes() {
        w.write_bits(b as u32, 8);
    }

    // Deblock -> inter reference (same as encode_slice_data).
    let ref_id: Vec<i32> = fe.ref_idx_y.iter().map(|&r| if r >= 0 { r } else { i32::MIN }).collect();
    let info = rusty_h264_common::deblock::BlockInfo {
        inter: &fe.inter_y,
        nnz: &fe.nnz_y,
        mv: &fe.mv_y,
        ref_id: &ref_id,
        mv1: &[],
        ref_id1: &[],
        w4: fe.mb_w * 4,
        t8x8: &[],
        bs: &[],
        };
    rusty_h264_common::deblock::filter_frame(
        &mut fe.rec_y, &mut fe.rec_u, &mut fe.rec_v, fe.mb_w, fe.mb_h, &mb_qpy, 0, 0, 0, &info,
    );
    let w4 = fe.mb_w * 4;
    crate::RefFrame {
        y: fe.rec_y,
        u: fe.rec_u,
        v: fe.rec_v,
        poc: 0,
        frame_num: 0,
        mv: fe.mv_y,
        ref_idx: fe.ref_idx_y,
        w4,
        // Filtered lazily on first sub-pel search use (see `RefFrame::hpel`).
        hpel: std::sync::OnceLock::new(),
    }
}

// ============================================================================
// CABAC B-slice entropy coding — inverse of the decoder's decode_slice_data_cabac
// B-slice path. Scope: the modes the encoder's B decision produces — B_Skip,
// B_Direct_16x16 (0), B_L0/L1/Bi_16x16 (1/2/3) — no sub_mb_type, no intra-in-B.
// The new piece vs P is the dual-list (L0 + L1) mvd/ref neighbour cache.
// ============================================================================

/// Fill one list's 30-entry mvd/ref neighbour cache from the per-MB export grids
/// (openh264 WelsFillCacheInterCabac). Shared by P (List-0) and B (both lists).
fn cb_fill_inter_cache(
    mb_ref: &[[i8; 16]],
    mb_mvd: &[[[i16; 2]; 16]],
    refc: &mut [i8; 30],
    mvdc: &mut [[i16; 2]; 30],
    top: Option<usize>,
    left: Option<usize>,
    addr: usize,
    mb_w: usize,
) {
    if let Some(l) = left {
        for (ci, bi) in [(6usize, 3usize), (12, 7), (18, 11), (24, 15)] {
            refc[ci] = mb_ref[l][bi];
            mvdc[ci] = mb_mvd[l][bi];
        }
    }
    if let Some(t) = top {
        for (ci, bi) in [(1usize, 12usize), (2, 13), (3, 14), (4, 15)] {
            refc[ci] = mb_ref[t][bi];
            mvdc[ci] = mb_mvd[t][bi];
        }
    }
    let mb_x = addr % mb_w;
    let mb_y = addr / mb_w;
    if mb_x > 0 && mb_y > 0 {
        let a = addr - mb_w - 1;
        (refc[0], mvdc[0]) = (mb_ref[a][15], mb_mvd[a][15]);
    }
    if mb_y > 0 && mb_x + 1 < mb_w {
        let a = addr - mb_w + 1;
        (refc[5], mvdc[5]) = (mb_ref[a][12], mb_mvd[a][12]);
    }
}

/// B-slice `mb_type` for the encoder's B modes (0 = B_Direct_16x16, 1 = B_L0_16x16,
/// 2 = B_L1_16x16, 3 = B_Bi_16x16) — inverse of `parse_mb_type_b_cabac` (ctx 27).
fn cb_mb_type_b(cab: &mut CabacEncoder, ctx_inc: usize, dir: u8) {
    const B: usize = 27;
    match dir {
        0 => cab.encode_decision(B + ctx_inc, 0), // B_Direct_16x16
        1 => {
            cab.encode_decision(B + ctx_inc, 1);
            cab.encode_decision(B + 3, 0);
            cab.encode_decision(B + 5, 0); // L0
        }
        2 => {
            cab.encode_decision(B + ctx_inc, 1);
            cab.encode_decision(B + 3, 0);
            cab.encode_decision(B + 5, 1); // L1
        }
        _ => {
            // dir == 3 (B_Bi_16x16): m = 0 → return m+3 = 3
            cab.encode_decision(B + ctx_inc, 1);
            cab.encode_decision(B + 3, 1);
            cab.encode_decision(B + 4, 0);
            cab.encode_decision(B + 5, 0);
            cab.encode_decision(B + 5, 0);
            cab.encode_decision(B + 5, 0);
        }
    }
}

const CB_ALL16: [usize; 16] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15];

/// Emit one planned INTER B macroblock (mb_skip_flag already coded 0). `dir` is the
/// B direction 0/1/2/3; `plan.mvds` holds mvd_l0 then mvd_l1 (per used list).
fn emit_mb_cabac_b(
    fe: &mut FrameEncoder,
    cab: &mut CabacEncoder,
    cs: &mut CabacState,
    dir: u8,
    plan: &InterPlan,
    mb_x: usize,
    mb_y: usize,
) {
    let mb_w = fe.mb_w;
    let addr = mb_y * mb_w + mb_x;
    let top = if mb_y > 0 { Some(addr - mb_w) } else { None };
    let left = if mb_x > 0 { Some(addr - 1) } else { None };

    let bci = left.map_or(0, |a| (!cs.mb_direct[a]) as usize)
        + top.map_or(0, |a| (!cs.mb_direct[a]) as usize);
    cb_mb_type_b(cab, bci, dir);

    // Dual-list mvd/ref caches (L0 = mb_ref/mb_mvd, L1 = mb_ref1/mb_mvd1).
    let mut mvdc0 = [[0i16; 2]; 30];
    let mut refc0 = [-1i8; 30];
    let mut mvdc1 = [[0i16; 2]; 30];
    let mut refc1 = [-1i8; 30];
    cb_fill_inter_cache(&cs.mb_ref, &cs.mb_mvd, &mut refc0, &mut mvdc0, top, left, addr, mb_w);
    cb_fill_inter_cache(&cs.mb_ref1, &cs.mb_mvd1, &mut refc1, &mut mvdc1, top, left, addr, mb_w);
    let mut mmvd0 = [[0i16; 2]; 16];
    let mut mref0 = [-1i8; 16];
    let mut mmvd1 = [[0i16; 2]; 16];
    let mut mref1 = [-1i8; 16];
    let (use0, use1) = (dir == 1 || dir == 3, dir == 2 || dir == 3);
    if dir == 0 {
        // B_Direct_16x16: no coded motion; ref 0 in both lists (mvd stays 0) so a
        // later MB's mvd ctxInc sums |0|.
        mref0 = [0i8; 16];
        mref1 = [0i8; 16];
    } else {
        // mvd parse order: list-major (L0 then L1); a single 16x16 partition (idx 0).
        let mut k = 0;
        if use0 {
            cb_emit_mvd_partition(cab, 0, &CB_ALL16, &mut mvdc0, &mut refc0, &mut mmvd0, &mut mref0, plan.mvds[k], 0);
            k += 1;
        }
        if use1 {
            cb_emit_mvd_partition(cab, 0, &CB_ALL16, &mut mvdc1, &mut refc1, &mut mmvd1, &mut mref1, plan.mvds[k], 0);
        }
    }
    cs.mb_mvd[addr] = mmvd0;
    cs.mb_ref[addr] = mref0;
    cs.mb_mvd1[addr] = mmvd1;
    cs.mb_ref1[addr] = mref1;
    cs.mb_direct[addr] = dir == 0;
    cs.cat[addr] = 100;
    cb_emit_inter_residual(fe, cab, cs, plan, mb_x, mb_y, addr, top, left);
}

/// Emit a B_Skip macroblock's mb_skip_flag = 1 (ctx 24 base) + neighbour state. The
/// direct motion was committed by `commit_direct_motion`; ref 0 in both lists, mvd 0
/// (matching the decoder's decode_b_skip handling).
fn emit_b_skip_cabac(cab: &mut CabacEncoder, cs: &mut CabacState, addr: usize, top: Option<usize>, left: Option<usize>) {
    let sctx = 24
        + left.map_or(0, |a| (!cs.mb_skip[a]) as usize)
        + top.map_or(0, |a| (!cs.mb_skip[a]) as usize);
    let t0 = if crate::bitacct::enabled() { cab.pos() } else { 0 };
    cb_mb_skip(cab, sctx, true);
    if crate::bitacct::enabled() {
        crate::bitacct::add(crate::bitacct::B::SkipFlag, cab.pos() - t0);
    }
    cs.mb_skip[addr] = true;
    cs.cat[addr] = 100;
    cs.mb_direct[addr] = true;
    cs.mb_ref[addr] = [0i8; 16];
    cs.mb_ref1[addr] = [0i8; 16];
    cs.last_delta_qp = 0;
}

/// CABAC B-slice data coder. Mirrors `encode_slice_data_b`'s B_Skip-free check +
/// L0/L1/Bi/Direct RD decision verbatim; only the emit differs (per-MB
/// mb_skip_flag + CABAC + per-MB terminate). B is non-reference → no deblock/return.
#[allow(clippy::too_many_arguments)]
pub fn encode_slice_data_cabac_b(
    w: &mut BitWriter,
    cfg: &EncoderConfig,
    frame: &YuvFrame,
    qp: u8,
    poc: i32,
    l0: &crate::RefFrame,
    l1: &crate::RefFrame,
    qpo: &[i32],
) {
    let mut fe = FrameEncoder::new(cfg);
    fe.qp = qp;
    fe.qpc = chroma_qp(qp);
    fe.cur_qp = qp;
    if cfg.cabac_dz_div > 0 {
        fe.idz = cfg.cabac_dz_div; // CABAC-specific dead-zone override
    }
    fe.bi_w = implicit_bi_weights(poc, l0.poc, l1.poc);
    let (sy, su, sv) = coded_source(cfg, frame);
    let lambda = 0.85 * fe.tune_lambda_scale * 2f64.powf((qp as f64 - 12.0) / 3.0);
    let lme = lambda.sqrt() * cfg.cabac_lambda_scale;
    let refs = std::slice::from_ref(l0);
    if fe.satd_q > 0.0 {
        let mut vars: Vec<i64> = (0..fe.mb_h)
            .flat_map(|my| (0..fe.mb_w).map(move |mx| (mx, my)))
            .map(|(mx, my)| mb_variance(&sy, fe.cw, mx, my))
            .collect();
        vars.sort_unstable();
        let idx = (((1.0 - fe.satd_q) * vars.len() as f64) as usize).min(vars.len() - 1);
        fe.satd_var_thresh = vars[idx];
    }
    let mut aq_qp = aq_qp_map(&sy, fe.cw, fe.mb_w, fe.mb_h, qp, fe.aq_strength);
    apply_mbtree_qpo(&mut aq_qp, qpo); // mb-tree temporal AQ (empty = byte-identical)
    fe.cur_qp = qp;

    let mut cab = CabacEncoder::new(qp as i32, cfg.cabac_init_idc, false);
    let mut cs = CabacState::new(fe.mb_w * fe.mb_h);
    let total = fe.mb_w * fe.mb_h;

    for mb_y in 0..fe.mb_h {
        for mb_x in 0..fe.mb_w {
            let mb_idx = mb_y * fe.mb_w + mb_x;
            let addr = mb_idx;
            let top = if mb_y > 0 { Some(addr - fe.mb_w) } else { None };
            let left = if mb_x > 0 { Some(addr - 1) } else { None };
            fe.qp = aq_qp[mb_idx];
            fe.qpc = chroma_qp(aq_qp[mb_idx]);
            let (lx, ly) = (mb_x * 16, mb_y * 16);
            let (pbx, pby) = (mb_x as isize * 4, mb_y as isize * 4);
            fe.mb_use_satd =
                fe.satd_q > 0.0 && mb_variance(&sy, fe.cw, mb_x, mb_y) >= fe.satd_var_thresh;
            let n0 = fe.mv_neighbors_block_list(pbx, pby, 4, 0);
            let n1 = fe.mv_neighbors_block_list(pbx, pby, 4, 1);
            let pmv0 = predict_partition_mv(0, 0, n0[0], n0[1], n0[2], 0);
            let pmv1 = predict_partition_mv(0, 0, n1[0], n1[1], n1[2], 0);
            let (dp, dc, dmotion) = fe.b_direct(l0, l1, mb_x, mb_y);
            // B_Skip: free direct prediction → mb_skip_flag = 1.
            if fe.skip_luma_is_free(&sy, mb_x, mb_y, &dp)
                && fe.skip_chroma_is_free(&su, &sv, mb_x, mb_y, &dc)
            {
                fe.commit_direct_motion(mb_x, mb_y, &dmotion);
                emit_b_skip_cabac(&mut cab, &mut cs, addr, top, left);
                {
                    let tt = if crate::bitacct::enabled() { cab.pos() } else { 0 };
                    {
                let tt = if crate::bitacct::enabled() { cab.pos() } else { 0 };
                cab.encode_terminate(mb_idx + 1 == total);
                if crate::bitacct::enabled() {
                    crate::bitacct::add(crate::bitacct::B::Terminate, cab.pos() - tt);
                }
            }
                    if crate::bitacct::enabled() {
                        crate::bitacct::add(crate::bitacct::B::Terminate, cab.pos() - tt);
                    }
                }
                continue;
            }
            let d_direct = fe.pred_dist(&sy, lx, ly, &dp);
            let (mv0, j0) = fe.motion_search(l0, &sy, lx, ly, 16, 16, &[pmv0], lme, None);
            let (mv1, j1) = fe.motion_search(l1, &sy, lx, ly, 16, 16, &[pmv1], lme, None);
            let d_bi = fe.bi_dist(l0, l1, &sy, lx, ly, mv0, mv1);
            let r_bi = mvd_bits(mv0.0 - pmv0.0) + mvd_bits(mv0.1 - pmv0.1)
                + mvd_bits(mv1.0 - pmv1.0) + mvd_bits(mv1.1 - pmv1.1);
            let j_bi = d_bi + (lme * r_bi as f64) as i64;
            let (mut dir, mut best) = (0u8, d_direct);
            if j0 < best { dir = 1; best = j0; }
            if j1 < best { dir = 2; best = j1; }
            if j_bi < best { dir = 3; best = j_bi; }
            let _ = best;
            // mb_skip_flag = 0, then the coded B MB.
            let sctx = 24
                + left.map_or(0, |a| (!cs.mb_skip[a]) as usize)
                + top.map_or(0, |a| (!cs.mb_skip[a]) as usize);
            let tskip = if crate::bitacct::enabled() { cab.pos() } else { 0 };
            cb_mb_skip(&mut cab, sctx, false);
            if crate::bitacct::enabled() {
                crate::bitacct::add(crate::bitacct::B::SkipFlag, cab.pos() - tskip);
            }
            cs.mb_skip[addr] = false;
            let bspec = BInter { dir, l1, mv0, mv1 };
            let plan = fe.plan_inter_mb(refs, &sy, &su, &sv, mb_x, mb_y, 0, &[], Some(bspec));
            emit_mb_cabac_b(&mut fe, &mut cab, &mut cs, dir, &plan, mb_x, mb_y);
            {
                let tt = if crate::bitacct::enabled() { cab.pos() } else { 0 };
                cab.encode_terminate(mb_idx + 1 == total);
                if crate::bitacct::enabled() {
                    crate::bitacct::add(crate::bitacct::B::Terminate, cab.pos() - tt);
                }
            }
        }
    }

    while !w.is_byte_aligned() {
        w.write_bit(true);
    }
    for b in cab.into_bytes() {
        w.write_bits(b as u32, 8);
    }
    // B is non-reference: no deblock, no RefFrame (the decoder deblocks for display).
}

/// Minimal all-B_Skip CABAC B-slice (the rare no-bracketing-anchor fallback in
/// `code_picture`): every MB is mb_skip_flag = 1. B is non-reference so the recon
/// is irrelevant; this only needs to be a legal CABAC slice.
pub fn encode_all_skip_b_cabac(w: &mut BitWriter, cfg: &EncoderConfig, qp: u8, n: usize) {
    let mut cab = CabacEncoder::new(qp as i32, cfg.cabac_init_idc, false);
    for i in 0..n {
        // ctxInc = 24 + (left avail & not-skip) + (top avail & not-skip). Every
        // neighbour is either a skip (contributes 0) or unavailable (0) → always 24.
        cab.encode_decision(24, 1);
        cab.encode_terminate(i + 1 == n);
    }
    while !w.is_byte_aligned() {
        w.write_bit(true);
    }
    for b in cab.into_bytes() {
        w.write_bits(b as u32, 8);
    }
}