hermes-tokenizer 1.8.102

//! Shared infrastructure for mask-scanner pretokenizers.
//!
//! A mask-scanner pretokenizer processes 64-byte batches: SIMD classifies
//! every byte, bitmask algebra derives "a token starts here" bits, and
//! `next()` pops one bit per token — no per-token dispatch branches, which
//! is what makes it ~2x the serial scalar scanners (see
//! pretokenizer_optimization_log.md step 15).
//!
//! A scheme plugs in two functions ([`MaskScheme`]):
//! - `advance`: the scalar ground truth (also the no-SIMD iterator),
//! - `batch_masks`: `(usable, bad)` bitmasks for a 64-byte batch. `usable`
//!   bits are trustworthy token starts; `bad` marks zones (non-ASCII the
//!   scheme doesn't classify in-mask, batch-edge ambiguities) that
//!   [`MaskState`] re-derives through `advance`, never emitting a token
//!   across an unresolved zone.
//!
//! Layering, bottom to top:
//! 1. Platform SIMD primitives (`movemask64`, `ascii_masks` on NEON;
//!    `ascii_masks_avx512` / `ascii_masks_avx2` on x86-64) — the only
//!    per-platform code.
//! 2. Bit-domain helpers shared across schemes — platform-independent
//!    u64 algebra and per-char table classification
//!    (`classify_uni_chars`, `char_through`, `nn_at_full`,
//!    `digit_run_splits3`), parameterized by each scheme's codepoint
//!    classifier.
//! 3. Per-scheme `batch_masks` boundary algebra (in the scheme's module).
//! 4. [`MaskState`] — the scheme-agnostic batch walker: segments, bad-zone
//!    gaps, scalar tail, one-batch-ahead precompute; scalar overruns stay
//!    on the 64-byte grid so the precompute survives them.
//! 5. [`MaskState::fill_spans_two_phase`] — the chunked pull the encode
//!    loop uses: the same masks and trust rules as `next_span`, but
//!    harvested a chunk at a time into a flat boundary buffer and emitted
//!    in a branch-free counted loop.

use crate::pretokenize::unicode::{self, CharClass};

// -----------------------------------------------------------------------
// Platform SIMD primitives: aarch64 NEON (compile-time, always present)
// and x86_64 AVX-512 or AVX2 (runtime-detected; scalar fallback
// otherwise).
// -----------------------------------------------------------------------

/// Does this x86_64 CPU have the full AVX-512 tier (Zen 4/5, Ice
/// Lake+)? Schemes dispatch their batch classifier on this: the AVX-512
/// front-end when true, the AVX2 one otherwise.
#[cfg(target_arch = "x86_64")]
#[inline]
pub(crate) fn avx512_scanner_available() -> bool {
    // std's feature cache makes this an atomic load + bit test after the
    // first call.
    std::arch::is_x86_feature_detected!("avx512f")
        && std::arch::is_x86_feature_detected!("avx512bw")
        && std::arch::is_x86_feature_detected!("avx512vl")
        && std::arch::is_x86_feature_detected!("bmi1")
        && std::arch::is_x86_feature_detected!("bmi2")
        && std::arch::is_x86_feature_detected!("lzcnt")
        && std::arch::is_x86_feature_detected!("popcnt")
}

/// Does this x86_64 CPU also have AVX-512 VBMI2 (native 512-bit
/// `vpcompressb`: Zen 4/5, Ice Lake+ — i.e. nearly every AVX-512 CPU,
/// but the bit is detected, not assumed: Skylake-X lacks it and stays on
/// the plain AVX-512 tier)? Gates the `X86_TIER_AVX512_VBMI2` fill tier
/// ([`MaskState::fill_spans_two_phase`]'s `_avx512_vbmi2_crc` wrapper),
/// whose `flatten_bits_avx512` needs VBMI2 on top of the scanner tier.
#[cfg(target_arch = "x86_64")]
#[inline]
pub(crate) fn avx512_fill_available() -> bool {
    avx512_scanner_available() && std::arch::is_x86_feature_detected!("avx512vbmi2")
}

/// Does this x86_64 CPU have the AVX2 tier (Haswell+, all Zen)? The bit
/// features (BMI1/2, LZCNT, POPCNT) arrived with or before AVX2 on every
/// AVX2 CPU, but are detected explicitly since the boundary algebra's
/// codegen relies on them.
#[cfg(target_arch = "x86_64")]
#[inline]
pub(crate) fn avx2_scanner_available() -> bool {
    std::arch::is_x86_feature_detected!("avx2")
        && std::arch::is_x86_feature_detected!("bmi1")
        && std::arch::is_x86_feature_detected!("bmi2")
        && std::arch::is_x86_feature_detected!("lzcnt")
        && std::arch::is_x86_feature_detected!("popcnt")
}

/// Is the SIMD mask scanner usable on this machine? aarch64 always has
/// NEON; x86_64 requires AVX-512 (Zen 4/5, Ice Lake+) or AVX2 (Haswell+,
/// Zen 1-3), detected at runtime. When this returns false, [`MaskState`]
/// runs every token through the scheme's scalar `advance`.
#[cfg(target_arch = "x86_64")]
#[inline]
pub(crate) fn simd_scanner_available() -> bool {
    avx512_scanner_available() || avx2_scanner_available()
}

#[cfg(not(target_arch = "x86_64"))]
#[inline]
pub(crate) fn simd_scanner_available() -> bool {
    cfg!(target_arch = "aarch64")
}

// The x86-64 batch classifiers are annotated
// `#[target_feature(enable = "avx512f,avx512bw,avx512vl,bmi1,bmi2,lzcnt,popcnt")]`
// (AVX-512 tier) or `#[target_feature(enable = "avx2,bmi1,bmi2,lzcnt,popcnt")]`
// (AVX2 tier). Besides the wide byte ops, the scalar-visible bit features
// (BMI1/2, LZCNT, POPCNT) are enabled so the boundary algebra inlined
// into those functions compiles to tzcnt/lzcnt/blsr instead of
// baseline-x86 bsf sequences. The sets must stay in sync with
// [`avx512_scanner_available`] / [`avx2_scanner_available`].

/// simdjson-style movemask: 4 mask vectors (64 lanes of 0x00/0xFF) -> u64,
/// bit i = lane i.
#[cfg(target_arch = "aarch64")]
#[inline(always)]
pub(crate) unsafe fn movemask64(
    v0: std::arch::aarch64::uint8x16_t,
    v1: std::arch::aarch64::uint8x16_t,
    v2: std::arch::aarch64::uint8x16_t,
    v3: std::arch::aarch64::uint8x16_t,
) -> u64 {
    use std::arch::aarch64::*;
    unsafe {
        const W: [u8; 16] = [1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128];
        let w = vld1q_u8(W.as_ptr());
        let mut a0 = vandq_u8(v0, w);
        let a1 = vandq_u8(v1, w);
        let mut a2 = vandq_u8(v2, w);
        let a3 = vandq_u8(v3, w);
        // The 4-`addp` reduction tree (simdjson's arm64 movemask), pinned
        // as asm. Written with `vpaddq_u8`, LLVM rewrites every pairwise
        // add into a uzp1/uzp2/orr triple — adjacent weighted lanes have
        // disjoint bits, so add == or, and the canonical or-form never
        // re-forms addp — inflating each call from 9 to 17 vector ops
        // (4-7 calls per 64-byte batch across the schemes). The weighted
        // `and`s stay outside so the scheduler still interleaves
        // neighboring calls. `addp(x, x)` lane 0..7 equals the old
        // `addp(x, zero)` lanes 0..7; only lane u64 0 is read.
        core::arch::asm!(
            "addp {a0:v}.16b, {a0:v}.16b, {a1:v}.16b",
            "addp {a2:v}.16b, {a2:v}.16b, {a3:v}.16b",
            "addp {a0:v}.16b, {a0:v}.16b, {a2:v}.16b",
            "addp {a0:v}.16b, {a0:v}.16b, {a0:v}.16b",
            a0 = inout(vreg) a0,
            a1 = in(vreg) a1,
            a2 = inout(vreg) a2,
            a3 = in(vreg) a3,
            options(pure, nomem, nostack, preserves_flags),
        );
        vgetq_lane_u64::<0>(vreinterpretq_u64_u8(a0))
    }
}

/// One u64 mask (bit i = byte scan+i) per byte predicate, for 64 bytes.
/// The working currency of scheme boundary algebra: everything after this
/// is platform-independent u64 bit math.
#[derive(Clone, Copy, Default)]
pub(crate) struct AsciiMasks {
    /// ASCII letters.
    pub l: u64,
    /// ASCII digits.
    pub d: u64,
    /// Space (0x20) only.
    pub s: u64,
    /// Non-newline ASCII whitespace: \t, \x0b, \x0c.
    pub wt: u64,
    /// Newlines: \r, \n.
    pub n: u64,
    /// Non-ASCII bytes (>= 0x80).
    pub hi: u64,
    /// ASCII apostrophes.
    pub ap: u64,
}

/// Classify `bytes[scan..scan+64]` (requires `scan + 64 <= bytes.len()`).
#[cfg(target_arch = "aarch64")]
#[inline(always)]
pub(crate) fn ascii_masks(bytes: &[u8], scan: usize) -> AsciiMasks {
    use std::arch::aarch64::*;
    unsafe {
        let p = bytes.as_ptr().add(scan);
        let mut l = [vdupq_n_u8(0); 4];
        let mut d = [vdupq_n_u8(0); 4];
        let mut s = [vdupq_n_u8(0); 4];
        let mut wt = [vdupq_n_u8(0); 4];
        let mut n = [vdupq_n_u8(0); 4];
        let mut hi = [vdupq_n_u8(0); 4];
        let mut ap = [vdupq_n_u8(0); 4];
        for i in 0..4 {
            let v = vld1q_u8(p.add(16 * i));
            let lowered = vorrq_u8(v, vdupq_n_u8(0x20));
            l[i] = vcleq_u8(vsubq_u8(lowered, vdupq_n_u8(b'a')), vdupq_n_u8(25));
            d[i] = vcleq_u8(vsubq_u8(v, vdupq_n_u8(b'0')), vdupq_n_u8(9));
            s[i] = vceqq_u8(v, vdupq_n_u8(b' '));
            n[i] = vorrq_u8(
                vceqq_u8(v, vdupq_n_u8(b'\r')),
                vceqq_u8(v, vdupq_n_u8(b'\n')),
            );
            // \t (9), \x0b (11), \x0c (12): ascii ws minus \r\n and space.
            wt[i] = vbicq_u8(vcleq_u8(vsubq_u8(v, vdupq_n_u8(9)), vdupq_n_u8(4)), n[i]);
            hi[i] = vcltzq_s8(vreinterpretq_s8_u8(v));
            ap[i] = vceqq_u8(v, vdupq_n_u8(b'\''));
        }
        AsciiMasks {
            l: movemask64(l[0], l[1], l[2], l[3]),
            d: movemask64(d[0], d[1], d[2], d[3]),
            s: movemask64(s[0], s[1], s[2], s[3]),
            wt: movemask64(wt[0], wt[1], wt[2], wt[3]),
            n: movemask64(n[0], n[1], n[2], n[3]),
            hi: movemask64(hi[0], hi[1], hi[2], hi[3]),
            ap: movemask64(ap[0], ap[1], ap[2], ap[3]),
        }
    }
}

/// Classify `bytes[scan..scan+64]` with AVX-512 (requires
/// `scan + 64 <= bytes.len()`). One 64-byte load and one k-register
/// compare per predicate: a `__mmask64` IS the u64 the bit algebra wants,
/// so there is no movemask ladder and no lazy any-tests — every field
/// (including `hi` and `ap`) is computed unconditionally.
///
/// Runtime-gated: callers reach this only after
/// [`simd_scanner_available`] reported AVX-512 support (enforced by
/// [`MaskState`], which otherwise never leaves the scalar path).
#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "avx512f,avx512bw,avx512vl,bmi1,bmi2,lzcnt,popcnt")]
#[inline]
pub(crate) fn ascii_masks_avx512(bytes: &[u8], scan: usize) -> AsciiMasks {
    use std::arch::x86_64::*;
    unsafe {
        let v = _mm512_loadu_si512(bytes.as_ptr().add(scan) as *const _);
        let lowered = _mm512_or_si512(v, _mm512_set1_epi8(0x20));
        let l = _mm512_cmple_epu8_mask(
            _mm512_sub_epi8(lowered, _mm512_set1_epi8(b'a' as i8)),
            _mm512_set1_epi8(25),
        );
        let d = _mm512_cmple_epu8_mask(
            _mm512_sub_epi8(v, _mm512_set1_epi8(b'0' as i8)),
            _mm512_set1_epi8(9),
        );
        let s = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8(b' ' as i8));
        let n = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8(b'\r' as i8))
            | _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8(b'\n' as i8));
        // \t (9), \x0b (11), \x0c (12): ascii ws minus \r\n and space.
        let wt =
            _mm512_cmple_epu8_mask(_mm512_sub_epi8(v, _mm512_set1_epi8(9)), _mm512_set1_epi8(4))
                & !n;
        let hi = _mm512_movepi8_mask(v) as u64;
        let ap = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8(b'\'' as i8));
        AsciiMasks {
            l,
            d,
            s,
            wt,
            n,
            hi,
            ap,
        }
    }
}

/// Classify `bytes[scan..scan+64]` with AVX2 (requires
/// `scan + 64 <= bytes.len()`). Two 32-byte loads; each predicate is one
/// vector compare per half plus a `vpmovmskb` ladder into the u64 the bit
/// algebra wants — more mask-extraction traffic than the AVX-512 version
/// (whose k-register compares ARE the u64s), but the output currency is
/// identical, so everything downstream is shared. AVX2 has no unsigned
/// byte compare; `x <= lim` is `min_epu8(x, lim) == x`.
///
/// Runtime-gated: callers reach this only after
/// [`avx2_scanner_available`] reported AVX2 support (enforced by the
/// schemes' dispatch, behind [`MaskState`]'s `simd_scanner_available`
/// gate).
///
/// `#[inline(never)]` is load-bearing: inlined, LLVM's vector combiner
/// sees the compare vectors behind the returned u64s and pulls the
/// caller's scalar boundary algebra back into the byte-vector domain,
/// expanding every mask<->vector crossing into vpinsrb/vpextrb ladders
/// (~240 byte ops per batch, measured 3.5x slower end to end on Zen 2).
/// The AVX-512 tier has no such domain to return to (k-register compares
/// ARE the u64s), so it stays inline.
#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "avx2,bmi1,bmi2,lzcnt,popcnt")]
#[inline(never)]
pub(crate) fn ascii_masks_avx2(bytes: &[u8], scan: usize) -> AsciiMasks {
    use std::arch::x86_64::*;
    unsafe {
        // Closures inherit the enclosing fn's target features.
        let le =
            |v: __m256i, lim: __m256i| -> __m256i { _mm256_cmpeq_epi8(_mm256_min_epu8(v, lim), v) };
        let mm = |m0: __m256i, m1: __m256i| -> u64 {
            (_mm256_movemask_epi8(m0) as u32 as u64)
                | ((_mm256_movemask_epi8(m1) as u32 as u64) << 32)
        };

        let p = bytes.as_ptr().add(scan);
        let v0 = _mm256_loadu_si256(p as *const _);
        let v1 = _mm256_loadu_si256(p.add(32) as *const _);

        let x20 = _mm256_set1_epi8(0x20);
        let ca = _mm256_set1_epi8(b'a' as i8);
        let c25 = _mm256_set1_epi8(25);
        let l = mm(
            le(_mm256_sub_epi8(_mm256_or_si256(v0, x20), ca), c25),
            le(_mm256_sub_epi8(_mm256_or_si256(v1, x20), ca), c25),
        );
        let c0 = _mm256_set1_epi8(b'0' as i8);
        let c9 = _mm256_set1_epi8(9);
        let d = mm(
            le(_mm256_sub_epi8(v0, c0), c9),
            le(_mm256_sub_epi8(v1, c0), c9),
        );
        let sp = _mm256_set1_epi8(b' ' as i8);
        let s = mm(_mm256_cmpeq_epi8(v0, sp), _mm256_cmpeq_epi8(v1, sp));
        let cr = _mm256_set1_epi8(b'\r' as i8);
        let lf = _mm256_set1_epi8(b'\n' as i8);
        let n = mm(
            _mm256_or_si256(_mm256_cmpeq_epi8(v0, cr), _mm256_cmpeq_epi8(v0, lf)),
            _mm256_or_si256(_mm256_cmpeq_epi8(v1, cr), _mm256_cmpeq_epi8(v1, lf)),
        );
        // \t (9), \x0b (11), \x0c (12): ascii ws minus \r\n and space.
        let c4 = _mm256_set1_epi8(4);
        let wt = mm(
            le(_mm256_sub_epi8(v0, c9), c4),
            le(_mm256_sub_epi8(v1, c9), c4),
        ) & !n;
        let hi = mm(v0, v1); // vpmovmskb takes the sign bit directly
        let apc = _mm256_set1_epi8(b'\'' as i8);
        let ap = mm(_mm256_cmpeq_epi8(v0, apc), _mm256_cmpeq_epi8(v1, apc));
        AsciiMasks {
            l,
            d,
            s,
            wt,
            n,
            hi,
            ap,
        }
    }
}

// -----------------------------------------------------------------------
// Bit-domain helpers (platform-independent)
// -----------------------------------------------------------------------

/// Is the char starting at `idx` NOT whitespace (`\S` for a `(?!\S)`
/// lookahead)? Full answer via the packed table.
///
/// # Safety
///
/// `idx < bytes.len()`, and when `bytes[idx]` is non-ASCII,
/// `idx + 4 <= bytes.len()` (the guardless [`decode_cp_inbounds`] read).
/// The batch classifiers' `scan + 70 <= len` guard covers every call
/// site's worst case (`idx = scan + 64`).
#[inline(always)]
pub(crate) unsafe fn nn_at_full(bytes: &[u8], idx: usize) -> bool {
    use super::{decode_cp_inbounds, is_ascii_ws};
    let b = bytes[idx];
    if b < 0x80 {
        return !is_ascii_ws(b);
    }
    // SAFETY: caller guarantees idx + 4 <= len for a non-ASCII byte here
    // (this fn's contract).
    let (cp, _) = unsafe { decode_cp_inbounds(bytes, idx) };
    unicode::class_of(cp) != CharClass::Whitespace
}

/// The char containing byte `pos - 1` (`pos > 0`, valid UTF-8): its
/// class, lead index, and end (exclusive). `end > pos` iff the char
/// straddles across `pos`. ASCII classifies with the byte predicates;
/// multi-byte chars walk back to their lead (at most 3 bytes) and use
/// the packed table — this is what lets a batch after a unicode char
/// compute true boundary carries instead of deferring to a bad zone.
/// `class`: the scheme's codepoint classifier (`unicode::class_of`, or a
/// mark-folding view like `unicode::class_of_marks_join`).
///
/// # Safety
///
/// `pos > 0`, and when `bytes[pos - 1]` is non-ASCII,
/// `pos + 3 <= bytes.len()`: the walk-back lead `j` satisfies
/// `j <= pos - 1`, so the guardless [`decode_cp_inbounds`] read needs
/// `j + 4 <= pos + 3` in-bounds bytes. The batch classifiers' `scan + 70
/// <= len` guard covers every call site (`pos <= scan + 64`).
#[inline(always)]
pub(crate) unsafe fn char_through(
    bytes: &[u8],
    pos: usize,
    class: impl Fn(u32) -> CharClass,
) -> (CharClass, usize, usize) {
    use super::{decode_cp_inbounds, is_ascii_ws, is_digit, is_letter};
    let b = bytes[pos - 1];
    if b < 0x80 {
        let cls = if is_letter(b) {
            CharClass::Letter
        } else if is_digit(b) {
            CharClass::Number
        } else if is_ascii_ws(b) {
            CharClass::Whitespace
        } else {
            CharClass::Other
        };
        return (cls, pos - 1, pos);
    }
    let mut j = pos - 1;
    while j > 0 && bytes[j] & 0xC0 == 0x80 {
        j -= 1;
    }
    // SAFETY: j < pos and pos + 3 <= len (this fn's contract), so
    // j + 4 <= len.
    let (cp, l) = unsafe { decode_cp_inbounds(bytes, j) };
    (class(cp), j, j + l)
}

/// Per-byte class masks for a batch's unicode chars, classified with the
/// packed table (`unicode::class_of`) — the same lookup the scalar paths
/// do. Every byte of a classified char carries the char's class, so
/// byte-adjacency == char-adjacency and the schemes' u64 boundary
/// algebra applies unchanged.
#[derive(Clone, Copy, Default)]
pub(crate) struct UniClasses {
    /// Letter / number / other / whitespace bytes.
    pub l: u64,
    pub n: u64,
    pub o: u64,
    pub ws: u64,
    /// Whitespace lead bits by char length, for the char-length-aware
    /// `(?!\S)` shift tests. Deferred ws chars (see `resid`) are not
    /// included.
    pub w2: u64,
    pub w3: u64,
    /// Lead bits of all classified chars by length, for schemes that
    /// shift a test by the previous char's length (the cl100k family's
    /// two-chars-back rule).
    pub lead2: u64,
    pub lead3: u64,
    pub lead4: u64,
    /// Continuation bytes of classified chars.
    pub cont: u64,
    /// Bytes only the scalar path can decide: whitespace chars straddling
    /// the batch end (their run-split bookkeeping crosses the boundary),
    /// number chars when `NUMBERS` is false, and stray continuation
    /// bytes. Class masks stay truthful for these bytes so neighbors'
    /// algebra is exact; callers turn `resid` into bad zones (±1 smear).
    pub resid: u64,
}

/// Classify every unicode char whose lead bit is in `m` (typically
/// `hi & !claimed-straddle-in-bytes`) for `bytes[scan..scan+64]`.
/// A char spilling off the batch end is classified via the lookahead
/// bytes; only its in-batch bytes get class bits, and the next
/// batch's `char_through` walk-back covers the remainder. `NUMBERS`:
/// false for schemes whose digit grouping is char-counted (`\p{N}{1,3}`
/// byte masks can't express multi-byte chars), true otherwise.
/// `LEADS`: whether to fill the per-length lead masks (only schemes with
/// a shift-by-prev-char-length rule need them).
///
/// The loop stays branchy on purpose: a branchless csel-selected
/// decode/classify body measured 0.986x (predicted branches beat data
/// chains, log step 13/17). 2-byte chars (nearly all non-ASCII in western
/// corpora) take a dedicated lane with an inline decode; 3/4-byte chars
/// pay the general ladder.
///
/// # Safety
///
/// `scan + 70 <= bytes.len()` (the batch classifiers' lookahead guard):
/// a lead bit at position 63 puts the guardless [`decode_cp_inbounds`]
/// read at `scan + 63`, which may touch through `scan + 67`.
#[inline(always)]
pub(crate) unsafe fn classify_uni_chars<const NUMBERS: bool, const LEADS: bool>(
    bytes: &[u8],
    scan: usize,
    mut m: u64,
    class: impl Fn(u32) -> CharClass,
) -> UniClasses {
    use super::decode_cp_inbounds;
    let mut u = UniClasses::default();
    while m != 0 {
        let i = m.trailing_zeros() as usize;
        m &= m - 1;
        let b = bytes[scan + i];
        if b < 0xE0 {
            // 2-byte lane (leads 0xC2..0xDF, cp < 0x800): nearly every
            // non-ASCII char in western corpora, so this branch predicts
            // taken and skips the length ladder + general decode.
            if b < 0xC2 {
                u.resid |= 1 << i; // stray continuation byte (invalid UTF-8)
                continue;
            }
            let lead = 1u64 << i;
            let chm = 3u64 << i; // in-batch bytes (excess drops at bit 63)
            // SAFETY: scan + 70 <= len (this fn's # Safety contract),
            // i <= 63, so scan + i + 1 <= scan + 64 < len.
            let b1 = unsafe { *bytes.get_unchecked(scan + i + 1) };
            let cp = ((b as u32 & 0x1F) << 6) | (b1 as u32 & 0x3F);
            match class(cp) {
                CharClass::Letter => u.l |= chm,
                CharClass::Number => {
                    u.n |= chm;
                    if !NUMBERS {
                        u.resid |= chm;
                    }
                }
                CharClass::Other => u.o |= chm,
                CharClass::Whitespace => {
                    u.ws |= chm;
                    if i + 2 > 64 {
                        // Straddling-out ws stays a bad zone; its true
                        // class marks keep neighbors' `(?!\S)` tests
                        // exact.
                        u.resid |= chm;
                    } else {
                        u.w2 |= lead;
                    }
                }
            }
            if LEADS {
                u.lead2 |= lead;
            }
            u.cont |= chm & !lead;
            m &= !chm;
            continue;
        }
        let l = if b < 0xF0 { 3 } else { 4 };
        let chm = ((1u64 << l) - 1) << i; // in-batch bytes (excess drops)
        let lead = 1u64 << i;
        // SAFETY: scan + 70 <= len (this fn's # Safety contract), i <= 63,
        // so scan + i + 4 <= len even for a 4-byte lead at bit 63.
        let (cp, _) = unsafe { decode_cp_inbounds(bytes, scan + i) };
        match class(cp) {
            CharClass::Letter => u.l |= chm,
            CharClass::Number => {
                u.n |= chm;
                if !NUMBERS {
                    u.resid |= chm;
                }
            }
            CharClass::Other => u.o |= chm,
            CharClass::Whitespace => {
                u.ws |= chm;
                if i + l > 64 || l == 4 {
                    // Straddling-out ws (and defensively: no 4-byte cp
                    // is ws in Unicode) stays a bad zone; its true class
                    // marks keep neighbors' `(?!\S)` tests exact.
                    u.resid |= chm;
                } else {
                    u.w3 |= lead;
                }
            }
        }
        if LEADS {
            if l == 3 {
                u.lead3 |= lead;
            } else {
                u.lead4 |= lead;
            }
        }
        u.cont |= chm & !lead;
        m &= !chm;
    }
    u
}

/// Token-start bits inside ASCII digit runs for `\p{N}{1,3}`: each run
/// splits into 3-char tokens, so boundaries sit at run start + 3k. (For a
/// plain `\p{N}` scheme every digit is a start — no helper needed.)
#[inline(always)]
pub(crate) fn digit_run_splits3(d: u64) -> u64 {
    let mut b = d & !(d << 1); // run starts
    // A start at p re-arms at p+3 while the run continues: hop condition
    // c = "p..p+3 all digits". Log-doubling covers 64-bit runs in 5 steps.
    let mut c = d & (d >> 1) & (d >> 2) & (d >> 3);
    let mut sh = 3u32;
    while sh < 64 {
        b |= (b & c) << sh;
        c &= c >> sh;
        sh <<= 1;
    }
    b
}

// -----------------------------------------------------------------------
// The batch walker
// -----------------------------------------------------------------------

/// The two per-scheme hooks of a mask-scanner pretokenizer.
pub(crate) trait MaskScheme {
    /// Scalar ground truth: end of the token starting at `pos`
    /// (`pos < bytes.len()`, `pos` on a token boundary).
    fn advance(bytes: &[u8], pos: usize) -> usize;

    /// `(usable, bad)` for `bytes[scan..scan+64]` (`scan+64 <= len`):
    /// `usable` bit k = trustworthy token start at scan+k; `bad` bit k =
    /// byte scan+k needs the scalar path. `usable & bad` must be 0.
    #[cfg(target_arch = "aarch64")]
    fn batch_masks(bytes: &[u8], scan: usize) -> (u64, u64);

    /// The x86_64 batch classifier, monomorphized on the SIMD tier
    /// (`AVX512` = true → the AVX-512 front-end, false → AVX2); same
    /// `(usable, bad)` contract as the aarch64 `batch_masks`. The fill
    /// wrappers instantiate this inside a matching `#[target_feature]`
    /// region, so the tier function inlines into the fill loop and no
    /// per-batch dispatch survives (the codegen a `-C target-cpu=native`
    /// build gets).
    ///
    /// # Safety
    ///
    /// The selected tier must have been runtime-detected:
    /// [`avx512_scanner_available`] for `AVX512` = true,
    /// [`avx2_scanner_available`] for `AVX512` = false.
    #[cfg(target_arch = "x86_64")]
    unsafe fn batch_masks_x86<const AVX512: bool>(bytes: &[u8], scan: usize) -> (u64, u64);

    /// Runtime-dispatched form of [`Self::batch_masks_x86`] for call
    /// sites outside a tier-monomorphized region (`next_span`): a cached
    /// tier check plus a non-inlined call per batch into a per-tier
    /// `#[target_feature]` wrapper ([`batch_masks_dyn_avx512`] /
    /// [`batch_masks_dyn_avx2`]), so the classifier body still compiles
    /// under the full tier feature set. Must only be called when
    /// [`simd_scanner_available`] is true — [`MaskState`] guarantees this
    /// by never leaving the scalar path otherwise.
    #[cfg(target_arch = "x86_64")]
    #[inline(always)]
    fn batch_masks(bytes: &[u8], scan: usize) -> (u64, u64)
    where
        Self: Sized,
    {
        debug_assert!(simd_scanner_available());
        // The tier check is a cached atomic load + bit test and the
        // branch is perfectly predicted, so it is noise next to the
        // batch classification it selects.
        if avx512_scanner_available() {
            // SAFETY: runtime AVX-512 detection right above.
            unsafe { batch_masks_dyn_avx512::<Self>(bytes, scan) }
        } else {
            // SAFETY: MaskState enables the mask-scanner path only after
            // runtime detection (simd_scanner_available); without AVX-512
            // that detection was the AVX2 tier's.
            unsafe { batch_masks_dyn_avx2::<Self>(bytes, scan) }
        }
    }
}

/// AVX-512 feature region for the runtime-dispatched
/// `MaskScheme::batch_masks`: the scheme's `#[inline(always)]`
/// `batch_masks_x86` body fuses into this wrapper, so the per-batch call
/// `next_span` pays runs full-tier codegen (without this region the body
/// would inline into the plain-feature caller, where the inner
/// `#[target_feature]` mask classifiers can't inline and the boundary
/// algebra loses BMI/LZCNT codegen — measured ~25% slower).
///
/// # Safety
///
/// The CPU must support the AVX-512 scanner tier
/// ([`avx512_scanner_available`]).
#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "avx512f,avx512bw,avx512vl,bmi1,bmi2,lzcnt,popcnt")]
#[inline]
unsafe fn batch_masks_dyn_avx512<S: MaskScheme>(bytes: &[u8], scan: usize) -> (u64, u64) {
    // SAFETY: the caller detected the AVX-512 tier (fn contract).
    unsafe { S::batch_masks_x86::<true>(bytes, scan) }
}

/// AVX2 counterpart of [`batch_masks_dyn_avx512`].
///
/// # Safety
///
/// The CPU must support the AVX2 scanner tier
/// ([`avx2_scanner_available`]).
#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "avx2,bmi1,bmi2,lzcnt,popcnt")]
#[inline]
unsafe fn batch_masks_dyn_avx2<S: MaskScheme>(bytes: &[u8], scan: usize) -> (u64, u64) {
    // SAFETY: the caller detected the AVX2 tier (fn contract).
    unsafe { S::batch_masks_x86::<false>(bytes, scan) }
}

/// x86 SIMD-tier selector for the monomorphized fill bodies
/// ([`MaskState::fill_spans_two_phase_impl`]): `DYN` keeps the per-batch
/// runtime dispatch of the provided `MaskScheme::batch_masks`; `AVX2` /
/// `AVX512` pin the tier, chosen once per fill inside a matching
/// `#[target_feature]` wrapper. `AVX512_VBMI2` is the AVX-512 tier plus
/// VBMI2 ([`avx512_fill_available`]): same batch classifiers, but phase
/// A's flatten runs `vpcompressb` (`flatten_bits_avx512`) — its only
/// divergence. Meaningless (and always `DYN`) off x86_64.
pub(crate) const X86_TIER_DYN: u8 = 0;
pub(crate) const X86_TIER_AVX2: u8 = 1;
pub(crate) const X86_TIER_AVX512: u8 = 2;
pub(crate) const X86_TIER_AVX512_VBMI2: u8 = 3;

/// Scheme-agnostic mask-scanner state: pops trusted boundary bits, walks
/// bad zones through the scheme's scalar `advance`, runs the buffer tail
/// scalar, and precomputes one batch ahead so the SIMD chain retires under
/// the previous batch's pops. Without SIMD support (non-aarch64/x86_64
/// targets, or an x86_64 CPU without AVX-512 or AVX2) `scalar_until`
/// starts at `usize::MAX`, so every token takes the scalar path.
pub(crate) struct MaskState {
    /// Start of the pending (not yet emitted) token.
    pub pos: usize,
    /// Base of the next batch to scan.
    scan: usize,
    /// Base the `rem`/`batch_*` bits refer to.
    mask_base: usize,
    /// Boundary bits of the current segment (trusted, pop-ready).
    rem: u64,
    /// Full usable mask of the current batch (later segments).
    batch_usable: u64,
    /// Bad zones of the current batch not yet passed.
    batch_bad: u64,
    /// Emit tokens via the scalar advance while `pos < scalar_until`.
    scalar_until: usize,
    /// Eagerly computed masks for the batch at `pre_base` (usize::MAX =
    /// none).
    pre_base: usize,
    pre_usable: u64,
    pre_bad: u64,
}

impl MaskState {
    #[inline]
    pub(crate) fn new(pos: usize) -> Self {
        let scalar_until = if simd_scanner_available() {
            pos
        } else {
            usize::MAX
        };
        Self {
            pos,
            scan: pos,
            mask_base: pos,
            rem: 0,
            batch_usable: 0,
            batch_bad: 0,
            scalar_until,
            pre_base: usize::MAX,
            pre_usable: 0,
            pre_bad: 0,
        }
    }

    /// Load the segment of `batch_usable` bits in [from_bit, next bad run)
    /// into `rem` and aim `scalar_until` past that bad run at the next
    /// trusted boundary (or the batch end).
    #[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))]
    #[inline(always)]
    fn load_segment(&mut self, from_bit: u32) {
        let live = u64::MAX << from_bit;
        let seg_bad = self.batch_bad & live;
        if seg_bad == 0 {
            self.rem = self.batch_usable & live;
            self.batch_bad = 0;
        } else {
            let nb = seg_bad.trailing_zeros();
            self.rem = self.batch_usable & live & ((1u64 << nb) - 1);
            let rest = self.batch_usable & (u64::MAX << nb);
            self.scalar_until = if rest != 0 {
                self.mask_base + rest.trailing_zeros() as usize
            } else {
                self.mask_base + 64
            };
        }
        // A bit at the pending token's own start is not an end. Branchless:
        // whether the pending token starts exactly at this segment's first
        // bit is a ~20% coin flip on natural text.
        let at_start = self.pos == self.mask_base + from_bit as usize;
        self.rem &= !(u64::from(at_start) << from_bit);
    }

    /// The next token's byte range, or None at end of input.
    #[inline(always)]
    pub(crate) fn next_span<S: MaskScheme>(&mut self, bytes: &[u8]) -> Option<(usize, usize)> {
        let len = bytes.len();
        loop {
            if self.rem != 0 {
                let tz = self.rem.trailing_zeros() as usize;
                let end = self.mask_base + tz;
                self.rem &= self.rem - 1;
                let start = self.pos;
                self.pos = end;
                return Some((start, end));
            }
            if self.pos < self.scalar_until {
                if self.pos >= len {
                    return None;
                }
                let start = self.pos;
                let end = S::advance(bytes, start);
                self.pos = end;
                return Some((start, end));
            }
            #[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))]
            {
                // Continue with the current batch's next trusted segment
                // after a scalar gap (each batch is computed exactly once).
                if self.batch_bad != 0 && self.pos < self.mask_base + 64 {
                    self.load_segment((self.pos - self.mask_base) as u32);
                    continue;
                }
                self.batch_bad = 0;
                // Resume after a scalar overrun WITHOUT leaving the
                // 64-byte grid: the precomputed next batch (and the
                // prefetch chain behind it) stays valid, where rebasing
                // to the token boundary invalidated it on every bad-zone
                // overrun — a large part of a deferral's ~800-cycle
                // cost. Grid bits below `pos` may be stale run-internal
                // bits (a ws or digit run the scalar walked through can
                // cross the grid base); they are masked by the
                // `from_bit` passed to load_segment below, and every
                // path that puts `pos` inside such a run goes through a
                // deferral first, so those bits are never trusted.
                while self.scan + 64 <= self.pos {
                    self.scan += 64;
                }
                if self.scan + 64 > len {
                    // Tail: scalar to the end of the buffer.
                    self.scalar_until = usize::MAX;
                    continue;
                }
                let (usable, bad) = if self.pre_base == self.scan {
                    (self.pre_usable, self.pre_bad)
                } else {
                    S::batch_masks(bytes, self.scan)
                };
                self.mask_base = self.scan;
                self.scan += 64;
                self.batch_usable = usable;
                self.batch_bad = bad;
                // Kick off the next batch now; its SIMD chain overlaps this
                // batch's pops instead of stalling the next refill. Also
                // done for dirty batches: a scalar overrun past the batch
                // end just leaves the precompute unused (`pre_base` misses),
                // while gaps that resolve inside the batch — the common
                // case — keep the pipeline primed. Dirty batches used to
                // skip this, and paying the whole SIMD chain latency at the
                // next refill was a large part of their ~270-cycle cost.
                if self.scan + 64 <= len {
                    let (u2, b2) = S::batch_masks(bytes, self.scan);
                    self.pre_base = self.scan;
                    self.pre_usable = u2;
                    self.pre_bad = b2;
                } else {
                    self.pre_base = usize::MAX;
                }
                // An overrun may have left `pos` inside this grid batch;
                // start from its bit so stale bits below never pop. The
                // no-overrun case keeps the constant argument (and its
                // folded codegen) — schemes with few bad zones take that
                // branch essentially always.
                if self.pos > self.mask_base {
                    self.load_segment((self.pos - self.mask_base) as u32);
                } else {
                    self.load_segment(0);
                }
            }
            #[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))]
            {
                // Unreachable: scalar_until is usize::MAX on this arch.
                self.scalar_until = usize::MAX;
            }
        }
    }
}

// -----------------------------------------------------------------------
// Two-phase chunked span fill
// -----------------------------------------------------------------------

/// Set-bit positions of a byte, packed in 8 u16 lanes (unused lanes 0,
/// never read). 4 KB, L1-resident alongside the unicode class table.
#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))]
static BIT_POS: [[u16; 8]; 256] = {
    let mut t = [[0u16; 8]; 256];
    let mut b = 1usize;
    while b < 256 {
        let mut j = 0;
        let mut w = 0;
        while j < 8 {
            if b >> j & 1 == 1 {
                t[b][w] = j as u16;
                w += 1;
            }
            j += 1;
        }
        b += 1;
    }
    t
};

/// Append the set-bit positions of `m`, offset by `rel` (wrapping), to
/// `out[0..popcount]` with no data-dependent branch: 8 fixed iterations,
/// one unconditional 8-lane store each at that octet's exclusive-prefix
/// popcount, so 64 bits cost the same straight-line code regardless of
/// population. Scribbles up to `out[popcount + 7]`; callers reserve the
/// slack. Returns the popcount.
#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))]
#[inline(always)]
unsafe fn flatten_bits(m: u64, rel: u16, out: *mut u16) -> usize {
    // Per-octet popcounts (SWAR); one multiply turns them into inclusive
    // prefix sums, and a byte shift makes them exclusive write offsets —
    // the 8 stores below are mutually independent.
    let mut x = m;
    x -= (x >> 1) & 0x5555_5555_5555_5555;
    x = (x & 0x3333_3333_3333_3333) + ((x >> 2) & 0x3333_3333_3333_3333);
    x = (x + (x >> 4)) & 0x0F0F_0F0F_0F0F_0F0F;
    let incl = x.wrapping_mul(0x0101_0101_0101_0101);
    let excl = incl << 8;
    #[cfg(target_arch = "aarch64")]
    unsafe {
        use std::arch::aarch64::*;
        for j in 0..8 {
            let b = (m >> (8 * j)) as u8 as usize;
            let w = (excl >> (8 * j)) as u8 as usize;
            let v = vld1q_u16(BIT_POS[b].as_ptr());
            let v = vaddq_u16(v, vdupq_n_u16(rel.wrapping_add(8 * j as u16)));
            vst1q_u16(out.add(w), v);
        }
    }
    #[cfg(not(target_arch = "aarch64"))]
    unsafe {
        for j in 0..8 {
            let b = (m >> (8 * j)) as u8 as usize;
            let w = (excl >> (8 * j)) as u8 as usize;
            let e = &BIT_POS[b];
            let base = rel.wrapping_add(8 * j as u16);
            // Fixed 8-lane copy: autovectorizes to one 16-byte store.
            for (t, &offset) in e.iter().enumerate() {
                out.add(w + t).write(offset.wrapping_add(base));
            }
        }
    }
    (incl >> 56) as usize
}

/// [`flatten_bits`] via AVX-512 VBMI2: `vpcompressb` packs the set-bit
/// positions of `m` (as compressed iota-byte lanes) in one op, replacing
/// the 8-octet BIT_POS LUT walk (~3.8% of warm cycles on Zen 5,
/// profiling/zen5_st_profile.md §5.5). Widen both halves to u16, add the
/// broadcast `rel` (wrapping, as the scalar version), two unconditional
/// 64-byte stores. Scribbles `out[0..128]` regardless of popcount — a
/// wider scribble than the scalar version's `out[popcount + 7]`; BOUND_BUF
/// reserves the 128-lane slack past every call site's worst-case cursor.
///
/// # Safety
///
/// The CPU must support AVX-512 F/BW/VBMI2 (reached only from the
/// `fill_spans_two_phase_avx512_vbmi2_crc` wrapper, gated on
/// [`avx512_fill_available`]), and `out[0..128]` must be writable.
#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "avx512f,avx512bw,avx512vbmi2")]
#[inline]
unsafe fn flatten_bits_avx512(m: u64, rel: u16, out: *mut u16) -> usize {
    use std::arch::x86_64::*;
    const IOTA: [u8; 64] = {
        let mut a = [0u8; 64];
        let mut i = 0;
        while i < 64 {
            a[i] = i as u8;
            i += 1;
        }
        a
    };
    unsafe {
        let iota = _mm512_loadu_si512(IOTA.as_ptr() as *const _);
        let comp = _mm512_maskz_compress_epi8(m, iota);
        let relv = _mm512_set1_epi16(rel as i16);
        let lo = _mm512_add_epi16(_mm512_cvtepu8_epi16(_mm512_castsi512_si256(comp)), relv);
        let hi = _mm512_add_epi16(
            _mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64::<1>(comp)),
            relv,
        );
        _mm512_storeu_si512(out as *mut _, lo);
        _mm512_storeu_si512(out.add(32) as *mut _, hi);
    }
    m.count_ones() as usize
}

/// [`flatten_bits`], monomorphized on the fill's x86 tier: the const
/// comparison folds at compile time (no per-call branch survives), and
/// the VBMI2 variant is only instantiated live inside the
/// `#[target_feature]` `_avx512_vbmi2_crc` wrapper, so its intrinsics
/// inline there.
///
/// # Safety
///
/// As [`flatten_bits`]; with `X86_TIER = X86_TIER_AVX512_VBMI2`,
/// additionally [`flatten_bits_avx512`]'s contract (VBMI2 CPU, 128
/// writable lanes).
#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))]
#[inline(always)]
unsafe fn flatten_bits_dispatch<const X86_TIER: u8>(m: u64, rel: u16, out: *mut u16) -> usize {
    #[cfg(target_arch = "x86_64")]
    if X86_TIER == X86_TIER_AVX512_VBMI2 {
        // SAFETY: forwarded from this fn's contract.
        return unsafe { flatten_bits_avx512(m, rel, out) };
    }
    // SAFETY: forwarded from this fn's contract.
    unsafe { flatten_bits(m, rel, out) }
}

/// [`pack_mask_halves`](crate::pretokenize::pack_mask_halves) — the single
/// source of the mask math — evaluated for each clamped length `m` in
/// 1..=15 (entry 0 unused), as one 16-byte row so the phase-B emission
/// loop loads both halves with a single `ldp`. That loop is
/// issue-width-bound (~34 instructions/span before, at 4 stores + 2 loads
/// it is nowhere near the load/store port limits), so trading the 7-op
/// per-half shift/select chain for 1 always-L1-hot load (256 B, 4 lines)
/// is a straight instruction-count cut. The ALU form stays in
/// `pack_mask_halves` for the latency-chained per-span paths — see its
/// docs for the measured cost of a dependent load on those chains.
#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))]
static PACK_MASK_TABLE: [[u64; 2]; 16] = {
    let mut t = [[0u64; 2]; 16];
    let mut n = 1;
    while n <= 15 {
        let (lo, hi) = crate::pretokenize::pack_mask_halves(n);
        t[n] = [lo, hi];
        n += 1;
    }
    t
};

/// Boundary scratch of one fill: PRETOKEN_CHUNK live entries, one batch of
/// overshoot from the last harvested batch (64 in-batch boundaries plus one
/// scalar-overrun end), and the flatten scribble slack — 128 lanes for
/// [`flatten_bits_avx512`]'s two unconditional 64-byte stores (the widest
/// path; [`flatten_bits`]' 8-lane scribble is subsumed), with margin.
/// Worst-case cursor at a flatten call: needed - 1 (= 255) at batch entry
/// plus up to 64 in-batch boundaries already written = 319; 319 + 128 =
/// 447 <= 464.
#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))]
const BOUND_BUF: usize = crate::pretokenize::PRETOKEN_CHUNK + 208;

/// Boundary offsets are u16-relative to the fill base; a batch is only
/// harvested while every position it can contribute (base + 63, or the
/// tail's `len`, both < base + 64) still fits.
#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))]
const REL_LIMIT: isize = u16::MAX as isize - 127;

#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))]
impl MaskState {
    /// Two-phase `fill_spans_keyed` body: phase A harvests one chunk's
    /// boundary positions into a flat buffer (branchless [`flatten_bits`]
    /// per clean batch; the scheme's scalar `advance` through bad zones,
    /// with `next_span`'s exact segment/overrun/tail trust rules), then
    /// phase B turns consecutive boundary pairs into batch entries
    /// in a counted loop with no data-dependent branch. The per-span
    /// refill ladder, pop-exit and pack mispredicts of the fused
    /// `next_span` loop (the dominant share of encode's 25% discarded
    /// issue bandwidth) collapse into one predictable branch per 64-byte
    /// batch.
    ///
    /// Boundary sets are identical to `next_span`'s by construction: the
    /// same `batch_masks` bits, the same scalar re-derivation for bad
    /// zones. Leftover boundaries past the chunk are discarded and `scan`
    /// rewound to the grid batch containing `pos` — masks are pure
    /// functions of the bytes, so the ~1 recomputed batch per fill buys
    /// carry-free fills, and a later `next_span` (which only ever advances
    /// `scan`) cannot skip the discarded bits. All other fields are reset
    /// so iterator and chunked pulls compose in any order.
    ///
    /// Callers must ensure [`simd_scanner_available`] (the scheme's
    /// `batch_masks` is unsafe to call otherwise on x86_64).
    #[inline(always)]
    pub(crate) fn fill_spans_two_phase<'a, S: MaskScheme>(
        &mut self,
        bytes: &'a [u8],
        batch: &mut crate::pretokenize::SpanBatch<'a>,
        prefetch: &impl Fn(u64),
    ) -> usize {
        // Tier + hash-arm dispatch, once per fill (≤ PRETOKEN_CHUNK
        // spans), on process-immutable bits (see `fill_span_hash` for the
        // hash-arm contract). Inside the tier wrappers the scheme's batch
        // classifier inlines into the harvest loop and the bit-scan loops
        // get BMI/LZCNT codegen — the per-batch tier branch and call that
        // the provided `MaskScheme::batch_masks` pays (~2% of end-to-end
        // encode) exist only in the DYN instantiation, which real
        // hardware never takes: fill callers require a SIMD tier, and
        // every AVX2/AVX-512 CPU has SSE4.2.
        #[cfg(target_arch = "x86_64")]
        if crate::pretokenize::crc_hash_selected() {
            // Feature detection (including the VBMI2 bit) stays in this
            // once-per-fill dispatch: `is_x86_feature_detected!` does NOT
            // const-fold inside a matching `#[target_feature]` fn (it
            // stays an atomic load), so testing it any deeper would put
            // the load in the loop.
            if avx512_fill_available() {
                // SAFETY: `avx512_fill_available` verified the AVX-512
                // scanner tier plus VBMI2; every such CPU has SSE4.2
                // (also implied by `crc_hash_selected` above).
                return unsafe {
                    self.fill_spans_two_phase_avx512_vbmi2_crc::<S>(bytes, batch, prefetch)
                };
            }
            if avx512_scanner_available() {
                // SAFETY: AVX-512 tier + SSE4.2 detected right above.
                return unsafe {
                    self.fill_spans_two_phase_avx512_crc::<S>(bytes, batch, prefetch)
                };
            }
            if avx2_scanner_available() {
                // SAFETY: AVX2 tier + SSE4.2 detected right above.
                return unsafe { self.fill_spans_two_phase_avx2_crc::<S>(bytes, batch, prefetch) };
            }
            // SAFETY: `crc_hash_selected` verified SSE4.2 support.
            return unsafe { self.fill_spans_two_phase_crc::<S>(bytes, batch, prefetch) };
        }
        self.fill_spans_two_phase_impl::<S, false, X86_TIER_DYN>(bytes, batch, prefetch)
    }

    /// The AVX-512 + VBMI2 tier, CRC-hash monomorphization of
    /// [`Self::fill_spans_two_phase`] (Zen 4/5, Ice Lake+): the AVX-512
    /// tier wrapper plus `avx512vbmi2` for `flatten_bits_avx512` — its
    /// only divergence from [`Self::fill_spans_two_phase_avx512_crc`],
    /// which stays the tier for AVX-512 CPUs without VBMI2 (Skylake-X).
    /// An AVX-512 phase-B key pack measured a large regression, see the
    /// phase-B loop's comment.
    ///
    /// # Safety
    ///
    /// The CPU must support the AVX-512 scanner tier plus VBMI2
    /// ([`avx512_fill_available`]) and SSE4.2 (`crc_hash_selected`).
    #[cfg(target_arch = "x86_64")]
    #[target_feature(
        enable = "avx512f,avx512bw,avx512vl,avx512vbmi2,bmi1,bmi2,lzcnt,popcnt,sse4.2"
    )]
    unsafe fn fill_spans_two_phase_avx512_vbmi2_crc<'a, S: MaskScheme>(
        &mut self,
        bytes: &'a [u8],
        batch: &mut crate::pretokenize::SpanBatch<'a>,
        prefetch: &impl Fn(u64),
    ) -> usize {
        self.fill_spans_two_phase_impl::<S, true, X86_TIER_AVX512_VBMI2>(bytes, batch, prefetch)
    }

    /// The AVX-512-tier, CRC-hash monomorphization of
    /// [`Self::fill_spans_two_phase`]. The feature set is the AVX-512
    /// scanner tier plus `sse4.2` for the CRC hash arm (implied by
    /// `avx512f`, spelled out because the `X86_CRC = true` body requires
    /// it — see `fill_span_hash`'s reachability contract).
    ///
    /// # Safety
    ///
    /// The CPU must support the AVX-512 scanner tier
    /// ([`avx512_scanner_available`]) and SSE4.2 (`crc_hash_selected`).
    #[cfg(target_arch = "x86_64")]
    #[target_feature(enable = "avx512f,avx512bw,avx512vl,bmi1,bmi2,lzcnt,popcnt,sse4.2")]
    unsafe fn fill_spans_two_phase_avx512_crc<'a, S: MaskScheme>(
        &mut self,
        bytes: &'a [u8],
        batch: &mut crate::pretokenize::SpanBatch<'a>,
        prefetch: &impl Fn(u64),
    ) -> usize {
        self.fill_spans_two_phase_impl::<S, true, X86_TIER_AVX512>(bytes, batch, prefetch)
    }

    /// The AVX2-tier, CRC-hash monomorphization of
    /// [`Self::fill_spans_two_phase`] (Haswell+, Zen 1-3; `sse4.2` is
    /// implied by `avx` but spelled out for the CRC arm's contract).
    ///
    /// # Safety
    ///
    /// The CPU must support the AVX2 scanner tier
    /// ([`avx2_scanner_available`]) and SSE4.2 (`crc_hash_selected`).
    #[cfg(target_arch = "x86_64")]
    #[target_feature(enable = "avx2,bmi1,bmi2,lzcnt,popcnt,sse4.2")]
    unsafe fn fill_spans_two_phase_avx2_crc<'a, S: MaskScheme>(
        &mut self,
        bytes: &'a [u8],
        batch: &mut crate::pretokenize::SpanBatch<'a>,
        prefetch: &impl Fn(u64),
    ) -> usize {
        self.fill_spans_two_phase_impl::<S, true, X86_TIER_AVX2>(bytes, batch, prefetch)
    }

    /// The SSE4.2-only (CRC-hash, per-batch tier dispatch)
    /// monomorphization of [`Self::fill_spans_two_phase`]: unreachable on
    /// real hardware (every AVX2/AVX-512 CPU has SSE4.2, so one of the
    /// tier wrappers wins), kept for CPUID-masking hypervisors.
    ///
    /// # Safety
    ///
    /// The CPU must support SSE4.2 (`crc_hash_selected` must have
    /// returned true). The caller must also uphold
    /// [`Self::fill_spans_two_phase`]'s own precondition
    /// ([`simd_scanner_available`]).
    #[cfg(target_arch = "x86_64")]
    #[target_feature(enable = "sse4.2")]
    unsafe fn fill_spans_two_phase_crc<'a, S: MaskScheme>(
        &mut self,
        bytes: &'a [u8],
        batch: &mut crate::pretokenize::SpanBatch<'a>,
        prefetch: &impl Fn(u64),
    ) -> usize {
        self.fill_spans_two_phase_impl::<S, true, X86_TIER_DYN>(bytes, batch, prefetch)
    }

    /// [`Self::fill_spans_two_phase`]'s body, monomorphized on the hash
    /// arm (`X86_CRC` — see `fill_span_hash`'s reachability contract) and
    /// the x86 SIMD tier (`X86_TIER` — the `X86_TIER_*` constants; the
    /// AVX2/AVX-512/VBMI2 instantiations are only reachable through the
    /// matching `#[target_feature]` wrappers above, whose feature sets
    /// cover every intrinsic their tier's arms use).
    #[inline(always)]
    fn fill_spans_two_phase_impl<'a, S: MaskScheme, const X86_CRC: bool, const X86_TIER: u8>(
        &mut self,
        bytes: &'a [u8],
        batch: &mut crate::pretokenize::SpanBatch<'a>,
        prefetch: &impl Fn(u64),
    ) -> usize {
        use crate::pretokenize::{PRETOKEN_CHUNK, fill_span_hash, pack_pretoken_key};
        debug_assert!(simd_scanner_available());
        let len = bytes.len();
        let mut pending = self.pos;
        let mut scan = self.scan;
        // Rewind onto the grid batch containing `pending` after iterator
        // pops (next_span keeps consumed batches' bits in `rem`, which
        // this path recomputes). The forward direction is normalized at
        // each refill below.
        if scan > pending {
            scan -= 64 * (scan - pending).div_ceil(64);
        }
        let mut n = 0usize;
        // Opaque table base: LLVM rematerializes the static's address as
        // an adrp+add pair inside the per-span emission loop (constant
        // addresses are "free to recompute" to the register allocator);
        // pinning it here keeps the loop at one indexed ldp per span.
        let pack_masks: *const [u64; 2] = std::hint::black_box(PACK_MASK_TABLE.as_ptr());

        'refill: while n < PRETOKEN_CHUNK && pending < len {
            // Skip grid batches wholly behind `pending` (a direct-emitted
            // long span or a dropped overrun end can leave `scan` far
            // back); keeps `resume - base <= 63` for every batch below.
            if pending >= scan + 64 {
                scan += 64 * ((pending - scan) / 64);
            }
            let fill_base = pending;
            let needed = PRETOKEN_CHUNK - n;
            let mut buf = [std::mem::MaybeUninit::<u16>::uninit(); BOUND_BUF];
            let bufp = buf.as_mut_ptr() as *mut u16;
            let mut nb = 0usize;
            // Boundary bits at or below `resume` are settled: the pending
            // token's own start, or stale run-internal bits behind a
            // scalar overrun (see next_span's grid-keeping comment).
            let mut resume = pending;
            let mut exhausted = false;
            // A scalar end past the u16 window: dropped and re-derived
            // next fill, unless it is the fill's first boundary (emitted
            // directly below).
            let mut overflow_end: Option<usize> = None;

            // Phase A: harvest boundary positions.
            'harvest: while nb < needed {
                if scan.wrapping_sub(fill_base) as isize > REL_LIMIT {
                    break; // re-base: offsets would leave the u16 window
                }
                if scan + 64 > len {
                    // Scalar tail to end of input.
                    let mut p = if nb > 0 {
                        fill_base + unsafe { *bufp.add(nb - 1) } as usize
                    } else {
                        fill_base
                    };
                    while p < len && nb < needed {
                        p = S::advance(bytes, p);
                        // p <= len < scan + 64, within the u16 window per
                        // the REL_LIMIT check above.
                        unsafe { bufp.add(nb).write((p - fill_base) as u16) };
                        nb += 1;
                    }
                    exhausted = p >= len;
                    break;
                }
                let base = scan;
                #[cfg(target_arch = "x86_64")]
                let (usable, bad) = match X86_TIER {
                    // SAFETY: the tier wrappers instantiate these arms
                    // only after runtime tier detection (see
                    // `fill_spans_two_phase`).
                    // The VBMI2 tier runs the same AVX-512 classifiers;
                    // it only diverges in the flatten below.
                    X86_TIER_AVX512 | X86_TIER_AVX512_VBMI2 => unsafe {
                        S::batch_masks_x86::<true>(bytes, base)
                    },
                    X86_TIER_AVX2 => unsafe { S::batch_masks_x86::<false>(bytes, base) },
                    _ => S::batch_masks(bytes, base),
                };
                #[cfg(not(target_arch = "x86_64"))]
                let (usable, bad) = S::batch_masks(bytes, base);
                // At a resume point r (the pending token's start): usable
                // bits at or below r are dead (the pending start itself,
                // or stale run-internal bits behind a scalar overrun), but
                // a bad bit AT r must stay live — load_segment's
                // `live = MAX << from_bit` plus the at_start clear. A zone
                // starting exactly at r has to route r through the scalar
                // path, or the stale post-zone usable bit would be
                // trusted. Only the fill's first batch and post-overrun
                // batches have such bits.
                let (mut ulive, mut blive) = if resume >= base {
                    debug_assert!(resume - base < 64);
                    let k = resume - base;
                    ((u64::MAX << k) << 1, u64::MAX << k)
                } else {
                    (u64::MAX, u64::MAX)
                };
                let rel = base.wrapping_sub(fill_base) as u16;
                if bad & blive == 0 {
                    // Scribble bound: 128 lanes (VBMI2 tier) / popcount +
                    // 7 (scalar) — see BOUND_BUF's worst-case analysis.
                    debug_assert!(
                        nb + if X86_TIER == X86_TIER_AVX512_VBMI2 {
                            128
                        } else {
                            72
                        } <= BOUND_BUF
                    );
                    nb += unsafe {
                        flatten_bits_dispatch::<X86_TIER>(usable & ulive, rel, bufp.add(nb))
                    };
                    scan = base + 64;
                    continue;
                }
                // Dirty batch: per segment, trusted prefix bits then the
                // scheme's scalar advance through the zone up to the next
                // trusted boundary — load_segment's rules, emitting into
                // the buffer.
                loop {
                    let seg_bad = bad & blive;
                    if seg_bad == 0 {
                        debug_assert!(
                            nb + if X86_TIER == X86_TIER_AVX512_VBMI2 {
                                128
                            } else {
                                72
                            } <= BOUND_BUF
                        );
                        nb += unsafe {
                            flatten_bits_dispatch::<X86_TIER>(usable & ulive, rel, bufp.add(nb))
                        };
                        scan = base + 64;
                        break;
                    }
                    let fb = seg_bad.trailing_zeros();
                    let prefix = usable & ulive & !(u64::MAX << fb);
                    debug_assert!(
                        nb + if X86_TIER == X86_TIER_AVX512_VBMI2 {
                            128
                        } else {
                            72
                        } <= BOUND_BUF
                    );
                    nb += unsafe { flatten_bits_dispatch::<X86_TIER>(prefix, rel, bufp.add(nb)) };
                    let mut p = if nb > 0 {
                        fill_base + unsafe { *bufp.add(nb - 1) } as usize
                    } else {
                        fill_base
                    };
                    let rest = usable & (u64::MAX << fb);
                    let until = if rest != 0 {
                        base + rest.trailing_zeros() as usize
                    } else {
                        base + 64
                    };
                    // until <= base + 64 <= len, so `advance` stays in
                    // bounds; it may overrun `until` and the batch end.
                    while p < until {
                        p = S::advance(bytes, p);
                        let relp = p - fill_base;
                        if relp > u16::MAX as usize {
                            overflow_end = Some(p);
                            break 'harvest;
                        }
                        debug_assert!(nb < BOUND_BUF);
                        unsafe { bufp.add(nb).write(relp as u16) };
                        nb += 1;
                    }
                    if p >= base + 64 {
                        // Overrun past the batch: stay on the grid and
                        // resume in the batch containing p, bits at or
                        // below p masked (they can be stale run-internal
                        // bits, exactly as in next_span).
                        scan = base + 64 * ((p - base) / 64);
                        resume = p;
                        break;
                    }
                    // Resume inside the batch at p: same at-start/bad-bit
                    // split as the batch-entry masks above.
                    blive = u64::MAX << (p - base);
                    ulive = blive << 1;
                }
            }

            if nb == 0 {
                // No boundary inside the u16 window: a > 65 KB pretoken.
                // Emit it alone through the careful pack.
                debug_assert!(!exhausted);
                let end = overflow_end.unwrap_or_else(|| S::advance(bytes, fill_base));
                let span = &bytes[fill_base..end];
                let (key, h) = match pack_pretoken_key(span) {
                    Some(key) => (key, fill_span_hash::<X86_CRC>(key)),
                    None => (0, 0),
                };
                prefetch(h);
                let meta = if key != 0 { h } else { span.len() as u64 };
                batch.entries[n] = crate::pretokenize::BatchEntry {
                    key,
                    ptr: span.as_ptr(),
                    meta,
                };
                n += 1;
                pending = end;
                continue 'refill;
            }

            // Phase B: flat emission with no data-dependent branch. One
            // hoisted check proves every 16-byte key load in-bounds of the
            // input slice; only a fill reaching within 16 bytes of EOF
            // routes through the careful per-span pack.
            let emit_n = nb.min(needed);
            let last_end = unsafe { *bufp.add(emit_n - 1) } as usize;
            let entries = &mut batch.entries[n..n + emit_n];
            let base_ptr = unsafe { bytes.as_ptr().add(fill_base) };
            // `prev`/`end` in usize: the u16 boundary domain forced two
            // `& 0xffff` masks and a duplicated 15-compare per span (the
            // compiler cannot see end >= prev in u16 subtraction).
            let mut prev = 0usize;
            if fill_base + last_end + 16 <= len {
                // Every x86 tier shares this scalar key pack. An AVX-512
                // masked pack (`vmovdqu8 {k}{z}` under a tok_len-derived
                // kmask, vpextrq/vmovq into the CRC) measured −36% warm /
                // −30% cold on Zen 5 (5-round interleaved A/B, 1 GB
                // gpt2, tokens identical): this plain load's address
                // depends only on `prev`, so it issues early and the
                // table row + ANDs apply late, while the masked load's
                // kmask waits on the whole boundary→tok_len chain and the
                // extracts add a vector→GPR crossing before the CRC —
                // per-span work went from overlapping to serialized
                // (~90% of loop samples on the masked load + dependents).
                // Do not re-try; the VBMI2 tier's only divergence is
                // `flatten_bits_avx512` in phase A.
                for (i, e) in entries.iter_mut().enumerate() {
                    let end = unsafe { *bufp.add(i) } as usize;
                    let tok_len = end - prev;
                    let p = unsafe { base_ptr.add(prev) };
                    prev = end;
                    // SAFETY: p + 16 <= base_ptr + last_end + 16 <= end of
                    // the input slice (hoisted check above).
                    let raw = unsafe { (p as *const u128).read_unaligned() };
                    // Branchless pack_pretoken_key: one ldp from
                    // PACK_MASK_TABLE instead of the 7-op per-half ALU
                    // chain (see the table's docs). tok_len >= 1
                    // (boundaries are strictly increasing), so the clamped
                    // length is in the table's 1..=15 domain. Long spans
                    // take key 0 (pretoken_key_hash(0) == 0) through the
                    // `keep` AND-mask — an if/select here gets if-converted
                    // into a real branch (LLVM hoists it to skip the two
                    // loads), reintroducing the pattern-free n > 15 branch
                    // this loop exists to avoid.
                    let m = tok_len.min(15);
                    // SAFETY: m <= 15, in the 16-entry table.
                    let [mask_lo, mask_hi] = unsafe { *pack_masks.add(m) };
                    let keep = ((tok_len <= 15) as u64).wrapping_neg();
                    let klo = (raw as u64) & mask_lo & keep;
                    let khi = (((raw >> 64) as u64 & mask_hi) | ((m as u64) << 56)) & keep;
                    let key = (klo as u128) | ((khi as u128) << 64);
                    let hv = fill_span_hash::<X86_CRC>(key);
                    prefetch(hv);
                    // meta = hash for short spans, length for long ones
                    // (see `BatchEntry::meta`), in the same AND-mask style
                    // as the key routing — a select gets if-converted.
                    let meta = (hv & keep) | (tok_len as u64 & !keep);
                    e.key = key;
                    e.ptr = p;
                    e.meta = meta;
                }
            } else {
                for (i, e) in entries.iter_mut().enumerate() {
                    let end = unsafe { *bufp.add(i) } as usize;
                    let tok_len = end - prev;
                    let p = unsafe { base_ptr.add(prev) };
                    prev = end;
                    // SAFETY: as above for the span bounds.
                    let span = unsafe { std::slice::from_raw_parts(p, tok_len) };
                    let (key, hv) = match pack_pretoken_key(span) {
                        Some(key) => (key, fill_span_hash::<X86_CRC>(key)),
                        None => (0, 0),
                    };
                    prefetch(hv);
                    let meta = if key != 0 { hv } else { tok_len as u64 };
                    e.key = key;
                    e.ptr = p;
                    e.meta = meta;
                }
            }
            n += emit_n;
            pending = fill_base + prev;
            if exhausted {
                debug_assert_eq!(pending, len);
                break;
            }
        }

        // Rewind past discarded leftover boundaries and leave the state as
        // a fresh resume at `pending` for either pull style.
        if scan > pending {
            scan -= 64 * (scan - pending).div_ceil(64);
        }
        self.pos = pending;
        self.scan = scan;
        self.mask_base = scan;
        self.rem = 0;
        self.batch_usable = 0;
        self.batch_bad = 0;
        self.scalar_until = pending;
        self.pre_base = usize::MAX;
        n
    }
}