coremlit 0.1.0

use std::sync::{Mutex, OnceLock, PoisonError};

use super::*;
use tokenizers::{Model, ModelWrapper, PaddingDirection, PaddingParams, PaddingStrategy};

/// The cell production hands `chunk_long` for `mt` — [`TextEmbedder`] keeps
/// one per embedder, and the lane builds the table into it on its first
/// engagement with production's own `MergeTable::from_tokenizer` — cached
/// here per BPE vocabulary size so the artifact's table (~0.5 s to build) is
/// built once per process. Every non-BPE tokenizer shares one cell that
/// `from_tokenizer` leaves `None`, so a hermetic test with a tiny WordLevel
/// tokenizer never touches the staged artifact. A test that MUTATES the
/// artifact's merges must call `super::chunk_long` with a cell of its own.
fn table_cell(mt: &Tokenizer) -> &'static OnceLock<Option<MergeTable>> {
  type Cell = &'static OnceLock<Option<MergeTable>>;
  static NON_BPE: OnceLock<Option<MergeTable>> = OnceLock::new();
  static CELLS: Mutex<Vec<(usize, Cell)>> = Mutex::new(Vec::new());
  let ModelWrapper::BPE(bpe) = mt.get_model() else {
    return &NON_BPE;
  };
  let key = bpe.get_vocab_size();
  let mut cells = CELLS.lock().unwrap_or_else(PoisonError::into_inner);
  if let Some((_, cell)) = cells.iter().find(|(k, _)| *k == key) {
    return cell;
  }
  let cell: Cell = Box::leak(Box::new(OnceLock::new()));
  cells.push((key, cell));
  cell
}

/// The staged artifact's merge table, built once (see [`table_cell`]): the
/// tests that time the fast lane pre-build it so the one-time cost stays out
/// of the stopwatch.
fn merge_table() -> &'static MergeTable {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  table_cell(&mt)
    .get_or_init(|| MergeTable::from_tokenizer(&mt))
    .as_ref()
    .expect("the artifact's BPE is mirrorable")
}

/// [`super::chunk_long`] with the fast lane enabled — what production runs.
fn chunk_long(
  mt: &Tokenizer,
  text: &str,
  opts: &WindowOptions,
) -> Result<Vec<windit::split::Chunk>> {
  super::chunk_long(mt, LazyTable::new(table_cell(mt), mt), text, opts)
}

use super::test_artifact::{
  GOLDEN_SOURCE_TOKENIZER_SHA256, tokenizer_bytes as artifact_tokenizer_bytes,
  tokenizer_path as artifact_tokenizer_path_for_tests,
};

// ── Options ────────────────────────────────────────────────────────────────

#[test]
fn options_default_equals_new() {
  assert_eq!(TextEmbedderOptions::default(), TextEmbedderOptions::new());
  assert_eq!(TextEmbedderOptions::new().compute(), DEFAULT_COMPUTE);
  assert_eq!(DEFAULT_COMPUTE, ComputeUnits::All);
}

#[test]
fn options_with_and_set_compute() {
  let opts = TextEmbedderOptions::new().with_compute(ComputeUnits::CpuAndNeuralEngine);
  assert_eq!(opts.compute(), ComputeUnits::CpuAndNeuralEngine);
  let mut opts = TextEmbedderOptions::new();
  opts.set_compute(ComputeUnits::CpuOnly);
  assert_eq!(opts.compute(), ComputeUnits::CpuOnly);
}

#[test]
fn describe_renders_shape_and_dtype() {
  assert_eq!(describe(&[1, 512], Some(DataType::I32)), "[1, 512] int32");
  assert_eq!(describe(&[1, 384], None), "[1, 384] none");
}

#[cfg(feature = "serde")]
#[test]
fn options_serde_roundtrip() {
  let opts = TextEmbedderOptions::new().with_compute(ComputeUnits::CpuAndNeuralEngine);
  let json = serde_json::to_string(&opts).unwrap();
  assert!(json.contains("cpu_and_neural_engine"), "serialized: {json}");
  let back: TextEmbedderOptions = serde_json::from_str(&json).unwrap();
  assert_eq!(back, opts);
}

#[cfg(feature = "serde")]
#[test]
fn options_serde_missing_compute_defaults() {
  let opts: TextEmbedderOptions = serde_json::from_str("{}").unwrap();
  assert_eq!(opts.compute(), DEFAULT_COMPUTE);
}

// ── Tokenizer identity gate (hermetic; the real tokenizer seam) ─────────────

/// SHA-256 of the staged artifact tokenizer must equal the tokenizer that produced the
/// committed goldens (the source model repo revision) — byte-identity is the
/// foundation of token-id identity. Any drift in the staged
/// `tokenizer.json` fails here.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn artifact_tokenizer_sha_matches_golden_source_pin() {
  use sha2::{Digest, Sha256};
  let sha: String = Sha256::digest(artifact_tokenizer_bytes())
    .iter()
    .map(|b| format!("{b:02x}"))
    .collect();
  assert_eq!(
    sha, GOLDEN_SOURCE_TOKENIZER_SHA256,
    "the staged tokenizer.json diverged from the granite tokenizer that cut the goldens"
  );
}

/// The runtime identity const equals the pinned golden-source SHA literal —
/// hermetic, so the const ↔ literal half of the const ↔ literal ↔ artifact-bytes
/// chain still runs with no artifact staged. Its sibling
/// [`artifact_tokenizer_sha_matches_golden_source_pin`] ties bytes ↔ literal.
#[test]
fn tokenizer_sha_pin_matches_golden_source_literal() {
  assert_eq!(
    contract::TOKENIZER_SHA256_HEX,
    GOLDEN_SOURCE_TOKENIZER_SHA256,
    "the tokenizer-identity contract const must equal the pinned golden-source SHA"
  );
}

/// Encode `text` through granite's ACTUAL configured tokenizer seam (the same
/// path [`TextEmbedder::token_ids`] uses), hermetically (no model).
fn ids(text: &str) -> Vec<u32> {
  let tok =
    configured_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("configure tokenizer");
  tok.encode(text, true).expect("encode").get_ids().to_vec()
}

/// Token-id EXACT-equality against a pinned subset of the committed corpus. The
/// full 16-entry corpus identity gate is `tests/granite/tokenizer_identity.rs`;
/// these two hermetic sequences keep the in-lib seam honest without the fixture
/// file. `<|startoftext|>`=179934 (CLS, pooled) and `<|return|>`=179938 (EOS)
/// bracket every sequence.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn token_ids_match_pinned_golden_subset() {
  let cases: &[(&str, &[u32])] = &[
    ("hello world", &[179934, 24313, 2318, 179938]),
    (
      "how do I build a Rust CoreML inference library for text embeddings?",
      &[
        179934, 8775, 579, 317, 2966, 221, 54305, 15984, 4051, 86068, 11087, 355, 2145, 158816, 30,
        179938,
      ],
    ),
  ];
  for (text, expected) in cases {
    let got = ids(text);
    assert_eq!(&got, expected, "token-id drift for {text:?}");
  }
}

/// Truncation identity — the DIRECTION, not just the length, is gated.
///
/// A *non-repetitive* input longer than the 512-token window (ascending
/// integers, every token distinct) truncates to EXACTLY [`MAX_TOKENS`] without
/// overflowing the export sequence length, and — because the module configures
/// `TruncationDirection::Right` — the kept interior is the untruncated
/// encoding's PREFIX. A `Right → Left` flip (which keeps the SUFFIX instead)
/// changes the interior of distinct tokens and trips this.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn long_input_truncation_keeps_the_right_directional_prefix() {
  // Non-repetitive, comfortably over one window: "1 2 3 … 1000", all distinct.
  let long: String = (1..=1000)
    .map(|n| n.to_string())
    .collect::<Vec<_>>()
    .join(" ");

  let truncated = ids(&long);
  assert_eq!(
    truncated.len(),
    MAX_TOKENS,
    "truncation must cap ids at the window"
  );
  assert_eq!(truncated[0], 179934, "leading <|startoftext|> kept");
  assert_eq!(
    truncated[MAX_TOKENS - 1],
    179938,
    "trailing <|return|> kept"
  );

  // Untruncated reference: the SAME tokenizer bytes with truncation OFF.
  let full = tokenizers::Tokenizer::from_bytes(artifact_tokenizer_bytes())
    .expect("load tokenizer")
    .encode(long.as_str(), true)
    .expect("encode")
    .get_ids()
    .to_vec();
  assert!(
    full.len() > MAX_TOKENS,
    "reference must actually overflow the window (got {})",
    full.len()
  );

  // RIGHT truncation ⇒ the 510 interior ids equal the untruncated PREFIX. Under
  // `Left` the interior would be the untruncated SUFFIX, which (distinct tokens)
  // differs ⇒ red.
  assert_eq!(
    &truncated[1..MAX_TOKENS - 1],
    &full[1..MAX_TOKENS - 1],
    "Right-truncation interior must equal the untruncated first-510 content tokens"
  );

  // Measure-then-pin: the exact 512-id sequence nailed to a SHA-256 constant, so
  // the whole interior is pinned absolutely. Any tokenizer-artifact or
  // truncation-config drift changes it.
  use sha2::{Digest, Sha256};
  let mut hasher = Sha256::new();
  for id in &truncated {
    hasher.update(id.to_le_bytes());
  }
  let sha: String = hasher
    .finalize()
    .iter()
    .map(|b| format!("{b:02x}"))
    .collect();
  assert_eq!(
    sha, "aec64c84fc8328d01b518a7cb4e63b42a00a659ba5d39789fc10a272667416af",
    "truncated 512-id sequence drifted (tokenizer artifact or truncation config changed)"
  );
}

// ── Fixed-window contract: padding override + build_window (hermetic) ────────
//
// A caller-supplied `tokenizer.json` can carry a padding policy; if it survived
// into `token_ids`, `embed` would mask PAD positions as real (corrupt embedding),
// pool CLS off position 0 (left padding), or overflow the window (fixed padding
// beyond 512, a release panic). These prove `configure_tokenizer` neutralizes
// every such policy and `build_window` is a typed guard, not a panic — with no
// model.

/// "hello world" through the granite tokenizer: `<|startoftext|>`=179934 (CLS,
/// pooled), then `hello`/`world`, then `<|return|>`=179938 (EOS). The exact
/// sequence is pinned by `token_ids_match_pinned_golden_subset` above; sourced
/// from the module contract (single source of truth).
const HELLO_WORLD_IDS: [u32; 4] = contract::SENTINEL_IDS;

/// A fresh artifact tokenizer carrying an adversarial fixed-window padding policy
/// (the kind a caller-supplied tokenizer might inherit), BEFORE this module's
/// config runs.
fn artifact_tokenizer_with_padding(direction: PaddingDirection) -> Tokenizer {
  let mut tok =
    Tokenizer::from_bytes(artifact_tokenizer_bytes()).expect("load the artifact tokenizer");
  tok.with_padding(Some(PaddingParams {
    strategy: PaddingStrategy::Fixed(MAX_TOKENS),
    direction,
    ..Default::default()
  }));
  tok
}

/// Fixed-512 RIGHT padding — the corrupt-mask case. Without the override the
/// tokenizer pads a short input to the full window, so `embed`'s mask would mark
/// the trailing PADs as real tokens. `configure_tokenizer` must disable the
/// tokenizer's own padding so only the real ids survive, and the window then
/// masks EXACTLY those.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn configured_tokenizer_disables_fixed_right_padding_mask_stays_correct() {
  let mut tok = artifact_tokenizer_with_padding(PaddingDirection::Right);

  // Precondition: the adversarial policy really does pad to the full window.
  let padded = tok
    .encode("hello world", true)
    .expect("encode")
    .get_ids()
    .to_vec();
  assert_eq!(
    padded.len(),
    MAX_TOKENS,
    "adversarial fixture must actually pad to the window"
  );

  // Override strips the padding: token_ids sees only the real, unpadded ids.
  configure_tokenizer(&mut tok).expect("configure");
  let real = tok
    .encode("hello world", true)
    .expect("encode")
    .get_ids()
    .to_vec();
  assert_eq!(
    real, HELLO_WORLD_IDS,
    "padding must be stripped, real ids only"
  );

  // The fixed window masks EXACTLY the real tokens — no PAD marked real.
  let (input_ids, mask) = build_window(&real, 0).expect("build window");
  assert_eq!(
    mask.iter().sum::<i32>(),
    i32::try_from(real.len()).unwrap(),
    "attention mask must count only the real tokens"
  );
  assert!(
    mask[..real.len()].iter().all(|&m| m == 1),
    "real tokens masked 1"
  );
  assert!(
    mask[real.len()..].iter().all(|&m| m == 0),
    "pad positions masked 0"
  );
  assert_eq!(input_ids[0], 179934, "CLS at position 0");
}

/// Fixed-512 LEFT padding — the wrong-CLS-pooling case. Without the override the
/// leading PADs push CLS (`<|startoftext|>`) off position 0, so CLS pooling would
/// read a PAD. `configure_tokenizer` must disable padding so CLS stays at 0.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn configured_tokenizer_disables_left_padding_keeps_cls_at_zero() {
  let mut tok = artifact_tokenizer_with_padding(PaddingDirection::Left);

  // Precondition: left padding pushes CLS off position 0 (the hazard).
  let padded = tok
    .encode("hello world", true)
    .expect("encode")
    .get_ids()
    .to_vec();
  assert_eq!(padded.len(), MAX_TOKENS);
  assert_ne!(
    padded[0], 179934,
    "left padding must push CLS off position 0 (the hazard being defended)"
  );

  // Override removes the leading pads: CLS is back at position 0.
  configure_tokenizer(&mut tok).expect("configure");
  let real = tok
    .encode("hello world", true)
    .expect("encode")
    .get_ids()
    .to_vec();
  assert_eq!(real, HELLO_WORLD_IDS);
  assert_eq!(
    real[0], 179934,
    "CLS must be at position 0 after the override"
  );
  let (input_ids, _mask) = build_window(&real, 0).expect("build window");
  assert_eq!(
    input_ids[0], 179934,
    "CLS stays at position 0 in the window"
  );
}

/// An over-long input (real text past the window) truncates to exactly
/// [`MAX_TOKENS`] through the configured seam and fills the window with real
/// tokens — no panic, CLS still at position 0.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn overlong_input_truncates_and_fills_the_window_without_panic() {
  // Non-repetitive, comfortably over one window: "1 2 3 … 1000".
  let long: String = (1..=1000)
    .map(|n| n.to_string())
    .collect::<Vec<_>>()
    .join(" ");
  let real = ids(&long); // configured seam: truncation on, padding off.
  assert_eq!(
    real.len(),
    MAX_TOKENS,
    "over-long input truncates to the window"
  );

  let (input_ids, mask) = build_window(&real, 0).expect("full window must build, not panic");
  assert!(
    mask.iter().all(|&m| m == 1),
    "a full window is entirely real tokens"
  );
  assert_eq!(input_ids[0], 179934, "CLS stays at position 0");
}

/// `build_window` returns a typed [`Error::TokenCount`] — never the release
/// out-of-bounds panic the old `debug_assert!` hid — if a tokenizer ever yields
/// more ids than the window.
#[test]
fn build_window_rejects_overlong_ids_with_typed_error() {
  let overlong = vec![7u32; MAX_TOKENS + 1];
  match build_window(&overlong, 0) {
    Err(Error::TokenCount(count)) => {
      assert_eq!(count.got(), MAX_TOKENS + 1);
      assert_eq!(count.max(), MAX_TOKENS);
    }
    other => panic!("expected Err(TokenCount), got {other:?}"),
  }
}

/// `build_window` returns a typed [`Error::TokenIdRange`] — never a silently
/// wrapping cast — for a token id outside the model's int32 range.
#[test]
fn build_window_rejects_out_of_range_token_id() {
  match build_window(&[u32::MAX], 0) {
    Err(Error::TokenIdRange(id)) => assert_eq!(id, u32::MAX),
    other => panic!("expected Err(TokenIdRange), got {other:?}"),
  }
}

/// `build_window` on a short real sequence masks exactly the real prefix and
/// right-pads the remainder with `pad_id` (masked 0) — the internal fixed-window
/// pad, done correctly.
#[test]
fn build_window_masks_prefix_and_right_pads_remainder() {
  let (input_ids, mask) = build_window(&[10, 20, 30], 7).expect("build");
  assert_eq!(&input_ids[..3], &[10i32, 20, 30]);
  assert!(
    input_ids[3..].iter().all(|&x| x == 7),
    "remainder is pad_id"
  );
  assert_eq!(&mask[..3], &[1i32, 1, 1]);
  assert!(mask[3..].iter().all(|&m| m == 0), "pad positions masked 0");
}

/// A full window (exactly [`MAX_TOKENS`] real ids) is accepted and entirely
/// masked — the boundary the old guard treated as `<=` must remain valid.
#[test]
fn build_window_accepts_a_full_window() {
  let (_input_ids, mask) = build_window(&vec![1u32; MAX_TOKENS], 0).expect("full window builds");
  assert_eq!(mask.iter().sum::<i32>(), i32::try_from(MAX_TOKENS).unwrap());
}

// ── embed_long: content-aware chunk geometry (hermetic; measuring tokenizer,
//    no model). The CoreML aggregation path is proven model-gated in
//    tests/granite/embed_long.rs. ─────────────────────────────────────────────

/// A deterministic multi-paragraph document comfortably over several 512-token
/// windows: 24 paragraphs of 40 distinct words each, `\n\n`-separated.
fn long_doc() -> String {
  (0..24)
    .map(|p| {
      (0..40)
        .map(|w| format!("para{p}word{w}"))
        .collect::<Vec<_>>()
        .join(" ")
    })
    .collect::<Vec<_>>()
    .join("\n\n")
}

/// THE hazard regression (design correction #1): the CONFIGURED (production)
/// tokenizer truncates a long input's id count to exactly [`MAX_TOKENS`], while
/// the MEASURING tokenizer (truncation disabled) reports the true, larger count.
/// `embed_long`'s chunker MUST measure with the latter — measuring with the
/// former would judge EVERY long document to "fit one window" and silently
/// degenerate `embed_long` into a truncated `embed`.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn measuring_tokenizer_reports_untruncated_counts() {
  // Non-repetitive, comfortably over one window: "1 2 3 … 1000".
  let long: String = (1..=1000)
    .map(|n| n.to_string())
    .collect::<Vec<_>>()
    .join(" ");
  let configured = configured_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("configure");
  let measuring = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");

  let configured_count = configured
    .encode(long.as_str(), true)
    .expect("encode")
    .get_ids()
    .len();
  let measuring_count = measuring
    .encode(long.as_str(), true)
    .expect("encode")
    .get_ids()
    .len();

  assert_eq!(
    configured_count, MAX_TOKENS,
    "the production tokenizer saturates a long input at the window"
  );
  assert!(
    measuring_count > MAX_TOKENS,
    "the measuring tokenizer must see the true (untruncated) count, got {measuring_count}"
  );
}

/// A long document splits into multiple chunks that PARTITION the text under the
/// default (overlap-free) geometry — the first starts at byte 0, each begins
/// where the previous ended, the last ends at `doc.len()` — with every chunk
/// within the token budget. The partition triplet is the coverage regression:
/// pre-repair windit left `\n\n` gaps between chunks, so `chunk.start()` ran
/// strictly ahead of the previous end rather than meeting it.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn long_text_chunks_multi_window_within_budget() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let doc = long_doc();
  let chunks = chunk_long(&mt, &doc, &WindowOptions::new(MAX_TOKENS)).expect("chunk");

  assert!(
    chunks.len() > 1,
    "a document over several windows must split into multiple chunks, got {}",
    chunks.len()
  );
  assert_eq!(chunks[0].start(), 0, "the first chunk starts at byte 0");
  let mut prev_end = 0usize;
  for chunk in &chunks {
    let s = chunk
      .as_str(&doc)
      .expect("chunk falls on a char boundary of its own text");
    let count = mt.encode(s, true).expect("encode").get_ids().len();
    assert!(
      count <= MAX_TOKENS,
      "every chunk stays within the token budget, got {count}"
    );
    assert_eq!(
      chunk.start(),
      prev_end,
      "each chunk begins where the previous ended (no gap, no overlap)"
    );
    prev_end = chunk.end();
  }
  assert_eq!(prev_end, doc.len(), "the last chunk ends at doc.len()");
}

/// Every byte of the document survives chunking, and every paragraph separator
/// stays in the token stream exactly once. windit drops the `\n\n` runs that fall
/// on chunk boundaries; `attach_gaps` reattaches them, so (a) the chunks
/// concatenate back to the document byte-for-byte and (b) the ByteLevel separator
/// token appears once per `\n\n` across the union of the chunk encodings —
/// interior and reattached-boundary separators alike.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn boundary_separators_stay_in_the_token_stream() {
  // `\n\n` tokenizes to `[<|startoftext|>, ĊĊ, <|return|>]`, so id 239 is the
  // paragraph separator's sole content token; counting it counts separators.
  const PARAGRAPH_SEPARATOR_TOKEN: u32 = 239;
  assert_eq!(
    ids("\n\n"),
    vec![179934, PARAGRAPH_SEPARATOR_TOKEN, 179938],
    "the paragraph separator's token id is pinned"
  );

  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let doc = long_doc();
  let chunks = chunk_long(&mt, &doc, &WindowOptions::new(MAX_TOKENS)).expect("chunk");

  let concat: String = chunks
    .iter()
    .map(|c| {
      c.as_str(&doc)
        .expect("chunk falls on a char boundary of its own text")
    })
    .collect();
  assert_eq!(
    concat, doc,
    "the chunks must concatenate back to the document byte-for-byte"
  );

  let separators: usize = chunks
    .iter()
    .map(|c| {
      let s = c.as_str(&doc).expect("char boundary");
      mt.encode(s, true)
        .expect("encode")
        .get_ids()
        .iter()
        .filter(|&&id| id == PARAGRAPH_SEPARATOR_TOKEN)
        .count()
    })
    .sum();
  assert_eq!(
    separators,
    doc.matches("\n\n").count(),
    "every `\\n\\n` is tokenized exactly once across the chunks"
  );
}

/// The word-level fallback (an oversized sentence with no paragraph or sentence
/// break) excludes inter-word punctuation from its chunks; `attach_gaps`
/// reattaches it. One 400-term comma-separated sentence at window 128 partitions
/// into byte-exact chunks — every `", "` preserved, none over budget.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn word_fallback_punctuation_is_reattached() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let sentence = (0..400)
    .map(|w| format!("term{w}"))
    .collect::<Vec<_>>()
    .join(", ");
  let chunks = chunk_long(&mt, &sentence, &WindowOptions::new(128)).expect("chunk");

  assert!(
    chunks.len() > 1,
    "a 400-term sentence must split into multiple chunks, got {}",
    chunks.len()
  );
  assert_eq!(chunks[0].start(), 0, "the first chunk starts at byte 0");
  let mut prev_end = 0usize;
  for chunk in &chunks {
    let s = chunk.as_str(&sentence).expect("char boundary");
    assert_eq!(
      chunk.start(),
      prev_end,
      "each chunk begins where the previous ended"
    );
    assert!(
      mt.encode(s, true).expect("encode").get_ids().len() <= 128,
      "every chunk stays within the 128-token budget"
    );
    prev_end = chunk.end();
  }
  assert_eq!(
    prev_end,
    sentence.len(),
    "the last chunk ends at the text length"
  );

  let concat: String = chunks
    .iter()
    .map(|c| c.as_str(&sentence).expect("char boundary"))
    .collect();
  assert_eq!(
    concat, sentence,
    "the chunks reproduce the sentence byte-for-byte"
  );
}

/// Leading and trailing separators are covered too: a document wrapped in `\n\n`
/// still partitions — the first chunk starts at byte 0 despite the leading
/// separator and the last ends at the text length despite the trailing one
/// (`attach_gaps`' leading and trailing branches).
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn leading_and_trailing_separators_are_covered() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let doc = format!("\n\n{}\n\n", long_doc());
  let chunks = chunk_long(&mt, &doc, &WindowOptions::new(MAX_TOKENS)).expect("chunk");

  assert!(
    chunks.len() > 1,
    "the wrapped document still splits, got {}",
    chunks.len()
  );
  assert_eq!(
    chunks[0].start(),
    0,
    "the first chunk starts at 0 despite the leading separator"
  );
  let mut prev_end = 0usize;
  for chunk in &chunks {
    assert_eq!(
      chunk.start(),
      prev_end,
      "each chunk begins where the previous ended"
    );
    prev_end = chunk.end();
  }
  assert_eq!(
    prev_end,
    doc.len(),
    "the last chunk ends at len despite the trailing separator"
  );

  let concat: String = chunks
    .iter()
    .map(|c| c.as_str(&doc).expect("char boundary"))
    .collect();
  assert_eq!(concat, doc, "the chunks reproduce the wrapped document");
}

/// The overflow fallback chain — right-prepend, own-chunk, leading, trailing — is
/// unreachable with the real tokenizer on natural corpora (packed chunks never
/// sit exactly at the window), so pin it with a `char`-count measure that drives
/// `ContentAware` + `attach_gaps` directly. Each windit trace is checked by hand
/// against the pinned rev; each case asserts the exact repaired ranges, which are
/// a partition of the input.
#[test]
fn gap_attachment_falls_back_right_then_own_chunk() {
  use windit::split::ContentAware;

  let measure = |s: &str| -> usize { s.chars().count() };
  // windit's raw chunks for `text` at `window`, repaired by `attach_gaps`, as
  // (start, end) byte ranges. `attach_gaps` now takes a fallible RANGE measure
  // (byte offsets into `text`); a `char`-count never fails, so the own-chunks
  // (all far below MAX_TOKENS) attach exactly as before.
  let repair = |text: &str, window: usize| -> Vec<(usize, usize)> {
    let measure_checked = |a: usize, b: usize| -> Result<usize> { Ok(text[a..b].chars().count()) };
    let chunks = ContentAware::new(&measure)
      .chunk(text, &WindowOptions::new(window))
      .expect("chunk");
    attach_gaps(text, chunks, &measure_checked, window)
      .expect("own-chunks measure within MAX_TOKENS")
      .iter()
      .map(|c| (c.start(), c.end()))
      .collect()
  };

  let cases: &[(&str, &[(usize, usize)])] = &[
    // Left neighbor full (`aaaaa` = 5); the `\n\n` gap cannot append (`aaaaa\n\n`
    // = 7 > 5) but prepends to the right neighbor, which still fits (`\n\nbbb`
    // = 5). windit: [0,5),[7,10).
    ("aaaaa\n\nbbb", &[(0, 5), (5, 10)]),
    // Both neighbors full; neither can absorb the `\n\n`, so it becomes its own
    // chunk between them. windit: [0,5),[7,12).
    ("aaaaa\n\nbbbbb", &[(0, 5), (5, 7), (7, 12)]),
    // windit's lone chunk [2,7) omits the leading `\n\n` (the 1-chunk coverage
    // hole at micro scale); it cannot prepend (`\n\naaaaa` = 7 > 5), so the
    // leading run is its own chunk.
    ("\n\naaaaa", &[(0, 2), (2, 7)]),
    // The trailing `\n\n` cannot append (`aaaaa\n\n` = 7 > 5), so it is its own
    // chunk. windit: [0,5).
    ("aaaaa\n\n", &[(0, 5), (5, 7)]),
  ];

  for &(text, expected) in cases {
    let got = repair(text, 5);
    assert_eq!(got.as_slice(), expected, "repaired ranges for {text:?}");
    // The exact ranges above are a partition: first start 0, adjacent tiling,
    // last end == text length.
    assert_eq!(got.first().unwrap().0, 0, "{text:?}: first start 0");
    assert_eq!(
      got.last().unwrap().1,
      text.len(),
      "{text:?}: last end == text length"
    );
    for w in got.windows(2) {
      assert_eq!(w[0].1, w[1].0, "{text:?}: adjacent chunks tile");
    }
  }
}

/// Gap repair must not silently defeat the caller's `max_windows` work bound
/// (each chunk is one CoreML prediction): windit's own cap passes pre-repair,
/// but an unabsorbable separator run becomes an extra own-chunk, so the cap is
/// re-enforced on the FINAL chunk count. With the granite tokenizer at window
/// 3, `a`/`b` pack a window exactly (3 ids with specials) while `a\n\n` /
/// `\n\nb` measure 4, so a `\n\n` between them fits neither neighbor.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn gap_repair_cannot_exceed_max_windows() {
  use windit::WinditError;

  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");

  // windit passes at two content chunks; repair inserts the interior `\n\n` as
  // a third. `got` is the full repaired count.
  match chunk_long(&mt, "a\n\nb", &WindowOptions::new(3).with_max_windows(2)) {
    Err(Error::Windowing(WinditError::TooManyWindows { got, max })) => {
      assert_eq!(got, 3, "the full repaired chunk count is reported");
      assert_eq!(max, 2);
    }
    other => panic!("expected Err(Windowing(TooManyWindows)), got {other:?}"),
  }

  // Uncapped, the same geometry chunks fine — three covering chunks — so the
  // error above is the cap, not the geometry.
  let uncapped = chunk_long(&mt, "a\n\nb", &WindowOptions::new(3)).expect("uncapped");
  let ranges: Vec<_> = uncapped.iter().map(|c| (c.start(), c.end())).collect();
  assert_eq!(ranges, vec![(0, 1), (1, 3), (3, 4)]);

  // Leading, interior, and trailing insertions co-occur and are all counted:
  // windit yields `a` and `b` (2 content chunks, within the cap of 3, so
  // windit's own check passes); repair adds all three `\n\n` runs, so the
  // final count (5) exceeds `max + 1` (4) by one — `got` is the full
  // repaired count, not windit's abort-at-`max + 1` value.
  match chunk_long(
    &mt,
    "\n\na\n\nb\n\n",
    &WindowOptions::new(3).with_max_windows(3),
  ) {
    Err(Error::Windowing(WinditError::TooManyWindows { got, max })) => {
      assert_eq!(
        got, 5,
        "leading + interior + trailing insertions all counted"
      );
      assert_eq!(max, 3);
    }
    other => panic!("expected Err(Windowing(TooManyWindows)), got {other:?}"),
  }
}

/// Contentless (whitespace-only) nonempty text chunks to no content, yet
/// embedding it still costs one whole-input CoreML prediction; the cap must
/// see that cost. Cap 0 refuses before any model work with the true count;
/// cap 1 (and no cap) admits it as a single whole-input chunk — full
/// coverage, one prediction.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn whitespace_only_text_counts_one_window_against_the_cap() {
  use windit::WinditError;

  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");

  match chunk_long(
    &mt,
    "   ",
    &WindowOptions::new(MAX_TOKENS).with_max_windows(0),
  ) {
    Err(Error::Windowing(WinditError::TooManyWindows { got, max })) => {
      assert_eq!(got, 1, "the whole-input fallback counts as one window");
      assert_eq!(max, 0);
    }
    other => panic!("expected Err(Windowing(TooManyWindows)), got {other:?}"),
  }

  let capped = chunk_long(
    &mt,
    "   ",
    &WindowOptions::new(MAX_TOKENS).with_max_windows(1),
  )
  .expect("cap 1 admits the whole-input fallback");
  assert_eq!(
    capped
      .iter()
      .map(|c| (c.start(), c.end()))
      .collect::<Vec<_>>(),
    vec![(0, 3)],
    "one chunk spanning the whole input"
  );

  let uncapped =
    chunk_long(&mt, "   ", &WindowOptions::new(MAX_TOKENS)).expect("uncapped whitespace");
  assert_eq!(
    uncapped
      .iter()
      .map(|c| (c.start(), c.end()))
      .collect::<Vec<_>>(),
    vec![(0, 3)],
    "the fallback chunk is synthesized regardless of any cap"
  );
}

/// Contentless over-budget input is REFUSED, not silently truncated. windit
/// drops the whitespace-only text; the whole-input fallback would then embed it
/// through the truncating production tokenizer, dropping every token past the
/// 512-window. The measured fallback refuses it instead. Fixtures span spaces,
/// tabs, CRLF, NBSP, and a mixed run (em/thin spaces); `tokens` is compared to
/// the test's own untruncated encode so the pin is self-consistent under any
/// tokenizer.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn contentless_over_budget_input_is_refused_not_truncated() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let fixtures = [
    " ".repeat(100_000),
    "\t".repeat(100_000),
    "\r\n".repeat(50_000),
    "\u{00A0}".repeat(100_000),
    " \t\r\n\u{00A0}\u{2003}\u{2009}".repeat(15_000),
  ];
  for s in &fixtures {
    let expected_tokens = mt.encode(s.as_str(), true).expect("encode").get_ids().len();
    assert!(
      expected_tokens > MAX_TOKENS,
      "fixture must actually exceed the window (got {expected_tokens})"
    );
    match chunk_long(&mt, s, &WindowOptions::new(MAX_TOKENS)) {
      Err(Error::ContentlessInputOverBudget(over)) => {
        assert_eq!(over.start(), 0, "the whole input is the offending run");
        assert_eq!(over.end(), s.len());
        assert_eq!(
          over.tokens(),
          expected_tokens,
          "reported count is the untruncated measure"
        );
        assert_eq!(over.max(), MAX_TOKENS);
      }
      other => panic!("expected ContentlessInputOverBudget, got {other:?}"),
    }
  }
}

/// The at-budget boundary is embedded whole; the first over-budget count is
/// refused. Binary-searches the largest space run that fits the window, so the
/// boundary is exact regardless of the pinned tokenizer.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn contentless_input_at_or_under_budget_still_embeds_whole() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let measure = |n: usize| {
    mt.encode(" ".repeat(n).as_str(), true)
      .expect("encode")
      .get_ids()
      .len()
  };
  let mut lo = 1usize;
  let mut hi = 100_000usize;
  assert!(measure(lo) <= MAX_TOKENS, "one space fits");
  assert!(measure(hi) > MAX_TOKENS, "100k spaces overflow");
  while lo + 1 < hi {
    let mid = (lo + hi) / 2;
    if measure(mid) <= MAX_TOKENS {
      lo = mid;
    } else {
      hi = mid;
    }
  }
  // `lo` is the largest in-budget count; `hi == lo + 1` the first over-budget.
  let at_budget = " ".repeat(lo);
  let chunks = chunk_long(&mt, &at_budget, &WindowOptions::new(MAX_TOKENS))
    .expect("in-budget contentless input embeds whole");
  assert_eq!(
    chunks
      .iter()
      .map(|c| (c.start(), c.end()))
      .collect::<Vec<_>>(),
    vec![(0, at_budget.len())],
    "in-budget contentless input is one whole-input chunk"
  );
  let over = " ".repeat(hi);
  match chunk_long(&mt, &over, &WindowOptions::new(MAX_TOKENS)) {
    Err(Error::ContentlessInputOverBudget(_)) => {}
    other => panic!("expected ContentlessInputOverBudget just past the budget, got {other:?}"),
  }
}

/// A pure-separator gap between two content chunks that neither neighbor can
/// absorb (the `attach_gaps` own-chunk escape, interior case) is refused when
/// its run measures past the window. `a<100k spaces>b` at window 3 forces the
/// escape; the in-budget own-chunk escape (`a\n\nb`) still chunks Ok.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn separator_gap_over_budget_is_refused() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let text = format!("a{}b", " ".repeat(100_000));
  match chunk_long(&mt, &text, &WindowOptions::new(3)) {
    Err(Error::ContentlessInputOverBudget(over)) => {
      assert_eq!(over.start(), 1, "the gap starts right after `a`");
      assert_eq!(over.end(), 100_001, "the gap ends right before `b`");
      assert!(
        over.tokens() > MAX_TOKENS,
        "the gap run measures over the window"
      );
      assert_eq!(over.max(), MAX_TOKENS);
    }
    other => panic!("expected ContentlessInputOverBudget, got {other:?}"),
  }
  // Control: an in-budget own-chunk gap still chunks Ok — the escape stays.
  let ok = chunk_long(&mt, "a\n\nb", &WindowOptions::new(3)).expect("in-budget own-chunk escape");
  assert_eq!(
    ok.iter().map(|c| (c.start(), c.end())).collect::<Vec<_>>(),
    vec![(0, 1), (1, 3), (3, 4)]
  );
}

/// Leading and trailing pure-separator gaps that measure past the window are
/// refused with the correct byte span (the `attach_gaps` leading and trailing
/// branches, measuring the GAP itself, not the extended candidate).
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn leading_and_trailing_over_budget_gaps_are_refused() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");

  let leading = format!("{}a", " ".repeat(100_000));
  match chunk_long(&mt, &leading, &WindowOptions::new(MAX_TOKENS)) {
    Err(Error::ContentlessInputOverBudget(over)) => {
      assert_eq!(over.start(), 0, "leading gap starts at byte 0");
      assert_eq!(over.end(), 100_000, "leading gap ends right before `a`");
    }
    other => panic!("expected leading ContentlessInputOverBudget, got {other:?}"),
  }

  let trailing = format!("a{}", " ".repeat(100_000));
  match chunk_long(&mt, &trailing, &WindowOptions::new(MAX_TOKENS)) {
    Err(Error::ContentlessInputOverBudget(over)) => {
      assert_eq!(over.start(), 1, "trailing gap starts right after `a`");
      assert_eq!(over.end(), 100_001, "trailing gap ends at text length");
    }
    other => panic!("expected trailing ContentlessInputOverBudget, got {other:?}"),
  }
}

/// An encode failure surfaces as `Error::Tokenize`, NOT a bogus
/// `ContentlessInputOverBudget { tokens: usize::MAX }`. `chunk_long` now builds
/// the `TokenIndex` from one whole-input encode BEFORE any chunking, so a
/// tokenizer that cannot encode the input fails at that index-build seam — still
/// `Error::Tokenize`, one call earlier than the old per-chunk `token_ids` would.
/// A tiny WordLevel tokenizer whose unk token is absent from its vocab fails to
/// encode the whitespace-only input.
#[test]
fn tokenizer_failure_on_fallback_measure_keeps_tokenize_identity() {
  // WordLevel, no pre-tokenizer, unk token `<unk>` not in vocab: encoding the
  // whole-string `"   "` (not in vocab) errors instead of truncating.
  const TINY_NO_UNK: &[u8] = br#"{"version":"1.0","truncation":null,"padding":null,"added_tokens":[],"normalizer":null,"pre_tokenizer":null,"post_processor":null,"decoder":null,"model":{"type":"WordLevel","vocab":{"hello":0,"world":1},"unk_token":"<unk>"}}"#;
  let mt = tokenizers::Tokenizer::from_bytes(TINY_NO_UNK).expect("load tiny WordLevel");
  match chunk_long(&mt, "   ", &WindowOptions::new(MAX_TOKENS)) {
    Err(Error::Tokenize(_)) => {}
    other => panic!("expected Err(Tokenize), got {other:?}"),
  }
}

/// `""` costs zero predictions (`embed_long_with` delegates it to `embed`,
/// which fails `EmptyText` before the model), so no fallback chunk is
/// synthesized and even a cap of 0 passes chunking.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn empty_text_chunks_to_nothing_under_any_cap() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let chunks = chunk_long(&mt, "", &WindowOptions::new(MAX_TOKENS).with_max_windows(0))
    .expect("empty text chunks to nothing under a cap of 0");
  assert!(chunks.is_empty());
}

/// A short text that fits one window is a single chunk spanning the whole text.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn single_window_text_is_one_whole_chunk() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let text = "a compact sentence that fits comfortably inside one window";
  let chunks = chunk_long(&mt, text, &WindowOptions::new(MAX_TOKENS)).expect("chunk");
  assert_eq!(chunks.len(), 1, "short text is one chunk");
  assert_eq!(chunks[0].start(), 0);
  assert_eq!(chunks[0].end(), text.len());
}

/// The chunk geometry adapts to `WindowOptions` alone (the spec's genericity at
/// granite's seam): a smaller window yields more, smaller chunks, each within its
/// own budget.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn chunk_geometry_adapts_by_window_options_alone() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let doc = long_doc();
  let coarse = chunk_long(&mt, &doc, &WindowOptions::new(128)).expect("chunk @128");
  let fine = chunk_long(&mt, &doc, &WindowOptions::new(64)).expect("chunk @64");

  assert!(
    fine.len() > coarse.len(),
    "a smaller window yields more chunks: {} @64 vs {} @128",
    fine.len(),
    coarse.len()
  );
  for chunk in &coarse {
    let s = chunk.as_str(&doc).expect("char boundary");
    assert!(mt.encode(s, true).expect("encode").get_ids().len() <= 128);
  }
  for chunk in &fine {
    let s = chunk.as_str(&doc).expect("char boundary");
    assert!(mt.encode(s, true).expect("encode").get_ids().len() <= 64);
  }
}

/// With a non-zero overlap, consecutive chunks repeat a trailing region whose
/// measured length stays within the overlap token budget.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn overlap_repeats_trailing_tokens_within_budget() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let doc = long_doc();
  let opts = WindowOptions::new(128).with_overlap(16);
  let chunks = chunk_long(&mt, &doc, &opts).expect("chunk");

  assert!(chunks.len() > 1, "an overlapped long doc still splits");
  for pair in chunks.windows(2) {
    // Consecutive chunks share a trailing region…
    assert!(
      pair[1].start() < pair[0].end(),
      "consecutive chunks overlap: next start {} vs prev end {}",
      pair[1].start(),
      pair[0].end()
    );
    // …and that repeated text measures within the overlap budget (the exact text
    // the packer measured, with special tokens, is `<= 16`).
    let repeated = &doc[pair[1].start()..pair[0].end()];
    let n = mt.encode(repeated, true).expect("encode").get_ids().len();
    assert!(
      n <= 16,
      "repeated region within the 16-token overlap budget, got {n}"
    );
  }
}

/// `validate_long_input` (and thus `embed_long_with`) rejects a per-chunk budget
/// above the model's fixed window before any tokenization — hermetically, no
/// model.
#[test]
fn window_over_budget_is_rejected() {
  match validate_long_input(
    "any text",
    &LongTextOptions::from(WindowOptions::new(MAX_TOKENS + 1)),
  ) {
    Err(Error::WindowOverBudget(budget)) => {
      assert_eq!(budget.window(), MAX_TOKENS + 1);
      assert_eq!(budget.max(), MAX_TOKENS);
    }
    other => panic!("expected Err(WindowOverBudget), got {other:?}"),
  }
  // The exact budget is accepted.
  assert!(
    validate_long_input(
      "any text",
      &LongTextOptions::from(WindowOptions::new(MAX_TOKENS)),
    )
    .is_ok()
  );
}

// ── #2: LongTextOptions + input-size gate (hermetic) ─────────────────────────

/// `Default` == `new`, the documented defaults (full window, no byte limit), the
/// builder/setter round-trips, and the `From<WindowOptions>` geometry-only form.
#[test]
fn long_text_options_default_equals_new() {
  assert_eq!(LongTextOptions::default(), LongTextOptions::new());
  assert_eq!(
    LongTextOptions::new().window_options(),
    WindowOptions::new(MAX_TOKENS)
  );
  assert_eq!(LongTextOptions::new().max_input_bytes(), None);

  let built = LongTextOptions::new()
    .with_window_options(WindowOptions::new(64))
    .with_max_input_bytes(4096);
  assert_eq!(built.window_options(), WindowOptions::new(64));
  assert_eq!(built.max_input_bytes(), Some(4096));

  let mut set = LongTextOptions::new();
  set.set_window_options(WindowOptions::new(32));
  set.set_max_input_bytes(2048);
  assert_eq!(set.window_options(), WindowOptions::new(32));
  assert_eq!(set.max_input_bytes(), Some(2048));

  let from = LongTextOptions::from(WindowOptions::new(64));
  assert_eq!(from.window_options().window(), 64);
  assert_eq!(from.max_input_bytes(), None);
}

/// The tail-policy triple reaches windit's `TailPolicy` through granite's own
/// re-export, and sets exactly that one field.
///
/// The setter writes THROUGH the carried `WindowOptions` rather than replacing
/// it, so the window, hop and cap set beforehand survive — the failure mode a
/// `self.window_options = WindowOptions::new(..).with_tail(..)` implementation
/// would have. What it does NOT pin is the effect on chunking — that is
/// `a_drop_below_min_tail_moves_the_last_boundary_and_keeps_every_byte`'s job,
/// since windit 0.5's `ContentAware` reads the policy where 0.4's did not.
#[test]
fn long_text_options_tail_policy_is_carried_through_the_geometry() {
  // The windit default, reached through coremlit with no `windit` in scope.
  assert_eq!(LongTextOptions::new().tail_policy(), TailPolicy::default());
  assert_eq!(
    LongTextOptions::new().tail_policy(),
    TailPolicy::KeepWithCoverage
  );

  let built = LongTextOptions::new()
    .with_window_options(WindowOptions::new(64).with_hop(32).with_max_windows(7))
    .with_tail_policy(TailPolicy::DropBelowMin(8))
    .with_max_input_bytes(4096);
  assert_eq!(built.tail_policy(), TailPolicy::DropBelowMin(8));
  // Only `tail` moved.
  assert_eq!(built.window_options().window(), 64);
  assert_eq!(built.window_options().hop(), 32);
  assert_eq!(built.window_options().max_windows(), Some(7));
  assert_eq!(built.max_input_bytes(), Some(4096));
  // The whole geometry is the same value windit builds directly, so a caller
  // may compose either way.
  assert_eq!(
    built.window_options(),
    WindowOptions::new(64)
      .with_hop(32)
      .with_max_windows(7)
      .with_tail(TailPolicy::DropBelowMin(8))
  );

  let mut set = LongTextOptions::new();
  set.set_tail_policy(TailPolicy::PadFull);
  assert_eq!(set.tail_policy(), TailPolicy::PadFull);
  assert_eq!(set.window_options().window(), MAX_TOKENS);
}

/// [`LongTextOptions`]'s `Display`, pinned byte-exactly: this is the spelling a
/// downstream derivation fingerprint persists, so a silent respelling here
/// would silently invalidate every fingerprint built on it (see the doc on
/// `impl Display for LongTextOptions`).
///
/// Composes windit's OWN `Display` for [`WindowOptions`] verbatim, parenthesized
/// per the "nested options in parentheses" rule, rather than re-deriving it —
/// windit 0.5.1 is the one place that nested spelling can change, and a drift
/// there fails this test exactly as readily as one introduced here. Exercises
/// each [`TailPolicy`] variant once, plus both branches of the two `Option`
/// folds (`max_windows` and `max_input_bytes`), so the nesting and the two
/// `none` folds are each pinned independently.
#[test]
fn long_text_options_display_pins_the_composed_spelling() {
  // Default: `KeepWithCoverage`, no cap, no byte limit — both `Option`s fold
  // to `none`.
  assert_eq!(
    LongTextOptions::new().to_string(),
    "window_options=(window=512,hop=512,tail=keep_with_coverage,max_windows=none),\
     max_input_bytes=none"
  );

  // Every field distinct from its default: `DropBelowMin`'s payload and both
  // `Option`s populated.
  let built = LongTextOptions::new()
    .with_window_options(WindowOptions::new(64).with_hop(32).with_max_windows(7))
    .with_tail_policy(TailPolicy::DropBelowMin(8))
    .with_max_input_bytes(4096);
  assert_eq!(
    built.to_string(),
    "window_options=(window=64,hop=32,tail=drop_below_min(8),max_windows=7),\
     max_input_bytes=4096"
  );

  // `PadFull`, the third variant, on its own.
  let mut pad_full = LongTextOptions::new();
  pad_full.set_tail_policy(TailPolicy::PadFull);
  assert_eq!(
    pad_full.to_string(),
    "window_options=(window=512,hop=512,tail=pad_full,max_windows=none),\
     max_input_bytes=none"
  );
}

/// The [`LongTextOptions`] document, pinned byte-exactly in BOTH directions.
///
/// The nested geometry is windit's OWN document — coremlit writes none of it —
/// so this also pins that coremlit's `serde` feature really does reach
/// `windit?/serde`: without it this type has no impls and the test does not
/// compile. The `tail` shape is the adjacently tagged form windit 0.4
/// introduced, the same one `audio::ced` and `embeddings::clap` realigned their
/// own policies onto. windit 0.5 rewrote the READER of that shape (see
/// `long_text_options_round_trip_through_postcard`) and this test is what says
/// the WRITER did not move with it: the bytes below are the 0.4 ones.
#[cfg(feature = "serde")]
#[test]
fn long_text_options_document_form_is_pinned() {
  let doc = concat!(
    r#"{"window_options":{"window":512,"hop":512,"#,
    r#""tail":{"kind":"keep_with_coverage"},"max_windows":null},"#,
    r#""max_input_bytes":null}"#
  );
  let opts = LongTextOptions::new();
  assert_eq!(serde_json::to_string(&opts).unwrap(), doc);
  assert_eq!(serde_json::from_str::<LongTextOptions>(doc).unwrap(), opts);

  // A fully configured value, geometry and byte gate together.
  let doc = concat!(
    r#"{"window_options":{"window":256,"hop":192,"#,
    r#""tail":{"kind":"drop_below_min","value":8},"max_windows":32},"#,
    r#""max_input_bytes":4096}"#
  );
  let opts = LongTextOptions::new()
    .with_window_options(
      WindowOptions::new(256)
        .with_hop(192)
        .with_max_windows(32)
        .with_tail(TailPolicy::DropBelowMin(8)),
    )
    .with_max_input_bytes(4096);
  assert_eq!(serde_json::to_string(&opts).unwrap(), doc);
  assert_eq!(serde_json::from_str::<LongTextOptions>(doc).unwrap(), opts);

  // Every field defaults, so a partial config fills from `new` — the
  // convention `TextEmbedderOptions` and the doors' `WindowPlan`s follow.
  assert_eq!(
    serde_json::from_str::<LongTextOptions>("{}").unwrap(),
    LongTextOptions::new()
  );
  assert_eq!(
    serde_json::from_str::<LongTextOptions>(r#"{"max_input_bytes":16}"#).unwrap(),
    LongTextOptions::new().with_max_input_bytes(16)
  );
}

/// Every [`TailPolicy`] reachable through [`LongTextOptions::with_tail_policy`]
/// survives a NON-self-describing format.
///
/// The falsifier for the caveat the type's "Binary formats" section used to
/// carry. Under windit 0.4 this test is red for EVERY variant: `to_allocvec`
/// wrote the value, and `from_bytes` then failed with `WontImplement`, because
/// the derived adjacent tag asks postcard for `deserialize_any`. windit 0.5 asks
/// for the adjacent shape itself, so the bytes read back — and the shape they
/// read back through is pinned here, not only the round trip: the policy is one
/// byte at its old variant ordinal (`KeepWithCoverage` 0, `DropBelowMin` 1
/// carrying its minimum as a varint, `PadFull` 2), so a compact document written
/// by another windit consumer is the same bytes coremlit reads.
///
/// `DropBelowMin(0)` is included deliberately: it is the one minimum below
/// nothing, and a reader that shortcut a zero payload would lose the variant.
#[cfg(feature = "serde")]
#[test]
fn long_text_options_round_trip_through_postcard() {
  // Wildcard-free: a new windit variant fails to compile until its ordinal is
  // pinned here.
  for policy in [
    TailPolicy::KeepWithCoverage,
    TailPolicy::PadFull,
    TailPolicy::DropBelowMin(0),
    TailPolicy::DropBelowMin(8),
  ] {
    let expected: &[u8] = match policy {
      TailPolicy::KeepWithCoverage => &[0],
      TailPolicy::DropBelowMin(0) => &[1, 0],
      TailPolicy::DropBelowMin(_) => &[1, 8],
      TailPolicy::PadFull => &[2],
    };
    let bytes = postcard::to_allocvec(&policy).unwrap();
    assert_eq!(bytes, expected, "compact spelling of {policy:?} moved");
    assert_eq!(
      postcard::from_bytes::<TailPolicy>(&bytes).unwrap(),
      policy,
      "postcard round-trip lost {policy:?} (bytes {bytes:?})"
    );

    // …and through the granite options that compose it, the shape a caller
    // actually persists: geometry, policy and byte gate together.
    let opts = LongTextOptions::new()
      .with_window_options(WindowOptions::new(256).with_hop(192).with_max_windows(32))
      .with_tail_policy(policy)
      .with_max_input_bytes(4096);
    let bytes = postcard::to_allocvec(&opts).unwrap();
    assert_eq!(
      postcard::from_bytes::<LongTextOptions>(&bytes).unwrap(),
      opts,
      "postcard round-trip lost {opts:?} (bytes {bytes:?})"
    );
  }
  // The DEFAULT value too — the one every caller who never touches the knob
  // persists.
  let opts = LongTextOptions::new();
  let bytes = postcard::to_allocvec(&opts).unwrap();
  assert_eq!(
    postcard::from_bytes::<LongTextOptions>(&bytes).unwrap(),
    opts
  );
}

/// A MISSPELLED key is refused, not silently dropped.
///
/// Both fields default, so without `deny_unknown_fields`
/// `{"max_input_byte":4096}` (singular) deserializes to `max_input_bytes ==
/// None` and `embed_long_with` then runs with NO size gate at all — the one
/// bound that stops an untrusted text reaching the tokenizer. The refusal names
/// the key, so the operator can see which one they typed wrong.
#[cfg(feature = "serde")]
#[test]
fn a_misspelled_key_is_refused_rather_than_silently_unbounded() {
  let err = serde_json::from_str::<LongTextOptions>(r#"{"max_input_byte":4096}"#).unwrap_err();
  let text = err.to_string();
  assert!(
    text.contains("max_input_byte"),
    "the refusal must name the unknown key, got {text:?}"
  );
  // The same for a stray key beside a well-spelled one — a config file that
  // half-parses is the failure this closes.
  assert!(
    serde_json::from_str::<LongTextOptions>(r#"{"max_input_bytes":4096,"windowoptions":{}}"#)
      .is_err()
  );
  // TOML too: the format a node-options file actually carries.
  assert!(toml::from_str::<LongTextOptions>("max_input_byte = 4096\n").is_err());
}

/// The input-size gate refuses an oversized input reading only `text.len()` (no
/// tokenizer); at-limit passes (`>` rejects, `==` accepts), and a `None` limit
/// accepts the same oversized input.
#[test]
fn input_too_large_is_rejected_before_any_tokenizer_work() {
  let big = "x".repeat(8 * 1024 * 1024);
  let opts = LongTextOptions::new().with_max_input_bytes(1024 * 1024);
  match validate_long_input(&big, &opts) {
    Err(Error::InputTooLarge(large)) => {
      assert_eq!(large.got(), big.len());
      assert_eq!(large.max(), 1024 * 1024);
    }
    other => panic!("expected InputTooLarge, got {other:?}"),
  }
  // At-limit is accepted.
  let at_limit = "x".repeat(1024 * 1024);
  assert!(validate_long_input(&at_limit, &opts).is_ok());
  // No limit accepts the same 8 MiB input.
  assert!(validate_long_input(&big, &LongTextOptions::new()).is_ok());
}

/// The untrusted-input gate is the outermost shield: an oversized input AND an
/// over-budget window yields `InputTooLarge`, not `WindowOverBudget`.
#[test]
fn input_too_large_takes_precedence_over_window_budget() {
  let big = "x".repeat(2 * 1024 * 1024);
  let opts =
    LongTextOptions::from(WindowOptions::new(MAX_TOKENS + 1)).with_max_input_bytes(1024 * 1024);
  match validate_long_input(&big, &opts) {
    Err(Error::InputTooLarge(_)) => {}
    other => panic!("expected InputTooLarge to win over WindowOverBudget, got {other:?}"),
  }
}

// ── Special-token overhead vs the fixed window (hermetic) ────────────────────

/// A minimal WordLevel tokenizer: the overhead guard is about the
/// post-processor and the window, so the model underneath only has to tokenize.
const OVERHEAD_TINY_TOKENIZER: &str = r#"{"version":"1.0","truncation":null,"padding":null,"added_tokens":[],"normalizer":null,"pre_tokenizer":{"type":"Whitespace"},"post_processor":null,"decoder":null,"model":{"type":"WordLevel","vocab":{"<pad>":0,"a":1,"b":2},"unk_token":"<pad>"}}"#;

/// Serialize [`OVERHEAD_TINY_TOKENIZER`] with a `TemplateProcessing`
/// post-processor whose single-sequence template adds exactly `added` special
/// tokens — the shape a caller can hand [`TextEmbedder::from_files`], and the
/// number the tokenizers crate subtracts from the truncation window without
/// checking. One `SpecialToken` carrying `added` ids is used rather than `added`
/// template pieces because `TemplateProcessing::added_tokens` sums `ids.len()`
/// per piece, so both count the same and only this one scales to a 512-token
/// window.
fn tokenizer_bytes_with_special_overhead(added: usize) -> Vec<u8> {
  use tokenizers::processors::template::{SpecialToken, TemplateProcessing};

  let mut tokenizer =
    Tokenizer::from_bytes(OVERHEAD_TINY_TOKENIZER.as_bytes()).expect("load the tiny tokenizer");
  let special = SpecialToken::new(
    "<sp>".to_string(),
    vec![0u32; added],
    vec!["<pad>".to_string(); added],
  )
  .expect("ids and tokens are the same length");
  let template = TemplateProcessing::builder()
    .try_single("<sp> $A")
    .expect("single template")
    .try_pair("<sp> $A $B")
    .expect("pair template")
    .special_tokens(vec![special])
    .build()
    .expect("build the template post-processor");
  tokenizer.with_post_processor(Some(template));
  tokenizer
    .to_string(false)
    .expect("serialize the tokenizer")
    .into_bytes()
}

/// The helper actually installs the overhead it claims — otherwise every case
/// below would pass vacuously against a post-processor that adds nothing.
#[test]
fn overhead_fixture_installs_the_claimed_special_token_count() {
  use tokenizers::PostProcessor;
  for added in [1usize, MAX_TOKENS - 1, MAX_TOKENS, MAX_TOKENS + 1] {
    let bytes = tokenizer_bytes_with_special_overhead(added);
    let tok = Tokenizer::from_bytes(&bytes).expect("reload the fixture");
    let post = tok.get_post_processor().expect("the fixture has one");
    assert_eq!(post.added_tokens(false), added, "single-sequence overhead");
  }
}

/// `added > MAX_TOKENS` is the tokenizers crate's unchecked
/// `max_length - added_tokens` subtraction: it PANICS with "attempt to subtract
/// with overflow" under overflow checks and wraps to a near-`usize::MAX` window
/// in release. `configure_tokenizer` refuses the tokenizer first — and it must,
/// because the panic site is `with_truncation`, which runs BEFORE
/// [`validate_tokenizer_contract`]'s sentinel encode could reject the foreign
/// tokenizer on its own.
#[test]
fn configure_tokenizer_refuses_overhead_over_the_window() {
  let bytes = tokenizer_bytes_with_special_overhead(MAX_TOKENS + 1);
  match configured_tokenizer_from_bytes(&bytes) {
    Err(Error::SpecialTokenOverhead(overhead)) => {
      assert_eq!(overhead.added(), MAX_TOKENS + 1);
      assert_eq!(overhead.window(), MAX_TOKENS);
    }
    other => panic!("expected SpecialTokenOverhead, got {other:?}"),
  }
}

/// `added == MAX_TOKENS` does not overflow — and is refused anyway, because the
/// effective text window is then zero and every encoding would be the special
/// tokens alone.
#[test]
fn configure_tokenizer_refuses_overhead_equal_to_the_window() {
  let bytes = tokenizer_bytes_with_special_overhead(MAX_TOKENS);
  match configured_tokenizer_from_bytes(&bytes) {
    Err(Error::SpecialTokenOverhead(overhead)) => {
      assert_eq!(overhead.added(), MAX_TOKENS);
      assert_eq!(overhead.window(), MAX_TOKENS);
    }
    other => panic!("expected SpecialTokenOverhead, got {other:?}"),
  }
}

/// One token of room is enough: `added < MAX_TOKENS` configures, and the window
/// then holds the specials plus at least one real token. (The tokenizer is still
/// foreign, so `validate_tokenizer_contract` rejects it downstream — that is a
/// different check, and this one must not pre-empt it.)
#[test]
fn configure_tokenizer_accepts_overhead_below_the_window() {
  let bytes = tokenizer_bytes_with_special_overhead(MAX_TOKENS - 1);
  let tok = configured_tokenizer_from_bytes(&bytes).expect("511 specials fit a 512-token window");
  let ids = tok
    .encode("a b a b", true)
    .expect("encode")
    .get_ids()
    .to_vec();
  assert_eq!(ids.len(), MAX_TOKENS, "specials plus one real token");
  assert_eq!(ids[MAX_TOKENS - 1], 1, "the real token survives (`a`)");
  assert!(
    matches!(
      validate_tokenizer_contract(&tok),
      Err(Error::TokenizerContractMismatch(_))
    ),
    "the overhead guard does not stand in for the contract check"
  );
}

/// A tokenizer with no post-processor at all reads as zero overhead, so the
/// guard never fires on it — the ordinary path stays open.
#[test]
fn configure_tokenizer_accepts_a_tokenizer_without_a_post_processor() {
  configured_tokenizer_from_bytes(OVERHEAD_TINY_TOKENIZER.as_bytes())
    .expect("no post-processor is zero overhead");
}

// ── #6: tokenizer contract validation (hermetic) ─────────────────────────────

/// Parse the artifact tokenizer JSON, apply `mutate`, and re-serialize to bytes —
/// the raw input a caller-supplied constructor receives. `serde_json` is a
/// dev-dependency.
fn mutated_artifact_tokenizer_bytes(mutate: impl FnOnce(&mut serde_json::Value)) -> Vec<u8> {
  let mut value: serde_json::Value =
    serde_json::from_slice(artifact_tokenizer_bytes()).expect("parse the artifact tokenizer.json");
  mutate(&mut value);
  serde_json::to_vec(&value).expect("re-serialize mutated tokenizer.json")
}

/// Parse the artifact tokenizer JSON, apply `mutate`, re-serialize, and build the
/// module's CONFIGURED tokenizer from the result — so a contract check sees
/// exactly the production tokenization.
fn mutated_artifact_tokenizer(mutate: impl FnOnce(&mut serde_json::Value)) -> Tokenizer {
  configured_tokenizer_from_bytes(&mutated_artifact_tokenizer_bytes(mutate))
    .expect("configure mutated tokenizer")
}

/// The granite artifact tokenizer passes every contract check — the invariant on
/// which all keep-green constructor/golden behavior rests. If this fails the
/// contract constants are wrong; strengthen the constants, never the checks.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn tokenizer_contract_accepts_the_artifact_tokenizer() {
  let tok =
    configured_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("configure the artifact");
  validate_tokenizer_contract(&tok).expect("the artifact tokenizer must satisfy the contract");
}

/// A tiny foreign tokenizer with none of granite's specials fails the first
/// check (`<|startoftext|>`), reported as `missing`.
#[test]
fn tokenizer_contract_rejects_missing_specials() {
  const TINY: &[u8] = br#"{"version":"1.0","truncation":null,"padding":null,"added_tokens":[],"normalizer":null,"pre_tokenizer":null,"post_processor":null,"decoder":null,"model":{"type":"WordLevel","vocab":{"hello":0,"world":1},"unk_token":"<unk>"}}"#;
  let tok = configured_tokenizer_from_bytes(TINY).expect("configure tiny");
  match validate_tokenizer_contract(&tok) {
    Err(Error::TokenizerContractMismatch(mismatch)) => {
      let check = mismatch.check();
      assert!(
        check.contains("<|startoftext|>"),
        "check names the missing special: {check}"
      );
      assert_eq!(mismatch.actual(), "missing");
    }
    other => panic!("expected TokenizerContractMismatch, got {other:?}"),
  }
}

/// The artifact tokenizer with one trailing (non-special) added token removed
/// keeps granite's three specials but drops the vocabulary to 179999, so the
/// vocab-size check fires. The pinned `tokenizers` reassigns added-token ids
/// densely as `base + array_index` (ignoring the JSON `id` field), and the three
/// specials are the array's first entries, so removing the LAST entry leaves
/// their ids intact — the fixture edits structure, not a declared id.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn tokenizer_contract_rejects_wrong_vocab_size() {
  let tok = mutated_artifact_tokenizer(|value| {
    let added = value["added_tokens"]
      .as_array_mut()
      .expect("added_tokens array");
    added.pop().expect("added_tokens is non-empty");
  });
  match validate_tokenizer_contract(&tok) {
    Err(Error::TokenizerContractMismatch(mismatch)) => {
      let (expected, actual) = (mismatch.expected(), mismatch.actual());
      assert_eq!(mismatch.check(), "vocab size");
      assert!(
        expected.contains("180000"),
        "expected names the contract size: {expected}"
      );
      assert!(
        actual.contains("179999"),
        "actual names the reduced size: {actual}"
      );
    }
    other => panic!("expected TokenizerContractMismatch on vocab size, got {other:?}"),
  }
}

/// The artifact tokenizer with its highest BASE-vocab id pushed past the model's
/// table (to 180000) keeps the count at 180000 and the specials intact, so the
/// specials and vocab-size checks pass but the max-token-id gate fires — the
/// out-of-vocabulary case a larger foreign tokenizer would hit. (Added-token ids
/// are reassigned densely by this `tokenizers`, so an OOV id can only come from
/// the base vocab; the fixture moves a base id, leaving a hole the count check
/// tolerates.)
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn tokenizer_contract_rejects_out_of_model_vocab_id() {
  let tok = mutated_artifact_tokenizer(|value| {
    let vocab = value["model"]["vocab"]
      .as_object_mut()
      .expect("model.vocab object");
    let key = vocab
      .iter()
      .max_by_key(|(_, id)| id.as_u64().unwrap_or(0))
      .map(|(k, _)| k.clone())
      .expect("non-empty base vocab");
    vocab.insert(key, serde_json::json!(180_000));
  });
  match validate_tokenizer_contract(&tok) {
    Err(Error::TokenizerContractMismatch(mismatch)) => {
      let actual = mismatch.actual();
      assert_eq!(mismatch.check(), "max token id");
      assert!(
        actual.contains("180000"),
        "actual carries the offending id: {actual}"
      );
    }
    other => panic!("expected TokenizerContractMismatch on max token id, got {other:?}"),
  }
}

/// Two base-vocab entries with their ids swapped (`hello`↔`Ġworld`, 24313↔2318)
/// leave the specials, vocab size, and max id intact but change the sentinel
/// encoding, so only the final check fires.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn tokenizer_contract_rejects_divergent_encoding() {
  let tok = mutated_artifact_tokenizer(|value| {
    let vocab = value["model"]["vocab"]
      .as_object_mut()
      .expect("model.vocab object");
    let mut key_a = None;
    let mut key_b = None;
    for (key, id) in vocab.iter() {
      match id.as_u64() {
        Some(24_313) => key_a = Some(key.clone()),
        Some(2_318) => key_b = Some(key.clone()),
        _ => {}
      }
    }
    let key_a = key_a.expect("id 24313 present in base vocab");
    let key_b = key_b.expect("id 2318 present in base vocab");
    vocab.insert(key_a, serde_json::json!(2_318));
    vocab.insert(key_b, serde_json::json!(24_313));
  });
  match validate_tokenizer_contract(&tok) {
    Err(Error::TokenizerContractMismatch(mismatch)) => {
      assert_eq!(mismatch.check(), "sentinel encoding");
    }
    other => panic!("expected TokenizerContractMismatch on sentinel encoding, got {other:?}"),
  }
}

// ── #6: tokenizer BYTE-IDENTITY backstop (hermetic) ──────────────────────────
//
// The behavioral contract above is a spot-check (specials, count, max id, one
// sentinel), so a corrupted-but-behaviorally-valid supplied tokenizer slips
// through it. `validate_tokenizer_identity` closes that gap on the
// caller-supplied path by pinning the exact artifact SHA-256, fail-closed. These
// tests drive the digest + provenance seam directly (no model).

/// THE codex repro: two ordinary base-vocab entries with their ids swapped
/// (5000 ↔ 6000, neither a special nor a sentinel content id) sail through every
/// BEHAVIORAL check — specials, count, max id, and the `"hello world"` sentinel
/// are all untouched — yet any text using those two tokens would embed with wrong
/// ids. The byte-identity backstop REJECTS this behaviorally-valid but
/// non-identical tokenizer (the gap this fix closes).
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn tokenizer_identity_rejects_non_sentinel_vocab_corruption() {
  const SWAP_A: u64 = 5_000;
  const SWAP_B: u64 = 6_000;
  // Guard: the swapped ids must avoid the specials and sentinel content ids, so
  // the corruption stays invisible to every behavioral check.
  const RESERVED: [u64; 5] = [24_313, 2_318, 179_934, 179_935, 179_938];
  assert!(
    !RESERVED.contains(&SWAP_A) && !RESERVED.contains(&SWAP_B),
    "swap ids must avoid the specials and sentinel content ids"
  );

  let bytes = mutated_artifact_tokenizer_bytes(|value| {
    let vocab = value["model"]["vocab"]
      .as_object_mut()
      .expect("model.vocab object");
    let mut key_a = None;
    let mut key_b = None;
    for (key, id) in vocab.iter() {
      match id.as_u64() {
        Some(SWAP_A) => key_a = Some(key.clone()),
        Some(SWAP_B) => key_b = Some(key.clone()),
        _ => {}
      }
    }
    let key_a = key_a.expect("id 5000 present in base vocab");
    let key_b = key_b.expect("id 6000 present in base vocab");
    vocab.insert(key_a, serde_json::json!(SWAP_B));
    vocab.insert(key_b, serde_json::json!(SWAP_A));
  });

  // (a) finding pin: the BEHAVIORAL gate alone ACCEPTS this corruption (documents
  // the gap; a future behavioral check that starts catching it flags the fixture
  // for rework).
  let configured = configured_tokenizer_from_bytes(&bytes).expect("configure mutated");
  validate_tokenizer_contract(&configured)
    .expect("behavioral contract accepts non-sentinel vocab corruption");

  // (b) premise guard: the mutated bytes are not the pinned artifact.
  assert_ne!(
    bytes.as_slice(),
    artifact_tokenizer_bytes(),
    "the swap must actually change the bytes"
  );

  // (c) the identity backstop REJECTS it, naming the identity check with the
  // pinned expected / computed actual digests.
  let actual = sha256_hex(&bytes);
  match validate_tokenizer_identity(&TokenizerProvenance::Supplied(actual.clone())) {
    Err(Error::TokenizerContractMismatch(mismatch)) => {
      assert_eq!(mismatch.check(), "tokenizer identity (sha-256)");
      assert_eq!(mismatch.expected(), contract::TOKENIZER_SHA256_HEX);
      assert_eq!(mismatch.actual(), actual);
    }
    other => panic!("expected identity TokenizerContractMismatch, got {other:?}"),
  }
}

/// Policy pin: byte identity, NOT behavioral identity, on the supplied path. A
/// parse→re-serialize round-trip of the artifact JSON (no semantic change) stays
/// behaviorally valid yet is byte-different (serde_json reorders keys / drops the
/// artifact's formatting), so the backstop REJECTS it — callers must supply the
/// pinned artifact bytes, not a re-emitted equivalent.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn tokenizer_identity_rejects_reserialized_artifact_json() {
  let bytes = mutated_artifact_tokenizer_bytes(|_| {});

  let configured = configured_tokenizer_from_bytes(&bytes).expect("configure round-trip");
  validate_tokenizer_contract(&configured).expect("a re-serialized bundle is behaviorally valid");

  assert_ne!(
    bytes.as_slice(),
    artifact_tokenizer_bytes(),
    "a serde_json round-trip must actually differ from the pinned bytes"
  );

  let actual = sha256_hex(&bytes);
  match validate_tokenizer_identity(&TokenizerProvenance::Supplied(actual.clone())) {
    Err(Error::TokenizerContractMismatch(mismatch)) => {
      assert_eq!(mismatch.check(), "tokenizer identity (sha-256)");
      assert_eq!(mismatch.actual(), actual);
    }
    other => panic!("expected identity TokenizerContractMismatch, got {other:?}"),
  }
}

/// The backstop ACCEPTS the pinned artifact bytes through BOTH provenances —
/// `Supplied` (`from_memory` / `from_files`) and `Artifact` (the sidecar `load`
/// reads). Neither is exempt: with nothing compiled in, "identity by
/// construction" no longer exists, so both must actually hash and match.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn tokenizer_identity_accepts_the_pinned_bytes_through_both_provenances() {
  let sha256_hex = sha256_hex(artifact_tokenizer_bytes());
  validate_tokenizer_identity(&TokenizerProvenance::Supplied(sha256_hex.clone()))
    .expect("the pinned bytes are the identity, supplied");
  validate_tokenizer_identity(&TokenizerProvenance::Artifact(Artifact::new(
    artifact_tokenizer_path_for_tests(),
    sha256_hex,
  )))
  .expect("the pinned bytes are the identity, read from the artifact");
}

/// A corrupted SIDECAR is refused exactly like corrupted caller-supplied bytes,
/// and the diagnostic names the file — the regression that would otherwise make
/// an on-disk tokenizer strictly weaker than the embedded bytes it replaced.
#[test]
fn artifact_provenance_is_not_exempt_from_the_identity_pin() {
  let err = validate_tokenizer_identity(&TokenizerProvenance::Artifact(Artifact::new(
    std::path::PathBuf::from("/models/granite/tokenizer.json"),
    "0".repeat(64),
  )))
  .expect_err("a sidecar that is not the pinned artifact must be refused");
  match err {
    Error::TokenizerContractMismatch(mismatch) => {
      let actual = mismatch.actual();
      assert_eq!(mismatch.check(), "artifact tokenizer identity (sha-256)");
      assert_eq!(mismatch.expected(), contract::TOKENIZER_SHA256_HEX);
      assert!(
        actual.contains("/models/granite/tokenizer.json"),
        "the diagnostic must name the offending file, got {actual}"
      );
    }
    other => panic!("expected TokenizerContractMismatch, got {other:?}"),
  }
}

/// `load` resolves the tokenizer from the ARTIFACT ROOT — the directory
/// CONTAINING the `.mlmodelc`, where the published `CHECKSUMS.sha256` places it
/// (`./granite_97m_512.mlmodelc/...` with siblings at `./`). Hermetic: path
/// arithmetic only, no file needed.
#[test]
fn artifact_tokenizer_path_is_the_bundle_sibling() {
  assert_eq!(
    artifact_tokenizer_path(Path::new(
      "/m/granite-97m-multilingual-r2/granite_97m_512.mlmodelc"
    )),
    Path::new("/m/granite-97m-multilingual-r2/tokenizer.json"),
  );
  // A bare bundle name has an empty parent: the sidecar resolves in the current
  // directory, the same place the bundle itself would.
  assert_eq!(
    artifact_tokenizer_path(Path::new("granite_97m_512.mlmodelc")),
    Path::new("tokenizer.json"),
  );
}

// ── #3: single-pass chunking — layer-2 chunk differential + perf gates ────────
//
// The `TokenIndex`-backed `chunk_long` must produce byte-IDENTICAL `Vec<Chunk>`
// to a reference that re-encodes every candidate range directly (the old
// behaviour). Identical chunks + the unchanged embed tail ⇒ bit-identical
// embeddings, so output identity reduces to this equality. The perf gates prove
// the single pass replaced the old ~11× re-encode.

/// The windit + `attach_gaps` chunking pipeline over arbitrary measures — the
/// seam both the exact slow twin and the load-bearing perturbation drive.
fn run_pipeline<W, R>(
  text: &str,
  opts: &WindowOptions,
  win_measure: W,
  range_measure: R,
) -> Result<Vec<windit::split::Chunk>>
where
  W: Fn(&str) -> usize,
  R: Fn(usize, usize) -> Result<usize>,
{
  let chunks = windit::split::ContentAware::new(&win_measure)
    .chunk(text, opts)
    .map_err(Error::from)?;
  let mut repaired = attach_gaps(text, chunks, &range_measure, opts.window())?;
  if repaired.is_empty() && !text.is_empty() {
    let tokens = range_measure(0, text.len())?;
    if tokens > MAX_TOKENS {
      return Err(Error::ContentlessInputOverBudget(
        ContentlessInputOverBudget::new(0, text.len(), tokens, MAX_TOKENS),
      ));
    }
    repaired.push(windit::split::Chunk::new(0, text.len()));
  }
  if let Some(max) = opts.max_windows()
    && repaired.len() > max
  {
    return Err(Error::Windowing(windit::WinditError::TooManyWindows {
      got: repaired.len(),
      max,
    }));
  }
  Ok(repaired)
}

/// The reference twin: every candidate range measured by a DIRECT
/// `encode(&text[a..b], true)` — the exact behaviour before the single-pass
/// index. Slow by construction (re-encodes growing prefixes), used only to pin
/// the fast path.
fn chunk_long_slow(
  mt: &Tokenizer,
  text: &str,
  opts: &WindowOptions,
) -> Result<Vec<windit::split::Chunk>> {
  run_pipeline(
    text,
    opts,
    |s: &str| {
      mt.encode(s, true)
        .map(|e| e.get_ids().len())
        .unwrap_or(usize::MAX)
    },
    |a: usize, b: usize| {
      mt.encode(&text[a..b], true)
        .map(|e| e.get_ids().len())
        .map_err(Error::Tokenize)
    },
  )
}

/// What [`TailPolicy::DropBelowMin`] does to `chunk_long`'s output now that
/// windit's `ContentAware` honours it (0.5; 0.4 read every other geometry field
/// and skipped `tail`).
///
/// windit drops the short final chunk. `attach_gaps` then covers the bytes it
/// left, exactly as it covers a paragraph separator, so the granite-visible
/// change is a MOVED BOUNDARY and never a lost byte: the `\n\n` between the last
/// two chunks stops being absorbed leftwards into its predecessor and heads the
/// tail chunk instead. The chunk count is unchanged, which is what says the cost
/// in CoreML predictions is unchanged too — the repair cannot fuse the tail left,
/// because a dropped tail is by construction content its predecessor had no room
/// for.
///
/// Measured on the whitespace-word measurer rather than the granite tokenizer:
/// the geometry is then arithmetic, and the drop rule windit states — keep when
/// `len >= minimum || len == window` — is pinned at both of its edges (a tail of
/// 2 survives `min == 2` and dies at `min == 3`; a tail measuring exactly the
/// window survives any minimum).
#[test]
fn a_drop_below_min_tail_moves_the_last_boundary_and_keeps_every_byte() {
  let words = |s: &str| s.split_whitespace().count().max(1);
  let run = |text: &'static str, opts: &WindowOptions| -> Vec<(usize, usize)> {
    run_pipeline(text, opts, words, |a, b| Ok(words(&text[a..b])))
      .unwrap_or_else(|e| panic!("chunk {text:?}: {e}"))
      .iter()
      .map(|c| (c.start(), c.end()))
      .collect()
  };

  // Eight one-letter words, a paragraph break, then a two-word tail.
  const DOC: &str = "a b c d e f g h\n\ni j";
  let keep = WindowOptions::new(4);
  assert_eq!(run(DOC, &keep), [(0, 8), (8, 17), (17, 20)]);
  // The default and `PadFull` agree with each other: padding a byte range has no
  // meaning, so windit names it a gap in this chunker rather than implementing it.
  assert_eq!(keep.tail(), &TailPolicy::KeepWithCoverage);
  assert_eq!(
    run(DOC, &keep.with_tail(TailPolicy::PadFull)),
    [(0, 8), (8, 17), (17, 20)]
  );

  // The tail measures 2. At `min == 2` it is at the minimum and survives.
  assert_eq!(
    run(DOC, &keep.with_tail(TailPolicy::DropBelowMin(2))),
    [(0, 8), (8, 17), (17, 20)]
  );
  // At `min == 3` windit drops it — and `attach_gaps` gives it back, two bytes
  // longer at the front: the `\n\n` at 15..17 moved out of the middle chunk.
  // Same count, same coverage, different last boundary.
  for min in [3usize, 4, 50] {
    let moved = run(DOC, &keep.with_tail(TailPolicy::DropBelowMin(min)));
    assert_eq!(
      moved,
      [(0, 8), (8, 15), (15, 20)],
      "min={min} must move the boundary, not drop the tail"
    );
    assert_eq!(moved.len(), 3, "min={min} must not change the chunk count");
    assert_eq!(
      moved.iter().map(|&(a, b)| &DOC[a..b]).collect::<String>(),
      DOC,
      "min={min} must still cover every byte"
    );
  }

  // `|| len == window`: a tail measuring exactly the window is kept whatever the
  // minimum says, which is why the rule is not "did the measure reach the limit".
  const EVEN: &str = "a b c d\n\ne f g h";
  let even = run(EVEN, &keep);
  for min in [5usize, 50] {
    assert_eq!(
      run(EVEN, &keep.with_tail(TailPolicy::DropBelowMin(min))),
      even,
      "a tail filling the whole window must survive min={min}"
    );
  }

  // The shape windit documents as "a non-empty input can now yield no chunks at
  // all": one chunk, below the minimum. `chunk_long`'s non-empty fallback still
  // emits the whole input, so this knob cannot make a text embed to nothing.
  const SHORT: &str = "one two three";
  let one = WindowOptions::new(8);
  assert_eq!(
    windit::split::ContentAware::new(&words)
      .chunk(SHORT, &one.with_tail(TailPolicy::DropBelowMin(4)))
      .unwrap()
      .len(),
    0,
    "windit must drop the only chunk for this to be the case it claims to be"
  );
  assert_eq!(
    run(SHORT, &one.with_tail(TailPolicy::DropBelowMin(4))),
    [(0, SHORT.len())]
  );
  assert_eq!(run(SHORT, &one), [(0, SHORT.len())]);
}

/// The committed multilingual golden texts.
fn golden_texts() -> Vec<String> {
  const CORPUS: &str = include_str!("../../../tests/granite/fixtures/goldens/corpus.json");
  let v: serde_json::Value = serde_json::from_str(CORPUS).expect("parse corpus.json");
  v["entries"]
    .as_array()
    .expect("entries array")
    .iter()
    .map(|e| e["text"].as_str().expect("entry text").to_string())
    .collect()
}

/// The gate-2 corpus: the 16 goldens plus the adversarial shapes the design calls
/// out (paragraph doc, punctuation storm, digit storms, whitespace pathologies, a
/// no-space char-fallback word, multibyte/emoji). Kept ≤ ~1.5 KiB each so the
/// quadratic slow twin stays fast; the 4 MiB scale is the `#[ignore]` gate.
fn differential_texts() -> Vec<String> {
  let mut texts = golden_texts();
  // A compact multi-paragraph doc (distinct words → real word/sentence descent).
  let doc: String = (0..8)
    .map(|p| {
      (0..16)
        .map(|w| format!("para{p}word{w}"))
        .collect::<Vec<_>>()
        .join(" ")
    })
    .collect::<Vec<_>>()
    .join("\n\n");
  texts.push(format!("\n\n{doc}\n\n"));
  texts.push(doc);
  texts.push(
    (0..120)
      .map(|w| format!("term{w}"))
      .collect::<Vec<_>>()
      .join(", "),
  );
  texts.push("192.168.1.1 10.0.0.255 call 555-0142 order #A1234-99 on 2026-07-18. ".repeat(10));
  texts.push(" \t \u{00A0}\u{2009}mixed  ws\r\n\r\n runs\t here and there ".repeat(10));
  texts.push("x".repeat(2048));
  texts.push("café\u{0301} 你好 🍕 👨\u{200D}👩\u{200D}👧\u{200D}👦 tëst ".repeat(12));
  texts
}

/// The window/overlap/max_windows grid (overlap 16 only paired with windows above
/// it, so windit never rejects overlap >= window).
fn geometry_grid() -> Vec<WindowOptions> {
  let mut g = Vec::new();
  for &w in &[8usize, 32, 128, 512] {
    g.push(WindowOptions::new(w));
    if w > 16 {
      g.push(WindowOptions::new(w).with_overlap(16));
    }
  }
  g.push(WindowOptions::new(128).with_max_windows(4));
  g.push(WindowOptions::new(512).with_max_windows(2));
  g
}

/// GATE 2: the index-backed fast path and the direct-encode slow twin agree
/// EXACTLY as `Vec<Chunk>` across the corpus × geometry grid (both `Ok` with equal
/// chunks, or both `Err`). Identical chunks are the reduction of embedding
/// bit-identity.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn chunk_long_matches_slow_twin_over_corpus_and_geometry() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let grid = geometry_grid();
  for text in differential_texts() {
    for opts in &grid {
      let (window, overlap) = (opts.window(), opts.overlap());
      match (
        chunk_long(&mt, &text, opts),
        chunk_long_slow(&mt, &text, opts),
      ) {
        (Ok(fast), Ok(slow)) => assert_eq!(
          fast, slow,
          "fast/slow chunk mismatch (window={window}, overlap={overlap}) for {text:.40?}"
        ),
        (Err(_), Err(_)) => {}
        (fast, slow) => panic!(
          "fast/slow Ok-vs-Err disagreement (window={window}, overlap={overlap}) for {text:.40?}: \
           {fast:?} vs {slow:?}"
        ),
      }
    }
  }
}

/// The four killer classes that broke the pre-fix single-pass measure (whitespace
/// runs split by a non-glue follower, the punct `[\r\n/]*` CRLF tail, contraction
/// suffixes rejoining the next word, combining/Other_Alphabetic marks), each
/// repeated so it spans multiple windows, plus one big cross-class concatenation —
/// the shapes gate 2's corpus lacks, driven through the WHOLE chunk pipeline.
fn killer_chunk_texts() -> Vec<String> {
  let units = [
    "456  1 word ",
    "a\u{00A0}\u{00A0}9 sep ",
    "b\u{2009}\u{2009}🌿 emoji ",
    "a!\r\n\r\n Next term ",
    "end.\r\n\r\n\r\nStart here ",
    " it'station end ",
    "we'reunited now ",
    "cafe\u{0301}s re\u{0301}sume\u{0301} ",
    "192.168.100.254 ٠١٢٣ ０１２３ ",
    "http://a/b/c/d /usr/local/bin ",
    "👨\u{200D}👩\u{200D}👧\u{200D}👦 x ",
  ];
  let mut texts: Vec<String> = units.iter().map(|u| u.repeat(24)).collect();
  texts.push(units.concat().repeat(12));
  texts
}

/// GATE 2 (killer classes): `chunk_long` and the direct-encode slow twin produce
/// byte-IDENTICAL `Vec<Chunk>` across the killer texts × the geometry grid — the
/// chunk-level counterpart of the exhaustive `measure_range` sweep, on exactly the
/// classes that exposed the pre-fix divergences.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn chunk_long_matches_slow_twin_over_killer_classes() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let grid = geometry_grid();
  for text in killer_chunk_texts() {
    for opts in &grid {
      let (window, overlap) = (opts.window(), opts.overlap());
      match (
        chunk_long(&mt, &text, opts),
        chunk_long_slow(&mt, &text, opts),
      ) {
        (Ok(fast), Ok(slow)) => assert_eq!(
          fast, slow,
          "fast/slow killer chunk mismatch (window={window}, overlap={overlap}) for {text:.40?}"
        ),
        (Err(_), Err(_)) => {}
        (fast, slow) => panic!(
          "fast/slow Ok-vs-Err disagreement (window={window}, overlap={overlap}) for {text:.40?}: \
           {fast:?} vs {slow:?}"
        ),
      }
    }
  }
}

/// RED-FIRST (non-vacuity): a windit measure that over-counts every range by ONE
/// token packs one fewer atom wherever a chunk otherwise ended exactly at the
/// window, moving a boundary — so the `Vec<Chunk>` differs from the exact run. A
/// `measure_range` off by a single token would therefore red the gate above; the
/// equality is load-bearing, not trivially true.
///
/// Uses space-separated single letters, each exactly one token, so the packed
/// measure hits every integer and a chunk necessarily ends exactly at the window
/// (`window - 2` letters after the two template specials) — where `+1` tips the
/// threshold and drops a letter, a guaranteed boundary move.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn chunk_differential_is_load_bearing_against_a_shifted_measure() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let doc: String = "a b c d e f g h i j k l m n o p q r s t u v w x y z "
    .repeat(8)
    .trim_end()
    .to_string();
  let opts = WindowOptions::new(16);
  let exact_range = |a: usize, b: usize| {
    mt.encode(&doc[a..b], true)
      .map(|e| e.get_ids().len())
      .map_err(Error::Tokenize)
  };
  let correct = run_pipeline(
    &doc,
    &opts,
    |s: &str| {
      mt.encode(s, true)
        .map(|e| e.get_ids().len())
        .unwrap_or(usize::MAX)
    },
    exact_range,
  )
  .expect("exact pipeline");
  let shifted = run_pipeline(
    &doc,
    &opts,
    |s: &str| {
      mt.encode(s, true)
        .map(|e| e.get_ids().len() + 1)
        .unwrap_or(usize::MAX)
    },
    exact_range,
  )
  .expect("shifted pipeline");
  assert_ne!(
    correct, shifted,
    "a one-token measure over-count must move a chunk boundary — the fast==slow gate is \
     load-bearing"
  );
}

/// A deterministic natural-ish document: sentences of dictionary words with the
/// odd number, packed into `\n\n`-separated paragraphs, up to `target_bytes`.
fn natural_doc(target_bytes: usize) -> String {
  const WORDS: &[&str] = &[
    "the",
    "quantum",
    "entanglement",
    "system",
    "provides",
    "native",
    "on-device",
    "inference",
    "for",
    "text",
    "embeddings",
    "and",
    "retrieval",
    "across",
    "many",
    "languages",
    "with",
    "stable",
    "latency",
    "under",
    "load",
    "because",
    "model",
    "compiles",
    "efficiently",
    "into",
    "a",
    "fixed",
    "graph",
  ];
  let mut s = String::with_capacity(target_bytes + 64);
  let mut r: u64 = 0x9E37_79B9_7F4A_7C15;
  let mut step = || {
    r = r
      .wrapping_mul(6_364_136_223_846_793_005)
      .wrapping_add(1_442_695_040_888_963_407);
    (r >> 33) as usize
  };
  let mut sentences_in_para = 0u32;
  while s.len() < target_bytes {
    let words = 8 + step() % 9;
    for k in 0..words {
      if k > 0 {
        s.push(' ');
      }
      s.push_str(WORDS[step() % WORDS.len()]);
    }
    if step() % 4 == 0 {
      s.push_str(&format!(" {}", 1000 + step() % 90_000));
    }
    s.push('.');
    sentences_in_para += 1;
    if sentences_in_para >= 5 {
      s.push_str("\n\n");
      sentences_in_para = 0;
    } else {
      s.push(' ');
    }
  }
  s
}

/// PERF GATE (structural, hermetic, non-flaky): on a ~256 KiB natural document the
/// measurement path re-encodes at most 1.5× the input in bytes (the old per-range
/// closure re-encoded ~11×).
///
/// Scope: this holds for text whose pre-tokens are short — i.e. separated by
/// whitespace or punctuation, as `natural_doc` is. It does NOT generalize to text
/// the o200k Split regex parses into one document-spanning pre-token (unspaced CJK,
/// an unbroken letter run); there the index has no interior boundary to answer
/// from and every measure direct-encodes, driving the ratio into the hundreds.
/// `separatorless_text_is_one_pretoken` pins that structural cause and
/// `measure_path_reencode_ratio_on_separatorless_text` characterizes the cost.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn measure_path_reencodes_at_most_1_5x_input() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let doc = natural_doc(256 * 1024);
  super::token_index::encode_meter::reset();
  let chunks = chunk_long(&mt, &doc, &WindowOptions::new(MAX_TOKENS)).expect("chunk");
  let encoded = super::token_index::encode_meter::get();
  let ratio = encoded as f64 / doc.len() as f64;
  println!(
    "[byte-ratio] input={} bytes, encoded={} bytes, ratio={ratio:.3}x, chunks={}",
    doc.len(),
    encoded,
    chunks.len()
  );
  assert!(
    ratio <= 1.5,
    "measure path re-encoded {ratio:.3}x the input (> 1.5x) — the single-pass index regressed \
     toward the old per-range re-encode"
  );
}

/// Dense unspaced CJK — no ASCII space, no ASCII sentence terminator. windit's
/// `ContentAware` finds no paragraph, sentence, or word boundary and descends to
/// its `char` fallback.
fn cjk_doc(target_bytes: usize) -> String {
  const RUN: &str = "你好世界模型推理文本嵌入检索";
  let mut s = String::with_capacity(target_bytes + RUN.len());
  while s.len() < target_bytes {
    s.push_str(RUN);
  }
  s
}

/// The structural cause of the separatorless blow-up, pinned cheaply (one encode,
/// no chunking): the o200k Split regex's word branches glue an unbroken
/// `\p{L}`/`\p{M}` run into a SINGLE pre-token, so an unspaced CJK document —
/// however long — parses to exactly one pre-token.
///
/// `TokenIndex` is indexed BY pre-token, so a one-pre-token text gives it no
/// interior boundary: before #72 every `measure_range(a, b)` with `a` inside that
/// pre-token re-encoded `[a, b)` in full, the pre-index cost; the separatorless
/// fast lane (`token_index::suffix_session`) now answers those probes from the
/// chunk suffix's recorded merge process. The single-pass property the sibling
/// 1.5× gate asserts is still a property of the CORPUS (short pre-tokens); the
/// separatorless regime has its own 8× gate below.
///
/// Goes red if the pinned tokenizer's pre-tokenization ever splits such a run —
/// at which point the separatorless re-encode gate below measures a different
/// regime than it was written for.
/// "Cheap" is about the work, not about reach: like every gate that needs REAL
/// granite tokenization, it reads the staged artifact sidecar and is `#[ignore]`d
/// on it, so it runs where the sibling 1.5× gate runs and nowhere else.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn separatorless_text_is_one_pretoken() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  for (label, doc) in [("cjk", cjk_doc(4096)), ("ascii-letters", "x".repeat(4096))] {
    let enc = mt.encode(doc.as_str(), false).expect("encode");
    let pretokens = enc
      .get_word_ids()
      .iter()
      .flatten()
      .copied()
      .max()
      .map_or(0, |m| m as usize + 1);
    assert_eq!(
      pretokens,
      1,
      "{label}: expected one document-spanning pre-token over {} bytes, got {pretokens} — the \
       separatorless measure-cost characterization rests on this",
      doc.len()
    );
    assert!(
      enc.get_ids().len() > 1,
      "{label}: the one pre-token must still carry many BPE tokens"
    );
  }
}

/// GATE (separatorless, `#[ignore]` — the slow twin is quadratic, and the
/// staged tokenizer is needed): on text the Split regex glues into ONE
/// pre-token, the measurement path no longer re-encodes the growing prefix.
/// Every probe inside such a pre-token is answered from the recorded merge
/// process of the chunk's suffix (`token_index::suffix_session`, #72), so the
/// bytes the tokenizer encodes are the index build (1×) plus one bounded
/// suffix encode per chunk — the session's own cross-check against the crate —
/// and never a whole-range re-encode per atom.
///
/// Before the fix (this machine, release): CJK 15,666 B → 9 chunks at a
/// **565.9×** re-encode ratio, `"x"×7,833` → 2 chunks at **1,964×**, both at
/// 1.0× or WORSE than the direct-encode twin. The bound below is 8×: at most
/// one 8 KiB session encode per chunk over these ~1.7 KB chunks. Chunks stay
/// byte-identical to the twin's.
#[test]
#[ignore = "quadratic slow twin, and the staged granite tokenizer.json (run locally); \
            the separatorless re-encode gate"]
fn separatorless_measure_path_reencodes_at_most_8x_input() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let opts = WindowOptions::new(MAX_TOKENS);
  for (label, doc) in [
    ("cjk", cjk_doc(15_666)),
    ("ascii-letters", "x".repeat(7_833)),
  ] {
    // The merge table is a one-time cost per embedder (built on the lane's
    // first engagement); it is not what this gate times.
    let _ = merge_table();
    super::token_index::encode_meter::reset();
    super::token_index::build_meter::reset();
    let t0 = std::time::Instant::now();
    let fast = chunk_long(&mt, &doc, &opts).expect("fast chunk");
    let fast_ms = t0.elapsed().as_secs_f64() * 1e3;
    let ratio = super::token_index::encode_meter::get() as f64 / doc.len() as f64;
    let builds = super::token_index::build_meter::builds();
    let mut sizes = super::token_index::encode_meter::sizes();
    sizes.sort_unstable_by(|a, b| b.cmp(a));
    let big: Vec<usize> = sizes.iter().copied().filter(|&n| n > 512).collect();
    println!(
      "[separatorless:{label}:encodes] calls={} over512={} over512_bytes={} top={:?}",
      sizes.len(),
      big.len(),
      big.iter().sum::<usize>(),
      &sizes[..sizes.len().min(12)]
    );

    let t1 = std::time::Instant::now();
    let slow = chunk_long_slow(&mt, &doc, &opts).expect("slow chunk");
    let slow_ms = t1.elapsed().as_secs_f64() * 1e3;

    assert_eq!(fast, slow, "{label}: fast/slow chunk mismatch");
    println!(
      "[separatorless:{label}] bytes={} chunks={} fast={fast_ms:.1}ms slow={slow_ms:.1}ms \
       speedup={:.1}x reencode_ratio={ratio:.1}x sessions={} session_bytes={}",
      doc.len(),
      fast.len(),
      slow_ms / fast_ms,
      builds.len(),
      builds.iter().map(|&(s, e)| e - s).sum::<usize>(),
    );
    assert!(
      ratio <= 8.0,
      "{label}: measure path re-encoded {ratio:.1}x the input (> 8x) — the separatorless fast \
       lane stopped engaging"
    );
  }
}

/// GATE 4 (`#[ignore]`, run locally for the PR notes): fast == slow chunks on
/// 1 MiB / 4 MiB natural documents, with wall-clock speedup and the re-encode
/// byte ratio printed. The slow twin is too slow for CI (its classes are covered
/// hermetically by gate 2); this is the scale/perf witness.
#[test]
#[ignore = "4 MiB fast-vs-slow chunk differential + timing, and the staged granite \
            tokenizer.json (run locally for PR notes)"]
fn big_document_fast_matches_slow_with_timing() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let opts = WindowOptions::new(MAX_TOKENS);
  for size in [1usize << 20, 4usize << 20] {
    let doc = natural_doc(size);

    let _ = merge_table();
    super::token_index::encode_meter::reset();
    let t0 = std::time::Instant::now();
    let fast = chunk_long(&mt, &doc, &opts).expect("fast chunk");
    let fast_ms = t0.elapsed().as_secs_f64() * 1e3;
    let fast_bytes = super::token_index::encode_meter::get();

    let t1 = std::time::Instant::now();
    let slow = chunk_long_slow(&mt, &doc, &opts).expect("slow chunk");
    let slow_ms = t1.elapsed().as_secs_f64() * 1e3;

    assert_eq!(fast, slow, "fast/slow chunk mismatch at {size} bytes");
    let ratio = fast_bytes as f64 / doc.len() as f64;
    println!(
      "[big-diff] size={} chunks={} fast={fast_ms:.1}ms slow={slow_ms:.1}ms \
       speedup={:.1}x reencode_ratio={ratio:.3}x",
      doc.len(),
      fast.len(),
      slow_ms / fast_ms,
    );
    assert!(
      ratio <= 1.5,
      "reencode ratio {ratio:.3}x > 1.5x at {size} bytes"
    );
  }
}

// ── The tokenizer is judged before it is parsed ──────────────────────────────

/// A defective `TemplateProcessing` single template — one whose `SpecialToken`
/// id its own `special_tokens` map does not declare. The tokenizers crate's
/// deserializer skips its builder's `validate`, so this PARSES, and applying it
/// indexes that map: `no entry found for key`, a panic inside the dependency.
const DEFECTIVE_TEMPLATE_TOKENIZER: &[u8] = br#"{"version":"1.0","truncation":null,"padding":null,"added_tokens":[],"normalizer":null,"pre_tokenizer":{"type":"Whitespace"},"post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"<s>","type_id":0}},{"Sequence":{"id":"A","type_id":0}}],"pair":[{"Sequence":{"id":"A","type_id":0}}],"special_tokens":{}},"decoder":null,"model":{"type":"WordLevel","vocab":{"<pad>":0,"a":1,"b":2},"unk_token":"<pad>"}}"#;

/// This door pins ONE tokenizer artifact byte for byte, on every constructor, so
/// the identity gate runs on the RAW bytes BEFORE `Tokenizer::from_bytes` — and
/// that ordering is what makes the dependency's skipped template validation
/// unreachable here. Asserting the `check` FIELD is what pins the order: move
/// the gate back behind the parse and the failure becomes a behavioral one (or,
/// for a tokenizer that satisfies the behavioral contract, a panic inside
/// `validate_tokenizer_contract`'s sentinel encode).
///
/// The model path never exists, so reaching `Model::load` would surface as
/// `Error::Load` — anything else proves the gate fired first.
#[test]
fn from_memory_hashes_the_bytes_before_parsing_them() {
  for bytes in [
    // Not even JSON: the parser would fail loudly, with the wrong diagnosis.
    b"this is not json at all".as_slice(),
    // Parses, and panics on the first `encode`.
    DEFECTIVE_TEMPLATE_TOKENIZER,
  ] {
    match TextEmbedder::from_memory(
      "/nonexistent/model.mlmodelc",
      bytes,
      TextEmbedderOptions::new(),
    ) {
      Err(Error::TokenizerContractMismatch(mismatch)) => {
        assert_eq!(mismatch.check(), "tokenizer identity (sha-256)");
        assert_eq!(mismatch.expected(), contract::TOKENIZER_SHA256_HEX);
      }
      other => panic!("expected the identity gate to fire before the parse, got {other:?}"),
    }
  }
}

/// The same ordering on the artifact path: `load` hashes the sidecar it read and
/// refuses it before parsing, naming the file.
#[test]
fn load_hashes_the_sidecar_before_parsing_it() {
  let dir = tempfile::tempdir().expect("tempdir");
  let model_path = dir.path().join("granite_97m_512.mlmodelc");
  let tokenizer_path = dir.path().join(TOKENIZER_FILE_NAME);
  std::fs::write(&tokenizer_path, b"this is not json at all").expect("write sidecar");

  match TextEmbedder::load(&model_path, TextEmbedderOptions::new()) {
    Err(Error::TokenizerContractMismatch(mismatch)) => {
      assert_eq!(mismatch.check(), "artifact tokenizer identity (sha-256)");
      assert!(
        mismatch
          .actual()
          .contains(&tokenizer_path.display().to_string()),
        "the diagnostic names the file: {}",
        mismatch.actual()
      );
    }
    other => panic!("expected the identity gate to fire before the parse, got {other:?}"),
  }
}

/// Non-vacuity for the test above: the defective template really does panic when
/// it reaches the dependency, so the identity gate is preventing something.
#[test]
fn the_defective_template_fixture_really_panics_at_encode() {
  let tokenizer =
    Tokenizer::from_bytes(DEFECTIVE_TEMPLATE_TOKENIZER).expect("a defective template still parses");
  let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
    let _ = tokenizer.encode("a b", true);
  }))
  .is_err();
  assert!(
    panicked,
    "applying the template panics inside the dependency"
  );
}

// ─── #72 codex round 1: the early-stop floor and dropped bytes ──────────────

/// RED-FIRST: `"\0\0"` encodes to exactly the two template specials — the
/// pinned tokenizer has no ByteLevel symbol for NUL and no unk fallback, so
/// both bytes are DROPPED and contribute no content token. A floor that counted
/// bytes as at-least-one-token said 3 > window 2, refused the exact-fit chunk,
/// and windit split where the direct-encode twin keeps one chunk — with
/// `max_windows = 1` that surfaced as `TooManyWindows`. The floor now applies
/// only where the index proved byte coverage; the fast path must equal the twin
/// here in shape and in chunks.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn dropped_byte_text_chunks_like_the_slow_twin_at_an_exact_fit_window() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  assert_eq!(
    mt.encode("\0\0", true).expect("encode").get_ids().len(),
    2,
    "premise: both NULs are dropped, only the specials remain"
  );
  let opts = WindowOptions::new(2).with_max_windows(1);
  let fast = chunk_long(&mt, "\0\0", &opts);
  let slow = chunk_long_slow(&mt, "\0\0", &opts);
  match (&fast, &slow) {
    (Ok(f), Ok(s)) => assert_eq!(f, s, "fast/slow chunk mismatch on dropped bytes"),
    (f, s) => panic!("fast {f:?} vs slow {s:?}"),
  }
  assert_eq!(fast.expect("one chunk").len(), 1);
}

/// RED-FIRST: 65,281 NULs at the default window — one window on the twin (the
/// text has no content tokens at all), so one window here too.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn a_document_of_dropped_bytes_stays_one_window() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let text = "\0".repeat(65_281);
  let opts = WindowOptions::new(MAX_TOKENS);
  let fast = chunk_long(&mt, &text, &opts).expect("fast");
  let slow = chunk_long_slow(&mt, &text, &opts).expect("slow");
  assert_eq!(fast, slow);
  assert_eq!(fast.len(), 1);
}

/// RED-FIRST: NUL bytes interleaved in CJK. With dropped bytes DOMINATING —
/// runs of 300 NULs between single ideographs — a slice of two ideographs and
/// the run between them is 306 bytes for two content tokens: it measures 4
/// and fits window 4, while a byte-count floor said 2 + ceil(306 / 128) = 5
/// and refused it, so the old fast path split every ideograph apart where the
/// twin packs pairs. The index is `direct_only` (a dropped byte voids its
/// offset reconstruction), so every probe takes the exact path and the chunks
/// must equal the twin's here and across a geometry grid of the
/// single-NUL interleaving.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn dropped_bytes_interleaved_in_cjk_chunk_like_the_slow_twin() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let nul_run = "\0".repeat(300);
  let dominated: String = "你好世界模型推理"
    .chars()
    .map(|c| format!("{c}{nul_run}"))
    .collect();
  assert_eq!(
    mt.encode(&dominated[..306], true)
      .expect("encode")
      .get_ids()
      .len(),
    4,
    "premise: two ideographs and the dropped run between them are two content tokens"
  );
  for window in [4usize, 5, 8] {
    let opts = WindowOptions::new(window);
    let fast = chunk_long(&mt, &dominated, &opts);
    let slow = chunk_long_slow(&mt, &dominated, &opts);
    match (&fast, &slow) {
      (Ok(f), Ok(s)) => assert_eq!(f, s, "dominated, window {window}: fast/slow chunk mismatch"),
      (f, s) => panic!("dominated, window {window}: fast {f:?} vs slow {s:?}"),
    }
  }
  let single: String = "你好世界模型推理文本嵌入检索"
    .chars()
    .flat_map(|c| [c, '\0'])
    .collect::<String>()
    .repeat(40);
  for window in [2usize, 3, 7, 32, 128, MAX_TOKENS] {
    let opts = WindowOptions::new(window);
    let fast = chunk_long(&mt, &single, &opts);
    let slow = chunk_long_slow(&mt, &single, &opts);
    match (&fast, &slow) {
      (Ok(f), Ok(s)) => assert_eq!(f, s, "single, window {window}: fast/slow chunk mismatch"),
      (f, s) => panic!("single, window {window}: fast {f:?} vs slow {s:?}"),
    }
  }
}

/// Fast and slow twin on one document, both shapes compared.
#[track_caller]
fn assert_twin(mt: &Tokenizer, what: &str, text: &str, opts: &WindowOptions) {
  let fast = chunk_long(mt, text, opts);
  let slow = chunk_long_slow(mt, text, opts);
  match (&fast, &slow) {
    (Ok(f), Ok(s)) => assert_eq!(f, s, "{what}: fast/slow chunk mismatch"),
    (f, s) => panic!("{what}: fast {f:?} vs slow {s:?}"),
  }
}

/// RED-FIRST (Opus review, F1): CJK×k ++ `UCCESS` ++ CJK×1600 is ONE
/// pre-token (`HEAD* TAIL+`: `\p{Lo}` and `\p{Lu}` are both head chars and
/// the last ideograph is a tail char), and windit's word atoms end exactly at
/// the end of `UCCESS`, so the packer probes a prefix that ends inside the
/// uppercase run — which re-parses as `…中` and `UCCESS`, a vocabulary entry
/// the merge process does not reproduce. A lane engaged on that pre-token
/// over-counted the probe and closed the chunk a word early: at the default
/// window the first chunk ended at byte 2,439 instead of 2,445 (k = 813, 814,
/// 815), at window 16 (k = 20) and 32 (k = 45) there was one chunk more. The
/// gate must refuse the pre-token; the chunks must equal the twin's.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn an_uppercase_run_inside_cjk_chunks_like_the_slow_twin() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let cycle = "你好世界模型推理";
  let tail = cycle.repeat(200);
  for (window, ks) in [
    (MAX_TOKENS, &[813usize, 814, 815][..]),
    (32, &[45][..]),
    (16, &[20][..]),
  ] {
    for &k in ks {
      let head: String = cycle.chars().cycle().take(k).collect();
      let doc = format!("{head}UCCESS{tail}");
      assert_twin(
        &mt,
        &format!("window {window}, k {k}"),
        &doc,
        &WindowOptions::new(window),
      );
    }
  }
  // The reviewer's repeating patterns (an uppercase run followed by a
  // lowercase char, so the run sits strictly inside the pre-token) over a
  // window sweep.
  for pat in [
    "你好世界UCCESSa模型推理",
    "你UCCESSa",
    "你好UCCESSa模",
    "你好世WARRANTIESa模",
  ] {
    let doc = pat.repeat(3_000 / pat.len() + 1);
    for window in [8usize, 11, 16, 24, 32, 45, 64, 100, 128, 160] {
      assert_twin(
        &mt,
        &format!("{pat:?} at window {window}"),
        &doc,
        &WindowOptions::new(window),
      );
    }
  }
}

/// A seeded corpus that mixes `\p{Lu}` and `\p{Lt}` runs (`UCCESS`, `API`,
/// `Windows`, the titlecase digraphs and Greek titlecase vowels) into CJK and
/// lowercase Latin, with combining marks, digits, spaces, CJK punctuation and
/// an apostrophe suffix — pre-tokens of every letter-branch shape, some longer
/// than the lane's engagement length and some not — chunks like the twin at
/// every window.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn mixed_case_scripts_chunk_like_the_slow_twin() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let pieces: [&str; 28] = [
    "你好世界",
    "模型推理",
    "文本嵌入",
    "检索",
    "返回",
    "hello",
    "world",
    "granite",
    "embedding",
    "UCCESS",
    "WARRANTIES",
    "API",
    "ABC",
    "Windows",
    "SUCCESS",
    "Dž",
    "Lj",
    "ᾈ",
    "Dz",
    "\u{301}",
    "\u{5bf}",
    " ",
    " ",
    "12",
    "。",
    "'s",
    "\n",
    "αβγ",
  ];
  let mut seed = 0x9E37_79B9_7F4A_7C15u64;
  let mut next = move || {
    seed = seed
      .wrapping_mul(6_364_136_223_846_793_005)
      .wrapping_add(1_442_695_040_888_963_407);
    (seed >> 33) as usize
  };
  for round in 0..6 {
    let mut doc = String::new();
    // Alternate long separatorless stretches with short mixed ones.
    while doc.len() < 2_500 {
      let stretch = 1 + next() % 40;
      for _ in 0..stretch {
        doc.push_str(pieces[next() % pieces.len()]);
      }
      if next() % 3 == 0 {
        doc.push(' ');
      }
    }
    for window in [16usize, 32, 128, MAX_TOKENS] {
      assert_twin(
        &mt,
        &format!("round {round}, window {window}"),
        &doc,
        &WindowOptions::new(window),
      );
    }
  }
}

/// RED-FIRST (Opus re-review, F5): a pre-token longer than any token whose
/// chars can never qualify — an uppercase run, a rule of `=` or `-`, a
/// newline or space run — must not build the merge table (~0.4 s, ~74 MB
/// transient) on the first probe into it: the class test runs before the
/// build, on a tail-class regex the lane compiles for itself. The chunks
/// equal the twin's either way; a qualifying run (CJK) does build it.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn runs_that_cannot_qualify_never_build_the_table() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let opts = WindowOptions::new(64);
  for (label, doc) in [
    ("uppercase-4000", format!("intro {} body", "A".repeat(4000))),
    (
      "equals-rule-4000",
      format!("intro\n{}\nbody\n", "=".repeat(4000)),
    ),
    (
      "dash-rule-200",
      format!("intro text\n{}\nbody text\n", "-".repeat(200)),
    ),
    (
      "newline-run-4000",
      format!("intro{}body", "\n".repeat(4000)),
    ),
    ("space-run-4000", format!("intro{}body", " ".repeat(4000))),
    (
      "titlecase-then-upper",
      format!("x {}{} y", "Dž", "B".repeat(300)),
    ),
  ] {
    let cell: OnceLock<Option<MergeTable>> = OnceLock::new();
    let fast = super::chunk_long(&mt, LazyTable::new(&cell, &mt), &doc, &opts);
    assert!(
      cell.get().is_none(),
      "{label}: a pre-token that cannot qualify must not build the table"
    );
    let slow = chunk_long_slow(&mt, &doc, &opts);
    match (&fast, &slow) {
      (Ok(f), Ok(s)) => assert_eq!(f, s, "{label}: fast/slow chunk mismatch"),
      (f, s) => panic!("{label}: fast {f:?} vs slow {s:?}"),
    }
  }
  let cell: OnceLock<Option<MergeTable>> = OnceLock::new();
  let doc = "你好世界模型推理文本嵌入检索".repeat(60);
  let fast = super::chunk_long(&mt, LazyTable::new(&cell, &mt), &doc, &opts).expect("fast");
  assert!(
    cell.get().is_some_and(Option::is_some),
    "a qualifying separatorless pre-token engages the lane and builds the table"
  );
  assert_eq!(fast, chunk_long_slow(&mt, &doc, &opts).expect("slow"));
}

/// The end-to-end form of the F4 falsifier: with the added spelling present
/// the lane-on chunking equals the twin's.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn an_added_token_spelled_like_a_word_chunks_like_the_slow_twin() {
  let base = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  let spelled = merge_table().spell(b" zzqxjkw");
  let mut value: serde_json::Value =
    serde_json::from_slice(artifact_tokenizer_bytes()).expect("parse");
  let next_id = u64::try_from(base.get_vocab_size(true)).expect("id");
  value["added_tokens"]
    .as_array_mut()
    .expect("array")
    .push(serde_json::json!({
      "id": next_id, "content": spelled, "single_word": false, "lstrip": false,
      "rstrip": false, "normalized": false, "special": false
    }));
  let hacked = Tokenizer::from_bytes(serde_json::to_vec(&value).expect("serialize")).expect("load");
  let cell: OnceLock<Option<MergeTable>> = OnceLock::new();
  let doc = format!(
    "{} zzqxjkw{}",
    "你好世界模型推理".repeat(30),
    "你好世界".repeat(30)
  );
  for window in [16usize, 64, MAX_TOKENS] {
    let opts = WindowOptions::new(window);
    let fast = super::chunk_long(&hacked, LazyTable::new(&cell, &hacked), &doc, &opts);
    let slow = chunk_long_slow(&hacked, &doc, &opts);
    match (&fast, &slow) {
      (Ok(f), Ok(s)) => assert_eq!(f, s, "window {window}: fast/slow chunk mismatch"),
      (f, s) => panic!("window {window}: fast {f:?} vs slow {s:?}"),
    }
  }
  assert!(cell.get().is_some_and(Option::is_some), "the lane engaged");
}

/// RED-FIRST (codex round 3): the engagement guard keyed on the ENCLOSING
/// pre-token's length, so a tiny probe inside a qualifying pre-token of 129
/// bytes built the whole merge table — 129 caller bytes at a tiny window
/// forcing the ~74 MB transient build. Engagement is keyed on the PROBE now:
/// `"a" × 129` at window 3 chunks like the twin with the cell still empty,
/// while a probe longer than any token inside a qualifying pre-token still
/// builds it.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn a_small_probe_inside_a_long_word_never_builds_the_table() {
  let mt = measuring_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("measuring");
  for (label, doc, window, max_windows) in [
    ("129 a's, window 3", "a".repeat(129), 3usize, None),
    (
      "129 a's, window 3, no windows allowed",
      "a".repeat(129),
      3,
      Some(0usize),
    ),
    ("400 a's, window 8", "a".repeat(400), 8, None),
    (
      "CJK 129 bytes, window 4",
      "你好世界模型推理".repeat(6),
      4,
      None,
    ),
  ] {
    let mut opts = WindowOptions::new(window);
    if let Some(cap) = max_windows {
      opts = opts.with_max_windows(cap);
    }
    let cell: OnceLock<Option<MergeTable>> = OnceLock::new();
    let fast = super::chunk_long(&mt, LazyTable::new(&cell, &mt), &doc, &opts);
    assert!(
      cell.get().is_none(),
      "{label}: no probe longer than a token, so no table"
    );
    let slow = chunk_long_slow(&mt, &doc, &opts);
    match (&fast, &slow) {
      (Ok(f), Ok(s)) => assert_eq!(f, s, "{label}: fast/slow chunk mismatch"),
      (Err(f), Err(s)) => assert_eq!(format!("{f:?}"), format!("{s:?}"), "{label}: error shape"),
      (f, s) => panic!("{label}: fast {f:?} vs slow {s:?}"),
    }
  }
  // A probe longer than any token inside a qualifying pre-token builds it:
  // unspaced CJK at window 64 grows a char-fallback slice to ~190 bytes
  // before the window overflows. (A run of `a`s is no such control: 400 of
  // them merge into a few long tokens and fit one window outright.)
  let cell: OnceLock<Option<MergeTable>> = OnceLock::new();
  let doc = "你好世界模型推理文本嵌入检索".repeat(60);
  let opts = WindowOptions::new(64);
  let fast = super::chunk_long(&mt, LazyTable::new(&cell, &mt), &doc, &opts).expect("fast");
  assert!(
    cell.get().is_some_and(Option::is_some),
    "a long probe engages the lane"
  );
  assert_eq!(fast, chunk_long_slow(&mt, &doc, &opts).expect("slow"));
}

// ── #160: the window-level contract (hermetic + the artifact's template) ─────

/// [`WindowEmbedding`]'s accessors report exactly what was planned into it:
/// the ordinal, both byte ends and the range they form, the token span with its
/// count, and the embedding by reference and by value.
///
/// The type is a payload struct with private fields, so this is the only proof
/// that the getters are wired to the right fields — a `byte_start`/`byte_end`
/// swap or a `start()`/`len()` transposition inside `token_span` compiles.
#[test]
fn window_embedding_accessors_report_the_planned_geometry() {
  let mut v = [0.0f32; EMBEDDING_DIM];
  v[7] = 2.0;
  let embedding = Embedding::from_slice_normalizing(&v).expect("unit vector");
  let w = WindowEmbedding {
    ordinal: 3,
    byte_range: 17..42,
    token_span: windit::plan::Span::new(1_024, 300, MAX_TOKENS),
    embedding: embedding.clone(),
  };
  assert_eq!(w.ordinal(), 3);
  assert_eq!(w.byte_start(), 17);
  assert_eq!(w.byte_end(), 42);
  assert_eq!(w.byte_range(), 17..42);
  assert_eq!(
    w.token_span(),
    windit::plan::Span::new(1_024, 300, MAX_TOKENS)
  );
  assert_eq!(w.token_span().start(), 1_024);
  assert_eq!(w.token_count(), 300);
  assert_eq!(w.token_span().window(), MAX_TOKENS);
  assert!(w.embedding().is_close(&embedding, 0.0), "by reference");
  assert!(
    w.clone().into_embedding().is_close(&embedding, 0.0),
    "by value"
  );
  // The byte range is the form to slice the caller's own text with.
  let text = "x".repeat(64);
  assert_eq!(text[w.byte_range()].len(), 25);
}

/// [`CONTENT_TOKENS_PER_WINDOW`] is the real content budget of a
/// [`MAX_TOKENS`]-wide window under a post-processor adding
/// [`SPECIAL_TOKENS_PER_WINDOW`] tokens — MEASURED as the truncation boundary
/// the `tokenizers` crate actually applies, not restated as arithmetic.
///
/// The fixture installs the artifact's overhead on a tiny vocabulary, so the
/// boundary is exercisable without the 24 MB artifact, and its specials are id
/// `0` against a content id of `1` — the content count is read off the ids
/// rather than subtracted. Content up to the budget survives whole; one token
/// past it is truncated away and the window still admits exactly the budget.
/// What the artifact contributes — that its own overhead really is
/// [`SPECIAL_TOKENS_PER_WINDOW`] — is
/// `special_token_overhead_matches_the_pinned_template`.
///
/// `add_special_tokens = false` is deliberately NOT the control here: measured,
/// `tokenizers` reserves the overhead only when it is going to add it, so a
/// special-free encoding of the same text truncates at the full
/// [`MAX_TOKENS`] and is two tokens longer, not equal.
#[test]
fn content_tokens_per_window_is_the_measured_truncation_budget() {
  const CONTENT_ID: u32 = 1;

  let bytes = tokenizer_bytes_with_special_overhead(SPECIAL_TOKENS_PER_WINDOW);
  let tok = configured_tokenizer_from_bytes(&bytes).expect("the artifact's overhead configures");
  for (content, kept) in [
    (1, 1),
    (CONTENT_TOKENS_PER_WINDOW - 1, CONTENT_TOKENS_PER_WINDOW - 1),
    (CONTENT_TOKENS_PER_WINDOW, CONTENT_TOKENS_PER_WINDOW),
    (CONTENT_TOKENS_PER_WINDOW + 1, CONTENT_TOKENS_PER_WINDOW),
    (MAX_TOKENS * 2, CONTENT_TOKENS_PER_WINDOW),
  ] {
    let text = "a ".repeat(content);
    let ids = tok.encode(text.as_str(), true).expect("encode");
    let ids = ids.get_ids();
    assert_eq!(
      ids.iter().filter(|&&id| id == CONTENT_ID).count(),
      kept,
      "{content} content tokens through a {MAX_TOKENS}-token window"
    );
    assert!(
      ids.len() <= MAX_TOKENS,
      "{content} content tokens overran the window: {} ids",
      ids.len()
    );
  }
}

/// [`SPECIAL_TOKENS_PER_WINDOW`] is MEASURED from the pinned artifact
/// tokenizer, three independent ways, rather than asserted: the post-processor's
/// own `added_tokens(false)` (the number `tokenizers` subtracts from the
/// truncation length), the length of `encode("", true)` (the template applied to
/// nothing), and the difference between encoding a text with and without
/// specials. The ids are the `<|startoftext|> A <|return|>` bracket the
/// artifact's `TemplateProcessing` declares.
#[test]
#[ignore = "requires the granite tokenizer.json staged beside the model bundle (EMBEDKIT_TEST_MODELS)"]
fn special_token_overhead_matches_the_pinned_template() {
  use tokenizers::PostProcessor;

  let tok = configured_tokenizer_from_bytes(artifact_tokenizer_bytes()).expect("configure");
  let post = tok
    .get_post_processor()
    .expect("the artifact declares a TemplateProcessing");
  assert_eq!(
    post.added_tokens(false),
    SPECIAL_TOKENS_PER_WINDOW,
    "the post-processor's single-sequence overhead"
  );

  let empty = tok.encode("", true).expect("encode the empty string");
  assert_eq!(
    empty.get_ids().len(),
    SPECIAL_TOKENS_PER_WINDOW,
    "the template applied to no content is the specials alone"
  );
  assert_eq!(
    empty.get_ids(),
    [contract::CLS_ID, contract::EOS_ID],
    "the single-sequence template is <|startoftext|> A <|return|>"
  );

  let text = "hello world";
  let with = tok.encode(text, true).expect("with specials");
  let without = tok.encode(text, false).expect("without specials");
  assert_eq!(
    with.get_ids().len() - without.get_ids().len(),
    SPECIAL_TOKENS_PER_WINDOW,
    "the specials are the whole difference"
  );
  assert_eq!(with.get_ids().first(), Some(&contract::CLS_ID));
  assert_eq!(with.get_ids().last(), Some(&contract::EOS_ID));

  assert_eq!(
    MAX_TOKENS - post.added_tokens(false),
    CONTENT_TOKENS_PER_WINDOW,
    "the artifact's own content budget"
  );
}

// ── Cross-model review: the single-window aggregate identity (hermetic) ─────

/// A tiny linear congruential generator (Knuth's MMIX 64-bit constants) for
/// deterministic pseudo-random test data — no `rand` dependency, and enough
/// entropy that the bit-exactness pin below is not vacuously true on an
/// all-equal or arithmetic-sequence fixture.
fn lcg_row(seed: u64) -> [f32; EMBEDDING_DIM] {
  let mut state = seed;
  std::array::from_fn(|_| {
    state = state
      .wrapping_mul(6_364_136_223_846_793_005)
      .wrapping_add(1_442_695_040_888_963_407);
    // The top 24 bits of each draw, mapped to (-1.0, 1.0): mixed signs, no
    // near-zero components, plenty of entropy — never the all-zero vector
    // `from_slice_normalizing` would refuse.
    let top24 = (state >> 40) as u32;
    (top24 as f32 / (1u32 << 24) as f32) * 2.0 - 1.0
  })
}

/// Aggregating exactly one window is the BIT-EXACT identity of its input —
/// pinning the TRUE reason `embed_long_with`'s short-circuit skips
/// `windit::aggregate::aggregate` for a single window (an optimization, not a
/// behavior change; a previous revision of that comment claimed an
/// f64-rounding drift that measurement never found). Swept over four coverage
/// values — `len` 1, 2, 511, and 512 out of `MAX_TOKENS` (near-zero, small,
/// near-full, and exactly full) — because `WindowEmbedding::token_span`'s
/// `Span::window()` is always `MAX_TOKENS`, so every one of these is a real
/// coverage `embed_windows` can plan, not a hand-picked one.
#[test]
fn single_window_is_the_bit_exact_identity() {
  for (i, &len) in [1usize, 2, 511, 512].iter().enumerate() {
    let row = lcg_row(0x9E37_79B9_7F4A_7C15 ^ (i as u64 + 1));
    let embedding = Embedding::from_slice_normalizing(&row).expect("normalize a random row");
    let windowed = windit::windowed::Windowed::new(
      embedding.clone(),
      windit::plan::Span::new(0, len, MAX_TOKENS),
    );
    let agg = windit::aggregate::aggregate(
      &windit::aggregate::CoverageWeightedMean,
      std::slice::from_ref(&windowed),
    )
    .expect("aggregate a single window");
    assert_eq!(
      agg.as_slice(),
      embedding.as_slice(),
      "len={len} (coverage {len}/{MAX_TOKENS}): single-window aggregation must be bit-exact"
    );
  }
}