mlxrs 0.1.0

//! Direct coverage for `tokenizer::wrapper::Tokenizer` (the HF-`tokenizers`
//! binding) and the public `no_bos_or_eos` helper (issue #260).
//!
//! These complement `tokenizer.rs` / `tokenizer_core.rs` /
//! `tokenizer_encode_options.rs` by pinning the parts of the wrapper's
//! public contract those files do not exercise: the `no_bos_or_eos`
//! free function (previously ZERO coverage), `decode`'s
//! `skip_special_tokens` behavior, `decode_batch`, multi-item
//! `encode_batch`, `convert_token_to_id`/`convert_id_to_token` miss paths,
//! `add_eos_token` (string / numeric / unknown-token error),
//! `contains_eos_id` + `eos_token_ids_iter` ordering, BOS/EOS round-trips,
//! and `from_path` error paths (missing / malformed `tokenizer.json`).
//!
//! Gated on `tokenizer-config` (mirroring `tokenizer_encode_options.rs`):
//! the config-derived `bos_token`/`eos_token` accessors are needed to
//! assert BOS/EOS behavior, and that feature pulls in the full core
//! encode/decode/batch surface. Local fixtures only — no HF Hub download
//! (`from_path` reads `tokenizer.json` + `tokenizer_config.json` straight
//! off disk, matching the project's local-path-only constraint).
#![cfg(feature = "tokenizer-config")]

use std::io::Write;

use mlxrs::{
  Error,
  tokenizer::{Tokenizer, no_bos_or_eos},
};

/// The committed, schema-correct fixtures shared by every tokenizer test
/// (regenerated by `cargo xtask-codegen`, see `tokenizer.rs`). The
/// `tokenizer.json` is a WordLevel model (`<unk>`=0, `<s>`=1, `</s>`=2,
/// then `hello`=3 .. `fox`=8, `<think>`=9, `</think>`=10); the matching
/// `tokenizer_config.json` declares `bos_token`=`<s>`, `eos_token`=`</s>`,
/// `unk_token`=`<unk>`.
const TOKENIZER_JSON: &str = include_str!("fixtures/tokenizer.json");
const TOKENIZER_CONFIG_JSON: &str = include_str!("fixtures/tokenizer_config.json");

/// A second `tokenizer.json` fixture identical to `TOKENIZER_JSON` (same
/// WordLevel vocab + special `added_tokens`) EXCEPT it carries a
/// `TemplateProcessing` post_processor that wraps single sequences as
/// `<s> $A </s>` (and pairs as `<s> $A </s> $B </s>`). With this
/// post_processor, HF `encode(text, true)` injects BOS=1 / EOS=2 while
/// `encode(text, false)` does not — making the `add_special_tokens` flag's
/// effect OBSERVABLE (the shared `TOKENIZER_JSON` has `post_processor:
/// null`, so true vs false are identical there and cannot prove the flag is
/// wired). Paired with the same `TOKENIZER_CONFIG_JSON`.
const TOKENIZER_TEMPLATE_JSON: &str = include_str!("fixtures/tokenizer_template.json");

/// Write-once fixture dir (same pattern as the sibling tokenizer tests):
/// `cargo test` runs this binary's tests in parallel, but a constant
/// payload + `OnceLock` removes the rewrite race without serializing.
fn fixture_dir() -> std::path::PathBuf {
  static FIXTURE: std::sync::OnceLock<std::path::PathBuf> = std::sync::OnceLock::new();
  FIXTURE
    .get_or_init(|| {
      let dir =
        std::env::temp_dir().join(format!("mlxrs-tok-wrapper-fixture-{}", std::process::id()));
      std::fs::create_dir_all(&dir).unwrap();
      let mut f = std::fs::File::create(dir.join("tokenizer.json")).unwrap();
      f.write_all(TOKENIZER_JSON.as_bytes()).unwrap();
      let mut c = std::fs::File::create(dir.join("tokenizer_config.json")).unwrap();
      c.write_all(TOKENIZER_CONFIG_JSON.as_bytes()).unwrap();
      dir
    })
    .clone()
}

/// Write-once fixture dir for the `TemplateProcessing` variant (same
/// write-once `OnceLock` pattern as [`fixture_dir`], distinct path so it
/// never collides with the post-processor-less fixture). Used only by the
/// `add_special_tokens`-observability tests.
fn template_fixture_dir() -> std::path::PathBuf {
  static FIXTURE: std::sync::OnceLock<std::path::PathBuf> = std::sync::OnceLock::new();
  FIXTURE
    .get_or_init(|| {
      let dir = std::env::temp_dir().join(format!(
        "mlxrs-tok-wrapper-tmpl-fixture-{}",
        std::process::id()
      ));
      std::fs::create_dir_all(&dir).unwrap();
      let mut f = std::fs::File::create(dir.join("tokenizer.json")).unwrap();
      f.write_all(TOKENIZER_TEMPLATE_JSON.as_bytes()).unwrap();
      let mut c = std::fs::File::create(dir.join("tokenizer_config.json")).unwrap();
      c.write_all(TOKENIZER_CONFIG_JSON.as_bytes()).unwrap();
      dir
    })
    .clone()
}

/// Extract the `Err` from a `from_path` result without requiring
/// `Tokenizer: Debug` (the success type is not `Debug`, so the std
/// `Result::unwrap_err` bound does not hold). On the unexpected `Ok` arm
/// the test fails with a clear message instead of an opaque trait error.
fn expect_load_err(res: Result<Tokenizer, Error>) -> Error {
  match res {
    Ok(_) => panic!("expected Tokenizer::from_path to error, but it loaded"),
    Err(e) => e,
  }
}

/// A fresh, isolated dir seeded with arbitrary file contents. Each call
/// gets a unique path (pid + nanos) so error-path tests that plant
/// malformed fixtures never collide with each other or the shared dir.
fn unique_dir(tag: &str) -> std::path::PathBuf {
  let dir = std::env::temp_dir().join(format!(
    "mlxrs-tok-wrapper-{tag}-{}-{}",
    std::process::id(),
    std::time::SystemTime::now()
      .duration_since(std::time::UNIX_EPOCH)
      .map(|d| d.as_nanos())
      .unwrap_or(0)
  ));
  std::fs::create_dir_all(&dir).unwrap();
  dir
}

// ---------------------------------------------------------------------------
// no_bos_or_eos — the public free function (Python `no_bos_or_eos`).
// Previously ZERO coverage. Strips a leading `bos` and a trailing `eos`,
// each at most once and only when actually present.
// ---------------------------------------------------------------------------

#[test]
fn no_bos_or_eos_strips_both_ends() {
  // `[bos, a, b, c, eos]` -> `[a, b, c]`.
  let seq = [1u32, 3, 4, 5, 2];
  assert_eq!(no_bos_or_eos(&seq, 1, 2), vec![3, 4, 5]);
}

#[test]
fn no_bos_or_eos_strips_only_leading_bos() {
  // Trailing token is not `eos`, so only the leading `bos` is removed.
  let seq = [1u32, 3, 4, 5];
  assert_eq!(no_bos_or_eos(&seq, 1, 2), vec![3, 4, 5]);
}

#[test]
fn no_bos_or_eos_strips_only_trailing_eos() {
  // Leading token is not `bos`, so only the trailing `eos` is removed.
  let seq = [3u32, 4, 5, 2];
  assert_eq!(no_bos_or_eos(&seq, 1, 2), vec![3, 4, 5]);
}

#[test]
fn no_bos_or_eos_no_op_when_neither_present() {
  let seq = [3u32, 4, 5];
  assert_eq!(no_bos_or_eos(&seq, 1, 2), vec![3, 4, 5]);
}

#[test]
fn no_bos_or_eos_strips_each_at_most_once() {
  // Two leading bos / two trailing eos: only the OUTERMOST one is removed
  // at each end (one slice from the front, one `pop` from the back).
  let seq = [1u32, 1, 3, 2, 2];
  assert_eq!(no_bos_or_eos(&seq, 1, 2), vec![1, 3, 2]);
}

#[test]
fn no_bos_or_eos_single_bos_only_element() {
  // A lone `bos` is stripped from the front, leaving nothing; the empty
  // tail's `last()` is `None`, so the eos branch is a no-op (no
  // underflow). Pins the boundary where front-strip empties the slice.
  let seq = [1u32];
  assert_eq!(no_bos_or_eos(&seq, 1, 2), Vec::<u32>::new());
}

#[test]
fn no_bos_or_eos_single_eos_only_element() {
  let seq = [2u32];
  assert_eq!(no_bos_or_eos(&seq, 2, 2), Vec::<u32>::new());
}

#[test]
fn no_bos_or_eos_empty_input() {
  let seq: [u32; 0] = [];
  assert_eq!(no_bos_or_eos(&seq, 1, 2), Vec::<u32>::new());
}

#[test]
fn no_bos_or_eos_same_bos_eos_single_element_strips_front_only() {
  // When `bos == eos` and the sequence is a single matching id, the front
  // strip consumes it; the now-empty tail's `last()` is `None`, so the
  // eos `pop` does not run. (Documents the order: front-strip first, then
  // the trailing check operates on the already-sliced tail.)
  let seq = [7u32];
  assert_eq!(no_bos_or_eos(&seq, 7, 7), Vec::<u32>::new());
}

#[test]
fn no_bos_or_eos_round_trips_with_real_tokenizer_bos_eos() {
  // Integration with the wrapper: build `[bos, <encoded>, eos]` from the
  // tokenizer's own configured bos/eos ids and assert the helper recovers
  // exactly the encoded body.
  let tok = Tokenizer::from_path(fixture_dir(), None).unwrap();
  let bos = tok.bos_token_id().expect("fixture has a bos token");
  let eos = tok.eos_token_id().expect("fixture has an eos token");
  assert_eq!((bos, eos), (1, 2));

  let body = tok.encode("hello world the quick", false).unwrap();
  assert_eq!(body, vec![3, 4, 5, 6]);

  let mut framed = vec![bos];
  framed.extend_from_slice(&body);
  framed.push(eos);

  assert_eq!(no_bos_or_eos(&framed, bos, eos), body);
}

// ---------------------------------------------------------------------------
// encode / decode round-trip + special-token handling.
// ---------------------------------------------------------------------------

#[test]
fn encode_add_special_true_injects_bos_eos_false_omits() {
  // OBSERVABLE proof that `add_special_tokens` is wired through to the HF
  // tokenizer: against the `TemplateProcessing` fixture (single = `<s> $A
  // </s>`), `encode(text, true)` must FRAME the body with BOS=1 / EOS=2,
  // while `encode(text, false)` must return the bare body. A wrapper that
  // ignored the flag (always passed `false`, or dropped it) would fail the
  // `true` assertions below. Distinct from
  // `encode_add_special_noop_without_post_processor`, which pins that the
  // post-processor-less shared fixture legitimately leaves both identical.
  let tok = Tokenizer::from_path(template_fixture_dir(), None).unwrap();

  let without = tok.encode("hello world", false).unwrap();
  assert_eq!(without, vec![3, 4], "bare body, no specials");

  let with_special = tok.encode("hello world", true).unwrap();
  // Exact framing: [BOS, hello, world, EOS].
  assert_eq!(with_special, vec![1, 3, 4, 2]);
  // Spelled-out contract the exact-eq above subsumes, so a future template
  // tweak still documents intent: starts with BOS, ends with EOS, strictly
  // longer than the bare body, and the bare body never carries them.
  assert_eq!(with_special.first(), Some(&1), "starts with BOS");
  assert_eq!(with_special.last(), Some(&2), "ends with EOS");
  assert!(with_special.len() > without.len());
  assert!(!without.contains(&1) && !without.contains(&2));
}

#[test]
fn encode_batch_add_special_true_injects_bos_eos_false_omits() {
  // Batch variant of the observable add_special check: every item is framed
  // with BOS/EOS under `true` and bare under `false`. Pins that the flag is
  // forwarded by `encode_batch` (not just the scalar `encode`).
  let tok = Tokenizer::from_path(template_fixture_dir(), None).unwrap();
  let texts = || vec!["hello world".to_string(), "fox".to_string()];

  let without = tok.encode_batch(texts(), false).unwrap();
  assert_eq!(without, vec![vec![3u32, 4], vec![8]]);

  let with_special = tok.encode_batch(texts(), true).unwrap();
  assert_eq!(with_special, vec![vec![1u32, 3, 4, 2], vec![1, 8, 2]]);
  for (framed, bare) in with_special.iter().zip(without.iter()) {
    assert_eq!(framed.first(), Some(&1), "item starts with BOS");
    assert_eq!(framed.last(), Some(&2), "item ends with EOS");
    assert!(framed.len() > bare.len());
  }
}

#[test]
fn encode_add_special_noop_without_post_processor() {
  // The SHARED `tokenizer.json` fixture has a null post_processor, so even
  // with `add_special_tokens = true` no BOS/EOS is injected — the flag is a
  // legitimate no-op here. This is NOT proof the flag is wired (see
  // `encode_add_special_true_injects_bos_eos_false_omits` for that); it
  // pins the post-processor-less fixture's behavior and documents the
  // contrast that motivated the separate TemplateProcessing fixture.
  let tok = Tokenizer::from_path(fixture_dir(), None).unwrap();
  let with_special = tok.encode("hello world", true).unwrap();
  let without = tok.encode("hello world", false).unwrap();
  assert_eq!(without, vec![3, 4]);
  assert_eq!(
    with_special, without,
    "null post_processor: flag is a no-op"
  );
  assert!(!with_special.contains(&1) && !with_special.contains(&2));
}

#[test]
fn decode_skip_special_tokens_drops_specials() {
  // `[<s>, hello, world, </s>]`: with `skip_special_tokens = true` the
  // `<s>`/`</s>` markers are dropped from the rendered text; with `false`
  // they are emitted verbatim. The fixture's `<unk>`/`<s>`/`</s>` are
  // declared `special: true` in `added_tokens`, so HF treats them as
  // special.
  let tok = Tokenizer::from_path(fixture_dir(), None).unwrap();
  let ids = [1u32, 3, 4, 2];

  let kept = tok.decode(&ids, false).unwrap();
  assert!(kept.contains("<s>"), "kept='{kept}'");
  assert!(kept.contains("</s>"), "kept='{kept}'");
  assert!(kept.contains("hello") && kept.contains("world"));

  let skipped = tok.decode(&ids, true).unwrap();
  assert!(!skipped.contains("<s>"), "skipped='{skipped}'");
  assert!(!skipped.contains("</s>"), "skipped='{skipped}'");
  assert!(skipped.contains("hello") && skipped.contains("world"));
}

#[test]
fn encode_decode_round_trip_via_skip_special() {
  // Encode plain text, decode with specials skipped, and re-encode: the
  // ids are recovered (no special tokens to perturb the round-trip).
  let tok = Tokenizer::from_path(fixture_dir(), None).unwrap();
  let ids = tok.encode("the quick brown fox", false).unwrap();
  assert_eq!(ids, vec![5, 6, 7, 8]);
  let text = tok.decode(&ids, true).unwrap();
  assert_eq!(tok.encode(&text, false).unwrap(), ids);
}

#[test]
fn encode_unknown_word_maps_to_unk_id() {
  // A word absent from the WordLevel vocab falls back to `<unk>` (id 0).
  let tok = Tokenizer::from_path(fixture_dir(), None).unwrap();
  let ids = tok.encode("hello zzzznotaword world", false).unwrap();
  assert_eq!(ids, vec![3, 0, 4]);
}

// ---------------------------------------------------------------------------
// convert_token_to_id / convert_id_to_token — hit AND miss paths.
// ---------------------------------------------------------------------------

#[test]
fn convert_token_id_round_trip_and_misses() {
  let tok = Tokenizer::from_path(fixture_dir(), None).unwrap();
  // Known vocab entry.
  assert_eq!(tok.convert_token_to_id("quick"), Some(6));
  assert_eq!(tok.convert_id_to_token(6).as_deref(), Some("quick"));
  // Special token resolves too.
  assert_eq!(tok.convert_token_to_id("</s>"), Some(2));
  // Unknown token string -> None (NOT silently mapped to <unk> id).
  assert_eq!(tok.convert_token_to_id("zzzznotaword"), None);
  // Out-of-range id -> None.
  assert_eq!(tok.convert_id_to_token(9_999), None);
}

// ---------------------------------------------------------------------------
// batch encode / decode.
// ---------------------------------------------------------------------------

#[test]
fn encode_batch_multi_item_exact_ids() {
  // Multi-token items, in order; ragged lengths preserved.
  let tok = Tokenizer::from_path(fixture_dir(), None).unwrap();
  let batch = tok
    .encode_batch(
      vec![
        "hello world".into(),
        "the quick brown fox".into(),
        "fox".into(),
      ],
      false,
    )
    .unwrap();
  assert_eq!(batch, vec![vec![3u32, 4], vec![5, 6, 7, 8], vec![8]]);
}

#[test]
fn encode_batch_empty_input_is_empty_output() {
  let tok = Tokenizer::from_path(fixture_dir(), None).unwrap();
  let batch = tok.encode_batch(Vec::new(), false).unwrap();
  assert!(batch.is_empty());
}

#[test]
fn decode_batch_matches_per_item_decode() {
  // `decode_batch` over multiple id sequences equals decoding each
  // sequence independently, in order.
  let tok = Tokenizer::from_path(fixture_dir(), None).unwrap();
  let a: &[u32] = &[3, 4];
  let b: &[u32] = &[5, 6, 7, 8];
  let seqs: &[&[u32]] = &[a, b];

  let out = tok.decode_batch(seqs, true).unwrap();
  assert_eq!(out.len(), 2);
  assert_eq!(out[0], tok.decode(a, true).unwrap());
  assert_eq!(out[1], tok.decode(b, true).unwrap());
  assert!(out[0].contains("hello") && out[0].contains("world"));
  assert!(out[1].contains("fox"));
}

#[test]
fn decode_batch_skip_special_tokens_drops_specials() {
  let tok = Tokenizer::from_path(fixture_dir(), None).unwrap();
  let a: &[u32] = &[1, 3, 2]; // <s> hello </s>
  let seqs: &[&[u32]] = &[a];

  let skipped = tok.decode_batch(seqs, true).unwrap();
  assert!(!skipped[0].contains("<s>"), "got '{}'", skipped[0]);
  assert!(skipped[0].contains("hello"));

  let kept = tok.decode_batch(seqs, false).unwrap();
  assert!(kept[0].contains("<s>"), "got '{}'", kept[0]);
}

// ---------------------------------------------------------------------------
// eos-set accessors + add_eos_token (Python `add_eos_token`).
// ---------------------------------------------------------------------------

#[test]
fn eos_set_from_config_contains_primary() {
  // With no caller-supplied set, the eos set falls back to the
  // tokenizer-config `eos_token` ("</s>" -> id 2).
  let tok = Tokenizer::from_path(fixture_dir(), None).unwrap();
  assert!(tok.contains_eos_id(2));
  assert!(!tok.contains_eos_id(3));
  let eos_ids: Vec<u32> = tok.eos_token_ids_iter().collect();
  assert_eq!(eos_ids, vec![2]);
}

#[test]
fn eos_token_ids_iter_is_sorted_set() {
  // Caller-supplied multi-id set is exposed as a SORTED set (BTreeSet),
  // de-duplicated, regardless of input order.
  let tok = Tokenizer::from_path(fixture_dir(), Some(&[8, 2, 8, 0])).unwrap();
  let eos_ids: Vec<u32> = tok.eos_token_ids_iter().collect();
  assert_eq!(eos_ids, vec![0, 2, 8]);
  assert!(tok.contains_eos_id(0));
  assert!(tok.contains_eos_id(2));
  assert!(tok.contains_eos_id(8));
  assert!(!tok.contains_eos_id(5));
}

#[test]
fn add_eos_token_by_string_and_numeric_string() {
  let mut tok = Tokenizer::from_path(fixture_dir(), None).unwrap();
  // By vocab string: "fox" -> id 8.
  tok.add_eos_token("fox").unwrap();
  assert!(tok.contains_eos_id(8));
  // By numeric string: "5" parses directly to id 5 (no vocab lookup).
  tok.add_eos_token("5").unwrap();
  assert!(tok.contains_eos_id(5));
  // Original config eos (2) is retained alongside the additions.
  let eos_ids: Vec<u32> = tok.eos_token_ids_iter().collect();
  assert_eq!(eos_ids, vec![2, 5, 8]);
}

#[test]
fn add_eos_token_unknown_token_errors() {
  // A non-numeric string that is not in the vocab cannot resolve to an id.
  let mut tok = Tokenizer::from_path(fixture_dir(), None).unwrap();
  let err = tok.add_eos_token("definitely-not-a-token").unwrap_err();
  assert!(matches!(err, Error::Tokenizer(_)), "got {err:?}");
  let msg = format!("{err}");
  assert!(msg.contains("is not a token"), "{msg}");
}

#[test]
fn add_eos_token_establishes_primary_when_none_configured() {
  // No tokenizer_config.json => no config eos and no caller set, so the
  // eos set starts empty and `encode_with(add_eos=true)` would error.
  // The FIRST `add_eos_token` becomes the primary used by `encode_with`.
  use mlxrs::tokenizer::EncodeOptions;

  let dir = unique_dir("addeos-primary");
  std::fs::write(dir.join("tokenizer.json"), TOKENIZER_JSON).unwrap();
  let mut tok = Tokenizer::from_path(&dir, None).unwrap();
  assert!(tok.eos_token_ids_iter().next().is_none());

  // Before any eos is established, add_eos must error.
  let err = tok
    .encode_with("hello", &EncodeOptions::new().with_add_eos(true))
    .unwrap_err();
  assert!(format!("{err}").contains("eos"));

  // Establish "fox" (id 8) as the primary eos.
  tok.add_eos_token("fox").unwrap();
  let out = tok
    .encode_with(
      "hello world",
      &EncodeOptions::new()
        .with_add_special(false)
        .with_add_eos(true),
    )
    .unwrap();
  assert_eq!(out.ids(), &[3u32, 4, 8]);

  let _ = std::fs::remove_dir_all(&dir);
}

// ---------------------------------------------------------------------------
// config-derived special-token accessors (the BOS/EOS surface).
// ---------------------------------------------------------------------------

#[test]
fn config_special_token_accessors() {
  let tok = Tokenizer::from_path(fixture_dir(), None).unwrap();
  assert_eq!(tok.bos_token(), Some("<s>"));
  assert_eq!(tok.eos_token(), Some("</s>"));
  assert_eq!(tok.unk_token(), Some("<unk>"));
  // pad_token absent from the fixture config.
  assert_eq!(tok.pad_token(), None);

  assert_eq!(tok.bos_token_id(), Some(1));
  assert_eq!(tok.eos_token_id(), Some(2));
  assert_eq!(tok.unk_token_id(), Some(0));
  assert_eq!(tok.pad_token_id(), None);
}

#[test]
fn caller_supplied_eos_replaces_config_default() {
  // Python `TokenizerWrapper`: a supplied `eos_token_ids` REPLACES the
  // config default entirely (not unioned). Supplying `[8]` makes the eos
  // set exactly `{8}`, and the config `</s>` (id 2) is NOT in it. The
  // config-derived `eos_token()` string accessor is independent and still
  // reports "</s>".
  let tok = Tokenizer::from_path(fixture_dir(), Some(&[8])).unwrap();
  assert!(tok.contains_eos_id(8));
  assert!(
    !tok.contains_eos_id(2),
    "supplied set replaces, not unions, the config eos"
  );
  assert_eq!(tok.eos_token(), Some("</s>"));
}

#[test]
fn empty_eos_slice_suppresses_fallback() {
  // An explicitly EMPTY caller slice suppresses the config fallback,
  // leaving the eos set empty (documented behavior used to force
  // `encode_with(add_eos=true)` to error).
  let tok = Tokenizer::from_path(fixture_dir(), Some(&[])).unwrap();
  assert!(tok.eos_token_ids_iter().next().is_none());
  assert!(!tok.contains_eos_id(2));
}

// ---------------------------------------------------------------------------
// thinking inference (Python `_infer_thinking`) via the wrapper accessors.
// ---------------------------------------------------------------------------

#[test]
fn thinking_inferred_from_vocab() {
  // The fixture vocab has `<think>` (9) / `</think>` (10), so the wrapper
  // infers a thinking mode with those markers + token ids.
  let tok = Tokenizer::from_path(fixture_dir(), None).unwrap();
  assert!(tok.has_thinking());
  assert_eq!(tok.think_start(), Some("<think>"));
  assert_eq!(tok.think_end(), Some("</think>"));
  assert_eq!(tok.think_start_tokens(), Some(&[9u32][..]));
  assert_eq!(tok.think_end_tokens(), Some(&[10u32][..]));
}

// ---------------------------------------------------------------------------
// hf() accessor — exposes the underlying HF tokenizer.
// ---------------------------------------------------------------------------

#[test]
fn hf_accessor_exposes_underlying_tokenizer() {
  let tok = Tokenizer::from_path(fixture_dir(), None).unwrap();
  // The HF vocab size matches the fixture's 11 entries.
  assert_eq!(tok.hf().get_vocab_size(true), 11);
  assert_eq!(tok.hf().token_to_id("hello"), Some(3));
}

// ---------------------------------------------------------------------------
// from_path error paths — local-only, malformed / missing input.
// ---------------------------------------------------------------------------

#[test]
fn from_path_missing_tokenizer_json_errors() {
  // An empty dir (no tokenizer.json) must surface a typed Tokenizer error,
  // not panic.
  let dir = unique_dir("missing");
  let err = expect_load_err(Tokenizer::from_path(&dir, None));
  assert!(matches!(err, Error::Tokenizer(_)), "got {err:?}");
  let msg = format!("{err}");
  assert!(
    msg.contains("tokenizer.json"),
    "expected tokenizer.json in error, got: {msg}"
  );
  let _ = std::fs::remove_dir_all(&dir);
}

#[test]
fn from_path_malformed_tokenizer_json_errors() {
  // A syntactically invalid tokenizer.json must error (not panic). The HF
  // loader rejects it; `from_path` wraps the failure as a Tokenizer error.
  let dir = unique_dir("malformed-tok");
  std::fs::write(dir.join("tokenizer.json"), b"{ this is not valid json").unwrap();
  let err = expect_load_err(Tokenizer::from_path(&dir, None));
  assert!(matches!(err, Error::Tokenizer(_)), "got {err:?}");
  let _ = std::fs::remove_dir_all(&dir);
}

#[test]
fn from_path_malformed_tokenizer_config_errors() {
  // A VALID tokenizer.json but a malformed tokenizer_config.json must
  // surface a typed error from the config parse step (tokenizer-config
  // path), not panic.
  let dir = unique_dir("malformed-cfg");
  std::fs::write(dir.join("tokenizer.json"), TOKENIZER_JSON).unwrap();
  std::fs::write(dir.join("tokenizer_config.json"), b"{ not: valid").unwrap();
  let err = expect_load_err(Tokenizer::from_path(&dir, None));
  assert!(matches!(err, Error::Tokenizer(_)), "got {err:?}");
  let msg = format!("{err}");
  assert!(
    msg.contains("tokenizer_config.json"),
    "expected tokenizer_config.json in error, got: {msg}"
  );
  let _ = std::fs::remove_dir_all(&dir);
}

#[test]
fn from_path_absent_config_uses_empty_defaults() {
  // A valid tokenizer.json with NO tokenizer_config.json loads fine: the
  // config falls back to an empty object, so the config-derived accessors
  // report `None` (absent, not error). Core encode/decode still works.
  let dir = unique_dir("no-config");
  std::fs::write(dir.join("tokenizer.json"), TOKENIZER_JSON).unwrap();
  let tok = Tokenizer::from_path(&dir, None).unwrap();
  assert_eq!(tok.bos_token(), None);
  assert_eq!(tok.eos_token(), None);
  assert_eq!(tok.unk_token(), None);
  // Core encode still works without any config.
  assert_eq!(tok.encode("hello world", false).unwrap(), vec![3, 4]);
  let _ = std::fs::remove_dir_all(&dir);
}