use super::*;
#[test]
fn sensevoice_marker_after_twenty_thousand_tokens_is_detected() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("tokens.txt");
std::fs::write(&path, format!("{}<|zh|> 24884\n", "word 1\n".repeat(24884))).unwrap();
assert!(is_sense_voice_tokens(&path).unwrap());
}
#[cfg(feature = "vad-silero")]
#[test]
fn family_precheck_is_one_way_evidence() {
use crate::OfflineFamily;
let dir = tempfile::tempdir().unwrap();
let marked = dir.path().join("marked.txt");
std::fs::write(&marked, "foo 1\n<|zh|> 2\n<|itn|> 3\n").unwrap();
let markerless = dir.path().join("markerless.txt");
std::fs::write(&markerless, "IQ== 1\nJg== 2\nPGJsaz4= 60514\n").unwrap();
ensure_family_not_contradicted(OfflineFamily::SenseVoice, &marked).unwrap();
ensure_family_not_contradicted(OfflineFamily::SenseVoice, &markerless).unwrap();
ensure_family_not_contradicted(OfflineFamily::Paraformer, &markerless).unwrap();
let err = ensure_family_not_contradicted(OfflineFamily::Paraformer, &marked).unwrap_err();
assert!(err.contains("SenseVoice language markers"), "{err}");
let invalid_utf8 = dir.path().join("invalid.txt");
std::fs::write(&invalid_utf8, [0xFF, 0xFE]).unwrap();
assert!(ensure_family_not_contradicted(OfflineFamily::SenseVoice, &invalid_utf8).is_err());
assert!(ensure_family_not_contradicted(OfflineFamily::Paraformer, &invalid_utf8).is_err());
}
#[test]
fn directories_and_empty_models_are_rejected() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("tokens.txt"), "token 1").unwrap();
std::fs::create_dir(dir.path().join("model.onnx")).unwrap();
assert!(find_offline_model_files(dir.path()).is_err());
std::fs::write(dir.path().join("model.int8.onnx"), []).unwrap();
assert!(find_offline_model_files(dir.path()).is_err());
}
#[test]
fn equally_preferred_candidates_are_rejected_with_safe_names() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("encoder-a.int8.onnx"), [0u8; 8]).unwrap();
std::fs::write(dir.path().join("encoder-b.int8.onnx"), [0u8; 8]).unwrap();
std::fs::write(dir.path().join("decoder.onnx"), [0u8; 8]).unwrap();
std::fs::write(dir.path().join("joiner.int8.onnx"), [0u8; 8]).unwrap();
std::fs::write(dir.path().join("tokens.txt"), "token 1\n").unwrap();
let error = find_model_files(dir.path()).unwrap_err();
assert!(error.contains("encoder-a.int8.onnx"), "{error}");
assert!(error.contains("encoder-b.int8.onnx"), "{error}");
assert!(error.contains("keep exactly one"), "{error}");
assert!(
!error.contains(&dir.path().display().to_string()),
"{error}"
);
}
#[cfg(feature = "punct-sherpa")]
#[test]
fn punct_model_family_and_int8_preference_are_detected() {
let ct = tempfile::tempdir().unwrap();
std::fs::write(ct.path().join("model.onnx"), [0u8; 8]).unwrap();
std::fs::write(ct.path().join("model.int8.onnx"), [0u8; 8]).unwrap();
std::fs::write(ct.path().join("tokens.json"), "{}").unwrap();
let files = find_punct_model_files(ct.path()).unwrap();
assert!(files.model.ends_with("model.int8.onnx"));
assert!(files.vocab.is_none());
let en = tempfile::tempdir().unwrap();
std::fs::write(en.path().join("model.int8.onnx"), [0u8; 8]).unwrap();
std::fs::write(en.path().join("bpe.vocab"), "a 0\n").unwrap();
let files = find_punct_model_files(en.path()).unwrap();
assert!(files.vocab.is_some());
let wrapped = tempfile::tempdir().unwrap();
let child = wrapped.path().join("sherpa-onnx-punct-x");
std::fs::create_dir(&child).unwrap();
std::fs::write(child.join("model.int8.onnx"), [0u8; 8]).unwrap();
let files = find_punct_model_files(wrapped.path()).unwrap();
assert!(files.model.starts_with(&child));
let empty_vocab = tempfile::tempdir().unwrap();
std::fs::write(empty_vocab.path().join("model.int8.onnx"), [0u8; 8]).unwrap();
std::fs::write(empty_vocab.path().join("bpe.vocab"), []).unwrap();
let error = find_punct_model_files(empty_vocab.path()).unwrap_err();
assert!(error.contains("bpe.vocab is empty"), "{error}");
assert!(find_punct_model_files(tempfile::tempdir().unwrap().path()).is_err());
}
#[cfg(feature = "backend-sherpa")]
#[test]
fn hotword_modeling_unit_is_detected_from_bpe_vocab_and_tokens() {
let files = |dir: &Path, bpe_vocab: Option<PathBuf>| ModelFiles {
encoder: dir.join("encoder.onnx"),
decoder: dir.join("decoder.onnx"),
joiner: dir.join("joiner.onnx"),
tokens: dir.join("tokens.txt"),
bpe_vocab,
};
let bilingual = tempfile::tempdir().unwrap();
std::fs::write(bilingual.path().join("tokens.txt"), "中 1\n▁hello 2\n").unwrap();
std::fs::write(bilingual.path().join("bpe.vocab"), "▁hello 0\n").unwrap();
let (unit, vocab) = detect_hotword_modeling_unit(&files(
bilingual.path(),
find_bpe_vocab(bilingual.path()).unwrap(),
));
assert_eq!(unit, "cjkchar+bpe");
assert_eq!(
vocab.as_deref(),
Some(bilingual.path().join("bpe.vocab").as_path())
);
let english = tempfile::tempdir().unwrap();
std::fs::write(english.path().join("tokens.txt"), "▁hello 1\n").unwrap();
std::fs::write(english.path().join("bpe.vocab"), "▁hello 0\n").unwrap();
let (unit, _) = detect_hotword_modeling_unit(&files(
english.path(),
find_bpe_vocab(english.path()).unwrap(),
));
assert_eq!(unit, "bpe");
let chinese = tempfile::tempdir().unwrap();
std::fs::write(chinese.path().join("tokens.txt"), "中 1\n").unwrap();
let (unit, vocab) = detect_hotword_modeling_unit(&files(
chinese.path(),
find_bpe_vocab(chinese.path()).unwrap(),
));
assert_eq!(unit, "cjkchar");
assert_eq!(vocab, None);
let empty = tempfile::tempdir().unwrap();
std::fs::write(empty.path().join("tokens.txt"), "中 1\n").unwrap();
std::fs::write(empty.path().join("bpe.vocab"), []).unwrap();
assert!(find_bpe_vocab(empty.path()).is_err());
assert!(is_cjk_char('\u{20000}')); assert!(is_cjk_char('\u{2A700}')); assert!(is_cjk_char('\u{2CEB0}')); assert!(is_cjk_char('\u{31350}')); assert!(is_cjk_char('\u{F900}')); assert!(is_cjk_char('\u{2F800}')); assert!(!is_cjk_char('a'));
assert!(!is_cjk_char('。')); let rare = tempfile::tempdir().unwrap();
std::fs::write(rare.path().join("tokens.txt"), "𠀀 1\n").unwrap();
std::fs::write(rare.path().join("bpe.vocab"), "▁hello 0\n").unwrap();
let (unit, _) =
detect_hotword_modeling_unit(&files(rare.path(), find_bpe_vocab(rare.path()).unwrap()));
assert_eq!(unit, "cjkchar+bpe");
}
#[test]
fn qwen3_layout_needs_conv_frontend_encoder_decoder_and_tokenizer() {
let dir = tempfile::tempdir().unwrap();
assert!(find_qwen3_files(dir.path()).is_err());
std::fs::write(dir.path().join("conv_frontend.onnx"), [0u8; 8]).unwrap();
std::fs::write(dir.path().join("encoder.int8.onnx"), [0u8; 8]).unwrap();
std::fs::write(dir.path().join("decoder.int8.onnx"), [0u8; 8]).unwrap();
let tokenizer = dir.path().join("tokenizer");
std::fs::create_dir(&tokenizer).unwrap();
assert!(find_qwen3_files(dir.path()).is_err());
std::fs::write(tokenizer.join("merges.txt"), "#version").unwrap();
assert!(find_qwen3_files(dir.path()).is_err());
std::fs::write(tokenizer.join("vocab.json"), "{}").unwrap();
let files = find_qwen3_files(dir.path()).unwrap();
assert!(files.conv_frontend.ends_with("conv_frontend.onnx"));
assert!(files.encoder.ends_with("encoder.int8.onnx"));
assert!(files.decoder.ends_with("decoder.int8.onnx"));
assert_eq!(files.tokenizer, tokenizer);
let wrapped = tempfile::tempdir().unwrap();
let child = wrapped
.path()
.join("sherpa-onnx-qwen3-asr-0.6B-int8-2026-03-25");
std::fs::create_dir(&child).unwrap();
std::fs::write(child.join("conv_frontend.onnx"), [0u8; 8]).unwrap();
std::fs::write(child.join("encoder.int8.onnx"), [0u8; 8]).unwrap();
std::fs::write(child.join("decoder.int8.onnx"), [0u8; 8]).unwrap();
std::fs::create_dir(child.join("tokenizer")).unwrap();
std::fs::write(child.join("tokenizer/merges.txt"), "#version").unwrap();
std::fs::write(child.join("tokenizer/vocab.json"), "{}").unwrap();
assert!(find_qwen3_files(wrapped.path()).is_ok());
}
#[test]
fn funasr_nano_layout_needs_all_models_and_tokenizer_trio() {
let dir = tempfile::tempdir().unwrap();
assert!(find_funasr_nano_files(dir.path()).is_err());
std::fs::write(dir.path().join("encoder_adaptor.int8.onnx"), [0u8; 8]).unwrap();
std::fs::write(dir.path().join("llm.int8.onnx"), [0u8; 8]).unwrap();
std::fs::write(dir.path().join("embedding.onnx"), [0u8; 8]).unwrap();
std::fs::write(dir.path().join("embedding.int8.onnx"), [0u8; 8]).unwrap();
let tokenizer = dir.path().join("Qwen3-0.6B");
std::fs::create_dir(&tokenizer).unwrap();
assert!(find_funasr_nano_files(dir.path()).is_err());
std::fs::write(tokenizer.join("vocab.json"), "{}").unwrap();
assert!(find_funasr_nano_files(dir.path()).is_err());
std::fs::write(tokenizer.join("merges.txt"), "#version").unwrap();
assert!(find_funasr_nano_files(dir.path()).is_err());
std::fs::write(tokenizer.join("tokenizer.json"), []).unwrap();
assert!(find_funasr_nano_files(dir.path()).is_err());
std::fs::write(tokenizer.join("tokenizer.json"), "{}").unwrap();
let files = find_funasr_nano_files(dir.path()).unwrap();
assert!(files.encoder_adaptor.ends_with("encoder_adaptor.int8.onnx"));
assert!(files.llm.ends_with("llm.int8.onnx"));
assert!(files.embedding.ends_with("embedding.int8.onnx"));
assert_eq!(files.tokenizer, tokenizer);
let second = dir.path().join("another-tokenizer");
std::fs::create_dir(&second).unwrap();
for name in ["vocab.json", "merges.txt", "tokenizer.json"] {
std::fs::write(second.join(name), [0u8; 8]).unwrap();
}
assert!(find_funasr_nano_files(dir.path()).is_err());
}
#[test]
fn funasr_nano_archive_wrapped_in_single_child_is_detected() {
let wrapped = tempfile::tempdir().unwrap();
let child = wrapped
.path()
.join("sherpa-onnx-funasr-nano-int8-2025-12-30");
std::fs::create_dir(&child).unwrap();
for name in [
"encoder_adaptor.int8.onnx",
"llm.int8.onnx",
"embedding.int8.onnx",
] {
std::fs::write(child.join(name), [0u8; 8]).unwrap();
}
std::fs::create_dir(child.join("test_wavs")).unwrap();
let tokenizer = child.join("Qwen3-0.6B");
std::fs::create_dir(&tokenizer).unwrap();
for name in ["vocab.json", "merges.txt", "tokenizer.json"] {
std::fs::write(tokenizer.join(name), [0u8; 8]).unwrap();
}
let files = find_funasr_nano_files(wrapped.path()).unwrap();
assert!(files.encoder_adaptor.starts_with(&child));
assert!(files.tokenizer.starts_with(&child));
}
#[test]
fn fire_red_aed_layout_needs_encoder_decoder_and_tokens() {
let dir = tempfile::tempdir().unwrap();
assert!(find_fire_red_aed_files(dir.path()).is_err());
std::fs::write(dir.path().join("encoder.int8.onnx"), [0u8; 8]).unwrap();
assert!(find_fire_red_aed_files(dir.path()).is_err());
std::fs::write(dir.path().join("decoder.onnx"), [0u8; 8]).unwrap();
std::fs::write(dir.path().join("decoder.int8.onnx"), [0u8; 8]).unwrap();
assert!(find_fire_red_aed_files(dir.path()).is_err());
std::fs::write(dir.path().join("tokens.txt"), "a 1\n").unwrap();
let files = find_fire_red_aed_files(dir.path()).unwrap();
assert!(files.encoder.ends_with("encoder.int8.onnx"));
assert!(files.decoder.ends_with("decoder.int8.onnx"));
assert!(files.tokens.ends_with("tokens.txt"));
let wrapped = tempfile::tempdir().unwrap();
let child = wrapped
.path()
.join("sherpa-onnx-fire-red-asr2-zh_en-int8-2026-02-26");
std::fs::create_dir(&child).unwrap();
for name in ["encoder.int8.onnx", "decoder.int8.onnx", "tokens.txt"] {
std::fs::write(child.join(name), [0u8; 8]).unwrap();
}
std::fs::create_dir(child.join("test_wavs")).unwrap();
let files = find_fire_red_aed_files(wrapped.path()).unwrap();
assert!(files.encoder.starts_with(&child));
assert!(files.decoder.starts_with(&child));
assert!(files.tokens.starts_with(&child));
}
#[test]
fn fire_red_aed_marker_requires_absent_joiner() {
let dir = tempfile::tempdir().unwrap();
for name in [
"encoder.int8.onnx",
"decoder.int8.onnx",
"joiner.int8.onnx",
"tokens.txt",
] {
std::fs::write(dir.path().join(name), [0u8; 8]).unwrap();
}
assert!(!has_fire_red_aed_layout_marker(dir.path()));
assert!(find_model_files(dir.path()).is_ok());
std::fs::remove_file(dir.path().join("joiner.int8.onnx")).unwrap();
assert!(has_fire_red_aed_layout_marker(dir.path()));
assert!(find_model_files(dir.path()).is_err());
assert!(find_fire_red_aed_files(dir.path()).is_ok());
std::fs::write(dir.path().join("joiner.onnx"), [0u8; 8]).unwrap();
assert!(!has_fire_red_aed_layout_marker(dir.path()));
std::fs::remove_file(dir.path().join("joiner.onnx")).unwrap();
}
#[test]
fn fire_red_aed_rejects_transducer_layout_before_native_init() {
for names in [
[
"encoder.int8.onnx",
"decoder.int8.onnx",
"joiner.int8.onnx",
"tokens.txt",
],
[
"encoder-epoch-99-avg-1.onnx",
"decoder-epoch-99-avg-1.onnx",
"joiner-epoch-99-avg-1.onnx",
"tokens.txt",
],
[
"encoder.onnx",
"decoder.onnx",
"joiner.fp16.onnx",
"tokens.txt",
],
] {
let dir = tempfile::tempdir().unwrap();
for name in names {
std::fs::write(dir.path().join(name), [0u8; 8]).unwrap();
}
assert!(!has_fire_red_aed_layout_marker(dir.path()));
let error = find_fire_red_aed_files(dir.path()).unwrap_err();
assert!(
error.contains("joiner") && error.contains("transducer layout"),
"{error}"
);
}
}