use std::path::{Path, PathBuf};
#[derive(Debug, Clone)]
pub(crate) struct ModelFiles {
pub(crate) encoder: PathBuf,
pub(crate) decoder: PathBuf,
pub(crate) joiner: PathBuf,
pub(crate) tokens: PathBuf,
pub(crate) bpe_vocab: Option<PathBuf>,
}
pub(crate) fn find_model_files(dir: &Path) -> Result<ModelFiles, String> {
let dir = descend_into_single_child(dir);
let entries = list_dir_files(&dir)?;
let encoder = pick(&entries, "encoder", true)?;
let decoder = pick(&entries, "decoder", false)?;
let joiner = pick(&entries, "joiner", true)?;
let tokens = require_tokens(&dir)?;
Ok(ModelFiles {
encoder,
decoder,
joiner,
tokens,
bpe_vocab: find_bpe_vocab(&dir)?,
})
}
pub(crate) fn find_offline_model_files(dir: &Path) -> Result<ModelFiles, String> {
let dir = descend_into_single_child(dir);
let entries = list_dir_files(&dir)?;
let model = pick(&entries, "model", true)?;
let tokens = require_tokens(&dir)?;
Ok(ModelFiles {
encoder: model,
decoder: PathBuf::new(),
joiner: PathBuf::new(),
tokens,
bpe_vocab: find_bpe_vocab(&dir)?,
})
}
#[derive(Debug, Clone)]
#[cfg_attr(not(feature = "vad-silero"), allow(dead_code))]
pub(crate) struct Qwen3ModelFiles {
pub(crate) conv_frontend: PathBuf,
pub(crate) encoder: PathBuf,
pub(crate) decoder: PathBuf,
pub(crate) tokenizer: PathBuf,
}
pub(crate) fn has_conv_frontend(dir: &Path) -> bool {
dir.join("conv_frontend.onnx").is_file() || dir.join("conv_frontend.int8.onnx").is_file()
}
pub(crate) fn find_qwen3_files(dir: &Path) -> Result<Qwen3ModelFiles, String> {
let dir = descend_where(dir, has_conv_frontend);
let entries = list_dir_files(&dir)?;
let conv_frontend = pick(&entries, "conv_frontend", true)?;
let encoder = pick(&entries, "encoder", true)?;
let decoder = pick(&entries, "decoder", true)?;
let tokenizer = dir.join("tokenizer");
for required in ["merges.txt", "vocab.json"] {
let path = tokenizer.join(required);
if !path.is_file() || path.metadata().is_ok_and(|m| m.len() == 0) {
return Err(format!("tokenizer directory is missing {required}"));
}
}
Ok(Qwen3ModelFiles {
conv_frontend,
encoder,
decoder,
tokenizer,
})
}
#[derive(Debug, Clone)]
#[cfg_attr(not(feature = "vad-silero"), allow(dead_code))]
pub(crate) struct FunAsrNanoModelFiles {
pub(crate) encoder_adaptor: PathBuf,
pub(crate) llm: PathBuf,
pub(crate) embedding: PathBuf,
pub(crate) tokenizer: PathBuf,
}
pub(crate) fn has_encoder_adaptor(dir: &Path) -> bool {
dir.join("encoder_adaptor.onnx").is_file() || dir.join("encoder_adaptor.int8.onnx").is_file()
}
fn find_funasr_tokenizer_dir(dir: &Path) -> Result<PathBuf, String> {
let it = std::fs::read_dir(dir).map_err(|e| format!("failed to read model directory: {e}"))?;
let mut candidates: Vec<PathBuf> = it
.filter_map(|e| e.ok())
.map(|e| e.path())
.filter(|p| p.is_dir())
.filter(|p| {
["vocab.json", "merges.txt", "tokenizer.json"]
.iter()
.all(|name| {
let file = p.join(name);
file.is_file() && file.metadata().is_ok_and(|m| m.len() > 0)
})
})
.collect();
match candidates.len() {
1 => Ok(candidates.remove(0)),
0 => Err(
"model directory is missing a tokenizer directory with vocab.json, \
merges.txt and tokenizer.json"
.into(),
),
_ => Err(
"multiple tokenizer directories found; point the engine at a directory \
containing exactly one"
.into(),
),
}
}
pub(crate) fn find_funasr_nano_files(dir: &Path) -> Result<FunAsrNanoModelFiles, String> {
let dir = descend_where(dir, has_encoder_adaptor);
let entries = list_dir_files(&dir)?;
let encoder_adaptor = pick(&entries, "encoder_adaptor", true)?;
let llm = pick(&entries, "llm", true)?;
let embedding = pick(&entries, "embedding", true)?;
let tokenizer = find_funasr_tokenizer_dir(&dir)?;
Ok(FunAsrNanoModelFiles {
encoder_adaptor,
llm,
embedding,
tokenizer,
})
}
#[derive(Debug, Clone)]
#[cfg_attr(not(feature = "vad-silero"), allow(dead_code))]
pub(crate) struct FireRedAedModelFiles {
pub(crate) encoder: PathBuf,
pub(crate) decoder: PathBuf,
pub(crate) tokens: PathBuf,
}
pub(crate) fn has_fire_red_aed_layout_marker(dir: &Path) -> bool {
match list_dir_files(dir) {
Ok(entries) => any_prefix_onnx(&entries, "encoder") && !any_prefix_onnx(&entries, "joiner"),
Err(_) => false,
}
}
fn any_prefix_onnx(entries: &[PathBuf], prefix: &str) -> bool {
entries.iter().any(|path| {
path.is_file()
&& path
.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| name.starts_with(prefix) && name.ends_with(".onnx"))
})
}
pub(crate) fn find_fire_red_aed_files(dir: &Path) -> Result<FireRedAedModelFiles, String> {
let dir = descend_where(dir, has_fire_red_aed_layout_marker);
let entries = list_dir_files(&dir)?;
if any_prefix_onnx(&entries, "joiner") {
return Err(
"model directory contains joiner*.onnx; this is a transducer layout, not FireRedASR-AED"
.into(),
);
}
let encoder = pick(&entries, "encoder", true)?;
let decoder = pick(&entries, "decoder", true)?;
let tokens = require_tokens(&dir)?;
Ok(FireRedAedModelFiles {
encoder,
decoder,
tokens,
})
}
fn find_bpe_vocab(dir: &Path) -> Result<Option<PathBuf>, String> {
let vocab = dir.join("bpe.vocab");
if !vocab.is_file() {
return Ok(None);
}
if vocab
.metadata()
.map_or(true, |metadata| metadata.len() == 0)
{
return Err(
"bpe.vocab is empty; provide the model's non-empty vocabulary or remove the invalid file"
.into(),
);
}
Ok(Some(vocab))
}
#[derive(Debug, Clone)]
#[cfg(feature = "punct-sherpa")]
pub(crate) struct PunctModelFiles {
pub(crate) model: PathBuf,
pub(crate) vocab: Option<PathBuf>,
}
#[cfg(feature = "punct-sherpa")]
pub(crate) fn find_punct_model_files(dir: &Path) -> Result<PunctModelFiles, String> {
let dir = descend_where(dir, has_punct_model);
let entries = list_dir_files(&dir)?;
let model = pick(&entries, "model", true)?;
let vocab = find_bpe_vocab(&dir)?;
Ok(PunctModelFiles { model, vocab })
}
#[cfg(feature = "punct-sherpa")]
fn has_punct_model(dir: &Path) -> bool {
std::fs::read_dir(dir).is_ok_and(|it| {
it.filter_map(|e| e.ok())
.any(|e| e.file_name().to_string_lossy().ends_with(".onnx"))
})
}
pub(crate) fn is_sense_voice_tokens(tokens: &Path) -> Result<bool, String> {
std::fs::read_to_string(tokens)
.map(|s| {
s.lines()
.any(|l| l.split_whitespace().next() == Some("<|zh|>"))
})
.map_err(|e| format!("failed to read tokens: {e}"))
}
#[cfg(feature = "vad-silero")]
pub(crate) fn ensure_family_not_contradicted(
family: crate::OfflineFamily,
tokens: &Path,
) -> Result<(), String> {
let sense = is_sense_voice_tokens(tokens)?;
if !sense || family == crate::OfflineFamily::SenseVoice {
return Ok(());
}
Err(format!(
"tokens contain SenseVoice language markers but family is {family:?}"
))
}
#[cfg(feature = "backend-sherpa")]
pub(crate) fn is_cjk_char(c: char) -> bool {
matches!(c as u32,
0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xF900..=0xFAFF | 0x20000..=0x2A6DF | 0x2A700..=0x2B739 | 0x2B740..=0x2B81D | 0x2B820..=0x2CEA1 | 0x2CEB0..=0x2EBE0 | 0x2EBF0..=0x2EE5D | 0x31350..=0x323AF | 0x30000..=0x3134A | 0x2F800..=0x2FA1F )
}
#[cfg(feature = "backend-sherpa")]
pub(crate) fn detect_hotword_modeling_unit(files: &ModelFiles) -> (String, Option<PathBuf>) {
let Some(vocab) = &files.bpe_vocab else {
return ("cjkchar".into(), None);
};
let cjk = std::fs::read_to_string(&files.tokens)
.map(|s| s.chars().any(is_cjk_char))
.unwrap_or(false);
if cjk {
("cjkchar+bpe".into(), Some(vocab.clone()))
} else {
("bpe".into(), Some(vocab.clone()))
}
}
fn descend_into_single_child(dir: &Path) -> PathBuf {
descend_where(dir, |d| d.join("tokens.txt").is_file())
}
pub(crate) fn descend_where(dir: &Path, has_model: impl Fn(&Path) -> bool) -> PathBuf {
if has_model(dir) {
return dir.to_path_buf();
}
if let Ok(mut it) = std::fs::read_dir(dir) {
let dirs: Vec<PathBuf> = it
.by_ref()
.filter_map(|e| e.ok())
.filter(|e| e.path().is_dir())
.map(|e| e.path())
.collect();
if dirs.len() == 1 && has_model(&dirs[0]) {
return dirs[0].clone();
}
}
dir.to_path_buf()
}
fn list_dir_files(dir: &Path) -> Result<Vec<PathBuf>, String> {
let mut out = Vec::new();
let it = std::fs::read_dir(dir).map_err(|e| format!("failed to read model directory: {e}"))?;
for e in it.filter_map(|e| e.ok()) {
out.push(e.path());
}
Ok(out)
}
fn pick(candidates: &[PathBuf], prefix: &str, prefer_int8: bool) -> Result<PathBuf, String> {
let matched: Vec<PathBuf> = candidates
.iter()
.filter(|p| {
let name = p.file_name().and_then(|n| n.to_str()).unwrap_or("");
p.is_file() && name.starts_with(prefix) && name.ends_with(".onnx")
})
.cloned()
.collect();
if matched.is_empty() {
return Err(format!("model directory is missing {prefix}*.onnx"));
}
let is_preferred = |path: &Path| {
path.file_name()
.and_then(|n| n.to_str())
.map(|n| n.contains(".int8."))
.unwrap_or(false)
== prefer_int8
};
let preferred_exists = matched.iter().any(|path| is_preferred(path));
let mut selected: Vec<PathBuf> = matched
.into_iter()
.filter(|path| !preferred_exists || is_preferred(path))
.collect();
selected.sort_by_key(|path| path.file_name().map(ToOwned::to_owned));
if selected.len() > 1 {
let names = selected
.iter()
.filter_map(|path| path.file_name())
.map(|name| name.to_string_lossy())
.collect::<Vec<_>>()
.join(", ");
return Err(format!(
"multiple equally preferred {prefix} model files found: {names}; keep exactly one candidate in the model directory"
));
}
let path = selected.remove(0);
if path.metadata().map_or(true, |m| m.len() == 0) {
return Err(format!("model file is empty: {}", path.display()));
}
Ok(path)
}
fn require_tokens(dir: &Path) -> Result<PathBuf, String> {
let tokens = dir.join("tokens.txt");
if tokens.is_file() && tokens.metadata().is_ok_and(|m| m.len() > 0) {
Ok(tokens)
} else {
Err("model directory is missing tokens.txt".into())
}
}
#[cfg(test)]
pub(crate) mod fixtures;
#[cfg(test)]
mod tests;