use std::path::{Path, PathBuf};
use anyhow::{Context, Result};
use fastembed::{RerankerModel, TextRerank};
use ort::ep::{CPU, CUDA, ExecutionProvider, ExecutionProviderDispatch};
use ort::session::builder::GraphOptimizationLevel;
use ort::session::{Session, SessionInputValue};
use ort::value::Tensor;
use tokenizers::{AddedToken, PaddingParams, PaddingStrategy, Tokenizer, TruncationParams};
use super::rerank::RerankEp;
use crate::model_cache::{
CachedModel, cached_snapshot, model_cache_dir, reranker_model, snapshots_dir,
};
const LENGTH_BUCKET: usize = 16;
const DEFAULT_MAX_LENGTH: usize = 512;
pub(crate) struct OnnxSessionConfig {
pub(crate) intra_threads: usize,
pub(crate) inter_threads: usize,
pub(crate) opt_level: GraphOptimizationLevel,
pub(crate) ep: RerankEp,
pub(crate) max_length: Option<usize>,
}
pub(crate) fn default_opt_level() -> GraphOptimizationLevel {
GraphOptimizationLevel::Level3
}
pub(crate) struct OnnxReranker {
tokenizer: Tokenizer,
session: Session,
need_token_type_ids: bool,
}
fn snapshot_dir(model: &CachedModel) -> Result<PathBuf> {
let cache = model_cache_dir()?;
cached_snapshot(&cache, model).with_context(|| {
format!(
"no complete cached snapshot for '{}' under {} (expected {} to already be on \
disk; this backend does not download)",
model.code,
snapshots_dir(&cache, &model.code).display(),
model.files.join(", ")
)
})
}
#[derive(Debug)]
struct ModelFiles {
model_onnx: PathBuf,
tokenizer_json: PathBuf,
config_json: PathBuf,
tokenizer_config_json: PathBuf,
special_tokens_map_json: PathBuf,
}
fn resolve_model_files(model_name: RerankerModel) -> Result<ModelFiles> {
resolve_model_files_with(model_name, download_snapshot)
}
fn resolve_model_files_with(
model_name: RerankerModel,
download: impl FnOnce(RerankerModel) -> Result<()>,
) -> Result<ModelFiles> {
let info = TextRerank::get_model_info(&model_name);
let cached = reranker_model(&model_name);
let snap = match snapshot_dir(&cached) {
Ok(dir) => dir,
Err(_) => {
download(model_name)?;
snapshot_dir(&cached)?
}
};
Ok(ModelFiles {
model_onnx: snap.join(&info.model_file),
tokenizer_json: snap.join("tokenizer.json"),
config_json: snap.join("config.json"),
tokenizer_config_json: snap.join("tokenizer_config.json"),
special_tokens_map_json: snap.join("special_tokens_map.json"),
})
}
fn download_snapshot(model_name: RerankerModel) -> Result<()> {
let model = reranker_model(&model_name);
let api = hf_hub::api::sync::ApiBuilder::new()
.with_cache_dir(model_cache_dir()?)
.with_token(None)
.with_progress(true)
.build()
.context("building hf-hub api client")?;
let repo = api.model(model.code.clone());
for filename in &model.files {
repo.download(filename)
.with_context(|| format!("downloading {filename} for {}", model.code))?;
}
Ok(())
}
fn read_json(path: &Path) -> Result<serde_json::Value> {
let raw =
std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?;
serde_json::from_str(&raw).with_context(|| format!("parsing {} as json", path.display()))
}
fn load_tokenizer(files: &ModelFiles, requested_max_length: Option<usize>) -> Result<Tokenizer> {
let config = read_json(&files.config_json)?;
let tokenizer_config = read_json(&files.tokenizer_config_json)?;
let special_tokens_map = read_json(&files.special_tokens_map_json)?;
let mut tokenizer = Tokenizer::from_file(&files.tokenizer_json)
.map_err(|e| anyhow::anyhow!(e.to_string()))
.with_context(|| format!("loading {}", files.tokenizer_json.display()))?;
let model_max_length = tokenizer_config["model_max_length"]
.as_f64()
.context("tokenizer_config.json missing model_max_length")?
as usize;
let max_length = requested_max_length
.unwrap_or(DEFAULT_MAX_LENGTH)
.min(model_max_length);
let pad_id = config["pad_token_id"].as_u64().unwrap_or(0) as u32;
let pad_token = tokenizer_config["pad_token"]
.as_str()
.context("tokenizer_config.json missing pad_token")?
.to_string();
tokenizer
.with_padding(Some(PaddingParams {
strategy: PaddingStrategy::BatchLongest,
pad_token,
pad_id,
..Default::default()
}))
.with_truncation(Some(TruncationParams {
max_length,
..Default::default()
}))
.map_err(|e| anyhow::anyhow!(e.to_string()))
.context("configuring tokenizer padding/truncation")?;
if let serde_json::Value::Object(root) = special_tokens_map {
for value in root.values() {
if let Some(content) = value.as_str() {
tokenizer.add_special_tokens(&[AddedToken {
content: content.into(),
special: true,
..Default::default()
}]);
} else if let (
Some(content),
Some(single_word),
Some(lstrip),
Some(rstrip),
Some(normalized),
) = (
value["content"].as_str(),
value["single_word"].as_bool(),
value["lstrip"].as_bool(),
value["rstrip"].as_bool(),
value["normalized"].as_bool(),
) {
tokenizer.add_special_tokens(&[AddedToken {
content: content.into(),
special: true,
single_word,
lstrip,
rstrip,
normalized,
}]);
}
}
}
Ok(tokenizer)
}
fn builder_err(err: ort::Error<ort::session::builder::SessionBuilder>) -> anyhow::Error {
anyhow::Error::msg(err.to_string())
}
fn execution_providers(ep: RerankEp) -> Vec<ExecutionProviderDispatch> {
if ep != RerankEp::Cuda {
return vec![CPU::default().build()];
}
let cuda = CUDA::default();
let available = cuda.is_available().unwrap_or_else(|e| {
tracing::warn!("checking CUDA execution provider availability failed: {e}");
false
});
if available {
vec![cuda.build(), CPU::default().build()]
} else {
tracing::warn!(
"SECOND_BRAIN_RERANK_EP=cuda requested but onnxruntime in this build reports no \
CUDA support; falling back to cpu"
);
vec![CPU::default().build()]
}
}
fn build_session(model_path: &Path, config: &OnnxSessionConfig) -> Result<Session> {
Session::builder()
.context("creating onnx session builder")?
.with_execution_providers(execution_providers(config.ep))
.map_err(builder_err)
.context("registering execution providers")?
.with_optimization_level(config.opt_level)
.map_err(builder_err)
.context("setting graph optimization level")?
.with_intra_threads(config.intra_threads)
.map_err(builder_err)
.context("setting intra-op thread count")?
.with_inter_threads(config.inter_threads)
.map_err(builder_err)
.context("setting inter-op thread count")?
.commit_from_file(model_path)
.with_context(|| format!("committing onnx model {}", model_path.display()))
}
impl OnnxReranker {
pub(crate) fn load(model_name: RerankerModel, config: &OnnxSessionConfig) -> Result<Self> {
let files = resolve_model_files(model_name)?;
let tokenizer = load_tokenizer(&files, config.max_length)?;
let session = build_session(&files.model_onnx, config)?;
let need_token_type_ids = session
.inputs()
.iter()
.any(|input| input.name() == "token_type_ids");
Ok(Self {
tokenizer,
session,
need_token_type_ids,
})
}
pub(crate) fn rerank_batch(&mut self, query: &str, documents: &[&str]) -> Result<Vec<f32>> {
let mut scores = vec![0.0; documents.len()];
for bucket in length_buckets(documents, LENGTH_BUCKET) {
let docs: Vec<&str> = bucket.iter().map(|&i| documents[i]).collect();
for (i, score) in bucket.into_iter().zip(self.run_batch(query, &docs)?) {
scores[i] = score;
}
}
Ok(scores)
}
fn run_batch(&mut self, query: &str, batch: &[&str]) -> Result<Vec<f32>> {
let mut scores = Vec::with_capacity(batch.len());
{
let pairs: Vec<(&str, &str)> = batch.iter().map(|doc| (query, *doc)).collect();
let encodings = self
.tokenizer
.encode_batch(pairs, true)
.map_err(|e| anyhow::anyhow!(e.to_string()))
.context("tokenizing rerank batch")?;
let seq_len = encodings
.first()
.context("tokenizer returned an empty batch")?
.len();
let batch_size = batch.len();
let mut input_ids = Vec::with_capacity(batch_size * seq_len);
let mut attention_mask = Vec::with_capacity(batch_size * seq_len);
let mut token_type_ids = Vec::with_capacity(batch_size * seq_len);
for encoding in &encodings {
input_ids.extend(encoding.get_ids().iter().map(|&id| id as i64));
attention_mask.extend(encoding.get_attention_mask().iter().map(|&m| m as i64));
token_type_ids.extend(encoding.get_type_ids().iter().map(|&t| t as i64));
}
let shape = [batch_size, seq_len];
let mut inputs: Vec<(&str, SessionInputValue)> = vec![
("input_ids", Tensor::from_array((shape, input_ids))?.into()),
(
"attention_mask",
Tensor::from_array((shape, attention_mask))?.into(),
),
];
if self.need_token_type_ids {
inputs.push((
"token_type_ids",
Tensor::from_array((shape, token_type_ids))?.into(),
));
}
let outputs = self
.session
.run(inputs)
.context("running onnx rerank session")?;
let (out_shape, data) = outputs
.get("logits")
.context("onnx model output missing 'logits'")?
.try_extract_tensor::<f32>()
.context("extracting logits tensor")?;
let num_cols = out_shape.get(1).copied().unwrap_or(1).max(1) as usize;
for row in 0..batch_size {
scores.push(data[row * num_cols]);
}
}
Ok(scores)
}
}
fn length_buckets(documents: &[&str], size: usize) -> Vec<Vec<usize>> {
let mut order: Vec<usize> = (0..documents.len()).collect();
order.sort_by_key(|&i| documents[i].len());
order.chunks(size.max(1)).map(<[usize]>::to_vec).collect()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::model_cache::test_support::CacheDirOverride;
const JINA_REPO: &str = "models--jinaai--jina-reranker-v1-turbo-en";
fn jina() -> CachedModel {
reranker_model(&RerankerModel::JINARerankerV1TurboEn)
}
fn seed_files(snap: &Path, files: &[String]) {
for file in files {
let path = snap.join(file);
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(path, b"x").unwrap();
}
}
fn seed_snapshot(cache_root: &Path, revision: &str) -> PathBuf {
let snap = cache_root.join(JINA_REPO).join("snapshots").join(revision);
std::fs::create_dir_all(&snap).unwrap();
seed_files(&snap, &jina().files);
snap
}
fn seed_snapshot_without_tokenizer(cache_root: &Path, revision: &str) -> PathBuf {
let snap = seed_snapshot(cache_root, revision);
std::fs::remove_file(snap.join("tokenizer.json")).unwrap();
snap
}
#[test]
fn length_buckets_group_similar_lengths_and_cover_every_document_once() {
let docs = [
"a",
"a much longer document here",
"bb",
"ccc",
"a medium one",
];
let buckets = length_buckets(&docs, 2);
assert_eq!(buckets, vec![vec![0, 2], vec![3, 4], vec![1]]);
}
#[test]
fn snapshot_dir_finds_an_existing_cached_snapshot() {
let tmp = tempfile::tempdir().unwrap();
seed_snapshot(tmp.path(), "abc123");
let _env = CacheDirOverride::new(tmp.path());
let found = snapshot_dir(&jina());
assert_eq!(
found.unwrap(),
tmp.path().join(JINA_REPO).join("snapshots/abc123")
);
}
#[test]
fn snapshot_dir_errors_when_nothing_was_ever_downloaded() {
let tmp = tempfile::tempdir().unwrap();
let _env = CacheDirOverride::new(tmp.path());
let err = snapshot_dir(&jina()).unwrap_err();
assert!(err.to_string().contains("does not download"), "{err:#}");
}
#[test]
fn snapshot_dir_errors_when_snapshots_dir_exists_but_is_empty() {
let tmp = tempfile::tempdir().unwrap();
std::fs::create_dir_all(tmp.path().join(JINA_REPO).join("snapshots")).unwrap();
let _env = CacheDirOverride::new(tmp.path());
let err = snapshot_dir(&jina()).unwrap_err();
assert!(err.to_string().contains("does not download"));
}
#[test]
fn snapshot_dir_errors_when_the_cached_snapshot_lacks_tokenizer_json() {
let tmp = tempfile::tempdir().unwrap();
seed_snapshot_without_tokenizer(tmp.path(), "interrupted");
let _env = CacheDirOverride::new(tmp.path());
let err = snapshot_dir(&jina()).unwrap_err();
assert!(err.to_string().contains("does not download"), "{err:#}");
}
#[test]
fn resolve_model_files_with_downloads_once_when_snapshot_missing() {
let tmp = tempfile::tempdir().unwrap();
let _env = CacheDirOverride::new(tmp.path());
let cache_root = tmp.path().to_path_buf();
let calls = std::cell::Cell::new(0);
let result = resolve_model_files_with(RerankerModel::JINARerankerV1TurboEn, |_| {
calls.set(calls.get() + 1);
seed_snapshot(&cache_root, "downloaded-rev");
Ok(())
});
assert_eq!(calls.get(), 1);
assert!(result.is_ok());
}
#[test]
fn resolve_model_files_with_skips_download_when_snapshot_already_present() {
let tmp = tempfile::tempdir().unwrap();
seed_snapshot(tmp.path(), "already-cached");
let _env = CacheDirOverride::new(tmp.path());
let calls = std::cell::Cell::new(0);
let result = resolve_model_files_with(RerankerModel::JINARerankerV1TurboEn, |_| {
calls.set(calls.get() + 1);
Ok(())
});
assert_eq!(calls.get(), 0);
assert!(result.is_ok());
}
#[test]
fn resolve_model_files_with_downloads_again_when_the_cached_snapshot_is_partial() {
let tmp = tempfile::tempdir().unwrap();
let snap = seed_snapshot_without_tokenizer(tmp.path(), "rev1");
let _env = CacheDirOverride::new(tmp.path());
let calls = std::cell::Cell::new(0);
let result = resolve_model_files_with(RerankerModel::JINARerankerV1TurboEn, |_| {
calls.set(calls.get() + 1);
std::fs::write(snap.join("tokenizer.json"), b"x").unwrap();
Ok(())
});
assert_eq!(calls.get(), 1);
assert!(result.unwrap().tokenizer_json.is_file());
}
#[test]
fn resolve_model_files_with_propagates_download_error() {
let tmp = tempfile::tempdir().unwrap();
let _env = CacheDirOverride::new(tmp.path());
let result = resolve_model_files_with(RerankerModel::JINARerankerV1TurboEn, |_| {
anyhow::bail!("network unreachable")
});
let err = result.unwrap_err();
assert!(err.to_string().contains("network unreachable"));
}
}