use crate::detection::detect_model_type;
use crate::error::{Error, Result};
use crate::labels::load_labels_from_file;
use crate::postprocess::top_k_predictions;
use crate::types::{ExecutionProviderInfo, ModelConfig, ModelType, PredictionResult};
use ndarray::Array2;
use ort::session::Session;
use ort::value::Value;
use std::sync::{Arc, Mutex};
macro_rules! with_provider_method {
($fn_name:ident, $provider_struct:ident, $provider_enum:ident, $doc:expr) => {
#[doc = $doc]
#[must_use]
pub fn $fn_name(mut self) -> Self {
use ort::execution_providers::$provider_struct;
self.execution_providers
.push($provider_struct::default().into());
if self.requested_provider == ExecutionProviderInfo::Cpu {
self.requested_provider = ExecutionProviderInfo::$provider_enum;
}
self
}
};
}
#[derive(Debug)]
enum Labels {
Path(String),
InMemory(Vec<String>),
}
#[derive(Debug)]
pub struct ClassifierBuilder {
model_path: Option<String>,
labels: Option<Labels>,
model_type_override: Option<ModelType>,
execution_providers: Vec<ort::execution_providers::ExecutionProviderDispatch>,
requested_provider: ExecutionProviderInfo,
top_k: usize,
min_confidence: Option<f32>,
}
impl Default for ClassifierBuilder {
fn default() -> Self {
Self::new()
}
}
impl ClassifierBuilder {
#[must_use]
pub const fn new() -> Self {
Self {
model_path: None,
labels: None,
model_type_override: None,
execution_providers: Vec::new(),
requested_provider: ExecutionProviderInfo::Cpu,
top_k: 10,
min_confidence: None,
}
}
#[must_use]
pub fn model_path(mut self, path: impl Into<String>) -> Self {
self.model_path = Some(path.into());
self
}
#[must_use]
pub fn labels_path(mut self, path: impl Into<String>) -> Self {
self.labels = Some(Labels::Path(path.into()));
self
}
#[must_use]
pub fn labels(mut self, labels: Vec<String>) -> Self {
self.labels = Some(Labels::InMemory(labels));
self
}
#[must_use]
pub const fn model_type(mut self, model_type: ModelType) -> Self {
self.model_type_override = Some(model_type);
self
}
#[must_use]
pub fn execution_provider(
mut self,
provider: impl Into<ort::execution_providers::ExecutionProviderDispatch>,
) -> Self {
self.execution_providers.push(provider.into());
self
}
#[must_use]
pub const fn top_k(mut self, k: usize) -> Self {
self.top_k = k;
self
}
#[must_use]
pub const fn min_confidence(mut self, threshold: f32) -> Self {
self.min_confidence = Some(threshold);
self
}
with_provider_method!(
with_cuda,
CUDAExecutionProvider,
Cuda,
"Request CUDA execution provider (NVIDIA GPU)"
);
#[must_use]
pub fn with_tensorrt(mut self) -> Self {
use ort::execution_providers::TensorRTExecutionProvider;
let config = crate::tensorrt_config::TensorRTConfig::new();
let provider = config.apply_to(TensorRTExecutionProvider::default());
self.execution_providers.push(provider.into());
if self.requested_provider == ExecutionProviderInfo::Cpu {
self.requested_provider = ExecutionProviderInfo::TensorRt;
}
self
}
#[must_use]
pub fn with_tensorrt_config(mut self, config: crate::tensorrt_config::TensorRTConfig) -> Self {
use ort::execution_providers::TensorRTExecutionProvider;
let provider = config.apply_to(TensorRTExecutionProvider::default());
self.execution_providers.push(provider.into());
if self.requested_provider == ExecutionProviderInfo::Cpu {
self.requested_provider = ExecutionProviderInfo::TensorRt;
}
self
}
with_provider_method!(
with_directml,
DirectMLExecutionProvider,
DirectMl,
"Request `DirectML` execution provider (Windows GPU)"
);
with_provider_method!(
with_coreml,
CoreMLExecutionProvider,
CoreMl,
"Request `CoreML` execution provider (Apple Neural Engine)"
);
with_provider_method!(
with_rocm,
ROCmExecutionProvider,
Rocm,
"Request `ROCm` execution provider (AMD GPU)"
);
with_provider_method!(
with_openvino,
OpenVINOExecutionProvider,
OpenVino,
"Request `OpenVINO` execution provider (Intel accelerator)"
);
with_provider_method!(
with_onednn,
OneDNNExecutionProvider,
OneDnn,
"Request oneDNN execution provider (Intel accelerator)"
);
with_provider_method!(
with_qnn,
QNNExecutionProvider,
Qnn,
"Request QNN execution provider (Qualcomm NPU)"
);
with_provider_method!(
with_acl,
ACLExecutionProvider,
Acl,
"Request ACL execution provider (Arm Compute Library)"
);
with_provider_method!(
with_armnn,
ArmNNExecutionProvider,
ArmNn,
"Request `ArmNN` execution provider (Arm Neural Network)"
);
pub fn build(self) -> Result<Classifier> {
let model_path = self.model_path.ok_or(Error::ModelPathRequired)?;
let labels_source = self.labels.ok_or(Error::LabelsRequired)?;
let mut session_builder = Session::builder().map_err(Error::ModelLoad)?;
for provider in self.execution_providers {
session_builder = session_builder
.with_execution_providers([provider])
.map_err(Error::ModelLoad)?;
}
let session = session_builder
.commit_from_file(&model_path)
.map_err(Error::ModelLoad)?;
let input_shape = extract_input_shape(&session)?;
let output_shapes = extract_output_shapes(&session)?;
let config = detect_model_type(&input_shape, &output_shapes, self.model_type_override)?;
let labels = match labels_source {
Labels::Path(path) => load_labels_from_file(&path, config.model_type)?,
Labels::InMemory(labels) => labels,
};
if labels.len() != config.num_species {
return Err(Error::LabelCount {
expected: config.num_species,
got: labels.len(),
});
}
Ok(Classifier {
inner: Arc::new(ClassifierInner {
session: Mutex::new(session),
config,
labels,
requested_provider: self.requested_provider,
top_k: self.top_k,
min_confidence: self.min_confidence,
}),
})
}
}
fn extract_input_shape(session: &Session) -> Result<Vec<i64>> {
let inputs = session
.inputs
.first()
.ok_or_else(|| Error::ModelDetection {
reason: "model has no inputs".to_string(),
})?;
let shape = inputs
.input_type
.tensor_shape()
.ok_or_else(|| Error::ModelDetection {
reason: "input is not a tensor".to_string(),
})?;
Ok(shape.iter().copied().collect())
}
fn extract_output_shapes(session: &Session) -> Result<Vec<Vec<i64>>> {
session
.outputs
.iter()
.map(|output| {
let shape = output
.output_type
.tensor_shape()
.ok_or_else(|| Error::ModelDetection {
reason: "output is not a tensor".to_string(),
})?;
Ok(shape.iter().copied().collect())
})
.collect()
}
struct ClassifierInner {
session: Mutex<Session>,
config: ModelConfig,
labels: Vec<String>,
requested_provider: ExecutionProviderInfo,
top_k: usize,
min_confidence: Option<f32>,
}
#[derive(Clone)]
pub struct Classifier {
inner: Arc<ClassifierInner>,
}
impl std::fmt::Debug for Classifier {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Classifier")
.field("config", &self.inner.config)
.field("labels_count", &self.inner.labels.len())
.field("requested_provider", &self.inner.requested_provider)
.field("top_k", &self.inner.top_k)
.field("min_confidence", &self.inner.min_confidence)
.finish_non_exhaustive()
}
}
impl Classifier {
#[must_use]
pub const fn builder() -> ClassifierBuilder {
ClassifierBuilder::new()
}
#[must_use]
pub fn config(&self) -> &ModelConfig {
&self.inner.config
}
#[must_use]
pub fn labels(&self) -> &[String] {
&self.inner.labels
}
#[must_use]
pub fn requested_provider(&self) -> ExecutionProviderInfo {
self.inner.requested_provider
}
#[allow(clippy::significant_drop_tightening)]
pub fn predict(&self, segment: &[f32]) -> Result<PredictionResult> {
let expected = self.inner.config.sample_count;
if segment.len() != expected {
return Err(Error::InputSize {
expected,
got: segment.len(),
});
}
let input_array = Array2::from_shape_vec((1, segment.len()), segment.to_vec())
.map_err(|e| Error::Inference(format!("failed to create input array: {e}")))?;
let input_value = Value::from_array(input_array)
.map_err(|e| Error::Inference(format!("failed to create input tensor: {e}")))?;
let mut session = self
.inner
.session
.lock()
.map_err(|e| Error::Inference(format!("session lock poisoned: {e}")))?;
let outputs = session
.run(ort::inputs![input_value])
.map_err(|e| Error::Inference(e.to_string()))?;
self.process_outputs(&outputs)
}
#[allow(clippy::significant_drop_tightening)]
pub fn predict_batch(&self, segments: &[&[f32]]) -> Result<Vec<PredictionResult>> {
if segments.is_empty() {
return Ok(Vec::new());
}
let expected = self.inner.config.sample_count;
for (i, seg) in segments.iter().enumerate() {
if seg.len() != expected {
return Err(Error::BatchInputSize {
index: i,
expected,
got: seg.len(),
});
}
}
let batch_size = segments.len();
let mut batch_data = Vec::with_capacity(batch_size * expected);
for seg in segments {
batch_data.extend_from_slice(seg);
}
let input_array = Array2::from_shape_vec((batch_size, expected), batch_data)
.map_err(|e| Error::Inference(format!("failed to create batch array: {e}")))?;
let input_value = Value::from_array(input_array)
.map_err(|e| Error::Inference(format!("failed to create input tensor: {e}")))?;
let mut session = self
.inner
.session
.lock()
.map_err(|e| Error::Inference(format!("session lock poisoned: {e}")))?;
let outputs = session
.run(ort::inputs![input_value])
.map_err(|e| Error::Inference(e.to_string()))?;
self.process_batch_outputs(&outputs, batch_size)
}
fn process_outputs(&self, outputs: &ort::session::SessionOutputs) -> Result<PredictionResult> {
let model_type = self.inner.config.model_type;
let (embeddings, logits) = match model_type {
ModelType::BirdNetV24 => {
let logits = extract_tensor_data(outputs, 0)?;
(None, logits)
}
ModelType::BirdNetV30 => {
let embeddings = extract_tensor_data(outputs, 0)?;
let logits = extract_tensor_data(outputs, 1)?;
(Some(embeddings), logits)
}
ModelType::PerchV2 => {
let embeddings = extract_tensor_data(outputs, 0)?;
let logits = extract_tensor_data(outputs, 3)?;
(Some(embeddings), logits)
}
};
let predictions = top_k_predictions(
&logits,
&self.inner.labels,
self.inner.top_k,
self.inner.min_confidence,
);
Ok(PredictionResult {
model_type,
predictions,
embeddings,
raw_scores: logits,
})
}
fn process_batch_outputs(
&self,
outputs: &ort::session::SessionOutputs,
batch_size: usize,
) -> Result<Vec<PredictionResult>> {
let model_type = self.inner.config.model_type;
let num_species = self.inner.config.num_species;
match model_type {
ModelType::BirdNetV24 => {
let logits_flat = extract_tensor_data(outputs, 0)?;
(0..batch_size)
.map(|i| {
let start = i * num_species;
let end = start + num_species;
let logits = &logits_flat[start..end];
let predictions = top_k_predictions(
logits,
&self.inner.labels,
self.inner.top_k,
self.inner.min_confidence,
);
Ok(PredictionResult {
model_type,
predictions,
embeddings: None,
raw_scores: logits.to_vec(),
})
})
.collect()
}
ModelType::BirdNetV30 => {
let embedding_dim = self.inner.config.embedding_dim.ok_or_else(|| {
Error::Inference(
"embedding_dim missing for model that requires embeddings".into(),
)
})?;
let emb_flat = extract_tensor_data(outputs, 0)?;
let logits_flat = extract_tensor_data(outputs, 1)?;
(0..batch_size)
.map(|i| {
let emb_start = i * embedding_dim;
let emb_end = emb_start + embedding_dim;
let embeddings = emb_flat[emb_start..emb_end].to_vec();
let logits_start = i * num_species;
let logits_end = logits_start + num_species;
let logits = &logits_flat[logits_start..logits_end];
let predictions = top_k_predictions(
logits,
&self.inner.labels,
self.inner.top_k,
self.inner.min_confidence,
);
Ok(PredictionResult {
model_type,
predictions,
embeddings: Some(embeddings),
raw_scores: logits.to_vec(),
})
})
.collect()
}
ModelType::PerchV2 => {
let embedding_dim = self.inner.config.embedding_dim.ok_or_else(|| {
Error::Inference(
"embedding_dim missing for model that requires embeddings".into(),
)
})?;
let emb_flat = extract_tensor_data(outputs, 0)?;
let logits_flat = extract_tensor_data(outputs, 3)?;
(0..batch_size)
.map(|i| {
let emb_start = i * embedding_dim;
let emb_end = emb_start + embedding_dim;
let embeddings = emb_flat[emb_start..emb_end].to_vec();
let logits_start = i * num_species;
let logits_end = logits_start + num_species;
let logits = &logits_flat[logits_start..logits_end];
let predictions = top_k_predictions(
logits,
&self.inner.labels,
self.inner.top_k,
self.inner.min_confidence,
);
Ok(PredictionResult {
model_type,
predictions,
embeddings: Some(embeddings),
raw_scores: logits.to_vec(),
})
})
.collect()
}
}
}
}
fn extract_tensor_data(outputs: &ort::session::SessionOutputs, index: usize) -> Result<Vec<f32>> {
let output_names: Vec<_> = outputs.keys().collect();
let name = output_names
.get(index)
.ok_or_else(|| Error::Inference(format!("missing output tensor at index {index}")))?;
let tensor = outputs
.get(*name)
.ok_or_else(|| Error::Inference(format!("missing output tensor '{name}'")))?;
let (_, data) = tensor
.try_extract_tensor::<f32>()
.map_err(|e| Error::Inference(e.to_string()))?;
Ok(data.to_vec())
}
#[cfg(test)]
mod tests {
#![allow(clippy::disallowed_methods)]
use super::*;
#[test]
fn test_builder_missing_model_path() {
let result = ClassifierBuilder::new()
.labels(vec!["species1".to_string()])
.build();
assert!(matches!(result, Err(Error::ModelPathRequired)));
}
#[test]
fn test_builder_missing_labels() {
let result = ClassifierBuilder::new().model_path("model.onnx").build();
assert!(matches!(result, Err(Error::LabelsRequired)));
}
#[test]
fn test_builder_missing_both() {
let result = ClassifierBuilder::new().build();
assert!(matches!(result, Err(Error::ModelPathRequired)));
}
#[test]
fn test_builder_method_chaining() {
let builder = ClassifierBuilder::new()
.model_path("model.onnx")
.labels_path("labels.txt")
.top_k(5)
.min_confidence(0.5)
.model_type(ModelType::BirdNetV24);
assert_eq!(builder.top_k, 5);
assert_eq!(builder.min_confidence, Some(0.5));
assert_eq!(builder.model_type_override, Some(ModelType::BirdNetV24));
}
#[test]
fn test_builder_default_values() {
let builder = ClassifierBuilder::new();
assert_eq!(builder.top_k, 10); assert_eq!(builder.min_confidence, None);
assert_eq!(builder.model_type_override, None);
assert!(builder.execution_providers.is_empty());
assert_eq!(builder.requested_provider, ExecutionProviderInfo::Cpu); }
#[test]
fn test_builder_top_k_zero() {
let builder = ClassifierBuilder::new()
.model_path("model.onnx")
.labels(vec!["species1".to_string()])
.top_k(0);
assert_eq!(builder.top_k, 0);
}
#[test]
fn test_builder_min_confidence_boundaries() {
let builder = ClassifierBuilder::new().min_confidence(0.0);
assert_eq!(builder.min_confidence, Some(0.0));
let builder = ClassifierBuilder::new().min_confidence(1.0);
assert_eq!(builder.min_confidence, Some(1.0));
let builder = ClassifierBuilder::new().min_confidence(1.5);
assert_eq!(builder.min_confidence, Some(1.5));
let builder = ClassifierBuilder::new().min_confidence(-0.5);
assert_eq!(builder.min_confidence, Some(-0.5)); }
#[test]
fn test_builder_labels_path_vs_in_memory() {
let builder1 = ClassifierBuilder::new().labels_path("labels.txt");
assert!(matches!(builder1.labels, Some(Labels::Path(_))));
let builder2 = ClassifierBuilder::new().labels(vec!["species1".to_string()]);
assert!(matches!(builder2.labels, Some(Labels::InMemory(_))));
}
#[test]
fn test_builder_multiple_execution_providers() {
use ort::execution_providers::CPUExecutionProvider;
let builder = ClassifierBuilder::new()
.execution_provider(CPUExecutionProvider::default())
.execution_provider(CPUExecutionProvider::default());
assert_eq!(builder.execution_providers.len(), 2);
}
#[test]
fn test_builder_default_trait() {
let builder1 = ClassifierBuilder::new();
let builder2 = ClassifierBuilder::default();
assert_eq!(builder1.top_k, builder2.top_k);
assert_eq!(builder1.min_confidence, builder2.min_confidence);
}
#[test]
fn test_mock_input_size_validation() {
let expected_size = 144_000; let wrong_size = 160_000;
let segment = vec![0.0f32; wrong_size];
if segment.len() != expected_size {
let err = Error::InputSize {
expected: expected_size,
got: segment.len(),
};
assert!(matches!(err, Error::InputSize { .. }));
}
}
#[test]
fn test_mock_batch_input_validation() {
let expected_size = 144_000;
let segments = [
vec![0.0f32; expected_size],
vec![0.0f32; 160_000], vec![0.0f32; expected_size],
];
for (i, seg) in segments.iter().enumerate() {
if seg.len() != expected_size {
let err = Error::BatchInputSize {
index: i,
expected: expected_size,
got: seg.len(),
};
assert!(matches!(err, Error::BatchInputSize { index: 1, .. }));
assert_eq!(i, 1);
break;
}
}
}
#[test]
fn test_empty_batch_handling() {
let segments: Vec<&[f32]> = vec![];
assert!(segments.is_empty());
}
#[test]
fn test_labels_enum_debug() {
let labels_path = Labels::Path("test.txt".to_string());
let debug_str = format!("{labels_path:?}");
assert!(debug_str.contains("Path"));
let labels_mem = Labels::InMemory(vec!["test".to_string()]);
let debug_str = format!("{labels_mem:?}");
assert!(debug_str.contains("InMemory"));
}
#[test]
fn test_requested_provider_defaults_to_cpu() {
let builder = ClassifierBuilder::new();
assert_eq!(builder.requested_provider, ExecutionProviderInfo::Cpu);
}
#[test]
fn test_builder_debug_includes_requested_provider() {
let builder = ClassifierBuilder::new()
.model_path("test.onnx")
.labels(vec!["species1".to_string()]);
let debug_str = format!("{builder:?}");
assert!(debug_str.contains("requested_provider"));
assert!(debug_str.contains("Cpu"));
}
#[test]
fn test_with_cuda_sets_requested_provider() {
let builder = ClassifierBuilder::new().with_cuda();
assert_eq!(builder.requested_provider, ExecutionProviderInfo::Cuda);
assert_eq!(builder.execution_providers.len(), 1);
}
#[test]
fn test_with_tensorrt_sets_requested_provider() {
let builder = ClassifierBuilder::new().with_tensorrt();
assert_eq!(builder.requested_provider, ExecutionProviderInfo::TensorRt);
assert_eq!(builder.execution_providers.len(), 1);
}
#[test]
fn test_with_tensorrt_config_sets_requested_provider() {
use crate::TensorRTConfig;
let config = TensorRTConfig::new();
let builder = ClassifierBuilder::new().with_tensorrt_config(config);
assert_eq!(builder.requested_provider, ExecutionProviderInfo::TensorRt);
assert_eq!(builder.execution_providers.len(), 1);
}
#[test]
fn test_with_tensorrt_config_custom_settings() {
use crate::TensorRTConfig;
let config = TensorRTConfig::new()
.with_fp16(false)
.with_builder_optimization_level(5)
.with_device_id(1);
let builder = ClassifierBuilder::new().with_tensorrt_config(config);
assert_eq!(builder.requested_provider, ExecutionProviderInfo::TensorRt);
assert_eq!(builder.execution_providers.len(), 1);
}
#[test]
fn test_with_tensorrt_config_disable_optimizations() {
use crate::TensorRTConfig;
let config = TensorRTConfig::new()
.with_fp16(false)
.with_cuda_graph(false)
.with_engine_cache(false)
.with_timing_cache(false);
let builder = ClassifierBuilder::new().with_tensorrt_config(config);
assert_eq!(builder.requested_provider, ExecutionProviderInfo::TensorRt);
assert_eq!(builder.execution_providers.len(), 1);
}
#[test]
fn test_with_directml_sets_requested_provider() {
let builder = ClassifierBuilder::new().with_directml();
assert_eq!(builder.requested_provider, ExecutionProviderInfo::DirectMl);
assert_eq!(builder.execution_providers.len(), 1);
}
#[test]
fn test_with_coreml_sets_requested_provider() {
let builder = ClassifierBuilder::new().with_coreml();
assert_eq!(builder.requested_provider, ExecutionProviderInfo::CoreMl);
assert_eq!(builder.execution_providers.len(), 1);
}
#[test]
fn test_with_rocm_sets_requested_provider() {
let builder = ClassifierBuilder::new().with_rocm();
assert_eq!(builder.requested_provider, ExecutionProviderInfo::Rocm);
assert_eq!(builder.execution_providers.len(), 1);
}
#[test]
fn test_with_openvino_sets_requested_provider() {
let builder = ClassifierBuilder::new().with_openvino();
assert_eq!(builder.requested_provider, ExecutionProviderInfo::OpenVino);
assert_eq!(builder.execution_providers.len(), 1);
}
#[test]
fn test_with_onednn_sets_requested_provider() {
let builder = ClassifierBuilder::new().with_onednn();
assert_eq!(builder.requested_provider, ExecutionProviderInfo::OneDnn);
assert_eq!(builder.execution_providers.len(), 1);
}
#[test]
fn test_with_qnn_sets_requested_provider() {
let builder = ClassifierBuilder::new().with_qnn();
assert_eq!(builder.requested_provider, ExecutionProviderInfo::Qnn);
assert_eq!(builder.execution_providers.len(), 1);
}
#[test]
fn test_with_acl_sets_requested_provider() {
let builder = ClassifierBuilder::new().with_acl();
assert_eq!(builder.requested_provider, ExecutionProviderInfo::Acl);
assert_eq!(builder.execution_providers.len(), 1);
}
#[test]
fn test_with_armnn_sets_requested_provider() {
let builder = ClassifierBuilder::new().with_armnn();
assert_eq!(builder.requested_provider, ExecutionProviderInfo::ArmNn);
assert_eq!(builder.execution_providers.len(), 1);
}
#[test]
fn test_chaining_multiple_providers_first_wins() {
let builder = ClassifierBuilder::new().with_cuda().with_tensorrt();
assert_eq!(builder.requested_provider, ExecutionProviderInfo::Cuda);
assert_eq!(builder.execution_providers.len(), 2);
}
#[test]
fn test_chaining_three_providers_first_wins() {
let builder = ClassifierBuilder::new()
.with_cuda()
.with_tensorrt()
.with_directml();
assert_eq!(builder.requested_provider, ExecutionProviderInfo::Cuda);
assert_eq!(builder.execution_providers.len(), 3);
}
#[test]
fn test_provider_methods_can_chain_with_other_builders() {
let builder = ClassifierBuilder::new()
.model_path("model.onnx")
.labels_path("labels.txt")
.with_cuda()
.top_k(5)
.min_confidence(0.8);
assert_eq!(builder.requested_provider, ExecutionProviderInfo::Cuda);
assert_eq!(builder.top_k, 5);
assert_eq!(builder.min_confidence, Some(0.8));
assert_eq!(builder.execution_providers.len(), 1);
}
#[test]
fn test_provider_methods_return_self_for_chaining() {
let builder = ClassifierBuilder::new()
.with_cuda()
.with_tensorrt()
.with_directml()
.with_coreml()
.with_rocm()
.with_openvino()
.with_onednn()
.with_qnn()
.with_acl()
.with_armnn();
assert_eq!(builder.requested_provider, ExecutionProviderInfo::Cuda);
assert_eq!(builder.execution_providers.len(), 10);
}
}