use crate::error::{CharonError, Result};
#[cfg(feature = "candle-backend")]
use candle_core::{Device, Tensor};
use ndarray::Array2;
#[cfg(feature = "ort-backend")]
use ort::session::{
builder::{GraphOptimizationLevel, SessionBuilder},
Session,
};
use serde::{Deserialize, Serialize};
use std::path::{Path, PathBuf};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ModelBackend {
#[cfg(feature = "ort-backend")]
OnnxRuntime,
#[cfg(feature = "candle-backend")]
Candle,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelConfig {
pub model_path: PathBuf,
#[serde(skip, default)]
#[cfg(any(feature = "ort-backend", feature = "candle-backend"))]
pub backend: Option<ModelBackend>,
pub sample_rate: u32,
pub channels: usize,
pub sources: Vec<String>,
pub chunk_size: Option<usize>,
}
impl Default for ModelConfig {
fn default() -> Self {
Self {
model_path: PathBuf::from("model.onnx"),
#[cfg(any(feature = "ort-backend", feature = "candle-backend"))]
backend: None, sample_rate: 44100,
channels: 2,
sources: vec![
"drums".to_string(),
"bass".to_string(),
"vocals".to_string(),
"other".to_string(),
],
chunk_size: Some(441000), }
}
}
#[cfg(feature = "ort-backend")]
pub struct OnnxModel {
#[allow(dead_code)]
session: Session,
config: ModelConfig,
}
#[cfg(feature = "ort-backend")]
impl OnnxModel {
pub fn new(config: ModelConfig) -> Result<Self> {
let session = SessionBuilder::new()?
.with_optimization_level(GraphOptimizationLevel::Level3)?
.with_intra_threads(4)?
.commit_from_file(&config.model_path)?;
Ok(Self { session, config })
}
pub fn infer(&self, input: &Array2<f32>) -> Result<Vec<Array2<f32>>> {
let num_sources = self.config.sources.len();
let separated = vec![input.clone(); num_sources];
Ok(separated)
}
}
#[cfg(feature = "candle-backend")]
pub struct CandleModel {
device: Device,
config: ModelConfig,
model: Option<candle_nn::VarMap>,
}
#[cfg(feature = "candle-backend")]
impl CandleModel {
pub fn new(config: ModelConfig) -> Result<Self> {
use candle_core::safetensors;
let device = if cfg!(target_arch = "wasm32") {
Device::Cpu
} else {
Device::cuda_if_available(0).unwrap_or(Device::Cpu)
};
let model = if config.model_path.exists() {
let tensors = safetensors::load(&config.model_path, &device)?;
let mut varmap = candle_nn::VarMap::new();
for (name, tensor) in tensors {
varmap
.data()
.lock()
.unwrap()
.insert(name, candle_nn::Var::from_tensor(&tensor)?);
}
Some(varmap)
} else {
None
};
Ok(Self {
device,
config,
model,
})
}
pub fn infer(&self, input: &Array2<f32>) -> Result<Vec<Array2<f32>>> {
let (channels, samples) = (input.nrows(), input.ncols());
let data: Vec<f32> = input.t().iter().copied().collect();
let tensor = Tensor::from_vec(data, (samples, channels), &self.device)?;
let output = if let Some(ref _model) = self.model {
tensor.clone()
} else {
tensor.clone()
};
let output_data: Vec<f32> = output.flatten_all()?.to_vec1()?;
let num_sources = self.config.sources.len();
let samples_per_source = output_data.len() / num_sources;
let mut separated = Vec::new();
for i in 0..num_sources {
let start = i * samples_per_source;
let end = start + samples_per_source;
let source_data = &output_data[start..end];
let mut source_array = Array2::zeros((channels, samples));
for (idx, &val) in source_data.iter().enumerate() {
let ch = idx % channels;
let samp = idx / channels;
if samp < samples {
source_array[[ch, samp]] = val;
}
}
separated.push(source_array);
}
Ok(separated)
}
}
pub enum Model {
#[cfg(feature = "ort-backend")]
Onnx(OnnxModel),
#[cfg(feature = "candle-backend")]
Candle(CandleModel),
}
impl Model {
pub fn from_config(config: ModelConfig) -> Result<Self> {
#[cfg(any(feature = "ort-backend", feature = "candle-backend"))]
let backend = config.backend.or_else(|| {
if config.model_path.extension()?.to_str()? == "onnx" {
#[cfg(feature = "ort-backend")]
return Some(ModelBackend::OnnxRuntime);
}
#[cfg(feature = "candle-backend")]
return Some(ModelBackend::Candle);
#[allow(unreachable_code)]
None
});
#[cfg(feature = "ort-backend")]
if matches!(backend, Some(ModelBackend::OnnxRuntime)) {
return Ok(Model::Onnx(OnnxModel::new(config)?));
}
#[cfg(feature = "candle-backend")]
if matches!(backend, Some(ModelBackend::Candle)) {
return Ok(Model::Candle(CandleModel::new(config)?));
}
Err(CharonError::NotSupported(
"No ML backend enabled or auto-detected".to_string(),
))
}
#[allow(unreachable_patterns)]
pub fn infer(&self, input: &Array2<f32>) -> Result<Vec<Array2<f32>>> {
match self {
#[cfg(feature = "ort-backend")]
Model::Onnx(model) => model.infer(input),
#[cfg(feature = "candle-backend")]
Model::Candle(model) => model.infer(input),
#[allow(unreachable_patterns)]
_ => Err(CharonError::NotSupported(
"No model backend available".to_string(),
)),
}
}
#[allow(unreachable_patterns)]
pub fn config(&self) -> &ModelConfig {
match self {
#[cfg(feature = "ort-backend")]
Model::Onnx(model) => &model.config,
#[cfg(feature = "candle-backend")]
Model::Candle(model) => &model.config,
#[allow(unreachable_patterns)]
_ => panic!("No model backend available"),
}
}
}
pub struct ModelRegistry {
models_dir: PathBuf,
}
impl ModelRegistry {
pub fn new<P: AsRef<Path>>(models_dir: P) -> Self {
Self {
models_dir: models_dir.as_ref().to_path_buf(),
}
}
pub fn list_models(&self) -> Result<Vec<String>> {
let mut models = Vec::new();
if !self.models_dir.exists() {
return Ok(models);
}
for entry in std::fs::read_dir(&self.models_dir)? {
let entry = entry?;
let path = entry.path();
if path.is_file() {
if let Some(ext) = path.extension() {
if ext == "onnx" || ext == "safetensors" {
if let Some(name) = path.file_stem() {
models.push(name.to_string_lossy().to_string());
}
}
}
}
}
Ok(models)
}
pub fn get_model_path(&self, name: &str) -> Option<PathBuf> {
let onnx_path = self.models_dir.join(format!("{name}.onnx"));
if onnx_path.exists() {
return Some(onnx_path);
}
let safetensors_path = self.models_dir.join(format!("{name}.safetensors"));
if safetensors_path.exists() {
return Some(safetensors_path);
}
None
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_model_config_default() {
let config = ModelConfig::default();
assert_eq!(config.sample_rate, 44100);
assert_eq!(config.channels, 2);
assert_eq!(config.sources.len(), 4);
}
#[test]
#[cfg(all(feature = "ort-backend", feature = "candle-backend"))]
fn test_model_backend_types() {
assert_ne!(ModelBackend::OnnxRuntime, ModelBackend::Candle);
}
}