1#![deny(missing_docs)]
14#![doc = include_str!("../README.md")]
15use rand::Rng;
16use rubato::Resampler;
17#[cfg(feature = "codec")]
19pub use rusttorch_codec as codec;
20use rusttorch_core::{Device, Kind, Tensor};
21use rusttorch_data::{Collate, Dataset, MemoryFootprint, PinMemory, ResourceLimits, TaskContext};
22use std::{
23 fs::File,
24 io::Cursor,
25 path::{Path, PathBuf},
26};
27use symphonia::core::{
28 codecs::audio::AudioDecoderOptions,
29 formats::{FormatOptions, TrackType, probe::Hint},
30 io::MediaSourceStream,
31 meta::MetadataOptions,
32};
33#[derive(Debug, thiserror::Error)]
35#[non_exhaustive]
36pub enum AudioError {
37 #[error(transparent)]
39 Io(#[from] std::io::Error),
40 #[error(transparent)]
42 Decode(#[from] symphonia::core::errors::Error),
43 #[error("{0}")]
45 Invalid(String),
46 #[error(transparent)]
48 Limit(#[from] rusttorch_core::RustTorchError),
49 #[error(transparent)]
51 Tensor(#[from] tch::TchError),
52 #[error(transparent)]
54 Resampler(#[from] rubato::ResamplerConstructionError),
55 #[error(transparent)]
57 Resample(#[from] rubato::ResampleError),
58}
59pub type Result<T> = std::result::Result<T, AudioError>;
61#[derive(Clone, Debug, PartialEq, Eq)]
63pub enum ChannelLayout {
64 Mono,
66 Stereo,
68 Discrete(usize),
70}
71#[derive(Debug)]
73pub struct Waveform {
74 pub samples: Tensor,
76 pub sample_rate: u32,
78 pub channel_layout: ChannelLayout,
80 pub time_origin: f64,
82}
83impl Waveform {
84 pub fn new(samples: Tensor, sample_rate: u32, time_origin: f64) -> Result<Self> {
86 let c = samples.size().first().copied().unwrap_or(0);
87 let channel_layout = match c {
88 1 => ChannelLayout::Mono,
89 2 => ChannelLayout::Stereo,
90 _ => ChannelLayout::Discrete(c.max(0) as usize),
91 };
92 let wave = Self {
93 samples,
94 sample_rate,
95 channel_layout,
96 time_origin,
97 };
98 wave.validate()?;
99 Ok(wave)
100 }
101 pub fn validate(&self) -> Result<()> {
103 let s = self.samples.size();
104 let c = match self.channel_layout {
105 ChannelLayout::Mono => 1,
106 ChannelLayout::Stereo => 2,
107 ChannelLayout::Discrete(c) => c,
108 };
109 if s.len() != 2
110 || s[0] <= 0
111 || s[1] < 0
112 || s[0] as usize != c
113 || !matches!(self.samples.kind(), Kind::Float | Kind::Double)
114 || self.sample_rate == 0
115 || !self.time_origin.is_finite()
116 || self.time_origin < 0.
117 {
118 return Err(AudioError::Invalid("expected finite-origin Float/Double [channels,frames], positive rate and matching channel layout".into()));
119 }
120 Ok(())
121 }
122 pub fn frames(&self) -> usize {
124 self.samples.size().get(1).copied().unwrap_or(0).max(0) as usize
125 }
126 pub fn duration(&self) -> f64 {
128 self.frames() as f64 / self.sample_rate as f64
129 }
130 pub fn gain(mut self, decibels: f64) -> Result<Self> {
132 self.validate()?;
133 let factor = 10f64.powf(decibels / 20.);
134 if !decibels.is_finite() || !factor.is_finite() || (factor as f32).is_infinite() {
135 return Err(AudioError::Invalid(
136 "gain must have finite representable amplitude".into(),
137 ));
138 }
139 self.samples = self.samples.f_mul_scalar(factor)?;
140 Ok(self)
141 }
142 pub fn add_noise(
145 mut self,
146 amplitude: f32,
147 context: &TaskContext,
148 limits: ResourceLimits,
149 ) -> Result<Self> {
150 self.validate()?;
151 if !amplitude.is_finite() || amplitude < 0. {
152 return Err(AudioError::Invalid(
153 "noise amplitude must be finite and nonnegative".into(),
154 ));
155 }
156 context
157 .check()
158 .map_err(|_| AudioError::Invalid("audio transform cancelled".into()))?;
159 check_wave(
160 self.samples.size()[0] as usize,
161 self.frames(),
162 self.sample_rate,
163 limits,
164 )?;
165 let n = limits.tensor_elements(&[self.samples.size()[0] as usize, self.frames()])?;
166 let mut rng = context.rng();
167 let noise = (0..n)
168 .map(|_| {
169 if amplitude == 0. {
170 0.
171 } else {
172 rng.gen_range(-1f32..=1f32) * amplitude
173 }
174 })
175 .collect::<Vec<_>>();
176 self.samples = self.samples.f_add(
177 &Tensor::f_from_slice(&noise)?
178 .f_reshape(self.samples.size())?
179 .f_to_kind(self.samples.kind())?
180 .f_to_device(self.samples.device())?,
181 )?;
182 Ok(self)
183 }
184 pub fn time_shift(mut self, frames: i64) -> Result<Self> {
186 self.validate()?;
187 let n = self.frames() as i64;
188 let shift = frames.unsigned_abs().min(n as u64) as i64;
189 if shift == n {
190 self.samples = self.samples.f_zeros_like()?;
191 } else if frames >= 0 {
192 self.samples = self
193 .samples
194 .f_narrow(1, 0, n - shift)?
195 .f_constant_pad_nd([shift, 0])?;
196 } else {
197 self.samples = self
198 .samples
199 .f_narrow(1, shift, n - shift)?
200 .f_constant_pad_nd([0, shift])?;
201 }
202 Ok(self)
203 }
204 pub fn mix(mut self, other: &Self, scale: f64) -> Result<Self> {
207 self.validate()?;
208 other.validate()?;
209 if !scale.is_finite()
210 || self.sample_rate != other.sample_rate
211 || self.channel_layout != other.channel_layout
212 || self.time_origin != other.time_origin
213 || self.samples.size() != other.samples.size()
214 || self.samples.kind() != other.samples.kind()
215 || self.samples.device() != other.samples.device()
216 {
217 return Err(AudioError::Invalid(
218 "mix requires aligned waveforms and finite scale".into(),
219 ));
220 }
221 self.samples = self.samples.f_add(&other.samples.f_mul_scalar(scale)?)?;
222 Ok(self)
223 }
224 pub fn resample(self, new_rate: u32, limits: ResourceLimits) -> Result<Self> {
228 self.validate()?;
229 if new_rate == 0 {
230 return Err(AudioError::Invalid("resample rate must be positive".into()));
231 }
232 if self.samples.device() != Device::Cpu {
233 return Err(AudioError::Invalid(
234 "Rubato resampling requires a CPU waveform".into(),
235 ));
236 }
237 if self.samples.requires_grad() {
238 return Err(AudioError::Invalid(
239 "Rubato resampling cannot preserve autograd; detach explicitly".into(),
240 ));
241 }
242 if new_rate == self.sample_rate {
243 return Ok(self);
244 }
245 let channels = self.samples.size()[0] as usize;
246 let frames = self.frames();
247 check_wave(channels, frames, self.sample_rate, limits)?;
248 let expected = ((frames as u128) * (new_rate as u128)).div_ceil(self.sample_rate as u128);
249 let expected = usize::try_from(expected)
250 .map_err(|_| AudioError::Invalid("resampled length overflow".into()))?;
251 check_wave(channels, expected, new_rate, limits)?;
252 if frames == 0 {
253 return Self::new(self.samples, new_rate, self.time_origin);
254 }
255 let mut resampler = rubato::Async::<f64>::new_sinc(
256 new_rate as f64 / self.sample_rate as f64,
257 1.,
258 &rubato::SincInterpolationParameters::default(),
259 1024,
260 channels,
261 rubato::FixedAsync::Input,
262 )?;
263 let needed = resampler.process_all_needed_output_len(frames);
264 check_wave(channels, needed, new_rate, limits)?;
265 let data = Vec::<f64>::try_from(
266 &self
267 .samples
268 .f_to_kind(Kind::Double)?
269 .f_transpose(0, 1)?
270 .f_contiguous()?
271 .f_view([-1])?,
272 )?;
273 let input = audioadapter_buffers::owned::InterleavedOwned::new_from(data, channels, frames)
274 .map_err(|e| AudioError::Invalid(e.to_string()))?;
275 let output = resampler.process_all(&input, frames, None)?.take_data();
276 if output.len() != expected * channels {
277 return Err(AudioError::Invalid(
278 "Rubato returned unexpected output length".into(),
279 ));
280 }
281 let samples = Tensor::f_from_slice(&output)?
282 .f_reshape([expected as i64, channels as i64])?
283 .f_transpose(0, 1)?
284 .f_to_kind(self.samples.kind())?;
285 let mut value = Self::new(samples, new_rate, self.time_origin)?;
286 value.channel_layout = self.channel_layout;
287 Ok(value)
288 }
289 pub fn spectrogram(&self, config: SpectrogramConfig, limits: ResourceLimits) -> Result<Tensor> {
293 self.validate()?;
294 config.validate()?;
295 let frames = self.frames();
296 if frames < config.n_fft {
297 return Err(AudioError::Invalid(
298 "waveform shorter than FFT window".into(),
299 ));
300 }
301 let windows = 1 + (frames - config.n_fft) / config.hop_length;
302 let n = limits.tensor_elements(&[
303 self.samples.size()[0] as usize,
304 config.n_fft / 2 + 1,
305 windows,
306 ])?;
307 limits.check(
308 "spectrogram bytes",
309 n.checked_mul(16)
310 .ok_or_else(|| AudioError::Invalid("spectrogram size overflow".into()))?,
311 limits.max_decoded_bytes,
312 )?;
313 let window = Tensor::f_hann_window(
314 config.n_fft as i64,
315 (self.samples.kind(), self.samples.device()),
316 )?;
317 Ok(self
318 .samples
319 .f_stft_center(
320 config.n_fft as i64,
321 config.hop_length as i64,
322 config.n_fft as i64,
323 Some(&window),
324 false,
325 "reflect",
326 false,
327 true,
328 true,
329 false,
330 )?
331 .f_abs()?
332 .f_pow_tensor_scalar(config.power)?)
333 }
334 pub fn mel_spectrogram(
337 &self,
338 config: SpectrogramConfig,
339 n_mels: usize,
340 limits: ResourceLimits,
341 ) -> Result<Tensor> {
342 let spectrum = self.spectrogram(config, limits)?;
343 let output = limits.tensor_elements(&[
344 self.samples.size()[0] as usize,
345 n_mels,
346 spectrum.size()[2] as usize,
347 ])?;
348 limits.check(
349 "Mel output bytes",
350 output
351 .checked_mul(8)
352 .ok_or_else(|| AudioError::Invalid("Mel output size overflow".into()))?,
353 limits.max_decoded_bytes,
354 )?;
355 let filter = mel_filter_bank(config.n_fft, self.sample_rate, n_mels, limits)?
356 .f_to_kind(self.samples.kind())?
357 .f_to_device(self.samples.device())?;
358 Ok(filter.f_matmul(&spectrum)?)
359 }
360 pub fn mfcc(
363 &self,
364 config: SpectrogramConfig,
365 n_mels: usize,
366 n_mfcc: usize,
367 limits: ResourceLimits,
368 ) -> Result<Tensor> {
369 if n_mfcc == 0 || n_mfcc > n_mels {
370 return Err(AudioError::Invalid(
371 "MFCC count must be in 1..=n_mels".into(),
372 ));
373 }
374 let coefficients = limits.tensor_elements(&[n_mfcc, n_mels])?;
375 limits.check(
376 "DCT bytes",
377 coefficients
378 .checked_mul(8)
379 .ok_or_else(|| AudioError::Invalid("DCT size overflow".into()))?,
380 limits.max_decoded_bytes,
381 )?;
382 let mel = self
383 .mel_spectrogram(config, n_mels, limits)?
384 .f_clamp_min(1e-10)?
385 .f_log()?;
386 let mut dct = Vec::with_capacity(n_mfcc * n_mels);
387 for k in 0..n_mfcc {
388 for n in 0..n_mels {
389 let scale = if k == 0 {
390 (1. / n_mels as f64).sqrt()
391 } else {
392 (2. / n_mels as f64).sqrt()
393 };
394 dct.push(
395 scale
396 * (std::f64::consts::PI * (n as f64 + 0.5) * k as f64 / n_mels as f64)
397 .cos(),
398 );
399 }
400 }
401 Ok(Tensor::f_from_slice(&dct)?
402 .f_reshape([n_mfcc as i64, n_mels as i64])?
403 .f_to_kind(self.samples.kind())?
404 .f_to_device(self.samples.device())?
405 .f_matmul(&mel)?)
406 }
407}
408fn check_wave(channels: usize, frames: usize, rate: u32, limits: ResourceLimits) -> Result<()> {
409 if channels == 0 || channels > 64 || rate == 0 {
410 return Err(AudioError::Invalid(
411 "expected 1..=64 channels and positive sample rate".into(),
412 ));
413 }
414 let n = limits.tensor_elements(&[channels, frames])?;
415 limits.check(
416 "decoded waveform bytes",
417 n.checked_mul(8)
418 .ok_or_else(|| AudioError::Invalid("waveform byte overflow".into()))?,
419 limits.max_decoded_bytes,
420 )?;
421 if frames as u128 > (rate as u128) * (limits.max_media_seconds as u128) {
422 return Err(AudioError::Invalid(
423 "audio duration exceeds resource limit".into(),
424 ));
425 }
426 Ok(())
427}
428#[derive(Clone, Copy, Debug)]
430pub struct SpectrogramConfig {
431 pub n_fft: usize,
433 pub hop_length: usize,
435 pub power: f64,
437}
438impl Default for SpectrogramConfig {
439 fn default() -> Self {
440 Self {
441 n_fft: 400,
442 hop_length: 200,
443 power: 2.,
444 }
445 }
446}
447impl SpectrogramConfig {
448 fn validate(&self) -> Result<()> {
449 if self.n_fft < 2
450 || self.n_fft > i64::MAX as usize
451 || self.hop_length == 0
452 || self.hop_length > i64::MAX as usize
453 || !self.power.is_finite()
454 || self.power <= 0.
455 {
456 return Err(AudioError::Invalid("invalid FFT size, hop or power".into()));
457 }
458 Ok(())
459 }
460}
461pub fn mel_filter_bank(
463 n_fft: usize,
464 sample_rate: u32,
465 n_mels: usize,
466 limits: ResourceLimits,
467) -> Result<Tensor> {
468 if n_fft < 2 || sample_rate == 0 || n_mels == 0 {
469 return Err(AudioError::Invalid(
470 "positive mel dimensions and rate required".into(),
471 ));
472 }
473 let bins = n_fft / 2 + 1;
474 let n = limits.tensor_elements(&[n_mels, bins])?;
475 limits.check(
476 "Mel filter bytes",
477 n.checked_mul(8)
478 .ok_or_else(|| AudioError::Invalid("Mel filter byte overflow".into()))?,
479 limits.max_decoded_bytes,
480 )?;
481 let max_mel = 2595. * (1. + (sample_rate as f64 / 2.) / 700.).log10();
482 let points = (0..n_mels + 2)
483 .map(|i| 700. * (10f64.powf((i as f64 * max_mel / (n_mels + 1) as f64) / 2595.) - 1.))
484 .collect::<Vec<_>>();
485 let mut values = Vec::with_capacity(n);
486 for m in 0..n_mels {
487 for k in 0..bins {
488 let hz = k as f64 * sample_rate as f64 / n_fft as f64;
489 let up = (hz - points[m]) / (points[m + 1] - points[m]);
490 let down = (points[m + 2] - hz) / (points[m + 2] - points[m + 1]);
491 values.push(up.min(down).max(0.));
492 }
493 }
494 Ok(Tensor::f_from_slice(&values)?.f_reshape([n_mels as i64, bins as i64])?)
495}
496pub fn mask_axis(
499 input: &Tensor,
500 axis: usize,
501 start: usize,
502 width: usize,
503 value: f64,
504) -> Result<Tensor> {
505 let shape = input.size();
506 if axis >= shape.len()
507 || start > shape[axis] as usize
508 || width > shape[axis] as usize - start
509 || !value.is_finite()
510 {
511 return Err(AudioError::Invalid(
512 "invalid mask axis, interval or value".into(),
513 ));
514 }
515 let mut output = input.f_empty_like()?;
516 output.f_copy_(input)?;
517 let _ = output
518 .f_narrow(axis as i64, start as i64, width as i64)?
519 .f_fill_(value)?;
520 Ok(output)
521}
522pub fn decode_audio(path: impl AsRef<Path>, limits: ResourceLimits) -> Result<Waveform> {
525 let bytes = limits.read_encoded(File::open(path.as_ref())?)?;
526 let mss = MediaSourceStream::new(Box::new(Cursor::new(bytes)), Default::default());
527 let mut format = symphonia::default::get_probe().probe(
528 &Hint::new(),
529 mss,
530 FormatOptions::default(),
531 MetadataOptions::default(),
532 )?;
533 limits.check("audio streams", format.tracks().len(), limits.max_streams)?;
534 let track = format
535 .default_track(TrackType::Audio)
536 .ok_or_else(|| AudioError::Invalid("no audio track".into()))?;
537 let id = track.id;
538 let params = track
539 .codec_params
540 .as_ref()
541 .and_then(|p| p.audio())
542 .ok_or_else(|| AudioError::Invalid("missing audio codec parameters".into()))?;
543 let mut decoder = symphonia::default::get_codecs()
544 .make_audio_decoder(params, &AudioDecoderOptions::default())?;
545 let mut data = Vec::<f32>::new();
546 let mut spec = None;
547 while let Some(packet) = format.next_packet()? {
548 if packet.track_id != id {
549 continue;
550 }
551 let decoded = decoder.decode(&packet)?;
552 let rate = decoded.spec().rate();
553 let channels = decoded.spec().channels().count();
554 check_wave(channels, 0, rate, limits)?;
555 if let Some(previous) = spec {
556 if previous != (rate, channels) {
557 return Err(AudioError::Invalid(
558 "audio rate or channel count changes within stream".into(),
559 ));
560 }
561 } else {
562 spec = Some((rate, channels));
563 }
564 let next = data
565 .len()
566 .checked_add(decoded.samples_interleaved())
567 .ok_or_else(|| AudioError::Invalid("decoded sample count overflow".into()))?;
568 check_wave(channels, next / channels, rate, limits)?;
569 let old = data.len();
570 data.resize(next, 0.);
571 decoded.copy_to_slice_interleaved(&mut data[old..]);
572 }
573 let (rate, channels) =
574 spec.ok_or_else(|| AudioError::Invalid("audio contains no decoded frames".into()))?;
575 let frames = data.len() / channels;
576 Waveform::new(
577 Tensor::f_from_slice(&data)?
578 .f_reshape([frames as i64, channels as i64])?
579 .f_transpose(0, 1)?,
580 rate,
581 0.,
582 )
583}
584pub struct AudioFiles {
586 paths: Vec<PathBuf>,
587 limits: ResourceLimits,
588}
589impl AudioFiles {
590 pub fn new(paths: Vec<PathBuf>, limits: ResourceLimits) -> Result<Self> {
592 limits.check("audio records", paths.len(), limits.max_records)?;
593 Ok(Self { paths, limits })
594 }
595}
596impl Dataset for AudioFiles {
597 fn get_batch_with_context(
598 &self,
599 indices: &[usize],
600 context: &rusttorch_data::WorkerContext,
601 ) -> Result<Vec<Self::Sample>> {
602 indices
603 .iter()
604 .map(|&index| {
605 context.check().map_err(|_| {
606 AudioError::Invalid("sample loading cancelled or timed out".into())
607 })?;
608 self.get(index)
609 })
610 .collect()
611 }
612
613 type Sample = Waveform;
614 type Error = AudioError;
615 fn len(&self) -> usize {
616 self.paths.len()
617 }
618 fn get(&self, index: usize) -> Result<Waveform> {
619 decode_audio(
620 self.paths
621 .get(index)
622 .ok_or_else(|| AudioError::Invalid("audio index out of range".into()))?,
623 self.limits,
624 )
625 }
626}
627impl MemoryFootprint for Waveform {
628 fn resident_bytes(&self) -> usize {
629 self.samples
630 .resident_bytes()
631 .saturating_add(std::mem::size_of::<Self>())
632 }
633}
634impl PinMemory for Waveform {
635 fn pin_memory(mut self, d: Device) -> rusttorch_core::Result<Self> {
636 self.samples = self.samples.pin_memory(d)?;
637 Ok(self)
638 }
639}
640#[derive(Debug)]
642pub struct AudioBatch {
643 pub samples: Tensor,
645 pub mask: Tensor,
647 pub lengths: Vec<usize>,
649 pub sample_rate: u32,
651}
652impl MemoryFootprint for AudioBatch {
653 fn resident_bytes(&self) -> usize {
654 self.samples
655 .resident_bytes()
656 .saturating_add(self.mask.resident_bytes())
657 .saturating_add(
658 self.lengths
659 .capacity()
660 .saturating_mul(std::mem::size_of::<usize>()),
661 )
662 }
663}
664impl PinMemory for AudioBatch {
665 fn pin_memory(mut self, d: Device) -> rusttorch_core::Result<Self> {
666 self.samples = self.samples.pin_memory(d)?;
667 self.mask = self.mask.pin_memory(d)?;
668 Ok(self)
669 }
670}
671#[derive(Clone, Copy, Debug, Default)]
673pub struct PadAudio {
674 pub limits: ResourceLimits,
676}
677impl Collate<Waveform> for PadAudio {
678 type Batch = AudioBatch;
679 type Error = AudioError;
680 fn collate(&mut self, samples: Vec<Waveform>) -> Result<AudioBatch> {
681 self.limits.check(
682 "audio batch records",
683 samples.len(),
684 self.limits.max_records,
685 )?;
686 let first = samples
687 .first()
688 .ok_or_else(|| AudioError::Invalid("empty audio batch".into()))?;
689 first.validate()?;
690 let (rate, c, kind, device, layout) = (
691 first.sample_rate,
692 first.samples.size()[0],
693 first.samples.kind(),
694 first.samples.device(),
695 first.channel_layout.clone(),
696 );
697 let width = samples.iter().map(Waveform::frames).max().unwrap_or(0);
698 check_wave(c as usize, width, rate, self.limits)?;
699 let n = self
700 .limits
701 .tensor_elements(&[samples.len(), c as usize, width])?;
702 let mask_elements = self.limits.tensor_elements(&[samples.len(), width])?;
703 self.limits.check(
704 "padded audio bytes",
705 n.checked_mul(kind.elt_size_in_bytes())
706 .and_then(|bytes| bytes.checked_add(mask_elements))
707 .and_then(|bytes| {
708 samples
709 .len()
710 .checked_mul(std::mem::size_of::<usize>())
711 .and_then(|lengths| bytes.checked_add(lengths))
712 })
713 .ok_or_else(|| AudioError::Invalid("audio batch overflow".into()))?,
714 self.limits.max_decoded_bytes,
715 )?;
716 let mut audio = Vec::new();
717 let mut masks = Vec::new();
718 let mut lengths = Vec::new();
719 for s in samples {
720 s.validate()?;
721 if s.sample_rate != rate
722 || s.channel_layout != layout
723 || s.samples.kind() != kind
724 || s.samples.device() != device
725 {
726 return Err(AudioError::Invalid(
727 "audio batch rates/layouts/dtypes/devices differ".into(),
728 ));
729 }
730 let len = s.frames();
731 lengths.push(len);
732 audio.push(s.samples.f_constant_pad_nd([0, (width - len) as i64])?);
733 masks.push(Tensor::f_arange(width as i64, (Kind::Int64, device))?.f_lt(len as i64)?);
734 }
735 Ok(AudioBatch {
736 samples: Tensor::f_stack(&audio, 0)?,
737 mask: Tensor::f_stack(&masks, 0)?,
738 lengths,
739 sample_rate: rate,
740 })
741 }
742}