Skip to main content

rusttorch_audio/
lib.rs

1//! Decode local audio, resample on CPU, and compute differentiable spectral
2//! features with LibTorch. Samples use floating-point `[channels, frames]`.
3//!
4//! ```
5//! use rusttorch_audio::{Waveform, SpectrogramConfig};
6//! use rusttorch_core::Tensor;
7//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
8//! let wave = Waveform::new(Tensor::from_slice(&[0f32; 16]).reshape([1,16]), 8000, 0.)?;
9//! let power = wave.spectrogram(SpectrogramConfig { n_fft: 8, hop_length: 4, power: 2. }, Default::default())?;
10//! assert_eq!(power.size(), [1,5,3]);
11//! # Ok(()) }
12//! ```
13#![deny(missing_docs)]
14#![doc = include_str!("../README.md")]
15use rand::Rng;
16use rubato::Resampler;
17/// Optional native FFmpeg adapter for formats outside the Rust decoder profile.
18#[cfg(feature = "codec")]
19pub use rusttorch_codec as codec;
20use rusttorch_core::{Device, Kind, Tensor};
21use rusttorch_data::{Collate, Dataset, MemoryFootprint, PinMemory, ResourceLimits, TaskContext};
22use std::{
23    fs::File,
24    io::Cursor,
25    path::{Path, PathBuf},
26};
27use symphonia::core::{
28    codecs::audio::AudioDecoderOptions,
29    formats::{FormatOptions, TrackType, probe::Hint},
30    io::MediaSourceStream,
31    meta::MetadataOptions,
32};
33/// Audio decoding, configuration, resampling or tensor error.
34#[derive(Debug, thiserror::Error)]
35#[non_exhaustive]
36pub enum AudioError {
37    /// Local input failed.
38    #[error(transparent)]
39    Io(#[from] std::io::Error),
40    /// Preserved Symphonia error.
41    #[error(transparent)]
42    Decode(#[from] symphonia::core::errors::Error),
43    /// Invalid format, dimensions or transform parameters.
44    #[error("{0}")]
45    Invalid(String),
46    /// Resource ceiling or shared runtime error.
47    #[error(transparent)]
48    Limit(#[from] rusttorch_core::RustTorchError),
49    /// LibTorch operation failed.
50    #[error(transparent)]
51    Tensor(#[from] tch::TchError),
52    /// Rubato rejected resampler construction.
53    #[error(transparent)]
54    Resampler(#[from] rubato::ResamplerConstructionError),
55    /// Rubato rejected buffer processing.
56    #[error(transparent)]
57    Resample(#[from] rubato::ResampleError),
58}
59/// Result returned by audio operations.
60pub type Result<T> = std::result::Result<T, AudioError>;
61/// Channel ordering carried with decoded samples.
62#[derive(Clone, Debug, PartialEq, Eq)]
63pub enum ChannelLayout {
64    /// One channel.
65    Mono,
66    /// Conventional left/right channels.
67    Stereo,
68    /// Ordered channels without a claimed speaker-position mapping.
69    Discrete(usize),
70}
71/// Owned waveform metadata and floating-point samples.
72#[derive(Debug)]
73pub struct Waveform {
74    /// Float/Double tensor in `[channels,frames]`, usually normalized to `[-1,1]`.
75    pub samples: Tensor,
76    /// Frames per second; nonzero.
77    pub sample_rate: u32,
78    /// Channel ordering. Multi-channel decoding uses `Discrete` unless known.
79    pub channel_layout: ChannelLayout,
80    /// Time of frame zero in seconds, finite and nonnegative.
81    pub time_origin: f64,
82}
83impl Waveform {
84    /// Validates a floating-point waveform and infers mono/stereo channel layout.
85    pub fn new(samples: Tensor, sample_rate: u32, time_origin: f64) -> Result<Self> {
86        let c = samples.size().first().copied().unwrap_or(0);
87        let channel_layout = match c {
88            1 => ChannelLayout::Mono,
89            2 => ChannelLayout::Stereo,
90            _ => ChannelLayout::Discrete(c.max(0) as usize),
91        };
92        let wave = Self {
93            samples,
94            sample_rate,
95            channel_layout,
96            time_origin,
97        };
98        wave.validate()?;
99        Ok(wave)
100    }
101    /// Validates shape, rate, channel count and finite time origin.
102    pub fn validate(&self) -> Result<()> {
103        let s = self.samples.size();
104        let c = match self.channel_layout {
105            ChannelLayout::Mono => 1,
106            ChannelLayout::Stereo => 2,
107            ChannelLayout::Discrete(c) => c,
108        };
109        if s.len() != 2
110            || s[0] <= 0
111            || s[1] < 0
112            || s[0] as usize != c
113            || !matches!(self.samples.kind(), Kind::Float | Kind::Double)
114            || self.sample_rate == 0
115            || !self.time_origin.is_finite()
116            || self.time_origin < 0.
117        {
118            return Err(AudioError::Invalid("expected finite-origin Float/Double [channels,frames], positive rate and matching channel layout".into()));
119        }
120        Ok(())
121    }
122    /// Number of sample frames per channel.
123    pub fn frames(&self) -> usize {
124        self.samples.size().get(1).copied().unwrap_or(0).max(0) as usize
125    }
126    /// Duration in seconds, excluding the time origin.
127    pub fn duration(&self) -> f64 {
128        self.frames() as f64 / self.sample_rate as f64
129    }
130    /// Applies finite decibel gain without mutating the original tensor storage.
131    pub fn gain(mut self, decibels: f64) -> Result<Self> {
132        self.validate()?;
133        let factor = 10f64.powf(decibels / 20.);
134        if !decibels.is_finite() || !factor.is_finite() || (factor as f32).is_infinite() {
135            return Err(AudioError::Invalid(
136                "gain must have finite representable amplitude".into(),
137            ));
138        }
139        self.samples = self.samples.f_mul_scalar(factor)?;
140        Ok(self)
141    }
142    /// Adds deterministic uniform noise in `[-amplitude, amplitude]` using task
143    /// randomness. Does not read or reseed LibTorch's global random generator.
144    pub fn add_noise(
145        mut self,
146        amplitude: f32,
147        context: &TaskContext,
148        limits: ResourceLimits,
149    ) -> Result<Self> {
150        self.validate()?;
151        if !amplitude.is_finite() || amplitude < 0. {
152            return Err(AudioError::Invalid(
153                "noise amplitude must be finite and nonnegative".into(),
154            ));
155        }
156        context
157            .check()
158            .map_err(|_| AudioError::Invalid("audio transform cancelled".into()))?;
159        check_wave(
160            self.samples.size()[0] as usize,
161            self.frames(),
162            self.sample_rate,
163            limits,
164        )?;
165        let n = limits.tensor_elements(&[self.samples.size()[0] as usize, self.frames()])?;
166        let mut rng = context.rng();
167        let noise = (0..n)
168            .map(|_| {
169                if amplitude == 0. {
170                    0.
171                } else {
172                    rng.gen_range(-1f32..=1f32) * amplitude
173                }
174            })
175            .collect::<Vec<_>>();
176        self.samples = self.samples.f_add(
177            &Tensor::f_from_slice(&noise)?
178                .f_reshape(self.samples.size())?
179                .f_to_kind(self.samples.kind())?
180                .f_to_device(self.samples.device())?,
181        )?;
182        Ok(self)
183    }
184    /// Shifts by signed frames with zero fill, preserving duration and time origin.
185    pub fn time_shift(mut self, frames: i64) -> Result<Self> {
186        self.validate()?;
187        let n = self.frames() as i64;
188        let shift = frames.unsigned_abs().min(n as u64) as i64;
189        if shift == n {
190            self.samples = self.samples.f_zeros_like()?;
191        } else if frames >= 0 {
192            self.samples = self
193                .samples
194                .f_narrow(1, 0, n - shift)?
195                .f_constant_pad_nd([shift, 0])?;
196        } else {
197            self.samples = self
198                .samples
199                .f_narrow(1, shift, n - shift)?
200                .f_constant_pad_nd([0, shift])?;
201        }
202        Ok(self)
203    }
204    /// Mixes aligned waveforms with a finite scale. Rate, shape, layout, origin,
205    /// dtype and device must match; there is no implicit broadcast or resampling.
206    pub fn mix(mut self, other: &Self, scale: f64) -> Result<Self> {
207        self.validate()?;
208        other.validate()?;
209        if !scale.is_finite()
210            || self.sample_rate != other.sample_rate
211            || self.channel_layout != other.channel_layout
212            || self.time_origin != other.time_origin
213            || self.samples.size() != other.samples.size()
214            || self.samples.kind() != other.samples.kind()
215            || self.samples.device() != other.samples.device()
216        {
217            return Err(AudioError::Invalid(
218                "mix requires aligned waveforms and finite scale".into(),
219            ));
220        }
221        self.samples = self.samples.f_add(&other.samples.f_mul_scalar(scale)?)?;
222        Ok(self)
223    }
224    /// Resamples on CPU using Rubato's windowed-sinc interpolation, removing its
225    /// startup delay. This host conversion is not differentiable. Output has
226    /// `ceil(input_frames * new_rate / old_rate)` frames and preserves origin.
227    pub fn resample(self, new_rate: u32, limits: ResourceLimits) -> Result<Self> {
228        self.validate()?;
229        if new_rate == 0 {
230            return Err(AudioError::Invalid("resample rate must be positive".into()));
231        }
232        if self.samples.device() != Device::Cpu {
233            return Err(AudioError::Invalid(
234                "Rubato resampling requires a CPU waveform".into(),
235            ));
236        }
237        if self.samples.requires_grad() {
238            return Err(AudioError::Invalid(
239                "Rubato resampling cannot preserve autograd; detach explicitly".into(),
240            ));
241        }
242        if new_rate == self.sample_rate {
243            return Ok(self);
244        }
245        let channels = self.samples.size()[0] as usize;
246        let frames = self.frames();
247        check_wave(channels, frames, self.sample_rate, limits)?;
248        let expected = ((frames as u128) * (new_rate as u128)).div_ceil(self.sample_rate as u128);
249        let expected = usize::try_from(expected)
250            .map_err(|_| AudioError::Invalid("resampled length overflow".into()))?;
251        check_wave(channels, expected, new_rate, limits)?;
252        if frames == 0 {
253            return Self::new(self.samples, new_rate, self.time_origin);
254        }
255        let mut resampler = rubato::Async::<f64>::new_sinc(
256            new_rate as f64 / self.sample_rate as f64,
257            1.,
258            &rubato::SincInterpolationParameters::default(),
259            1024,
260            channels,
261            rubato::FixedAsync::Input,
262        )?;
263        let needed = resampler.process_all_needed_output_len(frames);
264        check_wave(channels, needed, new_rate, limits)?;
265        let data = Vec::<f64>::try_from(
266            &self
267                .samples
268                .f_to_kind(Kind::Double)?
269                .f_transpose(0, 1)?
270                .f_contiguous()?
271                .f_view([-1])?,
272        )?;
273        let input = audioadapter_buffers::owned::InterleavedOwned::new_from(data, channels, frames)
274            .map_err(|e| AudioError::Invalid(e.to_string()))?;
275        let output = resampler.process_all(&input, frames, None)?.take_data();
276        if output.len() != expected * channels {
277            return Err(AudioError::Invalid(
278                "Rubato returned unexpected output length".into(),
279            ));
280        }
281        let samples = Tensor::f_from_slice(&output)?
282            .f_reshape([expected as i64, channels as i64])?
283            .f_transpose(0, 1)?
284            .f_to_kind(self.samples.kind())?;
285        let mut value = Self::new(samples, new_rate, self.time_origin)?;
286        value.channel_layout = self.channel_layout;
287        Ok(value)
288    }
289    /// Computes one-sided periodic-Hann STFT magnitudes raised to `power`.
290    /// Frames are uncentered with no padding: only complete windows are emitted.
291    /// The result is `[channels, n_fft/2+1, windows]` and retains autograd.
292    pub fn spectrogram(&self, config: SpectrogramConfig, limits: ResourceLimits) -> Result<Tensor> {
293        self.validate()?;
294        config.validate()?;
295        let frames = self.frames();
296        if frames < config.n_fft {
297            return Err(AudioError::Invalid(
298                "waveform shorter than FFT window".into(),
299            ));
300        }
301        let windows = 1 + (frames - config.n_fft) / config.hop_length;
302        let n = limits.tensor_elements(&[
303            self.samples.size()[0] as usize,
304            config.n_fft / 2 + 1,
305            windows,
306        ])?;
307        limits.check(
308            "spectrogram bytes",
309            n.checked_mul(16)
310                .ok_or_else(|| AudioError::Invalid("spectrogram size overflow".into()))?,
311            limits.max_decoded_bytes,
312        )?;
313        let window = Tensor::f_hann_window(
314            config.n_fft as i64,
315            (self.samples.kind(), self.samples.device()),
316        )?;
317        Ok(self
318            .samples
319            .f_stft_center(
320                config.n_fft as i64,
321                config.hop_length as i64,
322                config.n_fft as i64,
323                Some(&window),
324                false,
325                "reflect",
326                false,
327                true,
328                true,
329                false,
330            )?
331            .f_abs()?
332            .f_pow_tensor_scalar(config.power)?)
333    }
334    /// Projects the spectrogram through HTK triangular Mel filters. Frequencies
335    /// span `[0, sample_rate/2]`, with no Slaney area normalization.
336    pub fn mel_spectrogram(
337        &self,
338        config: SpectrogramConfig,
339        n_mels: usize,
340        limits: ResourceLimits,
341    ) -> Result<Tensor> {
342        let spectrum = self.spectrogram(config, limits)?;
343        let output = limits.tensor_elements(&[
344            self.samples.size()[0] as usize,
345            n_mels,
346            spectrum.size()[2] as usize,
347        ])?;
348        limits.check(
349            "Mel output bytes",
350            output
351                .checked_mul(8)
352                .ok_or_else(|| AudioError::Invalid("Mel output size overflow".into()))?,
353            limits.max_decoded_bytes,
354        )?;
355        let filter = mel_filter_bank(config.n_fft, self.sample_rate, n_mels, limits)?
356            .f_to_kind(self.samples.kind())?
357            .f_to_device(self.samples.device())?;
358        Ok(filter.f_matmul(&spectrum)?)
359    }
360    /// Computes orthonormal DCT-II of natural-log Mel energies with floor `1e-10`.
361    /// This is the explicitly named log-Mel convention, without dB/top-db scaling.
362    pub fn mfcc(
363        &self,
364        config: SpectrogramConfig,
365        n_mels: usize,
366        n_mfcc: usize,
367        limits: ResourceLimits,
368    ) -> Result<Tensor> {
369        if n_mfcc == 0 || n_mfcc > n_mels {
370            return Err(AudioError::Invalid(
371                "MFCC count must be in 1..=n_mels".into(),
372            ));
373        }
374        let coefficients = limits.tensor_elements(&[n_mfcc, n_mels])?;
375        limits.check(
376            "DCT bytes",
377            coefficients
378                .checked_mul(8)
379                .ok_or_else(|| AudioError::Invalid("DCT size overflow".into()))?,
380            limits.max_decoded_bytes,
381        )?;
382        let mel = self
383            .mel_spectrogram(config, n_mels, limits)?
384            .f_clamp_min(1e-10)?
385            .f_log()?;
386        let mut dct = Vec::with_capacity(n_mfcc * n_mels);
387        for k in 0..n_mfcc {
388            for n in 0..n_mels {
389                let scale = if k == 0 {
390                    (1. / n_mels as f64).sqrt()
391                } else {
392                    (2. / n_mels as f64).sqrt()
393                };
394                dct.push(
395                    scale
396                        * (std::f64::consts::PI * (n as f64 + 0.5) * k as f64 / n_mels as f64)
397                            .cos(),
398                );
399            }
400        }
401        Ok(Tensor::f_from_slice(&dct)?
402            .f_reshape([n_mfcc as i64, n_mels as i64])?
403            .f_to_kind(self.samples.kind())?
404            .f_to_device(self.samples.device())?
405            .f_matmul(&mel)?)
406    }
407}
408fn check_wave(channels: usize, frames: usize, rate: u32, limits: ResourceLimits) -> Result<()> {
409    if channels == 0 || channels > 64 || rate == 0 {
410        return Err(AudioError::Invalid(
411            "expected 1..=64 channels and positive sample rate".into(),
412        ));
413    }
414    let n = limits.tensor_elements(&[channels, frames])?;
415    limits.check(
416        "decoded waveform bytes",
417        n.checked_mul(8)
418            .ok_or_else(|| AudioError::Invalid("waveform byte overflow".into()))?,
419        limits.max_decoded_bytes,
420    )?;
421    if frames as u128 > (rate as u128) * (limits.max_media_seconds as u128) {
422        return Err(AudioError::Invalid(
423            "audio duration exceeds resource limit".into(),
424        ));
425    }
426    Ok(())
427}
428/// Uncentered STFT configuration, with periodic Hann window and no normalization.
429#[derive(Clone, Copy, Debug)]
430pub struct SpectrogramConfig {
431    /// FFT/window length, at least two.
432    pub n_fft: usize,
433    /// Hop length, positive.
434    pub hop_length: usize,
435    /// Positive finite magnitude exponent, typically 1 or 2.
436    pub power: f64,
437}
438impl Default for SpectrogramConfig {
439    fn default() -> Self {
440        Self {
441            n_fft: 400,
442            hop_length: 200,
443            power: 2.,
444        }
445    }
446}
447impl SpectrogramConfig {
448    fn validate(&self) -> Result<()> {
449        if self.n_fft < 2
450            || self.n_fft > i64::MAX as usize
451            || self.hop_length == 0
452            || self.hop_length > i64::MAX as usize
453            || !self.power.is_finite()
454            || self.power <= 0.
455        {
456            return Err(AudioError::Invalid("invalid FFT size, hop or power".into()));
457        }
458        Ok(())
459    }
460}
461/// Builds Double `[n_mels,n_fft/2+1]` HTK triangular filters on CPU.
462pub fn mel_filter_bank(
463    n_fft: usize,
464    sample_rate: u32,
465    n_mels: usize,
466    limits: ResourceLimits,
467) -> Result<Tensor> {
468    if n_fft < 2 || sample_rate == 0 || n_mels == 0 {
469        return Err(AudioError::Invalid(
470            "positive mel dimensions and rate required".into(),
471        ));
472    }
473    let bins = n_fft / 2 + 1;
474    let n = limits.tensor_elements(&[n_mels, bins])?;
475    limits.check(
476        "Mel filter bytes",
477        n.checked_mul(8)
478            .ok_or_else(|| AudioError::Invalid("Mel filter byte overflow".into()))?,
479        limits.max_decoded_bytes,
480    )?;
481    let max_mel = 2595. * (1. + (sample_rate as f64 / 2.) / 700.).log10();
482    let points = (0..n_mels + 2)
483        .map(|i| 700. * (10f64.powf((i as f64 * max_mel / (n_mels + 1) as f64) / 2595.) - 1.))
484        .collect::<Vec<_>>();
485    let mut values = Vec::with_capacity(n);
486    for m in 0..n_mels {
487        for k in 0..bins {
488            let hz = k as f64 * sample_rate as f64 / n_fft as f64;
489            let up = (hz - points[m]) / (points[m + 1] - points[m]);
490            let down = (points[m + 2] - hz) / (points[m + 2] - points[m + 1]);
491            values.push(up.min(down).max(0.));
492        }
493    }
494    Ok(Tensor::f_from_slice(&values)?.f_reshape([n_mels as i64, bins as i64])?)
495}
496/// Masks a contiguous frequency/time axis with a finite constant, without
497/// changing the input's storage. Axis and interval must be in bounds.
498pub fn mask_axis(
499    input: &Tensor,
500    axis: usize,
501    start: usize,
502    width: usize,
503    value: f64,
504) -> Result<Tensor> {
505    let shape = input.size();
506    if axis >= shape.len()
507        || start > shape[axis] as usize
508        || width > shape[axis] as usize - start
509        || !value.is_finite()
510    {
511        return Err(AudioError::Invalid(
512            "invalid mask axis, interval or value".into(),
513        ));
514    }
515    let mut output = input.f_empty_like()?;
516    output.f_copy_(input)?;
517    let _ = output
518        .f_narrow(axis as i64, start as i64, width as i64)?
519        .f_fill_(value)?;
520    Ok(output)
521}
522/// Decodes bounded local WAV/FLAC through Symphonia into CPU Float planar audio.
523/// Decoder errors are returned, never skipped; sample-rate/layout changes fail.
524pub fn decode_audio(path: impl AsRef<Path>, limits: ResourceLimits) -> Result<Waveform> {
525    let bytes = limits.read_encoded(File::open(path.as_ref())?)?;
526    let mss = MediaSourceStream::new(Box::new(Cursor::new(bytes)), Default::default());
527    let mut format = symphonia::default::get_probe().probe(
528        &Hint::new(),
529        mss,
530        FormatOptions::default(),
531        MetadataOptions::default(),
532    )?;
533    limits.check("audio streams", format.tracks().len(), limits.max_streams)?;
534    let track = format
535        .default_track(TrackType::Audio)
536        .ok_or_else(|| AudioError::Invalid("no audio track".into()))?;
537    let id = track.id;
538    let params = track
539        .codec_params
540        .as_ref()
541        .and_then(|p| p.audio())
542        .ok_or_else(|| AudioError::Invalid("missing audio codec parameters".into()))?;
543    let mut decoder = symphonia::default::get_codecs()
544        .make_audio_decoder(params, &AudioDecoderOptions::default())?;
545    let mut data = Vec::<f32>::new();
546    let mut spec = None;
547    while let Some(packet) = format.next_packet()? {
548        if packet.track_id != id {
549            continue;
550        }
551        let decoded = decoder.decode(&packet)?;
552        let rate = decoded.spec().rate();
553        let channels = decoded.spec().channels().count();
554        check_wave(channels, 0, rate, limits)?;
555        if let Some(previous) = spec {
556            if previous != (rate, channels) {
557                return Err(AudioError::Invalid(
558                    "audio rate or channel count changes within stream".into(),
559                ));
560            }
561        } else {
562            spec = Some((rate, channels));
563        }
564        let next = data
565            .len()
566            .checked_add(decoded.samples_interleaved())
567            .ok_or_else(|| AudioError::Invalid("decoded sample count overflow".into()))?;
568        check_wave(channels, next / channels, rate, limits)?;
569        let old = data.len();
570        data.resize(next, 0.);
571        decoded.copy_to_slice_interleaved(&mut data[old..]);
572    }
573    let (rate, channels) =
574        spec.ok_or_else(|| AudioError::Invalid("audio contains no decoded frames".into()))?;
575    let frames = data.len() / channels;
576    Waveform::new(
577        Tensor::f_from_slice(&data)?
578            .f_reshape([frames as i64, channels as i64])?
579            .f_transpose(0, 1)?,
580        rate,
581        0.,
582    )
583}
584/// Local audio files decoded independently by loader workers.
585pub struct AudioFiles {
586    paths: Vec<PathBuf>,
587    limits: ResourceLimits,
588}
589impl AudioFiles {
590    /// Creates a bounded file list. Files are opened lazily in `get`.
591    pub fn new(paths: Vec<PathBuf>, limits: ResourceLimits) -> Result<Self> {
592        limits.check("audio records", paths.len(), limits.max_records)?;
593        Ok(Self { paths, limits })
594    }
595}
596impl Dataset for AudioFiles {
597    fn get_batch_with_context(
598        &self,
599        indices: &[usize],
600        context: &rusttorch_data::WorkerContext,
601    ) -> Result<Vec<Self::Sample>> {
602        indices
603            .iter()
604            .map(|&index| {
605                context.check().map_err(|_| {
606                    AudioError::Invalid("sample loading cancelled or timed out".into())
607                })?;
608                self.get(index)
609            })
610            .collect()
611    }
612
613    type Sample = Waveform;
614    type Error = AudioError;
615    fn len(&self) -> usize {
616        self.paths.len()
617    }
618    fn get(&self, index: usize) -> Result<Waveform> {
619        decode_audio(
620            self.paths
621                .get(index)
622                .ok_or_else(|| AudioError::Invalid("audio index out of range".into()))?,
623            self.limits,
624        )
625    }
626}
627impl MemoryFootprint for Waveform {
628    fn resident_bytes(&self) -> usize {
629        self.samples
630            .resident_bytes()
631            .saturating_add(std::mem::size_of::<Self>())
632    }
633}
634impl PinMemory for Waveform {
635    fn pin_memory(mut self, d: Device) -> rusttorch_core::Result<Self> {
636        self.samples = self.samples.pin_memory(d)?;
637        Ok(self)
638    }
639}
640/// Zero-padded equal-rate waveform batch.
641#[derive(Debug)]
642pub struct AudioBatch {
643    /// `[batch,channels,max_frames]` Float/Double audio.
644    pub samples: Tensor,
645    /// True for real frames, shape `[batch,max_frames]`.
646    pub mask: Tensor,
647    /// Original frames per sample.
648    pub lengths: Vec<usize>,
649    /// Common frame rate.
650    pub sample_rate: u32,
651}
652impl MemoryFootprint for AudioBatch {
653    fn resident_bytes(&self) -> usize {
654        self.samples
655            .resident_bytes()
656            .saturating_add(self.mask.resident_bytes())
657            .saturating_add(
658                self.lengths
659                    .capacity()
660                    .saturating_mul(std::mem::size_of::<usize>()),
661            )
662    }
663}
664impl PinMemory for AudioBatch {
665    fn pin_memory(mut self, d: Device) -> rusttorch_core::Result<Self> {
666        self.samples = self.samples.pin_memory(d)?;
667        self.mask = self.mask.pin_memory(d)?;
668        Ok(self)
669    }
670}
671/// Pads waveforms after validating equal rate/layout/dtype/device and byte limits.
672#[derive(Clone, Copy, Debug, Default)]
673pub struct PadAudio {
674    /// Limits for the resulting padded allocation.
675    pub limits: ResourceLimits,
676}
677impl Collate<Waveform> for PadAudio {
678    type Batch = AudioBatch;
679    type Error = AudioError;
680    fn collate(&mut self, samples: Vec<Waveform>) -> Result<AudioBatch> {
681        self.limits.check(
682            "audio batch records",
683            samples.len(),
684            self.limits.max_records,
685        )?;
686        let first = samples
687            .first()
688            .ok_or_else(|| AudioError::Invalid("empty audio batch".into()))?;
689        first.validate()?;
690        let (rate, c, kind, device, layout) = (
691            first.sample_rate,
692            first.samples.size()[0],
693            first.samples.kind(),
694            first.samples.device(),
695            first.channel_layout.clone(),
696        );
697        let width = samples.iter().map(Waveform::frames).max().unwrap_or(0);
698        check_wave(c as usize, width, rate, self.limits)?;
699        let n = self
700            .limits
701            .tensor_elements(&[samples.len(), c as usize, width])?;
702        let mask_elements = self.limits.tensor_elements(&[samples.len(), width])?;
703        self.limits.check(
704            "padded audio bytes",
705            n.checked_mul(kind.elt_size_in_bytes())
706                .and_then(|bytes| bytes.checked_add(mask_elements))
707                .and_then(|bytes| {
708                    samples
709                        .len()
710                        .checked_mul(std::mem::size_of::<usize>())
711                        .and_then(|lengths| bytes.checked_add(lengths))
712                })
713                .ok_or_else(|| AudioError::Invalid("audio batch overflow".into()))?,
714            self.limits.max_decoded_bytes,
715        )?;
716        let mut audio = Vec::new();
717        let mut masks = Vec::new();
718        let mut lengths = Vec::new();
719        for s in samples {
720            s.validate()?;
721            if s.sample_rate != rate
722                || s.channel_layout != layout
723                || s.samples.kind() != kind
724                || s.samples.device() != device
725            {
726                return Err(AudioError::Invalid(
727                    "audio batch rates/layouts/dtypes/devices differ".into(),
728                ));
729            }
730            let len = s.frames();
731            lengths.push(len);
732            audio.push(s.samples.f_constant_pad_nd([0, (width - len) as i64])?);
733            masks.push(Tensor::f_arange(width as i64, (Kind::Int64, device))?.f_lt(len as i64)?);
734        }
735        Ok(AudioBatch {
736            samples: Tensor::f_stack(&audio, 0)?,
737            mask: Tensor::f_stack(&masks, 0)?,
738            lengths,
739            sample_rate: rate,
740        })
741    }
742}