use burn::{
Tensor,
config::Config,
module::{
Module,
Param,
},
nn::{
LayerNorm,
LayerNormConfig,
PaddingConfig1d,
conv::Conv1dConfig,
},
prelude::s,
tensor::{
Device,
Distribution,
FloatDType,
},
};
use super::WHISPER_DEFAULT_D_MODEL;
use crate::{
blocks::{
activation::BunsenActivationConfig,
conv::{
ConvBlock1dConfig,
ConvSeq1d,
ConvSeq1dConfig,
ConvSeq1dMeta,
},
transformers::encoders::residual::{
ResidualEncoderAttentionBlock,
ResidualEncoderAttentionBlockConfig,
ResidualEncoderAttentionBlockMeta,
},
},
burner::module::{
CanonicalKeys,
CoercesInput,
InputDTypes,
ModuleInit,
Rekey,
ResultDType,
},
errors::BunsenResult,
};
pub const AUDIO_ENCODER_STRIDE: usize = 2;
pub trait AudioEncoderMeta {
fn n_mels(&self) -> usize;
fn max_context(&self) -> usize;
fn d_model(&self) -> usize;
fn n_heads(&self) -> usize;
fn n_layers(&self) -> usize;
}
#[derive(Config, Debug)]
pub struct AudioEncoderConfig {
pub n_mels: usize,
pub d_model: usize,
pub max_context: usize,
pub n_layers: usize,
#[config(defaul_value = "WHISPER_DEFAULT_D_MODEL")]
pub d_head: usize,
#[config(default = "BunsenActivationConfig::Gelu")]
pub head_activation: BunsenActivationConfig,
}
impl AudioEncoderMeta for AudioEncoderConfig {
fn n_mels(&self) -> usize {
self.n_mels
}
fn d_model(&self) -> usize {
self.d_model
}
fn max_context(&self) -> usize {
self.max_context
}
fn n_heads(&self) -> usize {
self.d_model / self.d_head
}
fn n_layers(&self) -> usize {
self.n_layers
}
}
impl ModuleInit<AudioEncoder> for AudioEncoderConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<AudioEncoder> {
let pos_ctx = self.max_context / AUDIO_ENCODER_STRIDE;
Ok(AudioEncoder {
head: ConvSeq1dConfig::new(vec![
ConvBlock1dConfig::new(
Conv1dConfig::new(self.n_mels, self.d_model, 3)
.with_padding(PaddingConfig1d::Explicit(1, 1)),
),
ConvBlock1dConfig::new(
Conv1dConfig::new(self.d_model, self.d_model, 3)
.with_padding(PaddingConfig1d::Explicit(1, 1))
.with_stride(AUDIO_ENCODER_STRIDE),
),
])
.with_act(self.head_activation.clone())
.try_init(device)?,
positional_embedding: Param::from_tensor(Tensor::random(
[pos_ctx, self.d_model],
Distribution::Normal(0.0, 1.0),
device,
)),
blocks: (0..self.n_layers)
.map(|_| {
ResidualEncoderAttentionBlockConfig::new(self.d_model)
.with_d_head(self.d_head)
.try_init(device)
})
.collect::<BunsenResult<Vec<ResidualEncoderAttentionBlock>>>()?,
ln_post: LayerNormConfig::new(self.d_model).init(device),
})
}
}
#[derive(Module, Debug)]
pub struct AudioEncoder {
pub head: ConvSeq1d,
pub positional_embedding: Param<Tensor<2>>,
pub blocks: Vec<ResidualEncoderAttentionBlock>,
pub ln_post: LayerNorm,
}
impl AudioEncoderMeta for AudioEncoder {
fn n_mels(&self) -> usize {
self.head.in_channels()
}
fn d_model(&self) -> usize {
self.blocks[0].d_model()
}
fn max_context(&self) -> usize {
let pos_ctx = self.positional_embedding.val().dims()[0];
pos_ctx * AUDIO_ENCODER_STRIDE
}
fn n_heads(&self) -> usize {
self.blocks[0].n_heads()
}
fn n_layers(&self) -> usize {
self.blocks.len()
}
}
impl InputDTypes for AudioEncoder {
fn preferred_dtype(&self) -> FloatDType {
FloatDType::from(self.positional_embedding.dtype())
}
}
impl CoercesInput for AudioEncoder {}
impl ResultDType for AudioEncoder {
fn result_dtype(
&self,
_input: FloatDType,
) -> FloatDType {
self.preferred_dtype()
}
}
impl AudioEncoder {
pub fn forward(
&self,
x: Tensor<3>,
) -> Tensor<3> {
let x = self.forward_head(x);
let mut x = self.embed(x);
for b in self.blocks.iter() {
x = b.forward(x);
}
self.ln_post.forward(x)
}
pub fn forward_head(
&self,
x: Tensor<3>,
) -> Tensor<3> {
let x = self.coerce_input(x);
#[cfg(any(debug_assertions, test))]
let [batch] = crate::contracts::unpack_shape_contract!(
["batch", "n_mels", "seq_len"],
&x,
&["batch"],
&[("n_mels", self.n_mels())],
);
let seq_len = x.dims()[2];
assert!(
seq_len <= self.max_context(),
"Audio length {} cannot exceed {}.",
seq_len,
self.max_context()
);
let x = self.head.forward(x);
let x = x.swap_dims(1, 2);
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "len", "d_model"],
&x,
&[
("batch", batch),
("len", self.head.try_output_length(seq_len).unwrap()),
("d_model", self.d_model()),
],
);
x
}
pub fn embed(
&self,
x: Tensor<3>,
) -> Tensor<3> {
let k = x.dims()[1];
x + self
.positional_embedding
.val()
.slice(s![0..k])
.unsqueeze::<3>()
}
}
impl CanonicalKeys for AudioEncoder {
fn canonical_rekey(&self) -> Rekey {
Rekey::keep_rest().mount_canonical("blocks.", &self.blocks)
}
}
#[cfg(test)]
mod tests {
use serial_test::serial;
use super::*;
use crate::{
contracts::assert_shape_contract,
support::testing::{
DeviceMemoryGuard,
performance_device,
},
};
#[test]
#[serial]
fn test_audio_encoder_forward() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let d_model = 128;
let n_mels = 256;
let max_audio_ctx = 100;
let n_audio_layers = 2;
let config = AudioEncoderConfig::new(n_mels, d_model, max_audio_ctx, n_audio_layers);
let n_audio_heads = config.n_heads();
assert_eq!(config.n_mels(), n_mels);
assert_eq!(config.d_model(), d_model);
assert_eq!(config.max_context(), max_audio_ctx);
assert_eq!(config.n_heads(), n_audio_heads);
assert_eq!(config.n_layers(), n_audio_layers);
let encoder: AudioEncoder = config.init(&device);
assert_eq!(encoder.n_mels(), n_mels);
assert_eq!(encoder.d_model(), d_model);
assert_eq!(encoder.max_context(), max_audio_ctx);
assert_eq!(encoder.n_heads(), n_audio_heads);
assert_eq!(encoder.n_layers(), n_audio_layers);
let batch = 2;
let k = max_audio_ctx / 2;
let x: Tensor<3> = Tensor::random([batch, n_mels, k], Default::default(), &device);
let output = encoder.forward(x.clone());
assert_shape_contract!(
["batch", "seq", "d_model"],
&output,
&[("batch", batch), ("seq", k / 2), ("d_model", d_model),],
);
}
#[test]
fn test_encoder_coerces_its_input() {
use burn::{
module::Module,
tensor::{
DType,
Distribution,
},
};
use crate::{
burner::module::DTypeMapper,
support::testing::{
assert_coerces_input,
cpu_device,
},
};
let device = cpu_device();
let encoder: AudioEncoder = AudioEncoderConfig::new(8, 64, 16, 1).init(&device);
let encoder = encoder.map(&mut DTypeMapper::new(DType::F16));
let mel: Tensor<3> = Tensor::random([1, 8, 16], Distribution::Default, &device);
assert_coerces_input(&encoder, mel, |e, x| e.forward(x), 0.0);
}
}