use burn::{
Tensor,
config::Config,
module::Module,
prelude::{
Backend,
Int,
},
};
use super::{
WHISPER_DEFAULT_D_MODEL,
WhisperFrontEndConfig,
WhisperTokenLayoutConfig,
};
use crate::{
burner::{
module::ModuleInit,
store::FixPytorchLoadMappers,
},
kits::speech::whisper::blocks::{
AudioEncoder,
AudioEncoderConfig,
AudioEncoderMeta,
TextDecoder,
TextDecoderConfig,
TextDecoderMeta,
},
};
#[derive(Config, Debug)]
pub struct WhisperApiConfig {
pub n_mels: usize,
#[config(default = "WhisperFrontEndConfig::new()")]
pub front_end: WhisperFrontEndConfig,
#[config(default = "WhisperTokenLayoutConfig::new()")]
pub token_layout: WhisperTokenLayoutConfig,
pub vocab_size: usize,
pub d_model: usize,
pub max_audio_ctx: usize,
pub n_encoder_layers: usize,
pub max_text_ctx: usize,
pub n_decoder_layers: usize,
#[config(default = "WHISPER_DEFAULT_D_MODEL")]
pub d_head: usize,
}
impl WhisperApiConfig {
pub fn to_structure(&self) -> WhisperStructuralConfig {
WhisperStructuralConfig {
front_end: self.front_end.clone(),
token_layout: self.token_layout.clone(),
encoder: AudioEncoderConfig::new(
self.n_mels,
self.d_model,
self.max_audio_ctx,
self.n_encoder_layers,
)
.with_d_head(self.d_head),
decoder: TextDecoderConfig::new(
self.vocab_size,
self.d_model,
self.max_text_ctx,
self.n_decoder_layers,
)
.with_d_head(self.d_head),
}
}
}
impl<B: Backend> ModuleInit<B, Whisper<B>> for WhisperApiConfig {
fn try_init(
&self,
device: &B::Device,
) -> crate::errors::BunsenResult<Whisper<B>> {
self.to_structure().try_init(device)
}
}
pub trait WhisperMeta {
fn front_end(&self) -> &WhisperFrontEndConfig;
fn token_layout(&self) -> &WhisperTokenLayoutConfig;
fn sample_rate(&self) -> usize {
self.front_end().sample_rate
}
fn n_mels(&self) -> usize {
self.encoder().n_mels()
}
fn vocab_size(&self) -> usize {
self.decoder().vocab_size()
}
fn d_model(&self) -> usize {
self.encoder().d_model()
}
fn max_audio_ctx(&self) -> usize {
self.encoder().max_context()
}
fn max_text_ctx(&self) -> usize {
self.decoder().max_context()
}
fn encoder(&self) -> &impl AudioEncoderMeta;
fn decoder(&self) -> &impl TextDecoderMeta;
}
#[derive(Config, Debug)]
pub struct WhisperStructuralConfig {
pub front_end: WhisperFrontEndConfig,
pub token_layout: WhisperTokenLayoutConfig,
pub encoder: AudioEncoderConfig,
pub decoder: TextDecoderConfig,
}
impl WhisperMeta for WhisperStructuralConfig {
fn front_end(&self) -> &WhisperFrontEndConfig {
&self.front_end
}
fn token_layout(&self) -> &WhisperTokenLayoutConfig {
&self.token_layout
}
fn encoder(&self) -> &impl AudioEncoderMeta {
&self.encoder
}
fn decoder(&self) -> &impl TextDecoderMeta {
&self.decoder
}
}
impl<B: Backend> ModuleInit<B, Whisper<B>> for WhisperStructuralConfig {
fn try_init(
&self,
device: &B::Device,
) -> crate::errors::BunsenResult<Whisper<B>> {
let encoder = self.encoder.init(device);
let decoder = self.decoder.init(device);
assert_eq!(encoder.d_model(), decoder.d_model());
Ok(Whisper {
front_end: self.front_end.clone(),
token_layout: self.token_layout.clone(),
encoder,
decoder,
})
}
}
#[derive(Module, Debug)]
pub struct Whisper<B: Backend> {
#[module(skip)]
front_end: WhisperFrontEndConfig,
#[module(skip)]
token_layout: WhisperTokenLayoutConfig,
pub encoder: AudioEncoder<B>,
pub decoder: TextDecoder<B>,
}
impl<B: Backend> FixPytorchLoadMappers for Whisper<B> {
fn fix_pytorch_load_mappers(mut self) -> Self {
self.encoder = self.encoder.fix_pytorch_load_mappers();
self.decoder = self.decoder.fix_pytorch_load_mappers();
self
}
}
impl<B: Backend> WhisperMeta for Whisper<B> {
fn front_end(&self) -> &WhisperFrontEndConfig {
&self.front_end
}
fn token_layout(&self) -> &WhisperTokenLayoutConfig {
&self.token_layout
}
fn encoder(&self) -> &impl AudioEncoderMeta {
&self.encoder
}
fn decoder(&self) -> &impl TextDecoderMeta {
&self.decoder
}
}
impl<B: Backend> Whisper<B> {
pub fn forward(
&self,
mel: Tensor<B, 3>,
tokens: Tensor<B, 2, Int>,
) -> Tensor<B, 3> {
self.forward_decoder(tokens, self.forward_encoder(mel))
}
pub fn forward_encoder(
&self,
mel: Tensor<B, 3>,
) -> Tensor<B, 3> {
self.encoder.forward(mel)
}
pub fn forward_decoder(
&self,
tokens: Tensor<B, 2, Int>,
encoder_output: Tensor<B, 3>,
) -> Tensor<B, 3> {
self.decoder.forward(tokens, encoder_output)
}
}
#[cfg(test)]
mod tests {
use burn::{
module::Param,
prelude::{
Device,
TensorData,
},
tensor::DType,
};
use serial_test::serial;
use super::*;
use crate::{
contracts::assert_shape_contract,
support::testing::{
CpuBackend,
PerformanceBackend,
param_load_mapping,
},
};
fn is_mapped(
param: &Param<Tensor<CpuBackend, 2>>,
device: &Device<CpuBackend>,
) -> bool {
let probe: Tensor<CpuBackend, 2> = Tensor::from_data(
TensorData::new((0..6).map(|v| v as f64).collect::<Vec<_>>(), [2, 3]),
device,
);
let flat = |t: Tensor<CpuBackend, 2>| -> Vec<f32> {
t.cast(DType::F32).to_data().to_vec().unwrap()
};
flat(param_load_mapping(param, probe.clone())) != flat(probe)
}
#[test]
fn test_fix_pytorch_load_mappers_covers_the_projections() {
type B = CpuBackend;
let device: Device<B> = Default::default();
let model: Whisper<B> = WhisperApiConfig::new(8, 16, 128, 16, 1, 16, 1)
.try_init(&device)
.unwrap();
assert!(!is_mapped(
&model.encoder.blocks[0].attn.query.weight,
&device
));
let model = model.fix_pytorch_load_mappers();
let encoder = &model.encoder.blocks[0];
let decoder = &model.decoder.blocks[0];
for weight in [
&encoder.attn.query.weight,
&encoder.attn.key.weight,
&encoder.attn.value.weight,
&encoder.attn.output.weight,
&encoder.mlp.linear1.weight,
&encoder.mlp.linear2.weight,
&decoder.attn.query.weight,
&decoder.attn.key.weight,
&decoder.attn.value.weight,
&decoder.attn.output.weight,
&decoder.cross_attn.query.weight,
&decoder.cross_attn.key.weight,
&decoder.cross_attn.value.weight,
&decoder.cross_attn.output.weight,
&decoder.mlp.linear1.weight,
&decoder.mlp.linear2.weight,
] {
assert!(is_mapped(weight, &device));
}
assert!(!is_mapped(&model.encoder.positional_embedding, &device));
assert!(!is_mapped(&model.decoder.positional_embedding, &device));
assert!(!is_mapped(&model.decoder.token_embedding.weight, &device));
}
#[test]
fn test_front_end_and_layout_propagate() {
type B = CpuBackend;
let device: Device<B> = Default::default();
let config = WhisperApiConfig::new(8, 16, 128, 16, 1, 16, 1);
assert_eq!(config.front_end, WhisperFrontEndConfig::new());
assert_eq!(config.token_layout, WhisperTokenLayoutConfig::new());
assert_eq!(config.to_structure().sample_rate(), 16_000);
let model: Whisper<B> = config.try_init(&device).unwrap();
assert_eq!(model.sample_rate(), 16_000);
assert_eq!(model.token_layout().languages.len(), 100);
let config = config
.with_front_end(WhisperFrontEndConfig::new().with_sample_rate(8_000))
.with_token_layout(WhisperTokenLayoutConfig::new().with_timestamp_tokens(751));
assert_eq!(config.to_structure().sample_rate(), 8_000);
let model: Whisper<B> = config.try_init(&device).unwrap();
assert_eq!(model.sample_rate(), 8_000);
assert_eq!(model.front_end().hop(), 80);
assert_eq!(model.token_layout().timestamp_tokens, 751);
}
#[test]
#[serial]
fn test_whisper_forward() {
type B = PerformanceBackend;
let device = Default::default();
let d_model = 128;
let n_mels = 80;
let vocab_size = 64;
let max_audio_ctx = 128;
let n_audio_layers = 2;
let max_text_context = 128;
let n_text_layers = 2;
let config = WhisperApiConfig::new(
n_mels,
vocab_size,
d_model,
max_audio_ctx,
n_audio_layers,
max_text_context,
n_text_layers,
);
let structural = config.to_structure();
let n_audio_heads = structural.encoder.n_heads();
let n_text_heads = structural.decoder.n_heads();
assert_eq!(structural.n_mels(), n_mels);
assert_eq!(structural.vocab_size(), vocab_size);
assert_eq!(structural.d_model(), d_model);
assert_eq!(structural.max_audio_ctx(), max_audio_ctx);
assert_eq!(structural.max_text_ctx(), max_text_context);
assert_eq!(structural.encoder().n_mels(), n_mels);
assert_eq!(structural.encoder().d_model(), d_model);
assert_eq!(structural.encoder().max_context(), max_audio_ctx);
assert_eq!(structural.encoder().n_heads(), n_audio_heads);
assert_eq!(structural.encoder().n_layers(), n_audio_layers);
assert_eq!(structural.decoder().vocab_size(), vocab_size);
assert_eq!(structural.decoder().d_model(), d_model);
assert_eq!(structural.decoder().max_context(), max_text_context);
assert_eq!(structural.decoder().n_heads(), n_text_heads);
assert_eq!(structural.decoder().n_layers(), n_text_layers);
let model: Whisper<B> = structural.try_init(&device).unwrap();
assert_eq!(model.n_mels(), n_mels);
assert_eq!(model.vocab_size(), vocab_size);
assert_eq!(model.d_model(), d_model);
assert_eq!(model.max_audio_ctx(), max_audio_ctx);
assert_eq!(model.max_text_ctx(), max_text_context);
assert_eq!(model.encoder().n_mels(), n_mels);
assert_eq!(model.encoder().d_model(), d_model);
assert_eq!(model.encoder().max_context(), max_audio_ctx);
assert_eq!(model.encoder().n_heads(), n_audio_heads);
assert_eq!(model.encoder().n_layers(), n_audio_layers);
assert_eq!(model.decoder().vocab_size(), vocab_size);
assert_eq!(model.decoder().d_model(), d_model);
assert_eq!(model.decoder().max_context(), max_text_context);
assert_eq!(model.decoder().n_heads(), n_text_heads);
assert_eq!(model.decoder().n_layers(), n_text_layers);
let batch = 2;
let audio_len = max_audio_ctx / 2;
let token_len = audio_len / 2;
let mel: Tensor<B, 3> =
Tensor::random([batch, n_mels, audio_len], Default::default(), &device);
let tokens: Tensor<B, 2, Int> = Tensor::zeros([batch, token_len], &device);
let encoder_output = model.forward_encoder(mel.clone());
assert_shape_contract!(
["batch", "seq", "d_model"],
&encoder_output,
&[
("batch", batch),
("seq", audio_len / 2),
("d_model", d_model),
],
);
let output = model.forward(mel, tokens);
assert_shape_contract!(
["batch", "seq", "vocab_size"],
&output,
&[
("batch", batch),
("seq", token_len),
("vocab_size", vocab_size)
],
);
}
}