use burn::{
Tensor,
config::Config,
module::Module,
nn::{
LayerNorm,
LayerNormConfig,
Linear,
LinearConfig,
PaddingConfig2d,
activation::ActivationConfig,
conv::Conv2dConfig,
},
prelude::{
Bool,
Int,
TensorData,
},
tensor::{
Device,
activation::gelu,
},
};
use crate::{
blocks::{
conv::{
ConvBlock2dConfig,
ConvSeq2d,
ConvSeq2dConfig,
},
transformers::encoders::residual::{
ResidualEncoderAttentionBlock,
ResidualEncoderAttentionBlockConfig,
},
},
burner::module::ModuleInit,
errors::BunsenResult,
ops::embedding::sinusoid_table,
};
pub fn post_conv_len(len: usize) -> usize {
(0..3).fold(len, |l, _| if l == 0 { 0 } else { (l - 1) / 2 + 1 })
}
pub trait Qwen3AsrAudioEncoderMeta {
fn d_model(&self) -> usize;
fn n_layers(&self) -> usize;
fn chunk_frames(&self) -> usize;
fn window_chunks(&self) -> usize;
fn chunk_out(&self) -> usize {
post_conv_len(self.chunk_frames())
}
fn window_frames(&self) -> usize {
self.chunk_out() * self.window_chunks()
}
fn output_len(
&self,
frames: usize,
) -> usize {
let chunk = self.chunk_frames();
(frames / chunk) * self.chunk_out() + post_conv_len(frames % chunk)
}
}
#[derive(Config, Debug)]
pub struct Qwen3AsrAudioEncoderConfig {
pub n_mels: usize,
pub d_model: usize,
pub n_heads: usize,
pub d_ff: usize,
pub n_layers: usize,
pub conv_channels: usize,
#[config(default = "50")]
pub n_window: usize,
#[config(default = "800")]
pub n_window_infer: usize,
}
impl Qwen3AsrAudioEncoderConfig {
pub fn qwen3_asr_0_6b() -> Self {
Self::new(128, 896, 14, 3584, 18, 480)
}
pub fn qwen3_asr_1_7b() -> Self {
Self::new(128, 1024, 16, 4096, 24, 480)
}
pub fn freq_out(&self) -> usize {
post_conv_len(self.n_mels)
}
}
impl Qwen3AsrAudioEncoderMeta for Qwen3AsrAudioEncoderConfig {
fn d_model(&self) -> usize {
self.d_model
}
fn n_layers(&self) -> usize {
self.n_layers
}
fn chunk_frames(&self) -> usize {
2 * self.n_window
}
fn window_chunks(&self) -> usize {
self.n_window_infer / (2 * self.n_window)
}
}
impl ModuleInit<Qwen3AsrAudioEncoder> for Qwen3AsrAudioEncoderConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<Qwen3AsrAudioEncoder> {
let conv = |c_in| {
ConvBlock2dConfig::new(
Conv2dConfig::new([c_in, self.conv_channels], [3, 3])
.with_stride([2, 2])
.with_padding(PaddingConfig2d::Explicit(1, 1, 1, 1)),
)
.with_act(Some(ActivationConfig::Gelu))
};
let layers = (0..self.n_layers)
.map(|_| {
ResidualEncoderAttentionBlockConfig::new(self.d_model)
.with_d_head(self.d_model / self.n_heads)
.with_expansion_factor(self.d_ff / self.d_model)
.with_key_bias(true)
.try_init(device)
})
.collect::<BunsenResult<_>>()?;
Ok(Qwen3AsrAudioEncoder {
stem: ConvSeq2dConfig::new(vec![
conv(1),
conv(self.conv_channels),
conv(self.conv_channels),
])
.try_init(device)?,
conv_out: LinearConfig::new(self.conv_channels * self.freq_out(), self.d_model)
.with_bias(false)
.init(device),
layers,
ln_post: LayerNormConfig::new(self.d_model).init(device),
n_window: self.n_window,
n_window_infer: self.n_window_infer,
})
}
}
#[derive(Module, Debug)]
pub struct Qwen3AsrAudioEncoder {
pub stem: ConvSeq2d,
pub conv_out: Linear,
pub layers: Vec<ResidualEncoderAttentionBlock>,
pub ln_post: LayerNorm,
pub n_window: usize,
pub n_window_infer: usize,
}
impl Qwen3AsrAudioEncoderMeta for Qwen3AsrAudioEncoder {
fn d_model(&self) -> usize {
self.ln_post.gamma.dims()[0]
}
fn n_layers(&self) -> usize {
self.layers.len()
}
fn chunk_frames(&self) -> usize {
2 * self.n_window
}
fn window_chunks(&self) -> usize {
self.n_window_infer / (2 * self.n_window)
}
}
impl Qwen3AsrAudioEncoder {
pub fn stem(
&self,
features: Tensor<2>,
valid: usize,
) -> Tensor<2> {
let [n_mels, frames] = crate::contracts::unpack_shape_contract!(
["n_mels", "frames"],
&features.dims(),
&["n_mels", "frames"]
);
let device = features.device();
let chunk = self.chunk_frames();
let chunks = valid.div_ceil(chunk).max(1);
let features = if frames < chunks * chunk {
Tensor::cat(
vec![
features,
Tensor::zeros([n_mels, chunks * chunk - frames], &device),
],
1,
)
} else {
features.slice_dim(1, 0..chunks * chunk)
};
let x = features
.reshape([n_mels, chunks, chunk])
.swap_dims(0, 1)
.unsqueeze_dim::<4>(1);
let x = self.stem.forward(x);
let [_, c, f, t] = x.dims();
let x = x.permute([0, 3, 1, 2]).reshape([chunks, t, c * f]);
let x = self.conv_out.forward(x)
+ sinusoid_table(t, self.d_model(), 10000.0, &device).unsqueeze_dim::<3>(0);
let keep: Vec<i64> = (0..chunks)
.flat_map(|k| {
let len = post_conv_len(valid.saturating_sub(k * chunk).min(chunk));
(0..len).map(move |j| (k * t + j) as i64)
})
.collect();
let n = keep.len();
let y = x.reshape([chunks * t, self.d_model()]).select(
0,
Tensor::<1, Int>::from_data(TensorData::new(keep, [n]), &device),
);
crate::contracts::assert_shape_contract_periodically!(
["n", "d_model"],
&y.dims(),
&[("n", self.output_len(valid)), ("d_model", self.d_model())]
);
y
}
pub fn window_mask(
&self,
n: usize,
device: &Device,
) -> Tensor<3, Bool> {
crate::ops::transformers::attention::window_block_mask(n, self.window_frames(), device)
.unsqueeze_dim(0)
}
pub fn forward(
&self,
features: Tensor<2>,
valid: usize,
) -> Tensor<2> {
let x = self.stem(features, valid);
let [n, d] = x.dims();
let mask = self.window_mask(n, &x.device());
let mut x = x.unsqueeze_dim::<3>(0);
for layer in &self.layers {
x = layer.forward_masked(x, Some(mask.clone()));
}
self.ln_post.forward(x).reshape([n, d])
}
}
pub trait Qwen3AsrProjectorMeta {
fn d_audio(&self) -> usize;
fn d_text(&self) -> usize;
}
#[derive(Config, Debug)]
pub struct Qwen3AsrProjectorConfig {
pub d_audio: usize,
pub d_text: usize,
}
impl Qwen3AsrProjectorMeta for Qwen3AsrProjectorConfig {
fn d_audio(&self) -> usize {
self.d_audio
}
fn d_text(&self) -> usize {
self.d_text
}
}
impl ModuleInit<Qwen3AsrProjector> for Qwen3AsrProjectorConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<Qwen3AsrProjector> {
Ok(Qwen3AsrProjector {
linear_1: LinearConfig::new(self.d_audio, self.d_audio).init(device),
linear_2: LinearConfig::new(self.d_audio, self.d_text).init(device),
})
}
}
#[derive(Module, Debug)]
pub struct Qwen3AsrProjector {
pub linear_1: Linear,
pub linear_2: Linear,
}
impl Qwen3AsrProjectorMeta for Qwen3AsrProjector {
fn d_audio(&self) -> usize {
self.linear_1.weight.dims()[0]
}
fn d_text(&self) -> usize {
self.linear_2.weight.dims()[1]
}
}
impl Qwen3AsrProjector {
pub fn forward(
&self,
x: Tensor<2>,
) -> Tensor<2> {
let [n] = crate::contracts::unpack_shape_contract!(
["n", "d_audio"],
&x.dims(),
&["n"],
&[("d_audio", self.d_audio())]
);
let y = self.linear_2.forward(gelu(self.linear_1.forward(x)));
crate::contracts::assert_shape_contract_periodically!(
["n", "d_text"],
&y.dims(),
&[("n", n), ("d_text", self.d_text())]
);
y
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::support::testing::cpu_device;
#[test]
fn test_geometry() {
assert_eq!(post_conv_len(100), 13);
assert_eq!(post_conv_len(50), 7);
assert_eq!(post_conv_len(1), 1);
assert_eq!(post_conv_len(0), 0);
let config = Qwen3AsrAudioEncoderConfig::qwen3_asr_0_6b();
assert_eq!(config.freq_out(), 16);
assert_eq!(config.chunk_out(), 13);
assert_eq!(config.window_frames(), 104);
assert_eq!(config.output_len(3000), 390);
assert_eq!(config.output_len(250), 33);
}
#[test]
fn test_meta_and_shapes() {
let config = Qwen3AsrAudioEncoderConfig::new(16, 32, 4, 64, 2, 8)
.with_n_window(25)
.with_n_window_infer(150);
let encoder: Qwen3AsrAudioEncoder = config.init(&cpu_device());
assert_eq!(encoder.d_model(), config.d_model());
assert_eq!(encoder.n_layers(), config.n_layers());
assert_eq!(encoder.chunk_frames(), config.chunk_frames());
assert_eq!(encoder.window_chunks(), config.window_chunks());
assert_eq!(encoder.window_frames(), config.window_frames());
assert_eq!(config.chunk_frames(), 50);
assert_eq!(config.window_chunks(), 3);
let y = encoder.forward(Tensor::zeros([16, 300], &cpu_device()), 250);
assert_eq!(y.dims(), [config.output_len(250), 32]);
let config = Qwen3AsrAudioEncoderConfig::new(16, 32, 4, 64, 2, 8).with_n_window_infer(200);
let encoder: Qwen3AsrAudioEncoder = config.init(&cpu_device());
assert_eq!(encoder.layers[0].mlp.linear1.weight.dims(), [32, 64]);
let y = encoder.forward(Tensor::zeros([16, 300], &cpu_device()), 250);
assert_eq!(y.dims(), [33, 32]);
let projector_config = Qwen3AsrProjectorConfig::new(32, 24);
let projector: Qwen3AsrProjector = projector_config.init(&cpu_device());
assert_eq!(projector.d_audio(), projector_config.d_audio());
assert_eq!(projector.d_text(), projector_config.d_text());
assert_eq!(projector.forward(y).dims(), [33, 24]);
}
}