use burn::{
Tensor,
config::Config,
module::Module,
nn::{
Linear,
LinearConfig,
},
tensor::Device,
};
use super::{
TdtJoint,
TdtJointConfig,
TdtPredictionNetwork,
TdtPredictionNetworkConfig,
};
use crate::{
blocks::{
signals::DwStridingSubsamplingConfig,
transformers::encoders::{
SubsampledEncoder,
SubsampledEncoderMeta,
conformer::{
ConformerBlockConfig,
ConformerEncoderConfig,
ConformerEncoderMeta,
},
},
},
burner::module::{
CanonicalKeys,
ModuleInit,
Rekey,
},
errors::{
BunsenResult,
ConstraintError,
Rule,
},
};
pub trait ParakeetTdtMeta {
fn d_model(&self) -> usize;
fn d_pred(&self) -> usize;
fn n_vocab(&self) -> usize;
fn durations(&self) -> &[usize];
fn subsampling_factor(&self) -> usize;
fn blank_id(&self) -> usize {
self.n_vocab() - 1
}
}
#[derive(Config, Debug)]
pub struct ParakeetTdtConfig {
pub encoder: ConformerEncoderConfig,
pub n_vocab: usize,
pub d_pred: usize,
#[config(default = "2")]
pub n_pred_layers: usize,
#[config(default = "vec![0, 1, 2, 3, 4]")]
pub durations: Vec<usize>,
}
impl ParakeetTdtConfig {
pub fn tdt_0_6b_v3() -> Self {
Self::new(
ConformerEncoderConfig::new(
DwStridingSubsamplingConfig::new(128, 256, 1024),
ConformerBlockConfig::new(1024, 8, 4096, 9)
.with_bias(false)
.with_conv_bias(false),
24,
),
8193,
640,
)
}
}
impl ParakeetTdtMeta for ParakeetTdtConfig {
fn d_model(&self) -> usize {
self.encoder.d_model()
}
fn d_pred(&self) -> usize {
self.d_pred
}
fn n_vocab(&self) -> usize {
self.n_vocab
}
fn durations(&self) -> &[usize] {
&self.durations
}
fn subsampling_factor(&self) -> usize {
self.encoder.subsampling_factor()
}
}
impl ModuleInit<ParakeetTdt> for ParakeetTdtConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<ParakeetTdt> {
if self.durations.is_empty() {
return Err(
ConstraintError::new("ParakeetTdtConfig", "durations", Rule::ZeroOrEmpty).into(),
);
}
Ok(ParakeetTdt {
encoder: self.encoder.try_init(device)?,
encoder_projector: LinearConfig::new(self.d_model(), self.d_pred).init(device),
decoder: TdtPredictionNetworkConfig::new(self.n_vocab, self.d_pred)
.with_n_layers(self.n_pred_layers)
.try_init(device)?,
joint: TdtJointConfig::new(self.d_pred, self.n_vocab, self.durations.len())
.try_init(device)?,
durations: self.durations.clone(),
})
}
}
#[derive(Module, Debug)]
pub struct ParakeetTdt {
pub encoder: SubsampledEncoder,
pub encoder_projector: Linear,
pub decoder: TdtPredictionNetwork,
pub joint: TdtJoint,
#[module(skip)]
pub durations: Vec<usize>,
}
impl ParakeetTdtMeta for ParakeetTdt {
fn d_model(&self) -> usize {
self.encoder.d_model()
}
fn d_pred(&self) -> usize {
self.encoder_projector.weight.dims()[1]
}
fn n_vocab(&self) -> usize {
use super::TdtJointMeta;
self.joint.n_vocab()
}
fn durations(&self) -> &[usize] {
&self.durations
}
fn subsampling_factor(&self) -> usize {
self.encoder.subsampling_factor()
}
}
impl ParakeetTdt {
pub fn encode(
&self,
features: Tensor<3>,
lengths: Option<&[usize]>,
) -> (Tensor<3>, Vec<usize>) {
#[cfg(any(debug_assertions, test))]
let [batch, time] = crate::contracts::unpack_shape_contract!(
["batch", "n_mels", "time"],
&features.dims(),
&["batch", "time"],
);
let (x, lengths) = self.encoder.forward(features, lengths);
let y = self.encoder_projector.forward(x);
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "time", "d_pred"],
&y.dims(),
&[
("batch", batch),
("time", self.encoder.output_len(time)),
("d_pred", self.d_pred()),
],
);
(y, lengths)
}
}
impl CanonicalKeys for ParakeetTdt {
fn canonical_rekey(&self) -> Rekey {
Rekey::keep_rest().mount_canonical("encoder.", &self.encoder)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::support::testing::cpu_device;
fn toy_config() -> ParakeetTdtConfig {
ParakeetTdtConfig::new(
ConformerEncoderConfig::new(
DwStridingSubsamplingConfig::new(24, 4, 16),
ConformerBlockConfig::new(16, 2, 32, 5)
.with_bias(false)
.with_conv_bias(false),
2,
),
11,
12,
)
.with_durations(vec![0, 1, 2, 3, 4])
}
#[test]
fn test_meta() {
let config = toy_config().with_durations(vec![0, 2, 4]);
let model: ParakeetTdt = config.init(&cpu_device());
assert_eq!(model.d_model(), config.d_model());
assert_eq!(model.d_pred(), config.d_pred());
assert_eq!(model.n_vocab(), config.n_vocab());
assert_eq!(model.durations(), config.durations());
assert_eq!(model.subsampling_factor(), config.subsampling_factor());
assert_eq!(model.blank_id(), 10);
}
#[test]
fn test_v3_geometry() {
let config = ParakeetTdtConfig::tdt_0_6b_v3();
assert_eq!(config.d_model(), 1024);
assert_eq!(config.blank_id(), 8192);
assert_eq!(config.subsampling_factor(), 8);
}
}