extern crate alloc;
use burn::prelude::*;
use burn::nn::LayerNorm;
use burn::nn::LayerNormConfig;
use burn::nn::Linear;
use burn::nn::LinearConfig;
use burn::nn::PaddingConfig1d;
use burn::nn::PaddingConfig2d;
use burn::nn::conv::Conv1d;
use burn::nn::conv::Conv1dConfig;
use burn::nn::conv::Conv2d;
use burn::nn::conv::Conv2dConfig;
#[derive(Module, Debug)]
pub struct Submodule1 {
constant302: burn::module::Param<Tensor<1, Int>>,
constant483: burn::module::Param<Tensor<1, Int>>,
conv2d1: Conv2d,
constant304: burn::module::Param<Tensor<1>>,
constant305: burn::module::Param<Tensor<1>>,
constant306: burn::module::Param<Tensor<1>>,
conv2d2: Conv2d,
conv2d3: Conv2d,
conv2d4: Conv2d,
conv2d5: Conv2d,
linear1: Linear,
#[module(skip)]
device: Device,
}
impl Submodule1 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let constant302: burn::module::Param<Tensor<1, Int>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
1,
Int,
>::from_data(
burn::tensor::TensorData::from([0i64]),
(device, burn::tensor::DType::I64),
),
device.clone(),
false,
[1].into(),
);
let constant483: burn::module::Param<Tensor<1, Int>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
1,
Int,
>::from_data(
burn::tensor::TensorData::from([139i64]),
(device, burn::tensor::DType::I64),
),
device.clone(),
false,
[1].into(),
);
let conv2d1 = Conv2dConfig::new([1, 256], [3, 3])
.with_stride([2, 2])
.with_padding(PaddingConfig2d::Explicit(2, 2, 1, 1))
.with_dilation([1, 1])
.with_groups(1)
.with_bias(true)
.init(device);
let constant304: burn::module::Param<Tensor<1>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
1,
>::from_data(
burn::tensor::TensorData::from([2f64]),
(device, burn::tensor::DType::F32),
),
device.clone(),
false,
[1].into(),
);
let constant305: burn::module::Param<Tensor<1>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
1,
>::from_data(
burn::tensor::TensorData::from([1f64]),
(device, burn::tensor::DType::F32),
),
device.clone(),
false,
[1].into(),
);
let constant306: burn::module::Param<Tensor<1>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
1,
>::from_data(
burn::tensor::TensorData::from([3f64]),
(device, burn::tensor::DType::F32),
),
device.clone(),
false,
[1].into(),
);
let conv2d2 = Conv2dConfig::new([256, 256], [3, 3])
.with_stride([2, 2])
.with_padding(PaddingConfig2d::Explicit(2, 2, 1, 1))
.with_dilation([1, 1])
.with_groups(256)
.with_bias(true)
.init(device);
let conv2d3 = Conv2dConfig::new([256, 256], [1, 1])
.with_stride([1, 1])
.with_padding(PaddingConfig2d::Valid)
.with_dilation([1, 1])
.with_groups(1)
.with_bias(true)
.init(device);
let conv2d4 = Conv2dConfig::new([256, 256], [3, 3])
.with_stride([2, 2])
.with_padding(PaddingConfig2d::Explicit(2, 2, 1, 1))
.with_dilation([1, 1])
.with_groups(256)
.with_bias(true)
.init(device);
let conv2d5 = Conv2dConfig::new([256, 256], [1, 1])
.with_stride([1, 1])
.with_padding(PaddingConfig2d::Valid)
.with_dilation([1, 1])
.with_groups(1)
.with_bias(true)
.init(device);
let linear1 = LinearConfig::new(4352, 512).with_bias(true).init(device);
Self {
constant302,
constant483,
conv2d1,
constant304,
constant305,
constant306,
conv2d2,
conv2d3,
conv2d4,
conv2d5,
linear1,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
audio_signal: Tensor<3>,
length: Tensor<1, Int>,
) -> (
Tensor<1, Int>,
Tensor<1, Int>,
i64,
[i64; 1],
Tensor<1>,
Tensor<3>,
Tensor<1, Int>,
) {
let shape1_out1: [i64; 1] = {
let axes = &audio_signal.clone().dims()[2..3];
let mut output = [0i64; 1];
for i in 0..1 {
output[i] = axes[i] as i64;
}
output
};
let squeeze1_out1 = shape1_out1[0] as i64;
let constant480_out1 = 2i64;
let div1_out1 = squeeze1_out1 / constant480_out1;
let constant303_out1 = 1i64;
let add1_out1 = constant303_out1 + div1_out1;
let div2_out1 = add1_out1 / constant480_out1;
let add2_out1 = constant303_out1 + div2_out1;
let div3_out1 = add2_out1 / constant480_out1;
let add3_out1 = constant303_out1 + div3_out1;
let constant511_out1 = -1i64;
let mul1_out1 = constant480_out1 * constant511_out1;
let mul2_out1 = add3_out1 * constant511_out1;
let max1_out1 = mul1_out1.max(mul2_out1);
let mul3_out1 = max1_out1 * constant511_out1;
let constant481_out1 = -1i64;
let mul4_out1 = constant481_out1 * mul3_out1;
let add4_out1 = constant303_out1 + mul4_out1;
let add5_out1 = add4_out1 + div3_out1;
let constant302_out1 = self.constant302.val();
let max2_out1 = constant302_out1.clamp_min(add5_out1);
let mul5_out1 = constant480_out1 * max2_out1.clone();
let constant483_out1 = self.constant483.val();
let add6_out1 = constant483_out1.add(mul5_out1);
let transpose1_out1 = audio_signal.permute([0, 2, 1]);
let unsqueeze1_out1: Tensor<4> = transpose1_out1.unsqueeze_dims::<4>(&[1]);
let cast1_out1 = length.float().cast(burn::tensor::DType::F32);
let cast2_out1 = cast1_out1.clone().int().cast(burn::tensor::DType::I64);
let range1_out1 = {
let start = 0i64;
let delta = 1i64;
assert!(delta != 0);
let n = (((squeeze1_out1 as i128 - start as i128) as f64) / delta as f64)
.ceil()
.max(0.0) as i64;
Tensor::arange(0..n, &self.device)
.cast(burn::tensor::DType::I64)
.mul_scalar(delta)
.add_scalar(start)
};
let constant310_out1: [i64; 1] = [1i64];
let concat1_out1: [i64; 2usize] = [&constant310_out1[..], &shape1_out1[..]]
.concat()
.try_into()
.unwrap();
let expand1_out1 = {
let onnx_shape: [i64; 2usize] = concat1_out1;
let input_dims = range1_out1.dims();
let mut shape = onnx_shape;
#[allow(clippy::needless_range_loop)]
for i in 0..1usize {
let dim_offset = 1usize + i;
if shape[dim_offset] == 1 && input_dims[i] > 1 {
shape[dim_offset] = input_dims[i] as i64;
}
}
range1_out1.expand(shape)
};
let unsqueeze2_out1: Tensor<2, Int> = cast2_out1.unsqueeze_dims::<2>(&[1]);
let less1_out1 = expand1_out1.lower(unsqueeze2_out1);
let unsqueeze3_out1: Tensor<3, Bool> = less1_out1.unsqueeze_dims::<3>(&[-1]);
let constant484_out1: [i64; 1] = [128i64];
let concat2_out1: [i64; 3usize] = [
&constant310_out1[..],
&shape1_out1[..],
&constant484_out1[..],
]
.concat()
.try_into()
.unwrap();
let expand2_out1 = {
let onnx_shape: [i64; 3usize] = concat2_out1;
let input_dims = unsqueeze3_out1.dims();
let mut shape = onnx_shape;
#[allow(clippy::needless_range_loop)]
for i in 0..3usize {
let dim_offset = i;
if shape[dim_offset] == 1 && input_dims[i] > 1 {
shape[dim_offset] = input_dims[i] as i64;
}
}
unsqueeze3_out1.expand(shape)
};
let cast3_out1 = expand2_out1.float().cast(burn::tensor::DType::F32);
let unsqueeze4_out1: Tensor<4> = cast3_out1.unsqueeze_dims::<4>(&[1]);
let mul6_out1 = unsqueeze1_out1.mul(unsqueeze4_out1);
let conv2d1_out1 = self.conv2d1.forward(mul6_out1);
let constant304_out1 = self.constant304.val();
let add7_out1 = cast1_out1.add(constant304_out1.clone());
let constant305_out1 = self.constant305.val();
let add8_out1 = add7_out1.add(constant305_out1.clone());
let constant306_out1 = self.constant306.val();
let sub1_out1 = add8_out1.sub(constant306_out1.clone());
let div4_out1 = sub1_out1.div(constant304_out1.clone());
let floor1_out1 = div4_out1.floor();
let add9_out1 = floor1_out1.add(constant305_out1.clone());
let cast4_out1 = add9_out1.clone().int().cast(burn::tensor::DType::I64);
let range2_out1 = {
let start = 0i64;
let delta = 1i64;
assert!(delta != 0);
let n = (((add1_out1 as i128 - start as i128) as f64) / delta as f64)
.ceil()
.max(0.0) as i64;
Tensor::arange(0..n, &self.device)
.cast(burn::tensor::DType::I64)
.mul_scalar(delta)
.add_scalar(start)
};
let unsqueeze5_out1 = [add1_out1 as i64];
let concat3_out1: [i64; 2usize] = [&constant310_out1[..], &unsqueeze5_out1[..]]
.concat()
.try_into()
.unwrap();
let expand3_out1 = {
let onnx_shape: [i64; 2usize] = concat3_out1;
let input_dims = range2_out1.dims();
let mut shape = onnx_shape;
#[allow(clippy::needless_range_loop)]
for i in 0..1usize {
let dim_offset = 1usize + i;
if shape[dim_offset] == 1 && input_dims[i] > 1 {
shape[dim_offset] = input_dims[i] as i64;
}
}
range2_out1.expand(shape)
};
let unsqueeze6_out1: Tensor<2, Int> = cast4_out1.unsqueeze_dims::<2>(&[1]);
let less2_out1 = expand3_out1.lower(unsqueeze6_out1);
let unsqueeze7_out1: Tensor<3, Bool> = less2_out1.unsqueeze_dims::<3>(&[-1]);
let constant486_out1: [i64; 1] = [65i64];
let concat4_out1: [i64; 3usize] = [
&constant310_out1[..],
&unsqueeze5_out1[..],
&constant486_out1[..],
]
.concat()
.try_into()
.unwrap();
let expand4_out1 = {
let onnx_shape: [i64; 3usize] = concat4_out1;
let input_dims = unsqueeze7_out1.dims();
let mut shape = onnx_shape;
#[allow(clippy::needless_range_loop)]
for i in 0..3usize {
let dim_offset = i;
if shape[dim_offset] == 1 && input_dims[i] > 1 {
shape[dim_offset] = input_dims[i] as i64;
}
}
unsqueeze7_out1.expand(shape)
};
let cast5_out1 = expand4_out1.float().cast(burn::tensor::DType::F32);
let unsqueeze8_out1: Tensor<4> = cast5_out1.unsqueeze_dims::<4>(&[1]);
let constant487_out1: [i64; 1] = [256i64];
let concat5_out1: [i64; 4usize] = [
&constant310_out1[..],
&constant487_out1[..],
&unsqueeze5_out1[..],
&constant486_out1[..],
]
.concat()
.try_into()
.unwrap();
let expand5_out1 = {
let onnx_shape: [i64; 4usize] = concat5_out1;
let input_dims = unsqueeze8_out1.dims();
let mut shape = onnx_shape;
#[allow(clippy::needless_range_loop)]
for i in 0..4usize {
let dim_offset = i;
if shape[dim_offset] == 1 && input_dims[i] > 1 {
shape[dim_offset] = input_dims[i] as i64;
}
}
unsqueeze8_out1.expand(shape)
};
let mul7_out1 = conv2d1_out1.mul(expand5_out1.clone());
let relu1_out1 = burn::tensor::activation::relu(mul7_out1);
let mul8_out1 = relu1_out1.mul(expand5_out1);
let conv2d2_out1 = self.conv2d2.forward(mul8_out1);
let add10_out1 = add9_out1.add(constant304_out1.clone());
let add11_out1 = add10_out1.add(constant305_out1.clone());
let sub2_out1 = add11_out1.sub(constant306_out1.clone());
let div5_out1 = sub2_out1.div(constant304_out1.clone());
let floor2_out1 = div5_out1.floor();
let add12_out1 = floor2_out1.add(constant305_out1.clone());
let cast6_out1 = add12_out1.clone().int().cast(burn::tensor::DType::I64);
let range3_out1 = {
let start = 0i64;
let delta = 1i64;
assert!(delta != 0);
let n = (((add2_out1 as i128 - start as i128) as f64) / delta as f64)
.ceil()
.max(0.0) as i64;
Tensor::arange(0..n, &self.device)
.cast(burn::tensor::DType::I64)
.mul_scalar(delta)
.add_scalar(start)
};
let unsqueeze9_out1 = [add2_out1 as i64];
let concat6_out1: [i64; 2usize] = [&constant310_out1[..], &unsqueeze9_out1[..]]
.concat()
.try_into()
.unwrap();
let expand6_out1 = {
let onnx_shape: [i64; 2usize] = concat6_out1;
let input_dims = range3_out1.dims();
let mut shape = onnx_shape;
#[allow(clippy::needless_range_loop)]
for i in 0..1usize {
let dim_offset = 1usize + i;
if shape[dim_offset] == 1 && input_dims[i] > 1 {
shape[dim_offset] = input_dims[i] as i64;
}
}
range3_out1.expand(shape)
};
let unsqueeze10_out1: Tensor<2, Int> = cast6_out1.unsqueeze_dims::<2>(&[1]);
let less3_out1 = expand6_out1.lower(unsqueeze10_out1);
let unsqueeze11_out1: Tensor<3, Bool> = less3_out1.unsqueeze_dims::<3>(&[-1]);
let constant488_out1: [i64; 1] = [33i64];
let concat7_out1: [i64; 3usize] = [
&constant310_out1[..],
&unsqueeze9_out1[..],
&constant488_out1[..],
]
.concat()
.try_into()
.unwrap();
let expand7_out1 = {
let onnx_shape: [i64; 3usize] = concat7_out1;
let input_dims = unsqueeze11_out1.dims();
let mut shape = onnx_shape;
#[allow(clippy::needless_range_loop)]
for i in 0..3usize {
let dim_offset = i;
if shape[dim_offset] == 1 && input_dims[i] > 1 {
shape[dim_offset] = input_dims[i] as i64;
}
}
unsqueeze11_out1.expand(shape)
};
let cast7_out1 = expand7_out1.float().cast(burn::tensor::DType::F32);
let unsqueeze12_out1: Tensor<4> = cast7_out1.unsqueeze_dims::<4>(&[1]);
let concat8_out1: [i64; 4usize] = [
&constant310_out1[..],
&constant487_out1[..],
&unsqueeze9_out1[..],
&constant488_out1[..],
]
.concat()
.try_into()
.unwrap();
let expand8_out1 = {
let onnx_shape: [i64; 4usize] = concat8_out1;
let input_dims = unsqueeze12_out1.dims();
let mut shape = onnx_shape;
#[allow(clippy::needless_range_loop)]
for i in 0..4usize {
let dim_offset = i;
if shape[dim_offset] == 1 && input_dims[i] > 1 {
shape[dim_offset] = input_dims[i] as i64;
}
}
unsqueeze12_out1.expand(shape)
};
let mul9_out1 = conv2d2_out1.mul(expand8_out1.clone());
let conv2d3_out1 = self.conv2d3.forward(mul9_out1);
let mul10_out1 = conv2d3_out1.mul(expand8_out1.clone());
let relu2_out1 = burn::tensor::activation::relu(mul10_out1);
let mul11_out1 = relu2_out1.mul(expand8_out1);
let conv2d4_out1 = self.conv2d4.forward(mul11_out1);
let add13_out1 = add12_out1.add(constant304_out1.clone());
let add14_out1 = add13_out1.add(constant305_out1.clone());
let sub3_out1 = add14_out1.sub(constant306_out1);
let div6_out1 = sub3_out1.div(constant304_out1.clone());
let floor3_out1 = div6_out1.floor();
let add15_out1 = floor3_out1.add(constant305_out1);
let cast8_out1 = add15_out1.int().cast(burn::tensor::DType::I64);
let range4_out1 = {
let start = 0i64;
let delta = 1i64;
assert!(delta != 0);
let n = (((add3_out1 as i128 - start as i128) as f64) / delta as f64)
.ceil()
.max(0.0) as i64;
Tensor::arange(0..n, &self.device)
.cast(burn::tensor::DType::I64)
.mul_scalar(delta)
.add_scalar(start)
};
let unsqueeze13_out1 = [add3_out1 as i64];
let concat9_out1: [i64; 2usize] = [&constant310_out1[..], &unsqueeze13_out1[..]]
.concat()
.try_into()
.unwrap();
let expand9_out1 = {
let onnx_shape: [i64; 2usize] = concat9_out1;
let input_dims = range4_out1.dims();
let mut shape = onnx_shape;
#[allow(clippy::needless_range_loop)]
for i in 0..1usize {
let dim_offset = 1usize + i;
if shape[dim_offset] == 1 && input_dims[i] > 1 {
shape[dim_offset] = input_dims[i] as i64;
}
}
range4_out1.expand(shape)
};
let unsqueeze14_out1: Tensor<2, Int> = cast8_out1
.clone()
.unsqueeze_dims::<2>(&[1]);
let less4_out1 = expand9_out1.lower(unsqueeze14_out1);
let unsqueeze15_out1: Tensor<3, Bool> = less4_out1.unsqueeze_dims::<3>(&[-1]);
let constant489_out1: [i64; 1] = [17i64];
let concat10_out1: [i64; 3usize] = [
&constant310_out1[..],
&unsqueeze13_out1[..],
&constant489_out1[..],
]
.concat()
.try_into()
.unwrap();
let expand10_out1 = {
let onnx_shape: [i64; 3usize] = concat10_out1;
let input_dims = unsqueeze15_out1.dims();
let mut shape = onnx_shape;
#[allow(clippy::needless_range_loop)]
for i in 0..3usize {
let dim_offset = i;
if shape[dim_offset] == 1 && input_dims[i] > 1 {
shape[dim_offset] = input_dims[i] as i64;
}
}
unsqueeze15_out1.expand(shape)
};
let cast9_out1 = expand10_out1.float().cast(burn::tensor::DType::F32);
let unsqueeze16_out1: Tensor<4> = cast9_out1.unsqueeze_dims::<4>(&[1]);
let concat11_out1: [i64; 4usize] = [
&constant310_out1[..],
&constant487_out1[..],
&unsqueeze13_out1[..],
&constant489_out1[..],
]
.concat()
.try_into()
.unwrap();
let expand11_out1 = {
let onnx_shape: [i64; 4usize] = concat11_out1;
let input_dims = unsqueeze16_out1.dims();
let mut shape = onnx_shape;
#[allow(clippy::needless_range_loop)]
for i in 0..4usize {
let dim_offset = i;
if shape[dim_offset] == 1 && input_dims[i] > 1 {
shape[dim_offset] = input_dims[i] as i64;
}
}
unsqueeze16_out1.expand(shape)
};
let mul12_out1 = conv2d4_out1.mul(expand11_out1.clone());
let conv2d5_out1 = self.conv2d5.forward(mul12_out1);
let mul13_out1 = conv2d5_out1.mul(expand11_out1.clone());
let relu3_out1 = burn::tensor::activation::relu(mul13_out1);
let mul14_out1 = relu3_out1.mul(expand11_out1);
let transpose2_out1 = mul14_out1.permute([0, 2, 1, 3]);
let reshape1_out1 = transpose2_out1.reshape([1, -1, 4352]);
let linear1_out1 = self.linear1.forward(reshape1_out1);
let slice1_out1 = linear1_out1.slice(s![.., 2.., ..]);
let sub4_out1 = cast8_out1.sub_scalar(constant480_out1);
(
sub4_out1,
max2_out1,
constant303_out1,
constant310_out1,
constant304_out1,
slice1_out1,
add6_out1,
)
}
}
#[derive(Module, Debug)]
pub struct Submodule2 {
constant477: burn::module::Param<Tensor<1, Int>>,
constant515: burn::module::Param<Tensor<1, Int>>,
constant491: burn::module::Param<Tensor<1, Int>>,
constant301: burn::module::Param<Tensor<3>>,
constant311: burn::module::Param<Tensor<1, Bool>>,
layernormalization1: LayerNorm,
linear2: Linear,
#[module(skip)]
device: Device,
}
impl Submodule2 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let constant477: burn::module::Param<Tensor<1, Int>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
1,
Int,
>::from_data(
burn::tensor::TensorData::from([70i64]),
(device, burn::tensor::DType::I64),
),
device.clone(),
false,
[1].into(),
);
let constant515: burn::module::Param<Tensor<1, Int>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
1,
Int,
>::from_data(
burn::tensor::TensorData::from([-1i64]),
(device, burn::tensor::DType::I64),
),
device.clone(),
false,
[1].into(),
);
let constant491: burn::module::Param<Tensor<1, Int>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
1,
Int,
>::from_data(
burn::tensor::TensorData::from([5000i64]),
(device, burn::tensor::DType::I64),
),
device.clone(),
false,
[1].into(),
);
let constant301: burn::module::Param<Tensor<3>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
3,
>::zeros([1, 9999, 512], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[1, 9999, 512].into(),
);
let constant311: burn::module::Param<Tensor<1, Bool>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
1,
Bool,
>::from_data(
burn::tensor::TensorData::from([true]),
(device, burn::tensor::DType::Bool(burn::tensor::BoolStore::Native)),
),
device.clone(),
false,
[1].into(),
);
let layernormalization1 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear2 = LinearConfig::new(512, 2048).with_bias(false).init(device);
Self {
constant477,
constant515,
constant491,
constant301,
constant311,
layernormalization1,
linear2,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
sub4_out1: Tensor<1, Int>,
max2_out1: Tensor<1, Int>,
cache_last_channel_len: Tensor<1, Int>,
constant303_out1: i64,
constant310_out1: [i64; 1],
constant304_out1: Tensor<1>,
cache_last_channel: Tensor<4>,
cache_last_time: Tensor<4>,
slice1_out1: Tensor<3>,
) -> (
Tensor<3>,
Tensor<3>,
Tensor<3>,
[i64; 1],
Tensor<3, Bool>,
Tensor<2, Bool>,
Tensor<3>,
Tensor<1, Int>,
) {
let clip1_out1 = sub4_out1.clamp_min(0f64);
let constant477_out1 = self.constant477.val();
let add16_out1 = max2_out1.add(constant477_out1.clone());
let add17_out1 = clip1_out1.clone().add(constant477_out1.clone());
let constant515_out1 = self.constant515.val();
let mul15_out1 = cache_last_channel_len.mul(constant515_out1);
let add18_out1 = mul15_out1.add(constant477_out1);
let constant491_out1 = self.constant491.val();
let sub5_out1 = constant491_out1.clone().sub(add16_out1.clone());
let add19_out1 = constant491_out1.add(add16_out1.clone());
let sub6_out1 = add19_out1.sub_scalar(constant303_out1);
let unsqueeze17_out1 = {
let v: i64 = sub5_out1.into_scalar::<i64>();
[v as i64]
};
let unsqueeze18_out1 = {
let v: i64 = sub6_out1.into_scalar::<i64>();
[v as i64]
};
let constant301_out1 = self.constant301.val();
let transpose3_out1 = constant301_out1.permute([1, 0, 2]);
let slice2_out1 = transpose3_out1
.slice(s![unsqueeze17_out1[0]..unsqueeze18_out1[0], .., ..]);
let transpose4_out1 = slice2_out1.permute([1, 0, 2]);
let unsqueeze19_out1 = {
let v: i64 = add16_out1.clone().into_scalar::<i64>();
[v as i64]
};
let concat12_out1: [i64; 3usize] = [
&constant310_out1[..],
&unsqueeze19_out1[..],
&unsqueeze19_out1[..],
]
.concat()
.try_into()
.unwrap();
let constant311_out1 = self.constant311.val();
let expand12_out1 = {
let shape: [i64; 3] = concat12_out1;
constant311_out1.expand(shape)
};
let cast10_out1 = add16_out1.clone().cast(burn::tensor::DType::I32);
let range5_out1 = {
let start = 0i32;
let limit = (cast10_out1).into_scalar::<i32>();
let delta = 1i32;
assert!(delta != 0);
let n = (((limit as i128 - start as i128) as f64) / delta as f64)
.ceil()
.max(0.0) as i64;
Tensor::arange(0..n, &self.device)
.cast(burn::tensor::DType::I32)
.mul_scalar(delta)
.add_scalar(start)
};
let cast11_out1 = range5_out1.float().cast(burn::tensor::DType::F32);
let div7_out1 = cast11_out1.div(constant304_out1);
let abs1_out1 = div7_out1.clone().abs();
let floor4_out1 = abs1_out1.floor();
let sign1_out1 = div7_out1.sign();
let mul16_out1 = floor4_out1.mul(sign1_out1);
let cast12_out1 = mul16_out1.int().cast(burn::tensor::DType::I32);
let unsqueeze20_out1: Tensor<2, Int> = cast12_out1
.clone()
.unsqueeze_dims::<2>(&[1]);
let unsqueeze21_out1: Tensor<2, Int> = cast12_out1.unsqueeze_dims::<2>(&[0]);
let sub7_out1 = unsqueeze20_out1.sub(unsqueeze21_out1);
let constant314_out1 = 35i32;
let lessorequal1_out1 = sub7_out1.clone().lower_equal_elem(constant314_out1);
let constant312_out1 = 0i32;
let greaterorequal1_out1 = sub7_out1.greater_equal_elem(constant312_out1);
let and1_out1 = lessorequal1_out1.bool_and(greaterorequal1_out1);
let unsqueeze22_out1: Tensor<3, Bool> = and1_out1.unsqueeze_dims::<3>(&[0]);
let and2_out1 = expand12_out1.bool_and(unsqueeze22_out1);
let range6_out1 = {
let start = 0i64;
let limit = (add16_out1).into_scalar::<i64>();
let delta = 1i64;
assert!(delta != 0);
let n = (((limit as i128 - start as i128) as f64) / delta as f64)
.ceil()
.max(0.0) as i64;
Tensor::arange(0..n, &self.device)
.cast(burn::tensor::DType::I64)
.mul_scalar(delta)
.add_scalar(start)
};
let expand13_out1 = {
let onnx_shape: [i64; 2usize] = [1, 1];
let input_dims = range6_out1.dims();
let mut shape = onnx_shape;
#[allow(clippy::needless_range_loop)]
for i in 0..1usize {
let dim_offset = 1usize + i;
if shape[dim_offset] == 1 && input_dims[i] > 1 {
shape[dim_offset] = input_dims[i] as i64;
}
}
range6_out1.expand(shape)
};
let unsqueeze23_out1: Tensor<2, Int> = add17_out1.unsqueeze_dims::<2>(&[-1]);
let less5_out1 = expand13_out1.clone().lower(unsqueeze23_out1);
let unsqueeze24_out1: Tensor<2, Int> = add18_out1.unsqueeze_dims::<2>(&[-1]);
let greaterorequal2_out1 = expand13_out1.greater_equal(unsqueeze24_out1);
let and3_out1 = greaterorequal2_out1.bool_and(less5_out1);
let unsqueeze25_out1: Tensor<3, Bool> = and3_out1
.clone()
.unsqueeze_dims::<3>(&[1]);
let expand14_out1 = {
let onnx_shape: [i64; 3usize] = [1, 1, 1];
let input_dims = unsqueeze25_out1.dims();
let mut shape = onnx_shape;
#[allow(clippy::needless_range_loop)]
for i in 0..3usize {
let dim_offset = i;
if shape[dim_offset] == 1 && input_dims[i] > 1 {
shape[dim_offset] = input_dims[i] as i64;
}
}
unsqueeze25_out1.expand(shape)
};
let concat13_out1: [i64; 3usize] = [
&constant310_out1[..],
&unsqueeze19_out1[..],
&constant310_out1[..],
]
.concat()
.try_into()
.unwrap();
let tile1_out1 = {
let repeats: alloc::vec::Vec<usize> = concat13_out1
.iter()
.map(|&v| v as usize)
.collect();
expand14_out1.repeat(&repeats)
};
let transpose5_out1 = tile1_out1.clone().permute([0, 2, 1]);
let and4_out1 = tile1_out1.bool_and(transpose5_out1);
let and5_out1 = and4_out1.bool_and(and2_out1);
let not1_out1 = and5_out1.bool_not();
let not2_out1 = and3_out1.bool_not();
let slice3_out1 = not2_out1.slice(s![.., 70..]);
let slice4_out1 = not1_out1.slice(s![.., 70.., ..]);
let gather1_out1 = {
let sliced = cache_last_channel.slice(s![0, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let gather2_out1 = {
let sliced = cache_last_time.slice(s![0, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let layernormalization1_out1 = {
let dtype = slice1_out1.dtype();
self.layernormalization1
.forward(slice1_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear2_out1 = self.linear2.forward(layernormalization1_out1);
let sigmoid1_out1 = burn::tensor::activation::sigmoid(linear2_out1.clone());
let mul17_out1 = linear2_out1.mul(sigmoid1_out1);
(
mul17_out1,
gather1_out1,
transpose4_out1,
unsqueeze19_out1,
slice4_out1,
slice3_out1,
gather2_out1,
clip1_out1,
)
}
}
#[derive(Module, Debug)]
pub struct Submodule3 {
linear3: Linear,
constant493: burn::module::Param<Tensor<1>>,
layernormalization2: LayerNorm,
linear4: Linear,
linear5: Linear,
linear6: Linear,
linear7: Linear,
constant23: burn::module::Param<Tensor<2>>,
constant24: burn::module::Param<Tensor<2>>,
constant496: burn::module::Param<Tensor<1>>,
linear8: Linear,
layernormalization3: LayerNorm,
conv1d1: Conv1d,
#[module(skip)]
device: Device,
}
impl Submodule3 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let linear3 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let constant493: burn::module::Param<Tensor<1>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
1,
>::from_data(
burn::tensor::TensorData::from([0.5f64]),
(device, burn::tensor::DType::F32),
),
device.clone(),
false,
[1].into(),
);
let layernormalization2 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear4 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear5 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear6 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear7 = LinearConfig::new(512, 512).with_bias(false).init(device);
let constant23: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let constant24: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let constant496: burn::module::Param<Tensor<1>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
1,
>::from_data(
burn::tensor::TensorData::from([0.125f64]),
(device, burn::tensor::DType::F32),
),
device.clone(),
false,
[1].into(),
);
let linear8 = LinearConfig::new(512, 512).with_bias(false).init(device);
let layernormalization3 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d1 = Conv1dConfig::new(512, 1024, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
Self {
linear3,
constant493,
layernormalization2,
linear4,
linear5,
linear6,
linear7,
constant23,
constant24,
constant496,
linear8,
layernormalization3,
conv1d1,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
mul17_out1: Tensor<3>,
slice1_out1: Tensor<3>,
gather1_out1: Tensor<3>,
max2_out1: Tensor<1, Int>,
transpose4_out1: Tensor<3>,
constant310_out1: [i64; 1],
add6_out1: Tensor<1, Int>,
unsqueeze19_out1: [i64; 1],
slice4_out1: Tensor<3, Bool>,
slice3_out1: Tensor<2, Bool>,
gather2_out1: Tensor<3>,
) -> (
Tensor<3>,
Tensor<3>,
Tensor<1>,
[i64; 1],
[i64; 4],
[i64; 4],
Tensor<1>,
Tensor<4, Bool>,
f32,
Tensor<4, Bool>,
f32,
Tensor<3, Bool>,
Tensor<3>,
) {
let linear3_out1 = self.linear3.forward(mul17_out1);
let constant493_out1 = self.constant493.val();
let mul18_out1 = linear3_out1
.mul((constant493_out1.clone()).unsqueeze_dims(&[0isize, 1isize]));
let add20_out1 = slice1_out1.add(mul18_out1);
let layernormalization2_out1 = {
let dtype = add20_out1.clone().dtype();
self.layernormalization2
.forward(add20_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let concat14_out1 = burn::tensor::Tensor::cat(
[gather1_out1.clone(), layernormalization2_out1.clone()].into(),
1,
);
let unsqueeze26_out1 = {
let v: i64 = max2_out1.into_scalar::<i64>();
[v as i64]
};
let transpose6_out1 = gather1_out1.permute([1, 0, 2]);
let slice5_out1 = transpose6_out1
.slice(s![unsqueeze26_out1[0]..9223372036854775807, .., ..]);
let transpose7_out1 = slice5_out1.permute([1, 0, 2]);
let concat15_out1 = burn::tensor::Tensor::cat(
[transpose7_out1, layernormalization2_out1.clone()].into(),
1,
);
let linear4_out1 = self.linear4.forward(layernormalization2_out1);
let reshape2_out1 = linear4_out1.reshape([1, -1, 8, 64]);
let linear5_out1 = self.linear5.forward(concat14_out1.clone());
let reshape3_out1 = linear5_out1.reshape([1, -1, 8, 64]);
let linear6_out1 = self.linear6.forward(concat14_out1);
let reshape4_out1 = linear6_out1.reshape([1, -1, 8, 64]);
let transpose8_out1 = reshape3_out1.permute([0, 2, 1, 3]);
let transpose9_out1 = reshape4_out1.permute([0, 2, 1, 3]);
let linear7_out1 = self.linear7.forward(transpose4_out1);
let reshape5_out1 = linear7_out1.reshape([1, -1, 8, 64]);
let constant23_out1 = self.constant23.val();
let add21_out1 = reshape2_out1
.clone()
.add((constant23_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose10_out1 = add21_out1.permute([0, 2, 1, 3]);
let constant24_out1 = self.constant24.val();
let add22_out1 = reshape2_out1
.add((constant24_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose11_out1 = add22_out1.permute([0, 2, 1, 3]);
let transpose12_out1 = reshape5_out1.permute([0, 2, 3, 1]);
let matmul8_out1 = transpose11_out1.matmul(transpose12_out1);
let pad1_out1 = matmul8_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize), (1usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let constant326_out1: [i64; 1] = [-1i64];
let constant495_out1: [i64; 1] = [8i64];
let concat16_out1: [i64; 4usize] = [
&constant310_out1[..],
&constant495_out1[..],
&constant326_out1[..],
&unsqueeze26_out1[..],
]
.concat()
.try_into()
.unwrap();
let reshape6_out1 = pad1_out1.reshape(concat16_out1);
let slice6_out1 = reshape6_out1.slice(s![.., .., 1.., ..]);
let unsqueeze27_out1 = {
let v: i64 = add6_out1.into_scalar::<i64>();
[v as i64]
};
let concat17_out1: [i64; 4usize] = [
&constant310_out1[..],
&constant495_out1[..],
&unsqueeze26_out1[..],
&unsqueeze27_out1[..],
]
.concat()
.try_into()
.unwrap();
let reshape7_out1 = slice6_out1.reshape(concat17_out1);
let transpose13_out1 = reshape7_out1.permute([3, 0, 1, 2]);
let slice7_out1 = transpose13_out1.slice(s![0..unsqueeze19_out1[0], .., .., ..]);
let transpose14_out1 = slice7_out1.permute([1, 2, 3, 0]);
let constant496_out1 = self.constant496.val();
let mul19_out1 = transpose14_out1
.mul((constant496_out1.clone()).unsqueeze_dims(&[0isize, 1isize, 2isize]));
let unsqueeze28_out1: Tensor<4, Bool> = slice4_out1.unsqueeze_dims::<4>(&[1]);
let constant325_out1 = -10000f32;
let where1_out1 = mul19_out1
.mask_fill(unsqueeze28_out1.clone(), constant325_out1);
let (attention1_out1,) = {
let q = transpose10_out1;
let k = transpose8_out1;
let v = transpose9_out1;
let score_bias = where1_out1;
let hidden_rows = score_bias
.clone()
.max_dim(3)
.equal_elem(f32::NEG_INFINITY);
let attention1_out1 = burn::tensor::module::attention(
q,
k,
v,
None,
Some(score_bias),
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention1_out1 = {
let dims = attention1_out1.dims();
attention1_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention1_out1,)
};
let cast13_out1 = unsqueeze28_out1.clone().int().cast(burn::tensor::DType::I64);
let reducemin1_out1 = {
cast13_out1.min_dim(3usize).squeeze_dims::<3usize>(&[3])
};
let cast14_out1 = reducemin1_out1.bool();
let unsqueeze29_out1: Tensor<4, Bool> = cast14_out1.unsqueeze_dims::<4>(&[-1]);
let expand15_out1 = {
let onnx_shape: [i64; 4usize] = [1, 8, 1, 64];
let input_dims = unsqueeze29_out1.dims();
let mut shape = onnx_shape;
#[allow(clippy::needless_range_loop)]
for i in 0..4usize {
let dim_offset = i;
if shape[dim_offset] == 1 && input_dims[i] > 1 {
shape[dim_offset] = input_dims[i] as i64;
}
}
unsqueeze29_out1.expand(shape)
};
let constant328_out1 = 0f32;
let where2_out1 = attention1_out1
.mask_fill(expand15_out1.clone(), constant328_out1);
let transpose15_out1 = where2_out1.permute([0, 2, 1, 3]);
let reshape8_out1 = transpose15_out1.reshape([1, -1, 512]);
let linear8_out1 = self.linear8.forward(reshape8_out1);
let add23_out1 = add20_out1.add(linear8_out1);
let layernormalization3_out1 = {
let dtype = add23_out1.clone().dtype();
self.layernormalization3
.forward(add23_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose16_out1 = layernormalization3_out1.permute([0, 2, 1]);
let conv1d1_out1 = self.conv1d1.forward(transpose16_out1);
let split_tensors = conv1d1_out1.split_with_sizes([512, 512].into(), 1);
let [split1_out1, split1_out2] = split_tensors.try_into().unwrap();
let sigmoid2_out1 = burn::tensor::activation::sigmoid(split1_out2);
let mul20_out1 = split1_out1.mul(sigmoid2_out1);
let unsqueeze30_out1: Tensor<3, Bool> = slice3_out1.unsqueeze_dims::<3>(&[1]);
let where3_out1 = mul20_out1
.mask_fill(unsqueeze30_out1.clone(), constant328_out1);
let pad2_out1 = where3_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let concat18_out1 = burn::tensor::Tensor::cat(
[gather2_out1, pad2_out1].into(),
2,
);
(
concat18_out1,
add23_out1,
constant493_out1,
unsqueeze26_out1,
concat16_out1,
concat17_out1,
constant496_out1,
unsqueeze28_out1,
constant325_out1,
expand15_out1,
constant328_out1,
unsqueeze30_out1,
concat15_out1,
)
}
}
#[derive(Module, Debug)]
pub struct Submodule4 {
conv1d2: Conv1d,
layernormalization4: LayerNorm,
conv1d3: Conv1d,
layernormalization5: LayerNorm,
linear9: Linear,
linear10: Linear,
layernormalization6: LayerNorm,
layernormalization7: LayerNorm,
linear11: Linear,
linear12: Linear,
layernormalization8: LayerNorm,
linear13: Linear,
linear14: Linear,
linear15: Linear,
linear16: Linear,
constant40: burn::module::Param<Tensor<2>>,
constant41: burn::module::Param<Tensor<2>>,
linear17: Linear,
#[module(skip)]
device: Device,
}
impl Submodule4 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let conv1d2 = Conv1dConfig::new(512, 512, 9)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(512)
.with_bias(false)
.init(device);
let layernormalization4 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d3 = Conv1dConfig::new(512, 512, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let layernormalization5 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear9 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear10 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization6 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let layernormalization7 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear11 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear12 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization8 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear13 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear14 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear15 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear16 = LinearConfig::new(512, 512).with_bias(false).init(device);
let constant40: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let constant41: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let linear17 = LinearConfig::new(512, 512).with_bias(false).init(device);
Self {
conv1d2,
layernormalization4,
conv1d3,
layernormalization5,
linear9,
linear10,
layernormalization6,
layernormalization7,
linear11,
linear12,
layernormalization8,
linear13,
linear14,
linear15,
linear16,
constant40,
constant41,
linear17,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
concat18_out1: Tensor<3>,
add23_out1: Tensor<3>,
constant493_out1: Tensor<1>,
cache_last_channel: Tensor<4>,
cache_last_time: Tensor<4>,
unsqueeze26_out1: [i64; 1],
transpose4_out1: Tensor<3>,
concat16_out1: [i64; 4],
concat17_out1: [i64; 4],
unsqueeze19_out1: [i64; 1],
constant496_out1: Tensor<1>,
unsqueeze28_out1: Tensor<4, Bool>,
constant325_out1: f32,
expand15_out1: Tensor<4, Bool>,
constant328_out1: f32,
) -> (Tensor<3>, Tensor<3>, Tensor<3>, Tensor<3>) {
let slice8_out1 = concat18_out1.clone().slice(s![.., .., - 8..]);
let conv1d2_out1 = self.conv1d2.forward(concat18_out1);
let transpose17_out1 = conv1d2_out1.permute([0, 2, 1]);
let layernormalization4_out1 = {
let dtype = transpose17_out1.dtype();
self.layernormalization4
.forward(transpose17_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose18_out1 = layernormalization4_out1.permute([0, 2, 1]);
let sigmoid3_out1 = burn::tensor::activation::sigmoid(transpose18_out1.clone());
let mul21_out1 = transpose18_out1.mul(sigmoid3_out1);
let conv1d3_out1 = self.conv1d3.forward(mul21_out1);
let transpose19_out1 = conv1d3_out1.permute([0, 2, 1]);
let add24_out1 = add23_out1.add(transpose19_out1);
let layernormalization5_out1 = {
let dtype = add24_out1.clone().dtype();
self.layernormalization5
.forward(add24_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear9_out1 = self.linear9.forward(layernormalization5_out1);
let sigmoid4_out1 = burn::tensor::activation::sigmoid(linear9_out1.clone());
let mul22_out1 = linear9_out1.mul(sigmoid4_out1);
let linear10_out1 = self.linear10.forward(mul22_out1);
let mul23_out1 = linear10_out1
.mul((constant493_out1.clone()).unsqueeze_dims(&[0isize, 1isize]));
let add25_out1 = add24_out1.add(mul23_out1);
let layernormalization6_out1 = {
let dtype = add25_out1.dtype();
self.layernormalization6
.forward(add25_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let gather3_out1 = {
let sliced = cache_last_channel.slice(s![1, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let gather4_out1 = {
let sliced = cache_last_time.slice(s![1, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let layernormalization7_out1 = {
let dtype = layernormalization6_out1.clone().dtype();
self.layernormalization7
.forward(layernormalization6_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear11_out1 = self.linear11.forward(layernormalization7_out1);
let sigmoid5_out1 = burn::tensor::activation::sigmoid(linear11_out1.clone());
let mul24_out1 = linear11_out1.mul(sigmoid5_out1);
let linear12_out1 = self.linear12.forward(mul24_out1);
let mul25_out1 = linear12_out1
.mul((constant493_out1).unsqueeze_dims(&[0isize, 1isize]));
let add26_out1 = layernormalization6_out1.add(mul25_out1);
let layernormalization8_out1 = {
let dtype = add26_out1.clone().dtype();
self.layernormalization8
.forward(add26_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let concat19_out1 = burn::tensor::Tensor::cat(
[gather3_out1.clone(), layernormalization8_out1.clone()].into(),
1,
);
let transpose20_out1 = gather3_out1.permute([1, 0, 2]);
let slice9_out1 = transpose20_out1
.slice(s![unsqueeze26_out1[0]..9223372036854775807, .., ..]);
let transpose21_out1 = slice9_out1.permute([1, 0, 2]);
let concat20_out1 = burn::tensor::Tensor::cat(
[transpose21_out1, layernormalization8_out1.clone()].into(),
1,
);
let linear13_out1 = self.linear13.forward(layernormalization8_out1);
let reshape9_out1 = linear13_out1.reshape([1, -1, 8, 64]);
let linear14_out1 = self.linear14.forward(concat19_out1.clone());
let reshape10_out1 = linear14_out1.reshape([1, -1, 8, 64]);
let linear15_out1 = self.linear15.forward(concat19_out1);
let reshape11_out1 = linear15_out1.reshape([1, -1, 8, 64]);
let transpose22_out1 = reshape10_out1.permute([0, 2, 1, 3]);
let transpose23_out1 = reshape11_out1.permute([0, 2, 1, 3]);
let linear16_out1 = self.linear16.forward(transpose4_out1);
let reshape12_out1 = linear16_out1.reshape([1, -1, 8, 64]);
let constant40_out1 = self.constant40.val();
let add27_out1 = reshape9_out1
.clone()
.add((constant40_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose24_out1 = add27_out1.permute([0, 2, 1, 3]);
let constant41_out1 = self.constant41.val();
let add28_out1 = reshape9_out1
.add((constant41_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose25_out1 = add28_out1.permute([0, 2, 1, 3]);
let transpose26_out1 = reshape12_out1.permute([0, 2, 3, 1]);
let matmul18_out1 = transpose25_out1.matmul(transpose26_out1);
let pad3_out1 = matmul18_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize), (1usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let reshape13_out1 = pad3_out1.reshape(concat16_out1);
let slice10_out1 = reshape13_out1.slice(s![.., .., 1.., ..]);
let reshape14_out1 = slice10_out1.reshape(concat17_out1);
let transpose27_out1 = reshape14_out1.permute([3, 0, 1, 2]);
let slice11_out1 = transpose27_out1
.slice(s![0..unsqueeze19_out1[0], .., .., ..]);
let transpose28_out1 = slice11_out1.permute([1, 2, 3, 0]);
let mul26_out1 = transpose28_out1
.mul((constant496_out1).unsqueeze_dims(&[0isize, 1isize, 2isize]));
let where4_out1 = mul26_out1.mask_fill(unsqueeze28_out1, constant325_out1);
let (attention2_out1,) = {
let q = transpose24_out1;
let k = transpose22_out1;
let v = transpose23_out1;
let score_bias = where4_out1;
let hidden_rows = score_bias
.clone()
.max_dim(3)
.equal_elem(f32::NEG_INFINITY);
let attention2_out1 = burn::tensor::module::attention(
q,
k,
v,
None,
Some(score_bias),
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention2_out1 = {
let dims = attention2_out1.dims();
attention2_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention2_out1,)
};
let where5_out1 = attention2_out1.mask_fill(expand15_out1, constant328_out1);
let transpose29_out1 = where5_out1.permute([0, 2, 1, 3]);
let reshape15_out1 = transpose29_out1.reshape([1, -1, 512]);
let linear17_out1 = self.linear17.forward(reshape15_out1);
let add29_out1 = add26_out1.add(linear17_out1);
(add29_out1, gather4_out1, concat20_out1, slice8_out1)
}
}
#[derive(Module, Debug)]
pub struct Submodule5 {
layernormalization9: LayerNorm,
conv1d4: Conv1d,
conv1d5: Conv1d,
layernormalization10: LayerNorm,
conv1d6: Conv1d,
layernormalization11: LayerNorm,
linear18: Linear,
linear19: Linear,
layernormalization12: LayerNorm,
layernormalization13: LayerNorm,
linear20: Linear,
linear21: Linear,
layernormalization14: LayerNorm,
linear22: Linear,
linear23: Linear,
linear24: Linear,
linear25: Linear,
constant57: burn::module::Param<Tensor<2>>,
constant58: burn::module::Param<Tensor<2>>,
linear26: Linear,
#[module(skip)]
device: Device,
}
impl Submodule5 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization9 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d4 = Conv1dConfig::new(512, 1024, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let conv1d5 = Conv1dConfig::new(512, 512, 9)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(512)
.with_bias(false)
.init(device);
let layernormalization10 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d6 = Conv1dConfig::new(512, 512, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let layernormalization11 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear18 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear19 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization12 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let layernormalization13 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear20 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear21 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization14 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear22 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear23 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear24 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear25 = LinearConfig::new(512, 512).with_bias(false).init(device);
let constant57: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let constant58: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let linear26 = LinearConfig::new(512, 512).with_bias(false).init(device);
Self {
layernormalization9,
conv1d4,
conv1d5,
layernormalization10,
conv1d6,
layernormalization11,
linear18,
linear19,
layernormalization12,
layernormalization13,
linear20,
linear21,
layernormalization14,
linear22,
linear23,
linear24,
linear25,
constant57,
constant58,
linear26,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add29_out1: Tensor<3>,
unsqueeze30_out1: Tensor<3, Bool>,
constant328_out1: f32,
gather4_out1: Tensor<3>,
constant493_out1: Tensor<1>,
cache_last_channel: Tensor<4>,
cache_last_time: Tensor<4>,
unsqueeze26_out1: [i64; 1],
transpose4_out1: Tensor<3>,
concat16_out1: [i64; 4],
concat17_out1: [i64; 4],
unsqueeze19_out1: [i64; 1],
constant496_out1: Tensor<1>,
unsqueeze28_out1: Tensor<4, Bool>,
constant325_out1: f32,
expand15_out1: Tensor<4, Bool>,
) -> (Tensor<3>, Tensor<3>, Tensor<3>, Tensor<3>) {
let layernormalization9_out1 = {
let dtype = add29_out1.clone().dtype();
self.layernormalization9
.forward(add29_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose30_out1 = layernormalization9_out1.permute([0, 2, 1]);
let conv1d4_out1 = self.conv1d4.forward(transpose30_out1);
let split_tensors = conv1d4_out1.split_with_sizes([512, 512].into(), 1);
let [split2_out1, split2_out2] = split_tensors.try_into().unwrap();
let sigmoid6_out1 = burn::tensor::activation::sigmoid(split2_out2);
let mul27_out1 = split2_out1.mul(sigmoid6_out1);
let where6_out1 = mul27_out1.mask_fill(unsqueeze30_out1, constant328_out1);
let pad4_out1 = where6_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let concat21_out1 = burn::tensor::Tensor::cat(
[gather4_out1, pad4_out1].into(),
2,
);
let slice12_out1 = concat21_out1.clone().slice(s![.., .., - 8..]);
let conv1d5_out1 = self.conv1d5.forward(concat21_out1);
let transpose31_out1 = conv1d5_out1.permute([0, 2, 1]);
let layernormalization10_out1 = {
let dtype = transpose31_out1.dtype();
self.layernormalization10
.forward(transpose31_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose32_out1 = layernormalization10_out1.permute([0, 2, 1]);
let sigmoid7_out1 = burn::tensor::activation::sigmoid(transpose32_out1.clone());
let mul28_out1 = transpose32_out1.mul(sigmoid7_out1);
let conv1d6_out1 = self.conv1d6.forward(mul28_out1);
let transpose33_out1 = conv1d6_out1.permute([0, 2, 1]);
let add30_out1 = add29_out1.add(transpose33_out1);
let layernormalization11_out1 = {
let dtype = add30_out1.clone().dtype();
self.layernormalization11
.forward(add30_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear18_out1 = self.linear18.forward(layernormalization11_out1);
let sigmoid8_out1 = burn::tensor::activation::sigmoid(linear18_out1.clone());
let mul29_out1 = linear18_out1.mul(sigmoid8_out1);
let linear19_out1 = self.linear19.forward(mul29_out1);
let mul30_out1 = linear19_out1
.mul((constant493_out1.clone()).unsqueeze_dims(&[0isize, 1isize]));
let add31_out1 = add30_out1.add(mul30_out1);
let layernormalization12_out1 = {
let dtype = add31_out1.dtype();
self.layernormalization12
.forward(add31_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let gather5_out1 = {
let sliced = cache_last_channel.slice(s![2, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let gather6_out1 = {
let sliced = cache_last_time.slice(s![2, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let layernormalization13_out1 = {
let dtype = layernormalization12_out1.clone().dtype();
self.layernormalization13
.forward(
layernormalization12_out1.clone().cast(burn::tensor::DType::F32),
)
.cast(dtype)
};
let linear20_out1 = self.linear20.forward(layernormalization13_out1);
let sigmoid9_out1 = burn::tensor::activation::sigmoid(linear20_out1.clone());
let mul31_out1 = linear20_out1.mul(sigmoid9_out1);
let linear21_out1 = self.linear21.forward(mul31_out1);
let mul32_out1 = linear21_out1
.mul((constant493_out1).unsqueeze_dims(&[0isize, 1isize]));
let add32_out1 = layernormalization12_out1.add(mul32_out1);
let layernormalization14_out1 = {
let dtype = add32_out1.clone().dtype();
self.layernormalization14
.forward(add32_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let concat22_out1 = burn::tensor::Tensor::cat(
[gather5_out1.clone(), layernormalization14_out1.clone()].into(),
1,
);
let transpose34_out1 = gather5_out1.permute([1, 0, 2]);
let slice13_out1 = transpose34_out1
.slice(s![unsqueeze26_out1[0]..9223372036854775807, .., ..]);
let transpose35_out1 = slice13_out1.permute([1, 0, 2]);
let concat23_out1 = burn::tensor::Tensor::cat(
[transpose35_out1, layernormalization14_out1.clone()].into(),
1,
);
let linear22_out1 = self.linear22.forward(layernormalization14_out1);
let reshape16_out1 = linear22_out1.reshape([1, -1, 8, 64]);
let linear23_out1 = self.linear23.forward(concat22_out1.clone());
let reshape17_out1 = linear23_out1.reshape([1, -1, 8, 64]);
let linear24_out1 = self.linear24.forward(concat22_out1);
let reshape18_out1 = linear24_out1.reshape([1, -1, 8, 64]);
let transpose36_out1 = reshape17_out1.permute([0, 2, 1, 3]);
let transpose37_out1 = reshape18_out1.permute([0, 2, 1, 3]);
let linear25_out1 = self.linear25.forward(transpose4_out1);
let reshape19_out1 = linear25_out1.reshape([1, -1, 8, 64]);
let constant57_out1 = self.constant57.val();
let add33_out1 = reshape16_out1
.clone()
.add((constant57_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose38_out1 = add33_out1.permute([0, 2, 1, 3]);
let constant58_out1 = self.constant58.val();
let add34_out1 = reshape16_out1
.add((constant58_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose39_out1 = add34_out1.permute([0, 2, 1, 3]);
let transpose40_out1 = reshape19_out1.permute([0, 2, 3, 1]);
let matmul28_out1 = transpose39_out1.matmul(transpose40_out1);
let pad5_out1 = matmul28_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize), (1usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let reshape20_out1 = pad5_out1.reshape(concat16_out1);
let slice14_out1 = reshape20_out1.slice(s![.., .., 1.., ..]);
let reshape21_out1 = slice14_out1.reshape(concat17_out1);
let transpose41_out1 = reshape21_out1.permute([3, 0, 1, 2]);
let slice15_out1 = transpose41_out1
.slice(s![0..unsqueeze19_out1[0], .., .., ..]);
let transpose42_out1 = slice15_out1.permute([1, 2, 3, 0]);
let mul33_out1 = transpose42_out1
.mul((constant496_out1).unsqueeze_dims(&[0isize, 1isize, 2isize]));
let where7_out1 = mul33_out1.mask_fill(unsqueeze28_out1, constant325_out1);
let (attention3_out1,) = {
let q = transpose38_out1;
let k = transpose36_out1;
let v = transpose37_out1;
let score_bias = where7_out1;
let hidden_rows = score_bias
.clone()
.max_dim(3)
.equal_elem(f32::NEG_INFINITY);
let attention3_out1 = burn::tensor::module::attention(
q,
k,
v,
None,
Some(score_bias),
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention3_out1 = {
let dims = attention3_out1.dims();
attention3_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention3_out1,)
};
let where8_out1 = attention3_out1.mask_fill(expand15_out1, constant328_out1);
let transpose43_out1 = where8_out1.permute([0, 2, 1, 3]);
let reshape22_out1 = transpose43_out1.reshape([1, -1, 512]);
let linear26_out1 = self.linear26.forward(reshape22_out1);
let add35_out1 = add32_out1.add(linear26_out1);
(add35_out1, gather6_out1, concat23_out1, slice12_out1)
}
}
#[derive(Module, Debug)]
pub struct Submodule6 {
layernormalization15: LayerNorm,
conv1d7: Conv1d,
conv1d8: Conv1d,
layernormalization16: LayerNorm,
conv1d9: Conv1d,
layernormalization17: LayerNorm,
linear27: Linear,
linear28: Linear,
layernormalization18: LayerNorm,
layernormalization19: LayerNorm,
linear29: Linear,
linear30: Linear,
layernormalization20: LayerNorm,
linear31: Linear,
linear32: Linear,
linear33: Linear,
linear34: Linear,
constant74: burn::module::Param<Tensor<2>>,
constant75: burn::module::Param<Tensor<2>>,
linear35: Linear,
#[module(skip)]
device: Device,
}
impl Submodule6 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization15 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d7 = Conv1dConfig::new(512, 1024, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let conv1d8 = Conv1dConfig::new(512, 512, 9)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(512)
.with_bias(false)
.init(device);
let layernormalization16 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d9 = Conv1dConfig::new(512, 512, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let layernormalization17 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear27 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear28 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization18 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let layernormalization19 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear29 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear30 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization20 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear31 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear32 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear33 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear34 = LinearConfig::new(512, 512).with_bias(false).init(device);
let constant74: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let constant75: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let linear35 = LinearConfig::new(512, 512).with_bias(false).init(device);
Self {
layernormalization15,
conv1d7,
conv1d8,
layernormalization16,
conv1d9,
layernormalization17,
linear27,
linear28,
layernormalization18,
layernormalization19,
linear29,
linear30,
layernormalization20,
linear31,
linear32,
linear33,
linear34,
constant74,
constant75,
linear35,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add35_out1: Tensor<3>,
unsqueeze30_out1: Tensor<3, Bool>,
constant328_out1: f32,
gather6_out1: Tensor<3>,
constant493_out1: Tensor<1>,
cache_last_channel: Tensor<4>,
cache_last_time: Tensor<4>,
unsqueeze26_out1: [i64; 1],
transpose4_out1: Tensor<3>,
concat16_out1: [i64; 4],
concat17_out1: [i64; 4],
unsqueeze19_out1: [i64; 1],
constant496_out1: Tensor<1>,
unsqueeze28_out1: Tensor<4, Bool>,
constant325_out1: f32,
expand15_out1: Tensor<4, Bool>,
) -> (Tensor<3>, Tensor<3>, Tensor<3>, Tensor<3>) {
let layernormalization15_out1 = {
let dtype = add35_out1.clone().dtype();
self.layernormalization15
.forward(add35_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose44_out1 = layernormalization15_out1.permute([0, 2, 1]);
let conv1d7_out1 = self.conv1d7.forward(transpose44_out1);
let split_tensors = conv1d7_out1.split_with_sizes([512, 512].into(), 1);
let [split3_out1, split3_out2] = split_tensors.try_into().unwrap();
let sigmoid10_out1 = burn::tensor::activation::sigmoid(split3_out2);
let mul34_out1 = split3_out1.mul(sigmoid10_out1);
let where9_out1 = mul34_out1.mask_fill(unsqueeze30_out1, constant328_out1);
let pad6_out1 = where9_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let concat24_out1 = burn::tensor::Tensor::cat(
[gather6_out1, pad6_out1].into(),
2,
);
let slice16_out1 = concat24_out1.clone().slice(s![.., .., - 8..]);
let conv1d8_out1 = self.conv1d8.forward(concat24_out1);
let transpose45_out1 = conv1d8_out1.permute([0, 2, 1]);
let layernormalization16_out1 = {
let dtype = transpose45_out1.dtype();
self.layernormalization16
.forward(transpose45_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose46_out1 = layernormalization16_out1.permute([0, 2, 1]);
let sigmoid11_out1 = burn::tensor::activation::sigmoid(transpose46_out1.clone());
let mul35_out1 = transpose46_out1.mul(sigmoid11_out1);
let conv1d9_out1 = self.conv1d9.forward(mul35_out1);
let transpose47_out1 = conv1d9_out1.permute([0, 2, 1]);
let add36_out1 = add35_out1.add(transpose47_out1);
let layernormalization17_out1 = {
let dtype = add36_out1.clone().dtype();
self.layernormalization17
.forward(add36_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear27_out1 = self.linear27.forward(layernormalization17_out1);
let sigmoid12_out1 = burn::tensor::activation::sigmoid(linear27_out1.clone());
let mul36_out1 = linear27_out1.mul(sigmoid12_out1);
let linear28_out1 = self.linear28.forward(mul36_out1);
let mul37_out1 = linear28_out1
.mul((constant493_out1.clone()).unsqueeze_dims(&[0isize, 1isize]));
let add37_out1 = add36_out1.add(mul37_out1);
let layernormalization18_out1 = {
let dtype = add37_out1.dtype();
self.layernormalization18
.forward(add37_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let gather7_out1 = {
let sliced = cache_last_channel.slice(s![3, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let gather8_out1 = {
let sliced = cache_last_time.slice(s![3, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let layernormalization19_out1 = {
let dtype = layernormalization18_out1.clone().dtype();
self.layernormalization19
.forward(
layernormalization18_out1.clone().cast(burn::tensor::DType::F32),
)
.cast(dtype)
};
let linear29_out1 = self.linear29.forward(layernormalization19_out1);
let sigmoid13_out1 = burn::tensor::activation::sigmoid(linear29_out1.clone());
let mul38_out1 = linear29_out1.mul(sigmoid13_out1);
let linear30_out1 = self.linear30.forward(mul38_out1);
let mul39_out1 = linear30_out1
.mul((constant493_out1).unsqueeze_dims(&[0isize, 1isize]));
let add38_out1 = layernormalization18_out1.add(mul39_out1);
let layernormalization20_out1 = {
let dtype = add38_out1.clone().dtype();
self.layernormalization20
.forward(add38_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let concat25_out1 = burn::tensor::Tensor::cat(
[gather7_out1.clone(), layernormalization20_out1.clone()].into(),
1,
);
let transpose48_out1 = gather7_out1.permute([1, 0, 2]);
let slice17_out1 = transpose48_out1
.slice(s![unsqueeze26_out1[0]..9223372036854775807, .., ..]);
let transpose49_out1 = slice17_out1.permute([1, 0, 2]);
let concat26_out1 = burn::tensor::Tensor::cat(
[transpose49_out1, layernormalization20_out1.clone()].into(),
1,
);
let linear31_out1 = self.linear31.forward(layernormalization20_out1);
let reshape23_out1 = linear31_out1.reshape([1, -1, 8, 64]);
let linear32_out1 = self.linear32.forward(concat25_out1.clone());
let reshape24_out1 = linear32_out1.reshape([1, -1, 8, 64]);
let linear33_out1 = self.linear33.forward(concat25_out1);
let reshape25_out1 = linear33_out1.reshape([1, -1, 8, 64]);
let transpose50_out1 = reshape24_out1.permute([0, 2, 1, 3]);
let transpose51_out1 = reshape25_out1.permute([0, 2, 1, 3]);
let linear34_out1 = self.linear34.forward(transpose4_out1);
let reshape26_out1 = linear34_out1.reshape([1, -1, 8, 64]);
let constant74_out1 = self.constant74.val();
let add39_out1 = reshape23_out1
.clone()
.add((constant74_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose52_out1 = add39_out1.permute([0, 2, 1, 3]);
let constant75_out1 = self.constant75.val();
let add40_out1 = reshape23_out1
.add((constant75_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose53_out1 = add40_out1.permute([0, 2, 1, 3]);
let transpose54_out1 = reshape26_out1.permute([0, 2, 3, 1]);
let matmul38_out1 = transpose53_out1.matmul(transpose54_out1);
let pad7_out1 = matmul38_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize), (1usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let reshape27_out1 = pad7_out1.reshape(concat16_out1);
let slice18_out1 = reshape27_out1.slice(s![.., .., 1.., ..]);
let reshape28_out1 = slice18_out1.reshape(concat17_out1);
let transpose55_out1 = reshape28_out1.permute([3, 0, 1, 2]);
let slice19_out1 = transpose55_out1
.slice(s![0..unsqueeze19_out1[0], .., .., ..]);
let transpose56_out1 = slice19_out1.permute([1, 2, 3, 0]);
let mul40_out1 = transpose56_out1
.mul((constant496_out1).unsqueeze_dims(&[0isize, 1isize, 2isize]));
let where10_out1 = mul40_out1.mask_fill(unsqueeze28_out1, constant325_out1);
let (attention4_out1,) = {
let q = transpose52_out1;
let k = transpose50_out1;
let v = transpose51_out1;
let score_bias = where10_out1;
let hidden_rows = score_bias
.clone()
.max_dim(3)
.equal_elem(f32::NEG_INFINITY);
let attention4_out1 = burn::tensor::module::attention(
q,
k,
v,
None,
Some(score_bias),
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention4_out1 = {
let dims = attention4_out1.dims();
attention4_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention4_out1,)
};
let where11_out1 = attention4_out1.mask_fill(expand15_out1, constant328_out1);
let transpose57_out1 = where11_out1.permute([0, 2, 1, 3]);
let reshape29_out1 = transpose57_out1.reshape([1, -1, 512]);
let linear35_out1 = self.linear35.forward(reshape29_out1);
let add41_out1 = add38_out1.add(linear35_out1);
(add41_out1, gather8_out1, concat26_out1, slice16_out1)
}
}
#[derive(Module, Debug)]
pub struct Submodule7 {
layernormalization21: LayerNorm,
conv1d10: Conv1d,
conv1d11: Conv1d,
layernormalization22: LayerNorm,
conv1d12: Conv1d,
layernormalization23: LayerNorm,
linear36: Linear,
linear37: Linear,
layernormalization24: LayerNorm,
layernormalization25: LayerNorm,
linear38: Linear,
linear39: Linear,
layernormalization26: LayerNorm,
linear40: Linear,
linear41: Linear,
linear42: Linear,
linear43: Linear,
constant91: burn::module::Param<Tensor<2>>,
constant92: burn::module::Param<Tensor<2>>,
linear44: Linear,
#[module(skip)]
device: Device,
}
impl Submodule7 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization21 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d10 = Conv1dConfig::new(512, 1024, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let conv1d11 = Conv1dConfig::new(512, 512, 9)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(512)
.with_bias(false)
.init(device);
let layernormalization22 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d12 = Conv1dConfig::new(512, 512, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let layernormalization23 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear36 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear37 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization24 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let layernormalization25 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear38 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear39 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization26 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear40 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear41 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear42 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear43 = LinearConfig::new(512, 512).with_bias(false).init(device);
let constant91: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let constant92: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let linear44 = LinearConfig::new(512, 512).with_bias(false).init(device);
Self {
layernormalization21,
conv1d10,
conv1d11,
layernormalization22,
conv1d12,
layernormalization23,
linear36,
linear37,
layernormalization24,
layernormalization25,
linear38,
linear39,
layernormalization26,
linear40,
linear41,
linear42,
linear43,
constant91,
constant92,
linear44,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add41_out1: Tensor<3>,
unsqueeze30_out1: Tensor<3, Bool>,
constant328_out1: f32,
gather8_out1: Tensor<3>,
constant493_out1: Tensor<1>,
cache_last_channel: Tensor<4>,
cache_last_time: Tensor<4>,
unsqueeze26_out1: [i64; 1],
transpose4_out1: Tensor<3>,
concat16_out1: [i64; 4],
concat17_out1: [i64; 4],
unsqueeze19_out1: [i64; 1],
constant496_out1: Tensor<1>,
unsqueeze28_out1: Tensor<4, Bool>,
constant325_out1: f32,
expand15_out1: Tensor<4, Bool>,
) -> (Tensor<3>, Tensor<3>, Tensor<3>, Tensor<3>) {
let layernormalization21_out1 = {
let dtype = add41_out1.clone().dtype();
self.layernormalization21
.forward(add41_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose58_out1 = layernormalization21_out1.permute([0, 2, 1]);
let conv1d10_out1 = self.conv1d10.forward(transpose58_out1);
let split_tensors = conv1d10_out1.split_with_sizes([512, 512].into(), 1);
let [split4_out1, split4_out2] = split_tensors.try_into().unwrap();
let sigmoid14_out1 = burn::tensor::activation::sigmoid(split4_out2);
let mul41_out1 = split4_out1.mul(sigmoid14_out1);
let where12_out1 = mul41_out1.mask_fill(unsqueeze30_out1, constant328_out1);
let pad8_out1 = where12_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let concat27_out1 = burn::tensor::Tensor::cat(
[gather8_out1, pad8_out1].into(),
2,
);
let slice20_out1 = concat27_out1.clone().slice(s![.., .., - 8..]);
let conv1d11_out1 = self.conv1d11.forward(concat27_out1);
let transpose59_out1 = conv1d11_out1.permute([0, 2, 1]);
let layernormalization22_out1 = {
let dtype = transpose59_out1.dtype();
self.layernormalization22
.forward(transpose59_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose60_out1 = layernormalization22_out1.permute([0, 2, 1]);
let sigmoid15_out1 = burn::tensor::activation::sigmoid(transpose60_out1.clone());
let mul42_out1 = transpose60_out1.mul(sigmoid15_out1);
let conv1d12_out1 = self.conv1d12.forward(mul42_out1);
let transpose61_out1 = conv1d12_out1.permute([0, 2, 1]);
let add42_out1 = add41_out1.add(transpose61_out1);
let layernormalization23_out1 = {
let dtype = add42_out1.clone().dtype();
self.layernormalization23
.forward(add42_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear36_out1 = self.linear36.forward(layernormalization23_out1);
let sigmoid16_out1 = burn::tensor::activation::sigmoid(linear36_out1.clone());
let mul43_out1 = linear36_out1.mul(sigmoid16_out1);
let linear37_out1 = self.linear37.forward(mul43_out1);
let mul44_out1 = linear37_out1
.mul((constant493_out1.clone()).unsqueeze_dims(&[0isize, 1isize]));
let add43_out1 = add42_out1.add(mul44_out1);
let layernormalization24_out1 = {
let dtype = add43_out1.dtype();
self.layernormalization24
.forward(add43_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let gather9_out1 = {
let sliced = cache_last_channel.slice(s![4, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let gather10_out1 = {
let sliced = cache_last_time.slice(s![4, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let layernormalization25_out1 = {
let dtype = layernormalization24_out1.clone().dtype();
self.layernormalization25
.forward(
layernormalization24_out1.clone().cast(burn::tensor::DType::F32),
)
.cast(dtype)
};
let linear38_out1 = self.linear38.forward(layernormalization25_out1);
let sigmoid17_out1 = burn::tensor::activation::sigmoid(linear38_out1.clone());
let mul45_out1 = linear38_out1.mul(sigmoid17_out1);
let linear39_out1 = self.linear39.forward(mul45_out1);
let mul46_out1 = linear39_out1
.mul((constant493_out1).unsqueeze_dims(&[0isize, 1isize]));
let add44_out1 = layernormalization24_out1.add(mul46_out1);
let layernormalization26_out1 = {
let dtype = add44_out1.clone().dtype();
self.layernormalization26
.forward(add44_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let concat28_out1 = burn::tensor::Tensor::cat(
[gather9_out1.clone(), layernormalization26_out1.clone()].into(),
1,
);
let transpose62_out1 = gather9_out1.permute([1, 0, 2]);
let slice21_out1 = transpose62_out1
.slice(s![unsqueeze26_out1[0]..9223372036854775807, .., ..]);
let transpose63_out1 = slice21_out1.permute([1, 0, 2]);
let concat29_out1 = burn::tensor::Tensor::cat(
[transpose63_out1, layernormalization26_out1.clone()].into(),
1,
);
let linear40_out1 = self.linear40.forward(layernormalization26_out1);
let reshape30_out1 = linear40_out1.reshape([1, -1, 8, 64]);
let linear41_out1 = self.linear41.forward(concat28_out1.clone());
let reshape31_out1 = linear41_out1.reshape([1, -1, 8, 64]);
let linear42_out1 = self.linear42.forward(concat28_out1);
let reshape32_out1 = linear42_out1.reshape([1, -1, 8, 64]);
let transpose64_out1 = reshape31_out1.permute([0, 2, 1, 3]);
let transpose65_out1 = reshape32_out1.permute([0, 2, 1, 3]);
let linear43_out1 = self.linear43.forward(transpose4_out1);
let reshape33_out1 = linear43_out1.reshape([1, -1, 8, 64]);
let constant91_out1 = self.constant91.val();
let add45_out1 = reshape30_out1
.clone()
.add((constant91_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose66_out1 = add45_out1.permute([0, 2, 1, 3]);
let constant92_out1 = self.constant92.val();
let add46_out1 = reshape30_out1
.add((constant92_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose67_out1 = add46_out1.permute([0, 2, 1, 3]);
let transpose68_out1 = reshape33_out1.permute([0, 2, 3, 1]);
let matmul48_out1 = transpose67_out1.matmul(transpose68_out1);
let pad9_out1 = matmul48_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize), (1usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let reshape34_out1 = pad9_out1.reshape(concat16_out1);
let slice22_out1 = reshape34_out1.slice(s![.., .., 1.., ..]);
let reshape35_out1 = slice22_out1.reshape(concat17_out1);
let transpose69_out1 = reshape35_out1.permute([3, 0, 1, 2]);
let slice23_out1 = transpose69_out1
.slice(s![0..unsqueeze19_out1[0], .., .., ..]);
let transpose70_out1 = slice23_out1.permute([1, 2, 3, 0]);
let mul47_out1 = transpose70_out1
.mul((constant496_out1).unsqueeze_dims(&[0isize, 1isize, 2isize]));
let where13_out1 = mul47_out1.mask_fill(unsqueeze28_out1, constant325_out1);
let (attention5_out1,) = {
let q = transpose66_out1;
let k = transpose64_out1;
let v = transpose65_out1;
let score_bias = where13_out1;
let hidden_rows = score_bias
.clone()
.max_dim(3)
.equal_elem(f32::NEG_INFINITY);
let attention5_out1 = burn::tensor::module::attention(
q,
k,
v,
None,
Some(score_bias),
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention5_out1 = {
let dims = attention5_out1.dims();
attention5_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention5_out1,)
};
let where14_out1 = attention5_out1.mask_fill(expand15_out1, constant328_out1);
let transpose71_out1 = where14_out1.permute([0, 2, 1, 3]);
let reshape36_out1 = transpose71_out1.reshape([1, -1, 512]);
let linear44_out1 = self.linear44.forward(reshape36_out1);
let add47_out1 = add44_out1.add(linear44_out1);
(add47_out1, gather10_out1, concat29_out1, slice20_out1)
}
}
#[derive(Module, Debug)]
pub struct Submodule8 {
layernormalization27: LayerNorm,
conv1d13: Conv1d,
conv1d14: Conv1d,
layernormalization28: LayerNorm,
conv1d15: Conv1d,
layernormalization29: LayerNorm,
linear45: Linear,
linear46: Linear,
layernormalization30: LayerNorm,
layernormalization31: LayerNorm,
linear47: Linear,
linear48: Linear,
layernormalization32: LayerNorm,
linear49: Linear,
linear50: Linear,
linear51: Linear,
linear52: Linear,
constant108: burn::module::Param<Tensor<2>>,
constant109: burn::module::Param<Tensor<2>>,
linear53: Linear,
#[module(skip)]
device: Device,
}
impl Submodule8 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization27 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d13 = Conv1dConfig::new(512, 1024, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let conv1d14 = Conv1dConfig::new(512, 512, 9)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(512)
.with_bias(false)
.init(device);
let layernormalization28 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d15 = Conv1dConfig::new(512, 512, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let layernormalization29 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear45 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear46 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization30 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let layernormalization31 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear47 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear48 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization32 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear49 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear50 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear51 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear52 = LinearConfig::new(512, 512).with_bias(false).init(device);
let constant108: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let constant109: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let linear53 = LinearConfig::new(512, 512).with_bias(false).init(device);
Self {
layernormalization27,
conv1d13,
conv1d14,
layernormalization28,
conv1d15,
layernormalization29,
linear45,
linear46,
layernormalization30,
layernormalization31,
linear47,
linear48,
layernormalization32,
linear49,
linear50,
linear51,
linear52,
constant108,
constant109,
linear53,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add47_out1: Tensor<3>,
unsqueeze30_out1: Tensor<3, Bool>,
constant328_out1: f32,
gather10_out1: Tensor<3>,
constant493_out1: Tensor<1>,
cache_last_channel: Tensor<4>,
cache_last_time: Tensor<4>,
unsqueeze26_out1: [i64; 1],
transpose4_out1: Tensor<3>,
concat16_out1: [i64; 4],
concat17_out1: [i64; 4],
unsqueeze19_out1: [i64; 1],
constant496_out1: Tensor<1>,
unsqueeze28_out1: Tensor<4, Bool>,
constant325_out1: f32,
expand15_out1: Tensor<4, Bool>,
) -> (Tensor<3>, Tensor<3>, Tensor<3>, Tensor<3>) {
let layernormalization27_out1 = {
let dtype = add47_out1.clone().dtype();
self.layernormalization27
.forward(add47_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose72_out1 = layernormalization27_out1.permute([0, 2, 1]);
let conv1d13_out1 = self.conv1d13.forward(transpose72_out1);
let split_tensors = conv1d13_out1.split_with_sizes([512, 512].into(), 1);
let [split5_out1, split5_out2] = split_tensors.try_into().unwrap();
let sigmoid18_out1 = burn::tensor::activation::sigmoid(split5_out2);
let mul48_out1 = split5_out1.mul(sigmoid18_out1);
let where15_out1 = mul48_out1.mask_fill(unsqueeze30_out1, constant328_out1);
let pad10_out1 = where15_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let concat30_out1 = burn::tensor::Tensor::cat(
[gather10_out1, pad10_out1].into(),
2,
);
let slice24_out1 = concat30_out1.clone().slice(s![.., .., - 8..]);
let conv1d14_out1 = self.conv1d14.forward(concat30_out1);
let transpose73_out1 = conv1d14_out1.permute([0, 2, 1]);
let layernormalization28_out1 = {
let dtype = transpose73_out1.dtype();
self.layernormalization28
.forward(transpose73_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose74_out1 = layernormalization28_out1.permute([0, 2, 1]);
let sigmoid19_out1 = burn::tensor::activation::sigmoid(transpose74_out1.clone());
let mul49_out1 = transpose74_out1.mul(sigmoid19_out1);
let conv1d15_out1 = self.conv1d15.forward(mul49_out1);
let transpose75_out1 = conv1d15_out1.permute([0, 2, 1]);
let add48_out1 = add47_out1.add(transpose75_out1);
let layernormalization29_out1 = {
let dtype = add48_out1.clone().dtype();
self.layernormalization29
.forward(add48_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear45_out1 = self.linear45.forward(layernormalization29_out1);
let sigmoid20_out1 = burn::tensor::activation::sigmoid(linear45_out1.clone());
let mul50_out1 = linear45_out1.mul(sigmoid20_out1);
let linear46_out1 = self.linear46.forward(mul50_out1);
let mul51_out1 = linear46_out1
.mul((constant493_out1.clone()).unsqueeze_dims(&[0isize, 1isize]));
let add49_out1 = add48_out1.add(mul51_out1);
let layernormalization30_out1 = {
let dtype = add49_out1.dtype();
self.layernormalization30
.forward(add49_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let gather11_out1 = {
let sliced = cache_last_channel.slice(s![5, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let gather12_out1 = {
let sliced = cache_last_time.slice(s![5, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let layernormalization31_out1 = {
let dtype = layernormalization30_out1.clone().dtype();
self.layernormalization31
.forward(
layernormalization30_out1.clone().cast(burn::tensor::DType::F32),
)
.cast(dtype)
};
let linear47_out1 = self.linear47.forward(layernormalization31_out1);
let sigmoid21_out1 = burn::tensor::activation::sigmoid(linear47_out1.clone());
let mul52_out1 = linear47_out1.mul(sigmoid21_out1);
let linear48_out1 = self.linear48.forward(mul52_out1);
let mul53_out1 = linear48_out1
.mul((constant493_out1).unsqueeze_dims(&[0isize, 1isize]));
let add50_out1 = layernormalization30_out1.add(mul53_out1);
let layernormalization32_out1 = {
let dtype = add50_out1.clone().dtype();
self.layernormalization32
.forward(add50_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let concat31_out1 = burn::tensor::Tensor::cat(
[gather11_out1.clone(), layernormalization32_out1.clone()].into(),
1,
);
let transpose76_out1 = gather11_out1.permute([1, 0, 2]);
let slice25_out1 = transpose76_out1
.slice(s![unsqueeze26_out1[0]..9223372036854775807, .., ..]);
let transpose77_out1 = slice25_out1.permute([1, 0, 2]);
let concat32_out1 = burn::tensor::Tensor::cat(
[transpose77_out1, layernormalization32_out1.clone()].into(),
1,
);
let linear49_out1 = self.linear49.forward(layernormalization32_out1);
let reshape37_out1 = linear49_out1.reshape([1, -1, 8, 64]);
let linear50_out1 = self.linear50.forward(concat31_out1.clone());
let reshape38_out1 = linear50_out1.reshape([1, -1, 8, 64]);
let linear51_out1 = self.linear51.forward(concat31_out1);
let reshape39_out1 = linear51_out1.reshape([1, -1, 8, 64]);
let transpose78_out1 = reshape38_out1.permute([0, 2, 1, 3]);
let transpose79_out1 = reshape39_out1.permute([0, 2, 1, 3]);
let linear52_out1 = self.linear52.forward(transpose4_out1);
let reshape40_out1 = linear52_out1.reshape([1, -1, 8, 64]);
let constant108_out1 = self.constant108.val();
let add51_out1 = reshape37_out1
.clone()
.add((constant108_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose80_out1 = add51_out1.permute([0, 2, 1, 3]);
let constant109_out1 = self.constant109.val();
let add52_out1 = reshape37_out1
.add((constant109_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose81_out1 = add52_out1.permute([0, 2, 1, 3]);
let transpose82_out1 = reshape40_out1.permute([0, 2, 3, 1]);
let matmul58_out1 = transpose81_out1.matmul(transpose82_out1);
let pad11_out1 = matmul58_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize), (1usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let reshape41_out1 = pad11_out1.reshape(concat16_out1);
let slice26_out1 = reshape41_out1.slice(s![.., .., 1.., ..]);
let reshape42_out1 = slice26_out1.reshape(concat17_out1);
let transpose83_out1 = reshape42_out1.permute([3, 0, 1, 2]);
let slice27_out1 = transpose83_out1
.slice(s![0..unsqueeze19_out1[0], .., .., ..]);
let transpose84_out1 = slice27_out1.permute([1, 2, 3, 0]);
let mul54_out1 = transpose84_out1
.mul((constant496_out1).unsqueeze_dims(&[0isize, 1isize, 2isize]));
let where16_out1 = mul54_out1.mask_fill(unsqueeze28_out1, constant325_out1);
let (attention6_out1,) = {
let q = transpose80_out1;
let k = transpose78_out1;
let v = transpose79_out1;
let score_bias = where16_out1;
let hidden_rows = score_bias
.clone()
.max_dim(3)
.equal_elem(f32::NEG_INFINITY);
let attention6_out1 = burn::tensor::module::attention(
q,
k,
v,
None,
Some(score_bias),
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention6_out1 = {
let dims = attention6_out1.dims();
attention6_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention6_out1,)
};
let where17_out1 = attention6_out1.mask_fill(expand15_out1, constant328_out1);
let transpose85_out1 = where17_out1.permute([0, 2, 1, 3]);
let reshape43_out1 = transpose85_out1.reshape([1, -1, 512]);
let linear53_out1 = self.linear53.forward(reshape43_out1);
let add53_out1 = add50_out1.add(linear53_out1);
(add53_out1, gather12_out1, concat32_out1, slice24_out1)
}
}
#[derive(Module, Debug)]
pub struct Submodule9 {
layernormalization33: LayerNorm,
conv1d16: Conv1d,
conv1d17: Conv1d,
layernormalization34: LayerNorm,
conv1d18: Conv1d,
layernormalization35: LayerNorm,
linear54: Linear,
linear55: Linear,
layernormalization36: LayerNorm,
layernormalization37: LayerNorm,
linear56: Linear,
linear57: Linear,
layernormalization38: LayerNorm,
linear58: Linear,
linear59: Linear,
linear60: Linear,
linear61: Linear,
constant125: burn::module::Param<Tensor<2>>,
constant126: burn::module::Param<Tensor<2>>,
linear62: Linear,
#[module(skip)]
device: Device,
}
impl Submodule9 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization33 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d16 = Conv1dConfig::new(512, 1024, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let conv1d17 = Conv1dConfig::new(512, 512, 9)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(512)
.with_bias(false)
.init(device);
let layernormalization34 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d18 = Conv1dConfig::new(512, 512, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let layernormalization35 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear54 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear55 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization36 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let layernormalization37 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear56 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear57 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization38 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear58 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear59 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear60 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear61 = LinearConfig::new(512, 512).with_bias(false).init(device);
let constant125: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let constant126: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let linear62 = LinearConfig::new(512, 512).with_bias(false).init(device);
Self {
layernormalization33,
conv1d16,
conv1d17,
layernormalization34,
conv1d18,
layernormalization35,
linear54,
linear55,
layernormalization36,
layernormalization37,
linear56,
linear57,
layernormalization38,
linear58,
linear59,
linear60,
linear61,
constant125,
constant126,
linear62,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add53_out1: Tensor<3>,
unsqueeze30_out1: Tensor<3, Bool>,
constant328_out1: f32,
gather12_out1: Tensor<3>,
constant493_out1: Tensor<1>,
cache_last_channel: Tensor<4>,
cache_last_time: Tensor<4>,
unsqueeze26_out1: [i64; 1],
transpose4_out1: Tensor<3>,
concat16_out1: [i64; 4],
concat17_out1: [i64; 4],
unsqueeze19_out1: [i64; 1],
constant496_out1: Tensor<1>,
unsqueeze28_out1: Tensor<4, Bool>,
constant325_out1: f32,
expand15_out1: Tensor<4, Bool>,
) -> (Tensor<3>, Tensor<3>, Tensor<3>, Tensor<3>) {
let layernormalization33_out1 = {
let dtype = add53_out1.clone().dtype();
self.layernormalization33
.forward(add53_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose86_out1 = layernormalization33_out1.permute([0, 2, 1]);
let conv1d16_out1 = self.conv1d16.forward(transpose86_out1);
let split_tensors = conv1d16_out1.split_with_sizes([512, 512].into(), 1);
let [split6_out1, split6_out2] = split_tensors.try_into().unwrap();
let sigmoid22_out1 = burn::tensor::activation::sigmoid(split6_out2);
let mul55_out1 = split6_out1.mul(sigmoid22_out1);
let where18_out1 = mul55_out1.mask_fill(unsqueeze30_out1, constant328_out1);
let pad12_out1 = where18_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let concat33_out1 = burn::tensor::Tensor::cat(
[gather12_out1, pad12_out1].into(),
2,
);
let slice28_out1 = concat33_out1.clone().slice(s![.., .., - 8..]);
let conv1d17_out1 = self.conv1d17.forward(concat33_out1);
let transpose87_out1 = conv1d17_out1.permute([0, 2, 1]);
let layernormalization34_out1 = {
let dtype = transpose87_out1.dtype();
self.layernormalization34
.forward(transpose87_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose88_out1 = layernormalization34_out1.permute([0, 2, 1]);
let sigmoid23_out1 = burn::tensor::activation::sigmoid(transpose88_out1.clone());
let mul56_out1 = transpose88_out1.mul(sigmoid23_out1);
let conv1d18_out1 = self.conv1d18.forward(mul56_out1);
let transpose89_out1 = conv1d18_out1.permute([0, 2, 1]);
let add54_out1 = add53_out1.add(transpose89_out1);
let layernormalization35_out1 = {
let dtype = add54_out1.clone().dtype();
self.layernormalization35
.forward(add54_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear54_out1 = self.linear54.forward(layernormalization35_out1);
let sigmoid24_out1 = burn::tensor::activation::sigmoid(linear54_out1.clone());
let mul57_out1 = linear54_out1.mul(sigmoid24_out1);
let linear55_out1 = self.linear55.forward(mul57_out1);
let mul58_out1 = linear55_out1
.mul((constant493_out1.clone()).unsqueeze_dims(&[0isize, 1isize]));
let add55_out1 = add54_out1.add(mul58_out1);
let layernormalization36_out1 = {
let dtype = add55_out1.dtype();
self.layernormalization36
.forward(add55_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let gather13_out1 = {
let sliced = cache_last_channel.slice(s![6, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let gather14_out1 = {
let sliced = cache_last_time.slice(s![6, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let layernormalization37_out1 = {
let dtype = layernormalization36_out1.clone().dtype();
self.layernormalization37
.forward(
layernormalization36_out1.clone().cast(burn::tensor::DType::F32),
)
.cast(dtype)
};
let linear56_out1 = self.linear56.forward(layernormalization37_out1);
let sigmoid25_out1 = burn::tensor::activation::sigmoid(linear56_out1.clone());
let mul59_out1 = linear56_out1.mul(sigmoid25_out1);
let linear57_out1 = self.linear57.forward(mul59_out1);
let mul60_out1 = linear57_out1
.mul((constant493_out1).unsqueeze_dims(&[0isize, 1isize]));
let add56_out1 = layernormalization36_out1.add(mul60_out1);
let layernormalization38_out1 = {
let dtype = add56_out1.clone().dtype();
self.layernormalization38
.forward(add56_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let concat34_out1 = burn::tensor::Tensor::cat(
[gather13_out1.clone(), layernormalization38_out1.clone()].into(),
1,
);
let transpose90_out1 = gather13_out1.permute([1, 0, 2]);
let slice29_out1 = transpose90_out1
.slice(s![unsqueeze26_out1[0]..9223372036854775807, .., ..]);
let transpose91_out1 = slice29_out1.permute([1, 0, 2]);
let concat35_out1 = burn::tensor::Tensor::cat(
[transpose91_out1, layernormalization38_out1.clone()].into(),
1,
);
let linear58_out1 = self.linear58.forward(layernormalization38_out1);
let reshape44_out1 = linear58_out1.reshape([1, -1, 8, 64]);
let linear59_out1 = self.linear59.forward(concat34_out1.clone());
let reshape45_out1 = linear59_out1.reshape([1, -1, 8, 64]);
let linear60_out1 = self.linear60.forward(concat34_out1);
let reshape46_out1 = linear60_out1.reshape([1, -1, 8, 64]);
let transpose92_out1 = reshape45_out1.permute([0, 2, 1, 3]);
let transpose93_out1 = reshape46_out1.permute([0, 2, 1, 3]);
let linear61_out1 = self.linear61.forward(transpose4_out1);
let reshape47_out1 = linear61_out1.reshape([1, -1, 8, 64]);
let constant125_out1 = self.constant125.val();
let add57_out1 = reshape44_out1
.clone()
.add((constant125_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose94_out1 = add57_out1.permute([0, 2, 1, 3]);
let constant126_out1 = self.constant126.val();
let add58_out1 = reshape44_out1
.add((constant126_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose95_out1 = add58_out1.permute([0, 2, 1, 3]);
let transpose96_out1 = reshape47_out1.permute([0, 2, 3, 1]);
let matmul68_out1 = transpose95_out1.matmul(transpose96_out1);
let pad13_out1 = matmul68_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize), (1usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let reshape48_out1 = pad13_out1.reshape(concat16_out1);
let slice30_out1 = reshape48_out1.slice(s![.., .., 1.., ..]);
let reshape49_out1 = slice30_out1.reshape(concat17_out1);
let transpose97_out1 = reshape49_out1.permute([3, 0, 1, 2]);
let slice31_out1 = transpose97_out1
.slice(s![0..unsqueeze19_out1[0], .., .., ..]);
let transpose98_out1 = slice31_out1.permute([1, 2, 3, 0]);
let mul61_out1 = transpose98_out1
.mul((constant496_out1).unsqueeze_dims(&[0isize, 1isize, 2isize]));
let where19_out1 = mul61_out1.mask_fill(unsqueeze28_out1, constant325_out1);
let (attention7_out1,) = {
let q = transpose94_out1;
let k = transpose92_out1;
let v = transpose93_out1;
let score_bias = where19_out1;
let hidden_rows = score_bias
.clone()
.max_dim(3)
.equal_elem(f32::NEG_INFINITY);
let attention7_out1 = burn::tensor::module::attention(
q,
k,
v,
None,
Some(score_bias),
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention7_out1 = {
let dims = attention7_out1.dims();
attention7_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention7_out1,)
};
let where20_out1 = attention7_out1.mask_fill(expand15_out1, constant328_out1);
let transpose99_out1 = where20_out1.permute([0, 2, 1, 3]);
let reshape50_out1 = transpose99_out1.reshape([1, -1, 512]);
let linear62_out1 = self.linear62.forward(reshape50_out1);
let add59_out1 = add56_out1.add(linear62_out1);
(add59_out1, gather14_out1, concat35_out1, slice28_out1)
}
}
#[derive(Module, Debug)]
pub struct Submodule10 {
layernormalization39: LayerNorm,
conv1d19: Conv1d,
conv1d20: Conv1d,
layernormalization40: LayerNorm,
conv1d21: Conv1d,
layernormalization41: LayerNorm,
linear63: Linear,
linear64: Linear,
layernormalization42: LayerNorm,
layernormalization43: LayerNorm,
linear65: Linear,
linear66: Linear,
layernormalization44: LayerNorm,
linear67: Linear,
linear68: Linear,
linear69: Linear,
linear70: Linear,
constant142: burn::module::Param<Tensor<2>>,
constant143: burn::module::Param<Tensor<2>>,
linear71: Linear,
#[module(skip)]
device: Device,
}
impl Submodule10 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization39 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d19 = Conv1dConfig::new(512, 1024, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let conv1d20 = Conv1dConfig::new(512, 512, 9)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(512)
.with_bias(false)
.init(device);
let layernormalization40 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d21 = Conv1dConfig::new(512, 512, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let layernormalization41 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear63 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear64 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization42 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let layernormalization43 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear65 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear66 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization44 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear67 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear68 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear69 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear70 = LinearConfig::new(512, 512).with_bias(false).init(device);
let constant142: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let constant143: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let linear71 = LinearConfig::new(512, 512).with_bias(false).init(device);
Self {
layernormalization39,
conv1d19,
conv1d20,
layernormalization40,
conv1d21,
layernormalization41,
linear63,
linear64,
layernormalization42,
layernormalization43,
linear65,
linear66,
layernormalization44,
linear67,
linear68,
linear69,
linear70,
constant142,
constant143,
linear71,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add59_out1: Tensor<3>,
unsqueeze30_out1: Tensor<3, Bool>,
constant328_out1: f32,
gather14_out1: Tensor<3>,
constant493_out1: Tensor<1>,
cache_last_channel: Tensor<4>,
cache_last_time: Tensor<4>,
unsqueeze26_out1: [i64; 1],
transpose4_out1: Tensor<3>,
concat16_out1: [i64; 4],
concat17_out1: [i64; 4],
unsqueeze19_out1: [i64; 1],
constant496_out1: Tensor<1>,
unsqueeze28_out1: Tensor<4, Bool>,
constant325_out1: f32,
expand15_out1: Tensor<4, Bool>,
) -> (Tensor<3>, Tensor<3>, Tensor<3>, Tensor<3>) {
let layernormalization39_out1 = {
let dtype = add59_out1.clone().dtype();
self.layernormalization39
.forward(add59_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose100_out1 = layernormalization39_out1.permute([0, 2, 1]);
let conv1d19_out1 = self.conv1d19.forward(transpose100_out1);
let split_tensors = conv1d19_out1.split_with_sizes([512, 512].into(), 1);
let [split7_out1, split7_out2] = split_tensors.try_into().unwrap();
let sigmoid26_out1 = burn::tensor::activation::sigmoid(split7_out2);
let mul62_out1 = split7_out1.mul(sigmoid26_out1);
let where21_out1 = mul62_out1.mask_fill(unsqueeze30_out1, constant328_out1);
let pad14_out1 = where21_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let concat36_out1 = burn::tensor::Tensor::cat(
[gather14_out1, pad14_out1].into(),
2,
);
let slice32_out1 = concat36_out1.clone().slice(s![.., .., - 8..]);
let conv1d20_out1 = self.conv1d20.forward(concat36_out1);
let transpose101_out1 = conv1d20_out1.permute([0, 2, 1]);
let layernormalization40_out1 = {
let dtype = transpose101_out1.dtype();
self.layernormalization40
.forward(transpose101_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose102_out1 = layernormalization40_out1.permute([0, 2, 1]);
let sigmoid27_out1 = burn::tensor::activation::sigmoid(
transpose102_out1.clone(),
);
let mul63_out1 = transpose102_out1.mul(sigmoid27_out1);
let conv1d21_out1 = self.conv1d21.forward(mul63_out1);
let transpose103_out1 = conv1d21_out1.permute([0, 2, 1]);
let add60_out1 = add59_out1.add(transpose103_out1);
let layernormalization41_out1 = {
let dtype = add60_out1.clone().dtype();
self.layernormalization41
.forward(add60_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear63_out1 = self.linear63.forward(layernormalization41_out1);
let sigmoid28_out1 = burn::tensor::activation::sigmoid(linear63_out1.clone());
let mul64_out1 = linear63_out1.mul(sigmoid28_out1);
let linear64_out1 = self.linear64.forward(mul64_out1);
let mul65_out1 = linear64_out1
.mul((constant493_out1.clone()).unsqueeze_dims(&[0isize, 1isize]));
let add61_out1 = add60_out1.add(mul65_out1);
let layernormalization42_out1 = {
let dtype = add61_out1.dtype();
self.layernormalization42
.forward(add61_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let gather15_out1 = {
let sliced = cache_last_channel.slice(s![7, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let gather16_out1 = {
let sliced = cache_last_time.slice(s![7, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let layernormalization43_out1 = {
let dtype = layernormalization42_out1.clone().dtype();
self.layernormalization43
.forward(
layernormalization42_out1.clone().cast(burn::tensor::DType::F32),
)
.cast(dtype)
};
let linear65_out1 = self.linear65.forward(layernormalization43_out1);
let sigmoid29_out1 = burn::tensor::activation::sigmoid(linear65_out1.clone());
let mul66_out1 = linear65_out1.mul(sigmoid29_out1);
let linear66_out1 = self.linear66.forward(mul66_out1);
let mul67_out1 = linear66_out1
.mul((constant493_out1).unsqueeze_dims(&[0isize, 1isize]));
let add62_out1 = layernormalization42_out1.add(mul67_out1);
let layernormalization44_out1 = {
let dtype = add62_out1.clone().dtype();
self.layernormalization44
.forward(add62_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let concat37_out1 = burn::tensor::Tensor::cat(
[gather15_out1.clone(), layernormalization44_out1.clone()].into(),
1,
);
let transpose104_out1 = gather15_out1.permute([1, 0, 2]);
let slice33_out1 = transpose104_out1
.slice(s![unsqueeze26_out1[0]..9223372036854775807, .., ..]);
let transpose105_out1 = slice33_out1.permute([1, 0, 2]);
let concat38_out1 = burn::tensor::Tensor::cat(
[transpose105_out1, layernormalization44_out1.clone()].into(),
1,
);
let linear67_out1 = self.linear67.forward(layernormalization44_out1);
let reshape51_out1 = linear67_out1.reshape([1, -1, 8, 64]);
let linear68_out1 = self.linear68.forward(concat37_out1.clone());
let reshape52_out1 = linear68_out1.reshape([1, -1, 8, 64]);
let linear69_out1 = self.linear69.forward(concat37_out1);
let reshape53_out1 = linear69_out1.reshape([1, -1, 8, 64]);
let transpose106_out1 = reshape52_out1.permute([0, 2, 1, 3]);
let transpose107_out1 = reshape53_out1.permute([0, 2, 1, 3]);
let linear70_out1 = self.linear70.forward(transpose4_out1);
let reshape54_out1 = linear70_out1.reshape([1, -1, 8, 64]);
let constant142_out1 = self.constant142.val();
let add63_out1 = reshape51_out1
.clone()
.add((constant142_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose108_out1 = add63_out1.permute([0, 2, 1, 3]);
let constant143_out1 = self.constant143.val();
let add64_out1 = reshape51_out1
.add((constant143_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose109_out1 = add64_out1.permute([0, 2, 1, 3]);
let transpose110_out1 = reshape54_out1.permute([0, 2, 3, 1]);
let matmul78_out1 = transpose109_out1.matmul(transpose110_out1);
let pad15_out1 = matmul78_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize), (1usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let reshape55_out1 = pad15_out1.reshape(concat16_out1);
let slice34_out1 = reshape55_out1.slice(s![.., .., 1.., ..]);
let reshape56_out1 = slice34_out1.reshape(concat17_out1);
let transpose111_out1 = reshape56_out1.permute([3, 0, 1, 2]);
let slice35_out1 = transpose111_out1
.slice(s![0..unsqueeze19_out1[0], .., .., ..]);
let transpose112_out1 = slice35_out1.permute([1, 2, 3, 0]);
let mul68_out1 = transpose112_out1
.mul((constant496_out1).unsqueeze_dims(&[0isize, 1isize, 2isize]));
let where22_out1 = mul68_out1.mask_fill(unsqueeze28_out1, constant325_out1);
let (attention8_out1,) = {
let q = transpose108_out1;
let k = transpose106_out1;
let v = transpose107_out1;
let score_bias = where22_out1;
let hidden_rows = score_bias
.clone()
.max_dim(3)
.equal_elem(f32::NEG_INFINITY);
let attention8_out1 = burn::tensor::module::attention(
q,
k,
v,
None,
Some(score_bias),
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention8_out1 = {
let dims = attention8_out1.dims();
attention8_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention8_out1,)
};
let where23_out1 = attention8_out1.mask_fill(expand15_out1, constant328_out1);
let transpose113_out1 = where23_out1.permute([0, 2, 1, 3]);
let reshape57_out1 = transpose113_out1.reshape([1, -1, 512]);
let linear71_out1 = self.linear71.forward(reshape57_out1);
let add65_out1 = add62_out1.add(linear71_out1);
(add65_out1, gather16_out1, concat38_out1, slice32_out1)
}
}
#[derive(Module, Debug)]
pub struct Submodule11 {
layernormalization45: LayerNorm,
conv1d22: Conv1d,
conv1d23: Conv1d,
layernormalization46: LayerNorm,
conv1d24: Conv1d,
layernormalization47: LayerNorm,
linear72: Linear,
linear73: Linear,
layernormalization48: LayerNorm,
layernormalization49: LayerNorm,
linear74: Linear,
linear75: Linear,
layernormalization50: LayerNorm,
linear76: Linear,
linear77: Linear,
linear78: Linear,
linear79: Linear,
constant159: burn::module::Param<Tensor<2>>,
constant160: burn::module::Param<Tensor<2>>,
linear80: Linear,
#[module(skip)]
device: Device,
}
impl Submodule11 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization45 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d22 = Conv1dConfig::new(512, 1024, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let conv1d23 = Conv1dConfig::new(512, 512, 9)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(512)
.with_bias(false)
.init(device);
let layernormalization46 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d24 = Conv1dConfig::new(512, 512, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let layernormalization47 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear72 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear73 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization48 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let layernormalization49 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear74 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear75 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization50 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear76 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear77 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear78 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear79 = LinearConfig::new(512, 512).with_bias(false).init(device);
let constant159: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let constant160: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let linear80 = LinearConfig::new(512, 512).with_bias(false).init(device);
Self {
layernormalization45,
conv1d22,
conv1d23,
layernormalization46,
conv1d24,
layernormalization47,
linear72,
linear73,
layernormalization48,
layernormalization49,
linear74,
linear75,
layernormalization50,
linear76,
linear77,
linear78,
linear79,
constant159,
constant160,
linear80,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add65_out1: Tensor<3>,
unsqueeze30_out1: Tensor<3, Bool>,
constant328_out1: f32,
gather16_out1: Tensor<3>,
constant493_out1: Tensor<1>,
cache_last_channel: Tensor<4>,
cache_last_time: Tensor<4>,
unsqueeze26_out1: [i64; 1],
transpose4_out1: Tensor<3>,
concat16_out1: [i64; 4],
concat17_out1: [i64; 4],
unsqueeze19_out1: [i64; 1],
constant496_out1: Tensor<1>,
unsqueeze28_out1: Tensor<4, Bool>,
constant325_out1: f32,
expand15_out1: Tensor<4, Bool>,
) -> (Tensor<3>, Tensor<3>, Tensor<3>, Tensor<3>) {
let layernormalization45_out1 = {
let dtype = add65_out1.clone().dtype();
self.layernormalization45
.forward(add65_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose114_out1 = layernormalization45_out1.permute([0, 2, 1]);
let conv1d22_out1 = self.conv1d22.forward(transpose114_out1);
let split_tensors = conv1d22_out1.split_with_sizes([512, 512].into(), 1);
let [split8_out1, split8_out2] = split_tensors.try_into().unwrap();
let sigmoid30_out1 = burn::tensor::activation::sigmoid(split8_out2);
let mul69_out1 = split8_out1.mul(sigmoid30_out1);
let where24_out1 = mul69_out1.mask_fill(unsqueeze30_out1, constant328_out1);
let pad16_out1 = where24_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let concat39_out1 = burn::tensor::Tensor::cat(
[gather16_out1, pad16_out1].into(),
2,
);
let slice36_out1 = concat39_out1.clone().slice(s![.., .., - 8..]);
let conv1d23_out1 = self.conv1d23.forward(concat39_out1);
let transpose115_out1 = conv1d23_out1.permute([0, 2, 1]);
let layernormalization46_out1 = {
let dtype = transpose115_out1.dtype();
self.layernormalization46
.forward(transpose115_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose116_out1 = layernormalization46_out1.permute([0, 2, 1]);
let sigmoid31_out1 = burn::tensor::activation::sigmoid(
transpose116_out1.clone(),
);
let mul70_out1 = transpose116_out1.mul(sigmoid31_out1);
let conv1d24_out1 = self.conv1d24.forward(mul70_out1);
let transpose117_out1 = conv1d24_out1.permute([0, 2, 1]);
let add66_out1 = add65_out1.add(transpose117_out1);
let layernormalization47_out1 = {
let dtype = add66_out1.clone().dtype();
self.layernormalization47
.forward(add66_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear72_out1 = self.linear72.forward(layernormalization47_out1);
let sigmoid32_out1 = burn::tensor::activation::sigmoid(linear72_out1.clone());
let mul71_out1 = linear72_out1.mul(sigmoid32_out1);
let linear73_out1 = self.linear73.forward(mul71_out1);
let mul72_out1 = linear73_out1
.mul((constant493_out1.clone()).unsqueeze_dims(&[0isize, 1isize]));
let add67_out1 = add66_out1.add(mul72_out1);
let layernormalization48_out1 = {
let dtype = add67_out1.dtype();
self.layernormalization48
.forward(add67_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let gather17_out1 = {
let sliced = cache_last_channel.slice(s![8, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let gather18_out1 = {
let sliced = cache_last_time.slice(s![8, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let layernormalization49_out1 = {
let dtype = layernormalization48_out1.clone().dtype();
self.layernormalization49
.forward(
layernormalization48_out1.clone().cast(burn::tensor::DType::F32),
)
.cast(dtype)
};
let linear74_out1 = self.linear74.forward(layernormalization49_out1);
let sigmoid33_out1 = burn::tensor::activation::sigmoid(linear74_out1.clone());
let mul73_out1 = linear74_out1.mul(sigmoid33_out1);
let linear75_out1 = self.linear75.forward(mul73_out1);
let mul74_out1 = linear75_out1
.mul((constant493_out1).unsqueeze_dims(&[0isize, 1isize]));
let add68_out1 = layernormalization48_out1.add(mul74_out1);
let layernormalization50_out1 = {
let dtype = add68_out1.clone().dtype();
self.layernormalization50
.forward(add68_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let concat40_out1 = burn::tensor::Tensor::cat(
[gather17_out1.clone(), layernormalization50_out1.clone()].into(),
1,
);
let transpose118_out1 = gather17_out1.permute([1, 0, 2]);
let slice37_out1 = transpose118_out1
.slice(s![unsqueeze26_out1[0]..9223372036854775807, .., ..]);
let transpose119_out1 = slice37_out1.permute([1, 0, 2]);
let concat41_out1 = burn::tensor::Tensor::cat(
[transpose119_out1, layernormalization50_out1.clone()].into(),
1,
);
let linear76_out1 = self.linear76.forward(layernormalization50_out1);
let reshape58_out1 = linear76_out1.reshape([1, -1, 8, 64]);
let linear77_out1 = self.linear77.forward(concat40_out1.clone());
let reshape59_out1 = linear77_out1.reshape([1, -1, 8, 64]);
let linear78_out1 = self.linear78.forward(concat40_out1);
let reshape60_out1 = linear78_out1.reshape([1, -1, 8, 64]);
let transpose120_out1 = reshape59_out1.permute([0, 2, 1, 3]);
let transpose121_out1 = reshape60_out1.permute([0, 2, 1, 3]);
let linear79_out1 = self.linear79.forward(transpose4_out1);
let reshape61_out1 = linear79_out1.reshape([1, -1, 8, 64]);
let constant159_out1 = self.constant159.val();
let add69_out1 = reshape58_out1
.clone()
.add((constant159_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose122_out1 = add69_out1.permute([0, 2, 1, 3]);
let constant160_out1 = self.constant160.val();
let add70_out1 = reshape58_out1
.add((constant160_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose123_out1 = add70_out1.permute([0, 2, 1, 3]);
let transpose124_out1 = reshape61_out1.permute([0, 2, 3, 1]);
let matmul88_out1 = transpose123_out1.matmul(transpose124_out1);
let pad17_out1 = matmul88_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize), (1usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let reshape62_out1 = pad17_out1.reshape(concat16_out1);
let slice38_out1 = reshape62_out1.slice(s![.., .., 1.., ..]);
let reshape63_out1 = slice38_out1.reshape(concat17_out1);
let transpose125_out1 = reshape63_out1.permute([3, 0, 1, 2]);
let slice39_out1 = transpose125_out1
.slice(s![0..unsqueeze19_out1[0], .., .., ..]);
let transpose126_out1 = slice39_out1.permute([1, 2, 3, 0]);
let mul75_out1 = transpose126_out1
.mul((constant496_out1).unsqueeze_dims(&[0isize, 1isize, 2isize]));
let where25_out1 = mul75_out1.mask_fill(unsqueeze28_out1, constant325_out1);
let (attention9_out1,) = {
let q = transpose122_out1;
let k = transpose120_out1;
let v = transpose121_out1;
let score_bias = where25_out1;
let hidden_rows = score_bias
.clone()
.max_dim(3)
.equal_elem(f32::NEG_INFINITY);
let attention9_out1 = burn::tensor::module::attention(
q,
k,
v,
None,
Some(score_bias),
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention9_out1 = {
let dims = attention9_out1.dims();
attention9_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention9_out1,)
};
let where26_out1 = attention9_out1.mask_fill(expand15_out1, constant328_out1);
let transpose127_out1 = where26_out1.permute([0, 2, 1, 3]);
let reshape64_out1 = transpose127_out1.reshape([1, -1, 512]);
let linear80_out1 = self.linear80.forward(reshape64_out1);
let add71_out1 = add68_out1.add(linear80_out1);
(add71_out1, gather18_out1, concat41_out1, slice36_out1)
}
}
#[derive(Module, Debug)]
pub struct Submodule12 {
layernormalization51: LayerNorm,
conv1d25: Conv1d,
conv1d26: Conv1d,
layernormalization52: LayerNorm,
conv1d27: Conv1d,
layernormalization53: LayerNorm,
linear81: Linear,
linear82: Linear,
layernormalization54: LayerNorm,
layernormalization55: LayerNorm,
linear83: Linear,
linear84: Linear,
layernormalization56: LayerNorm,
linear85: Linear,
linear86: Linear,
linear87: Linear,
linear88: Linear,
constant176: burn::module::Param<Tensor<2>>,
constant177: burn::module::Param<Tensor<2>>,
linear89: Linear,
#[module(skip)]
device: Device,
}
impl Submodule12 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization51 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d25 = Conv1dConfig::new(512, 1024, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let conv1d26 = Conv1dConfig::new(512, 512, 9)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(512)
.with_bias(false)
.init(device);
let layernormalization52 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d27 = Conv1dConfig::new(512, 512, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let layernormalization53 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear81 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear82 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization54 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let layernormalization55 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear83 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear84 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization56 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear85 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear86 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear87 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear88 = LinearConfig::new(512, 512).with_bias(false).init(device);
let constant176: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let constant177: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let linear89 = LinearConfig::new(512, 512).with_bias(false).init(device);
Self {
layernormalization51,
conv1d25,
conv1d26,
layernormalization52,
conv1d27,
layernormalization53,
linear81,
linear82,
layernormalization54,
layernormalization55,
linear83,
linear84,
layernormalization56,
linear85,
linear86,
linear87,
linear88,
constant176,
constant177,
linear89,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add71_out1: Tensor<3>,
unsqueeze30_out1: Tensor<3, Bool>,
constant328_out1: f32,
gather18_out1: Tensor<3>,
constant493_out1: Tensor<1>,
cache_last_channel: Tensor<4>,
cache_last_time: Tensor<4>,
unsqueeze26_out1: [i64; 1],
transpose4_out1: Tensor<3>,
concat16_out1: [i64; 4],
concat17_out1: [i64; 4],
unsqueeze19_out1: [i64; 1],
constant496_out1: Tensor<1>,
unsqueeze28_out1: Tensor<4, Bool>,
constant325_out1: f32,
expand15_out1: Tensor<4, Bool>,
) -> (Tensor<3>, Tensor<3>, Tensor<3>, Tensor<3>) {
let layernormalization51_out1 = {
let dtype = add71_out1.clone().dtype();
self.layernormalization51
.forward(add71_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose128_out1 = layernormalization51_out1.permute([0, 2, 1]);
let conv1d25_out1 = self.conv1d25.forward(transpose128_out1);
let split_tensors = conv1d25_out1.split_with_sizes([512, 512].into(), 1);
let [split9_out1, split9_out2] = split_tensors.try_into().unwrap();
let sigmoid34_out1 = burn::tensor::activation::sigmoid(split9_out2);
let mul76_out1 = split9_out1.mul(sigmoid34_out1);
let where27_out1 = mul76_out1.mask_fill(unsqueeze30_out1, constant328_out1);
let pad18_out1 = where27_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let concat42_out1 = burn::tensor::Tensor::cat(
[gather18_out1, pad18_out1].into(),
2,
);
let slice40_out1 = concat42_out1.clone().slice(s![.., .., - 8..]);
let conv1d26_out1 = self.conv1d26.forward(concat42_out1);
let transpose129_out1 = conv1d26_out1.permute([0, 2, 1]);
let layernormalization52_out1 = {
let dtype = transpose129_out1.dtype();
self.layernormalization52
.forward(transpose129_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose130_out1 = layernormalization52_out1.permute([0, 2, 1]);
let sigmoid35_out1 = burn::tensor::activation::sigmoid(
transpose130_out1.clone(),
);
let mul77_out1 = transpose130_out1.mul(sigmoid35_out1);
let conv1d27_out1 = self.conv1d27.forward(mul77_out1);
let transpose131_out1 = conv1d27_out1.permute([0, 2, 1]);
let add72_out1 = add71_out1.add(transpose131_out1);
let layernormalization53_out1 = {
let dtype = add72_out1.clone().dtype();
self.layernormalization53
.forward(add72_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear81_out1 = self.linear81.forward(layernormalization53_out1);
let sigmoid36_out1 = burn::tensor::activation::sigmoid(linear81_out1.clone());
let mul78_out1 = linear81_out1.mul(sigmoid36_out1);
let linear82_out1 = self.linear82.forward(mul78_out1);
let mul79_out1 = linear82_out1
.mul((constant493_out1.clone()).unsqueeze_dims(&[0isize, 1isize]));
let add73_out1 = add72_out1.add(mul79_out1);
let layernormalization54_out1 = {
let dtype = add73_out1.dtype();
self.layernormalization54
.forward(add73_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let gather19_out1 = {
let sliced = cache_last_channel.slice(s![9, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let gather20_out1 = {
let sliced = cache_last_time.slice(s![9, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let layernormalization55_out1 = {
let dtype = layernormalization54_out1.clone().dtype();
self.layernormalization55
.forward(
layernormalization54_out1.clone().cast(burn::tensor::DType::F32),
)
.cast(dtype)
};
let linear83_out1 = self.linear83.forward(layernormalization55_out1);
let sigmoid37_out1 = burn::tensor::activation::sigmoid(linear83_out1.clone());
let mul80_out1 = linear83_out1.mul(sigmoid37_out1);
let linear84_out1 = self.linear84.forward(mul80_out1);
let mul81_out1 = linear84_out1
.mul((constant493_out1).unsqueeze_dims(&[0isize, 1isize]));
let add74_out1 = layernormalization54_out1.add(mul81_out1);
let layernormalization56_out1 = {
let dtype = add74_out1.clone().dtype();
self.layernormalization56
.forward(add74_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let concat43_out1 = burn::tensor::Tensor::cat(
[gather19_out1.clone(), layernormalization56_out1.clone()].into(),
1,
);
let transpose132_out1 = gather19_out1.permute([1, 0, 2]);
let slice41_out1 = transpose132_out1
.slice(s![unsqueeze26_out1[0]..9223372036854775807, .., ..]);
let transpose133_out1 = slice41_out1.permute([1, 0, 2]);
let concat44_out1 = burn::tensor::Tensor::cat(
[transpose133_out1, layernormalization56_out1.clone()].into(),
1,
);
let linear85_out1 = self.linear85.forward(layernormalization56_out1);
let reshape65_out1 = linear85_out1.reshape([1, -1, 8, 64]);
let linear86_out1 = self.linear86.forward(concat43_out1.clone());
let reshape66_out1 = linear86_out1.reshape([1, -1, 8, 64]);
let linear87_out1 = self.linear87.forward(concat43_out1);
let reshape67_out1 = linear87_out1.reshape([1, -1, 8, 64]);
let transpose134_out1 = reshape66_out1.permute([0, 2, 1, 3]);
let transpose135_out1 = reshape67_out1.permute([0, 2, 1, 3]);
let linear88_out1 = self.linear88.forward(transpose4_out1);
let reshape68_out1 = linear88_out1.reshape([1, -1, 8, 64]);
let constant176_out1 = self.constant176.val();
let add75_out1 = reshape65_out1
.clone()
.add((constant176_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose136_out1 = add75_out1.permute([0, 2, 1, 3]);
let constant177_out1 = self.constant177.val();
let add76_out1 = reshape65_out1
.add((constant177_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose137_out1 = add76_out1.permute([0, 2, 1, 3]);
let transpose138_out1 = reshape68_out1.permute([0, 2, 3, 1]);
let matmul98_out1 = transpose137_out1.matmul(transpose138_out1);
let pad19_out1 = matmul98_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize), (1usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let reshape69_out1 = pad19_out1.reshape(concat16_out1);
let slice42_out1 = reshape69_out1.slice(s![.., .., 1.., ..]);
let reshape70_out1 = slice42_out1.reshape(concat17_out1);
let transpose139_out1 = reshape70_out1.permute([3, 0, 1, 2]);
let slice43_out1 = transpose139_out1
.slice(s![0..unsqueeze19_out1[0], .., .., ..]);
let transpose140_out1 = slice43_out1.permute([1, 2, 3, 0]);
let mul82_out1 = transpose140_out1
.mul((constant496_out1).unsqueeze_dims(&[0isize, 1isize, 2isize]));
let where28_out1 = mul82_out1.mask_fill(unsqueeze28_out1, constant325_out1);
let (attention10_out1,) = {
let q = transpose136_out1;
let k = transpose134_out1;
let v = transpose135_out1;
let score_bias = where28_out1;
let hidden_rows = score_bias
.clone()
.max_dim(3)
.equal_elem(f32::NEG_INFINITY);
let attention10_out1 = burn::tensor::module::attention(
q,
k,
v,
None,
Some(score_bias),
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention10_out1 = {
let dims = attention10_out1.dims();
attention10_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention10_out1,)
};
let where29_out1 = attention10_out1.mask_fill(expand15_out1, constant328_out1);
let transpose141_out1 = where29_out1.permute([0, 2, 1, 3]);
let reshape71_out1 = transpose141_out1.reshape([1, -1, 512]);
let linear89_out1 = self.linear89.forward(reshape71_out1);
let add77_out1 = add74_out1.add(linear89_out1);
(add77_out1, gather20_out1, concat44_out1, slice40_out1)
}
}
#[derive(Module, Debug)]
pub struct Submodule13 {
layernormalization57: LayerNorm,
conv1d28: Conv1d,
conv1d29: Conv1d,
layernormalization58: LayerNorm,
conv1d30: Conv1d,
layernormalization59: LayerNorm,
linear90: Linear,
linear91: Linear,
layernormalization60: LayerNorm,
layernormalization61: LayerNorm,
linear92: Linear,
linear93: Linear,
layernormalization62: LayerNorm,
linear94: Linear,
linear95: Linear,
linear96: Linear,
linear97: Linear,
constant193: burn::module::Param<Tensor<2>>,
constant194: burn::module::Param<Tensor<2>>,
linear98: Linear,
#[module(skip)]
device: Device,
}
impl Submodule13 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization57 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d28 = Conv1dConfig::new(512, 1024, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let conv1d29 = Conv1dConfig::new(512, 512, 9)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(512)
.with_bias(false)
.init(device);
let layernormalization58 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d30 = Conv1dConfig::new(512, 512, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let layernormalization59 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear90 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear91 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization60 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let layernormalization61 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear92 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear93 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization62 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear94 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear95 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear96 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear97 = LinearConfig::new(512, 512).with_bias(false).init(device);
let constant193: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let constant194: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let linear98 = LinearConfig::new(512, 512).with_bias(false).init(device);
Self {
layernormalization57,
conv1d28,
conv1d29,
layernormalization58,
conv1d30,
layernormalization59,
linear90,
linear91,
layernormalization60,
layernormalization61,
linear92,
linear93,
layernormalization62,
linear94,
linear95,
linear96,
linear97,
constant193,
constant194,
linear98,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add77_out1: Tensor<3>,
unsqueeze30_out1: Tensor<3, Bool>,
constant328_out1: f32,
gather20_out1: Tensor<3>,
constant493_out1: Tensor<1>,
cache_last_channel: Tensor<4>,
cache_last_time: Tensor<4>,
unsqueeze26_out1: [i64; 1],
transpose4_out1: Tensor<3>,
concat16_out1: [i64; 4],
concat17_out1: [i64; 4],
unsqueeze19_out1: [i64; 1],
constant496_out1: Tensor<1>,
unsqueeze28_out1: Tensor<4, Bool>,
constant325_out1: f32,
expand15_out1: Tensor<4, Bool>,
) -> (Tensor<3>, Tensor<3>, Tensor<3>, Tensor<3>) {
let layernormalization57_out1 = {
let dtype = add77_out1.clone().dtype();
self.layernormalization57
.forward(add77_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose142_out1 = layernormalization57_out1.permute([0, 2, 1]);
let conv1d28_out1 = self.conv1d28.forward(transpose142_out1);
let split_tensors = conv1d28_out1.split_with_sizes([512, 512].into(), 1);
let [split10_out1, split10_out2] = split_tensors.try_into().unwrap();
let sigmoid38_out1 = burn::tensor::activation::sigmoid(split10_out2);
let mul83_out1 = split10_out1.mul(sigmoid38_out1);
let where30_out1 = mul83_out1.mask_fill(unsqueeze30_out1, constant328_out1);
let pad20_out1 = where30_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let concat45_out1 = burn::tensor::Tensor::cat(
[gather20_out1, pad20_out1].into(),
2,
);
let slice44_out1 = concat45_out1.clone().slice(s![.., .., - 8..]);
let conv1d29_out1 = self.conv1d29.forward(concat45_out1);
let transpose143_out1 = conv1d29_out1.permute([0, 2, 1]);
let layernormalization58_out1 = {
let dtype = transpose143_out1.dtype();
self.layernormalization58
.forward(transpose143_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose144_out1 = layernormalization58_out1.permute([0, 2, 1]);
let sigmoid39_out1 = burn::tensor::activation::sigmoid(
transpose144_out1.clone(),
);
let mul84_out1 = transpose144_out1.mul(sigmoid39_out1);
let conv1d30_out1 = self.conv1d30.forward(mul84_out1);
let transpose145_out1 = conv1d30_out1.permute([0, 2, 1]);
let add78_out1 = add77_out1.add(transpose145_out1);
let layernormalization59_out1 = {
let dtype = add78_out1.clone().dtype();
self.layernormalization59
.forward(add78_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear90_out1 = self.linear90.forward(layernormalization59_out1);
let sigmoid40_out1 = burn::tensor::activation::sigmoid(linear90_out1.clone());
let mul85_out1 = linear90_out1.mul(sigmoid40_out1);
let linear91_out1 = self.linear91.forward(mul85_out1);
let mul86_out1 = linear91_out1
.mul((constant493_out1.clone()).unsqueeze_dims(&[0isize, 1isize]));
let add79_out1 = add78_out1.add(mul86_out1);
let layernormalization60_out1 = {
let dtype = add79_out1.dtype();
self.layernormalization60
.forward(add79_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let gather21_out1 = {
let sliced = cache_last_channel.slice(s![10, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let gather22_out1 = {
let sliced = cache_last_time.slice(s![10, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let layernormalization61_out1 = {
let dtype = layernormalization60_out1.clone().dtype();
self.layernormalization61
.forward(
layernormalization60_out1.clone().cast(burn::tensor::DType::F32),
)
.cast(dtype)
};
let linear92_out1 = self.linear92.forward(layernormalization61_out1);
let sigmoid41_out1 = burn::tensor::activation::sigmoid(linear92_out1.clone());
let mul87_out1 = linear92_out1.mul(sigmoid41_out1);
let linear93_out1 = self.linear93.forward(mul87_out1);
let mul88_out1 = linear93_out1
.mul((constant493_out1).unsqueeze_dims(&[0isize, 1isize]));
let add80_out1 = layernormalization60_out1.add(mul88_out1);
let layernormalization62_out1 = {
let dtype = add80_out1.clone().dtype();
self.layernormalization62
.forward(add80_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let concat46_out1 = burn::tensor::Tensor::cat(
[gather21_out1.clone(), layernormalization62_out1.clone()].into(),
1,
);
let transpose146_out1 = gather21_out1.permute([1, 0, 2]);
let slice45_out1 = transpose146_out1
.slice(s![unsqueeze26_out1[0]..9223372036854775807, .., ..]);
let transpose147_out1 = slice45_out1.permute([1, 0, 2]);
let concat47_out1 = burn::tensor::Tensor::cat(
[transpose147_out1, layernormalization62_out1.clone()].into(),
1,
);
let linear94_out1 = self.linear94.forward(layernormalization62_out1);
let reshape72_out1 = linear94_out1.reshape([1, -1, 8, 64]);
let linear95_out1 = self.linear95.forward(concat46_out1.clone());
let reshape73_out1 = linear95_out1.reshape([1, -1, 8, 64]);
let linear96_out1 = self.linear96.forward(concat46_out1);
let reshape74_out1 = linear96_out1.reshape([1, -1, 8, 64]);
let transpose148_out1 = reshape73_out1.permute([0, 2, 1, 3]);
let transpose149_out1 = reshape74_out1.permute([0, 2, 1, 3]);
let linear97_out1 = self.linear97.forward(transpose4_out1);
let reshape75_out1 = linear97_out1.reshape([1, -1, 8, 64]);
let constant193_out1 = self.constant193.val();
let add81_out1 = reshape72_out1
.clone()
.add((constant193_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose150_out1 = add81_out1.permute([0, 2, 1, 3]);
let constant194_out1 = self.constant194.val();
let add82_out1 = reshape72_out1
.add((constant194_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose151_out1 = add82_out1.permute([0, 2, 1, 3]);
let transpose152_out1 = reshape75_out1.permute([0, 2, 3, 1]);
let matmul108_out1 = transpose151_out1.matmul(transpose152_out1);
let pad21_out1 = matmul108_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize), (1usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let reshape76_out1 = pad21_out1.reshape(concat16_out1);
let slice46_out1 = reshape76_out1.slice(s![.., .., 1.., ..]);
let reshape77_out1 = slice46_out1.reshape(concat17_out1);
let transpose153_out1 = reshape77_out1.permute([3, 0, 1, 2]);
let slice47_out1 = transpose153_out1
.slice(s![0..unsqueeze19_out1[0], .., .., ..]);
let transpose154_out1 = slice47_out1.permute([1, 2, 3, 0]);
let mul89_out1 = transpose154_out1
.mul((constant496_out1).unsqueeze_dims(&[0isize, 1isize, 2isize]));
let where31_out1 = mul89_out1.mask_fill(unsqueeze28_out1, constant325_out1);
let (attention11_out1,) = {
let q = transpose150_out1;
let k = transpose148_out1;
let v = transpose149_out1;
let score_bias = where31_out1;
let hidden_rows = score_bias
.clone()
.max_dim(3)
.equal_elem(f32::NEG_INFINITY);
let attention11_out1 = burn::tensor::module::attention(
q,
k,
v,
None,
Some(score_bias),
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention11_out1 = {
let dims = attention11_out1.dims();
attention11_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention11_out1,)
};
let where32_out1 = attention11_out1.mask_fill(expand15_out1, constant328_out1);
let transpose155_out1 = where32_out1.permute([0, 2, 1, 3]);
let reshape78_out1 = transpose155_out1.reshape([1, -1, 512]);
let linear98_out1 = self.linear98.forward(reshape78_out1);
let add83_out1 = add80_out1.add(linear98_out1);
(add83_out1, gather22_out1, concat47_out1, slice44_out1)
}
}
#[derive(Module, Debug)]
pub struct Submodule14 {
layernormalization63: LayerNorm,
conv1d31: Conv1d,
conv1d32: Conv1d,
layernormalization64: LayerNorm,
conv1d33: Conv1d,
layernormalization65: LayerNorm,
linear99: Linear,
linear100: Linear,
layernormalization66: LayerNorm,
layernormalization67: LayerNorm,
linear101: Linear,
linear102: Linear,
layernormalization68: LayerNorm,
linear103: Linear,
linear104: Linear,
linear105: Linear,
linear106: Linear,
constant210: burn::module::Param<Tensor<2>>,
constant211: burn::module::Param<Tensor<2>>,
linear107: Linear,
#[module(skip)]
device: Device,
}
impl Submodule14 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization63 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d31 = Conv1dConfig::new(512, 1024, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let conv1d32 = Conv1dConfig::new(512, 512, 9)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(512)
.with_bias(false)
.init(device);
let layernormalization64 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d33 = Conv1dConfig::new(512, 512, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let layernormalization65 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear99 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear100 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization66 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let layernormalization67 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear101 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear102 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization68 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear103 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear104 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear105 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear106 = LinearConfig::new(512, 512).with_bias(false).init(device);
let constant210: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let constant211: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let linear107 = LinearConfig::new(512, 512).with_bias(false).init(device);
Self {
layernormalization63,
conv1d31,
conv1d32,
layernormalization64,
conv1d33,
layernormalization65,
linear99,
linear100,
layernormalization66,
layernormalization67,
linear101,
linear102,
layernormalization68,
linear103,
linear104,
linear105,
linear106,
constant210,
constant211,
linear107,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add83_out1: Tensor<3>,
unsqueeze30_out1: Tensor<3, Bool>,
constant328_out1: f32,
gather22_out1: Tensor<3>,
constant493_out1: Tensor<1>,
cache_last_channel: Tensor<4>,
cache_last_time: Tensor<4>,
unsqueeze26_out1: [i64; 1],
transpose4_out1: Tensor<3>,
concat16_out1: [i64; 4],
concat17_out1: [i64; 4],
unsqueeze19_out1: [i64; 1],
constant496_out1: Tensor<1>,
unsqueeze28_out1: Tensor<4, Bool>,
constant325_out1: f32,
expand15_out1: Tensor<4, Bool>,
) -> (Tensor<3>, Tensor<3>, Tensor<3>, Tensor<3>) {
let layernormalization63_out1 = {
let dtype = add83_out1.clone().dtype();
self.layernormalization63
.forward(add83_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose156_out1 = layernormalization63_out1.permute([0, 2, 1]);
let conv1d31_out1 = self.conv1d31.forward(transpose156_out1);
let split_tensors = conv1d31_out1.split_with_sizes([512, 512].into(), 1);
let [split11_out1, split11_out2] = split_tensors.try_into().unwrap();
let sigmoid42_out1 = burn::tensor::activation::sigmoid(split11_out2);
let mul90_out1 = split11_out1.mul(sigmoid42_out1);
let where33_out1 = mul90_out1.mask_fill(unsqueeze30_out1, constant328_out1);
let pad22_out1 = where33_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let concat48_out1 = burn::tensor::Tensor::cat(
[gather22_out1, pad22_out1].into(),
2,
);
let slice48_out1 = concat48_out1.clone().slice(s![.., .., - 8..]);
let conv1d32_out1 = self.conv1d32.forward(concat48_out1);
let transpose157_out1 = conv1d32_out1.permute([0, 2, 1]);
let layernormalization64_out1 = {
let dtype = transpose157_out1.dtype();
self.layernormalization64
.forward(transpose157_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose158_out1 = layernormalization64_out1.permute([0, 2, 1]);
let sigmoid43_out1 = burn::tensor::activation::sigmoid(
transpose158_out1.clone(),
);
let mul91_out1 = transpose158_out1.mul(sigmoid43_out1);
let conv1d33_out1 = self.conv1d33.forward(mul91_out1);
let transpose159_out1 = conv1d33_out1.permute([0, 2, 1]);
let add84_out1 = add83_out1.add(transpose159_out1);
let layernormalization65_out1 = {
let dtype = add84_out1.clone().dtype();
self.layernormalization65
.forward(add84_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear99_out1 = self.linear99.forward(layernormalization65_out1);
let sigmoid44_out1 = burn::tensor::activation::sigmoid(linear99_out1.clone());
let mul92_out1 = linear99_out1.mul(sigmoid44_out1);
let linear100_out1 = self.linear100.forward(mul92_out1);
let mul93_out1 = linear100_out1
.mul((constant493_out1.clone()).unsqueeze_dims(&[0isize, 1isize]));
let add85_out1 = add84_out1.add(mul93_out1);
let layernormalization66_out1 = {
let dtype = add85_out1.dtype();
self.layernormalization66
.forward(add85_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let gather23_out1 = {
let sliced = cache_last_channel.slice(s![11, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let gather24_out1 = {
let sliced = cache_last_time.slice(s![11, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let layernormalization67_out1 = {
let dtype = layernormalization66_out1.clone().dtype();
self.layernormalization67
.forward(
layernormalization66_out1.clone().cast(burn::tensor::DType::F32),
)
.cast(dtype)
};
let linear101_out1 = self.linear101.forward(layernormalization67_out1);
let sigmoid45_out1 = burn::tensor::activation::sigmoid(linear101_out1.clone());
let mul94_out1 = linear101_out1.mul(sigmoid45_out1);
let linear102_out1 = self.linear102.forward(mul94_out1);
let mul95_out1 = linear102_out1
.mul((constant493_out1).unsqueeze_dims(&[0isize, 1isize]));
let add86_out1 = layernormalization66_out1.add(mul95_out1);
let layernormalization68_out1 = {
let dtype = add86_out1.clone().dtype();
self.layernormalization68
.forward(add86_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let concat49_out1 = burn::tensor::Tensor::cat(
[gather23_out1.clone(), layernormalization68_out1.clone()].into(),
1,
);
let transpose160_out1 = gather23_out1.permute([1, 0, 2]);
let slice49_out1 = transpose160_out1
.slice(s![unsqueeze26_out1[0]..9223372036854775807, .., ..]);
let transpose161_out1 = slice49_out1.permute([1, 0, 2]);
let concat50_out1 = burn::tensor::Tensor::cat(
[transpose161_out1, layernormalization68_out1.clone()].into(),
1,
);
let linear103_out1 = self.linear103.forward(layernormalization68_out1);
let reshape79_out1 = linear103_out1.reshape([1, -1, 8, 64]);
let linear104_out1 = self.linear104.forward(concat49_out1.clone());
let reshape80_out1 = linear104_out1.reshape([1, -1, 8, 64]);
let linear105_out1 = self.linear105.forward(concat49_out1);
let reshape81_out1 = linear105_out1.reshape([1, -1, 8, 64]);
let transpose162_out1 = reshape80_out1.permute([0, 2, 1, 3]);
let transpose163_out1 = reshape81_out1.permute([0, 2, 1, 3]);
let linear106_out1 = self.linear106.forward(transpose4_out1);
let reshape82_out1 = linear106_out1.reshape([1, -1, 8, 64]);
let constant210_out1 = self.constant210.val();
let add87_out1 = reshape79_out1
.clone()
.add((constant210_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose164_out1 = add87_out1.permute([0, 2, 1, 3]);
let constant211_out1 = self.constant211.val();
let add88_out1 = reshape79_out1
.add((constant211_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose165_out1 = add88_out1.permute([0, 2, 1, 3]);
let transpose166_out1 = reshape82_out1.permute([0, 2, 3, 1]);
let matmul118_out1 = transpose165_out1.matmul(transpose166_out1);
let pad23_out1 = matmul118_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize), (1usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let reshape83_out1 = pad23_out1.reshape(concat16_out1);
let slice50_out1 = reshape83_out1.slice(s![.., .., 1.., ..]);
let reshape84_out1 = slice50_out1.reshape(concat17_out1);
let transpose167_out1 = reshape84_out1.permute([3, 0, 1, 2]);
let slice51_out1 = transpose167_out1
.slice(s![0..unsqueeze19_out1[0], .., .., ..]);
let transpose168_out1 = slice51_out1.permute([1, 2, 3, 0]);
let mul96_out1 = transpose168_out1
.mul((constant496_out1).unsqueeze_dims(&[0isize, 1isize, 2isize]));
let where34_out1 = mul96_out1.mask_fill(unsqueeze28_out1, constant325_out1);
let (attention12_out1,) = {
let q = transpose164_out1;
let k = transpose162_out1;
let v = transpose163_out1;
let score_bias = where34_out1;
let hidden_rows = score_bias
.clone()
.max_dim(3)
.equal_elem(f32::NEG_INFINITY);
let attention12_out1 = burn::tensor::module::attention(
q,
k,
v,
None,
Some(score_bias),
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention12_out1 = {
let dims = attention12_out1.dims();
attention12_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention12_out1,)
};
let where35_out1 = attention12_out1.mask_fill(expand15_out1, constant328_out1);
let transpose169_out1 = where35_out1.permute([0, 2, 1, 3]);
let reshape85_out1 = transpose169_out1.reshape([1, -1, 512]);
let linear107_out1 = self.linear107.forward(reshape85_out1);
let add89_out1 = add86_out1.add(linear107_out1);
(add89_out1, gather24_out1, concat50_out1, slice48_out1)
}
}
#[derive(Module, Debug)]
pub struct Submodule15 {
layernormalization69: LayerNorm,
conv1d34: Conv1d,
conv1d35: Conv1d,
layernormalization70: LayerNorm,
conv1d36: Conv1d,
layernormalization71: LayerNorm,
linear108: Linear,
linear109: Linear,
layernormalization72: LayerNorm,
layernormalization73: LayerNorm,
linear110: Linear,
linear111: Linear,
layernormalization74: LayerNorm,
linear112: Linear,
linear113: Linear,
linear114: Linear,
linear115: Linear,
constant227: burn::module::Param<Tensor<2>>,
constant228: burn::module::Param<Tensor<2>>,
linear116: Linear,
#[module(skip)]
device: Device,
}
impl Submodule15 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization69 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d34 = Conv1dConfig::new(512, 1024, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let conv1d35 = Conv1dConfig::new(512, 512, 9)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(512)
.with_bias(false)
.init(device);
let layernormalization70 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d36 = Conv1dConfig::new(512, 512, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let layernormalization71 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear108 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear109 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization72 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let layernormalization73 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear110 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear111 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization74 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear112 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear113 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear114 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear115 = LinearConfig::new(512, 512).with_bias(false).init(device);
let constant227: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let constant228: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let linear116 = LinearConfig::new(512, 512).with_bias(false).init(device);
Self {
layernormalization69,
conv1d34,
conv1d35,
layernormalization70,
conv1d36,
layernormalization71,
linear108,
linear109,
layernormalization72,
layernormalization73,
linear110,
linear111,
layernormalization74,
linear112,
linear113,
linear114,
linear115,
constant227,
constant228,
linear116,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add89_out1: Tensor<3>,
unsqueeze30_out1: Tensor<3, Bool>,
constant328_out1: f32,
gather24_out1: Tensor<3>,
constant493_out1: Tensor<1>,
cache_last_channel: Tensor<4>,
cache_last_time: Tensor<4>,
unsqueeze26_out1: [i64; 1],
transpose4_out1: Tensor<3>,
concat16_out1: [i64; 4],
concat17_out1: [i64; 4],
unsqueeze19_out1: [i64; 1],
constant496_out1: Tensor<1>,
unsqueeze28_out1: Tensor<4, Bool>,
constant325_out1: f32,
expand15_out1: Tensor<4, Bool>,
) -> (Tensor<3>, Tensor<3>, Tensor<3>, Tensor<3>) {
let layernormalization69_out1 = {
let dtype = add89_out1.clone().dtype();
self.layernormalization69
.forward(add89_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose170_out1 = layernormalization69_out1.permute([0, 2, 1]);
let conv1d34_out1 = self.conv1d34.forward(transpose170_out1);
let split_tensors = conv1d34_out1.split_with_sizes([512, 512].into(), 1);
let [split12_out1, split12_out2] = split_tensors.try_into().unwrap();
let sigmoid46_out1 = burn::tensor::activation::sigmoid(split12_out2);
let mul97_out1 = split12_out1.mul(sigmoid46_out1);
let where36_out1 = mul97_out1.mask_fill(unsqueeze30_out1, constant328_out1);
let pad24_out1 = where36_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let concat51_out1 = burn::tensor::Tensor::cat(
[gather24_out1, pad24_out1].into(),
2,
);
let slice52_out1 = concat51_out1.clone().slice(s![.., .., - 8..]);
let conv1d35_out1 = self.conv1d35.forward(concat51_out1);
let transpose171_out1 = conv1d35_out1.permute([0, 2, 1]);
let layernormalization70_out1 = {
let dtype = transpose171_out1.dtype();
self.layernormalization70
.forward(transpose171_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose172_out1 = layernormalization70_out1.permute([0, 2, 1]);
let sigmoid47_out1 = burn::tensor::activation::sigmoid(
transpose172_out1.clone(),
);
let mul98_out1 = transpose172_out1.mul(sigmoid47_out1);
let conv1d36_out1 = self.conv1d36.forward(mul98_out1);
let transpose173_out1 = conv1d36_out1.permute([0, 2, 1]);
let add90_out1 = add89_out1.add(transpose173_out1);
let layernormalization71_out1 = {
let dtype = add90_out1.clone().dtype();
self.layernormalization71
.forward(add90_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear108_out1 = self.linear108.forward(layernormalization71_out1);
let sigmoid48_out1 = burn::tensor::activation::sigmoid(linear108_out1.clone());
let mul99_out1 = linear108_out1.mul(sigmoid48_out1);
let linear109_out1 = self.linear109.forward(mul99_out1);
let mul100_out1 = linear109_out1
.mul((constant493_out1.clone()).unsqueeze_dims(&[0isize, 1isize]));
let add91_out1 = add90_out1.add(mul100_out1);
let layernormalization72_out1 = {
let dtype = add91_out1.dtype();
self.layernormalization72
.forward(add91_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let gather25_out1 = {
let sliced = cache_last_channel.slice(s![12, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let gather26_out1 = {
let sliced = cache_last_time.slice(s![12, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let layernormalization73_out1 = {
let dtype = layernormalization72_out1.clone().dtype();
self.layernormalization73
.forward(
layernormalization72_out1.clone().cast(burn::tensor::DType::F32),
)
.cast(dtype)
};
let linear110_out1 = self.linear110.forward(layernormalization73_out1);
let sigmoid49_out1 = burn::tensor::activation::sigmoid(linear110_out1.clone());
let mul101_out1 = linear110_out1.mul(sigmoid49_out1);
let linear111_out1 = self.linear111.forward(mul101_out1);
let mul102_out1 = linear111_out1
.mul((constant493_out1).unsqueeze_dims(&[0isize, 1isize]));
let add92_out1 = layernormalization72_out1.add(mul102_out1);
let layernormalization74_out1 = {
let dtype = add92_out1.clone().dtype();
self.layernormalization74
.forward(add92_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let concat52_out1 = burn::tensor::Tensor::cat(
[gather25_out1.clone(), layernormalization74_out1.clone()].into(),
1,
);
let transpose174_out1 = gather25_out1.permute([1, 0, 2]);
let slice53_out1 = transpose174_out1
.slice(s![unsqueeze26_out1[0]..9223372036854775807, .., ..]);
let transpose175_out1 = slice53_out1.permute([1, 0, 2]);
let concat53_out1 = burn::tensor::Tensor::cat(
[transpose175_out1, layernormalization74_out1.clone()].into(),
1,
);
let linear112_out1 = self.linear112.forward(layernormalization74_out1);
let reshape86_out1 = linear112_out1.reshape([1, -1, 8, 64]);
let linear113_out1 = self.linear113.forward(concat52_out1.clone());
let reshape87_out1 = linear113_out1.reshape([1, -1, 8, 64]);
let linear114_out1 = self.linear114.forward(concat52_out1);
let reshape88_out1 = linear114_out1.reshape([1, -1, 8, 64]);
let transpose176_out1 = reshape87_out1.permute([0, 2, 1, 3]);
let transpose177_out1 = reshape88_out1.permute([0, 2, 1, 3]);
let linear115_out1 = self.linear115.forward(transpose4_out1);
let reshape89_out1 = linear115_out1.reshape([1, -1, 8, 64]);
let constant227_out1 = self.constant227.val();
let add93_out1 = reshape86_out1
.clone()
.add((constant227_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose178_out1 = add93_out1.permute([0, 2, 1, 3]);
let constant228_out1 = self.constant228.val();
let add94_out1 = reshape86_out1
.add((constant228_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose179_out1 = add94_out1.permute([0, 2, 1, 3]);
let transpose180_out1 = reshape89_out1.permute([0, 2, 3, 1]);
let matmul128_out1 = transpose179_out1.matmul(transpose180_out1);
let pad25_out1 = matmul128_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize), (1usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let reshape90_out1 = pad25_out1.reshape(concat16_out1);
let slice54_out1 = reshape90_out1.slice(s![.., .., 1.., ..]);
let reshape91_out1 = slice54_out1.reshape(concat17_out1);
let transpose181_out1 = reshape91_out1.permute([3, 0, 1, 2]);
let slice55_out1 = transpose181_out1
.slice(s![0..unsqueeze19_out1[0], .., .., ..]);
let transpose182_out1 = slice55_out1.permute([1, 2, 3, 0]);
let mul103_out1 = transpose182_out1
.mul((constant496_out1).unsqueeze_dims(&[0isize, 1isize, 2isize]));
let where37_out1 = mul103_out1.mask_fill(unsqueeze28_out1, constant325_out1);
let (attention13_out1,) = {
let q = transpose178_out1;
let k = transpose176_out1;
let v = transpose177_out1;
let score_bias = where37_out1;
let hidden_rows = score_bias
.clone()
.max_dim(3)
.equal_elem(f32::NEG_INFINITY);
let attention13_out1 = burn::tensor::module::attention(
q,
k,
v,
None,
Some(score_bias),
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention13_out1 = {
let dims = attention13_out1.dims();
attention13_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention13_out1,)
};
let where38_out1 = attention13_out1.mask_fill(expand15_out1, constant328_out1);
let transpose183_out1 = where38_out1.permute([0, 2, 1, 3]);
let reshape92_out1 = transpose183_out1.reshape([1, -1, 512]);
let linear116_out1 = self.linear116.forward(reshape92_out1);
let add95_out1 = add92_out1.add(linear116_out1);
(add95_out1, gather26_out1, concat53_out1, slice52_out1)
}
}
#[derive(Module, Debug)]
pub struct Submodule16 {
layernormalization75: LayerNorm,
conv1d37: Conv1d,
conv1d38: Conv1d,
layernormalization76: LayerNorm,
conv1d39: Conv1d,
layernormalization77: LayerNorm,
linear117: Linear,
linear118: Linear,
layernormalization78: LayerNorm,
layernormalization79: LayerNorm,
linear119: Linear,
linear120: Linear,
layernormalization80: LayerNorm,
linear121: Linear,
linear122: Linear,
linear123: Linear,
linear124: Linear,
constant244: burn::module::Param<Tensor<2>>,
constant245: burn::module::Param<Tensor<2>>,
linear125: Linear,
#[module(skip)]
device: Device,
}
impl Submodule16 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization75 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d37 = Conv1dConfig::new(512, 1024, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let conv1d38 = Conv1dConfig::new(512, 512, 9)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(512)
.with_bias(false)
.init(device);
let layernormalization76 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d39 = Conv1dConfig::new(512, 512, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let layernormalization77 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear117 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear118 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization78 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let layernormalization79 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear119 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear120 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization80 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear121 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear122 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear123 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear124 = LinearConfig::new(512, 512).with_bias(false).init(device);
let constant244: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let constant245: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let linear125 = LinearConfig::new(512, 512).with_bias(false).init(device);
Self {
layernormalization75,
conv1d37,
conv1d38,
layernormalization76,
conv1d39,
layernormalization77,
linear117,
linear118,
layernormalization78,
layernormalization79,
linear119,
linear120,
layernormalization80,
linear121,
linear122,
linear123,
linear124,
constant244,
constant245,
linear125,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add95_out1: Tensor<3>,
unsqueeze30_out1: Tensor<3, Bool>,
constant328_out1: f32,
gather26_out1: Tensor<3>,
constant493_out1: Tensor<1>,
cache_last_channel: Tensor<4>,
cache_last_time: Tensor<4>,
unsqueeze26_out1: [i64; 1],
transpose4_out1: Tensor<3>,
concat16_out1: [i64; 4],
concat17_out1: [i64; 4],
unsqueeze19_out1: [i64; 1],
constant496_out1: Tensor<1>,
unsqueeze28_out1: Tensor<4, Bool>,
constant325_out1: f32,
expand15_out1: Tensor<4, Bool>,
) -> (Tensor<3>, Tensor<3>, Tensor<3>, Tensor<3>) {
let layernormalization75_out1 = {
let dtype = add95_out1.clone().dtype();
self.layernormalization75
.forward(add95_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose184_out1 = layernormalization75_out1.permute([0, 2, 1]);
let conv1d37_out1 = self.conv1d37.forward(transpose184_out1);
let split_tensors = conv1d37_out1.split_with_sizes([512, 512].into(), 1);
let [split13_out1, split13_out2] = split_tensors.try_into().unwrap();
let sigmoid50_out1 = burn::tensor::activation::sigmoid(split13_out2);
let mul104_out1 = split13_out1.mul(sigmoid50_out1);
let where39_out1 = mul104_out1.mask_fill(unsqueeze30_out1, constant328_out1);
let pad26_out1 = where39_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let concat54_out1 = burn::tensor::Tensor::cat(
[gather26_out1, pad26_out1].into(),
2,
);
let slice56_out1 = concat54_out1.clone().slice(s![.., .., - 8..]);
let conv1d38_out1 = self.conv1d38.forward(concat54_out1);
let transpose185_out1 = conv1d38_out1.permute([0, 2, 1]);
let layernormalization76_out1 = {
let dtype = transpose185_out1.dtype();
self.layernormalization76
.forward(transpose185_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose186_out1 = layernormalization76_out1.permute([0, 2, 1]);
let sigmoid51_out1 = burn::tensor::activation::sigmoid(
transpose186_out1.clone(),
);
let mul105_out1 = transpose186_out1.mul(sigmoid51_out1);
let conv1d39_out1 = self.conv1d39.forward(mul105_out1);
let transpose187_out1 = conv1d39_out1.permute([0, 2, 1]);
let add96_out1 = add95_out1.add(transpose187_out1);
let layernormalization77_out1 = {
let dtype = add96_out1.clone().dtype();
self.layernormalization77
.forward(add96_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear117_out1 = self.linear117.forward(layernormalization77_out1);
let sigmoid52_out1 = burn::tensor::activation::sigmoid(linear117_out1.clone());
let mul106_out1 = linear117_out1.mul(sigmoid52_out1);
let linear118_out1 = self.linear118.forward(mul106_out1);
let mul107_out1 = linear118_out1
.mul((constant493_out1.clone()).unsqueeze_dims(&[0isize, 1isize]));
let add97_out1 = add96_out1.add(mul107_out1);
let layernormalization78_out1 = {
let dtype = add97_out1.dtype();
self.layernormalization78
.forward(add97_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let gather27_out1 = {
let sliced = cache_last_channel.slice(s![13, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let gather28_out1 = {
let sliced = cache_last_time.slice(s![13, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let layernormalization79_out1 = {
let dtype = layernormalization78_out1.clone().dtype();
self.layernormalization79
.forward(
layernormalization78_out1.clone().cast(burn::tensor::DType::F32),
)
.cast(dtype)
};
let linear119_out1 = self.linear119.forward(layernormalization79_out1);
let sigmoid53_out1 = burn::tensor::activation::sigmoid(linear119_out1.clone());
let mul108_out1 = linear119_out1.mul(sigmoid53_out1);
let linear120_out1 = self.linear120.forward(mul108_out1);
let mul109_out1 = linear120_out1
.mul((constant493_out1).unsqueeze_dims(&[0isize, 1isize]));
let add98_out1 = layernormalization78_out1.add(mul109_out1);
let layernormalization80_out1 = {
let dtype = add98_out1.clone().dtype();
self.layernormalization80
.forward(add98_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let concat55_out1 = burn::tensor::Tensor::cat(
[gather27_out1.clone(), layernormalization80_out1.clone()].into(),
1,
);
let transpose188_out1 = gather27_out1.permute([1, 0, 2]);
let slice57_out1 = transpose188_out1
.slice(s![unsqueeze26_out1[0]..9223372036854775807, .., ..]);
let transpose189_out1 = slice57_out1.permute([1, 0, 2]);
let concat56_out1 = burn::tensor::Tensor::cat(
[transpose189_out1, layernormalization80_out1.clone()].into(),
1,
);
let linear121_out1 = self.linear121.forward(layernormalization80_out1);
let reshape93_out1 = linear121_out1.reshape([1, -1, 8, 64]);
let linear122_out1 = self.linear122.forward(concat55_out1.clone());
let reshape94_out1 = linear122_out1.reshape([1, -1, 8, 64]);
let linear123_out1 = self.linear123.forward(concat55_out1);
let reshape95_out1 = linear123_out1.reshape([1, -1, 8, 64]);
let transpose190_out1 = reshape94_out1.permute([0, 2, 1, 3]);
let transpose191_out1 = reshape95_out1.permute([0, 2, 1, 3]);
let linear124_out1 = self.linear124.forward(transpose4_out1);
let reshape96_out1 = linear124_out1.reshape([1, -1, 8, 64]);
let constant244_out1 = self.constant244.val();
let add99_out1 = reshape93_out1
.clone()
.add((constant244_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose192_out1 = add99_out1.permute([0, 2, 1, 3]);
let constant245_out1 = self.constant245.val();
let add100_out1 = reshape93_out1
.add((constant245_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose193_out1 = add100_out1.permute([0, 2, 1, 3]);
let transpose194_out1 = reshape96_out1.permute([0, 2, 3, 1]);
let matmul138_out1 = transpose193_out1.matmul(transpose194_out1);
let pad27_out1 = matmul138_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize), (1usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let reshape97_out1 = pad27_out1.reshape(concat16_out1);
let slice58_out1 = reshape97_out1.slice(s![.., .., 1.., ..]);
let reshape98_out1 = slice58_out1.reshape(concat17_out1);
let transpose195_out1 = reshape98_out1.permute([3, 0, 1, 2]);
let slice59_out1 = transpose195_out1
.slice(s![0..unsqueeze19_out1[0], .., .., ..]);
let transpose196_out1 = slice59_out1.permute([1, 2, 3, 0]);
let mul110_out1 = transpose196_out1
.mul((constant496_out1).unsqueeze_dims(&[0isize, 1isize, 2isize]));
let where40_out1 = mul110_out1.mask_fill(unsqueeze28_out1, constant325_out1);
let (attention14_out1,) = {
let q = transpose192_out1;
let k = transpose190_out1;
let v = transpose191_out1;
let score_bias = where40_out1;
let hidden_rows = score_bias
.clone()
.max_dim(3)
.equal_elem(f32::NEG_INFINITY);
let attention14_out1 = burn::tensor::module::attention(
q,
k,
v,
None,
Some(score_bias),
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention14_out1 = {
let dims = attention14_out1.dims();
attention14_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention14_out1,)
};
let where41_out1 = attention14_out1.mask_fill(expand15_out1, constant328_out1);
let transpose197_out1 = where41_out1.permute([0, 2, 1, 3]);
let reshape99_out1 = transpose197_out1.reshape([1, -1, 512]);
let linear125_out1 = self.linear125.forward(reshape99_out1);
let add101_out1 = add98_out1.add(linear125_out1);
(add101_out1, gather28_out1, concat56_out1, slice56_out1)
}
}
#[derive(Module, Debug)]
pub struct Submodule17 {
layernormalization81: LayerNorm,
conv1d40: Conv1d,
conv1d41: Conv1d,
layernormalization82: LayerNorm,
conv1d42: Conv1d,
layernormalization83: LayerNorm,
linear126: Linear,
linear127: Linear,
layernormalization84: LayerNorm,
layernormalization85: LayerNorm,
linear128: Linear,
linear129: Linear,
layernormalization86: LayerNorm,
linear130: Linear,
linear131: Linear,
linear132: Linear,
linear133: Linear,
constant261: burn::module::Param<Tensor<2>>,
constant262: burn::module::Param<Tensor<2>>,
linear134: Linear,
layernormalization87: LayerNorm,
conv1d43: Conv1d,
conv1d44: Conv1d,
layernormalization88: LayerNorm,
conv1d45: Conv1d,
layernormalization89: LayerNorm,
linear135: Linear,
linear136: Linear,
layernormalization90: LayerNorm,
layernormalization91: LayerNorm,
linear137: Linear,
linear138: Linear,
layernormalization92: LayerNorm,
linear139: Linear,
linear140: Linear,
linear141: Linear,
linear142: Linear,
constant278: burn::module::Param<Tensor<2>>,
constant279: burn::module::Param<Tensor<2>>,
linear143: Linear,
layernormalization93: LayerNorm,
conv1d46: Conv1d,
conv1d47: Conv1d,
layernormalization94: LayerNorm,
conv1d48: Conv1d,
layernormalization95: LayerNorm,
linear144: Linear,
linear145: Linear,
layernormalization96: LayerNorm,
layernormalization97: LayerNorm,
linear146: Linear,
linear147: Linear,
layernormalization98: LayerNorm,
linear148: Linear,
linear149: Linear,
linear150: Linear,
linear151: Linear,
constant295: burn::module::Param<Tensor<2>>,
constant296: burn::module::Param<Tensor<2>>,
linear152: Linear,
#[module(skip)]
device: Device,
}
impl Submodule17 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization81 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d40 = Conv1dConfig::new(512, 1024, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let conv1d41 = Conv1dConfig::new(512, 512, 9)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(512)
.with_bias(false)
.init(device);
let layernormalization82 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d42 = Conv1dConfig::new(512, 512, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let layernormalization83 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear126 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear127 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization84 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let layernormalization85 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear128 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear129 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization86 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear130 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear131 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear132 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear133 = LinearConfig::new(512, 512).with_bias(false).init(device);
let constant261: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let constant262: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let linear134 = LinearConfig::new(512, 512).with_bias(false).init(device);
let layernormalization87 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d43 = Conv1dConfig::new(512, 1024, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let conv1d44 = Conv1dConfig::new(512, 512, 9)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(512)
.with_bias(false)
.init(device);
let layernormalization88 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d45 = Conv1dConfig::new(512, 512, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let layernormalization89 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear135 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear136 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization90 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let layernormalization91 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear137 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear138 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization92 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear139 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear140 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear141 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear142 = LinearConfig::new(512, 512).with_bias(false).init(device);
let constant278: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let constant279: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let linear143 = LinearConfig::new(512, 512).with_bias(false).init(device);
let layernormalization93 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d46 = Conv1dConfig::new(512, 1024, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let conv1d47 = Conv1dConfig::new(512, 512, 9)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(512)
.with_bias(false)
.init(device);
let layernormalization94 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d48 = Conv1dConfig::new(512, 512, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let layernormalization95 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear144 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear145 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization96 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let layernormalization97 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear146 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear147 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization98 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear148 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear149 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear150 = LinearConfig::new(512, 512).with_bias(false).init(device);
let linear151 = LinearConfig::new(512, 512).with_bias(false).init(device);
let constant295: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let constant296: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([8, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[8, 64].into(),
);
let linear152 = LinearConfig::new(512, 512).with_bias(false).init(device);
Self {
layernormalization81,
conv1d40,
conv1d41,
layernormalization82,
conv1d42,
layernormalization83,
linear126,
linear127,
layernormalization84,
layernormalization85,
linear128,
linear129,
layernormalization86,
linear130,
linear131,
linear132,
linear133,
constant261,
constant262,
linear134,
layernormalization87,
conv1d43,
conv1d44,
layernormalization88,
conv1d45,
layernormalization89,
linear135,
linear136,
layernormalization90,
layernormalization91,
linear137,
linear138,
layernormalization92,
linear139,
linear140,
linear141,
linear142,
constant278,
constant279,
linear143,
layernormalization93,
conv1d46,
conv1d47,
layernormalization94,
conv1d48,
layernormalization95,
linear144,
linear145,
layernormalization96,
layernormalization97,
linear146,
linear147,
layernormalization98,
linear148,
linear149,
linear150,
linear151,
constant295,
constant296,
linear152,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add101_out1: Tensor<3>,
unsqueeze30_out1: Tensor<3, Bool>,
constant328_out1: f32,
gather28_out1: Tensor<3>,
constant493_out1: Tensor<1>,
cache_last_channel: Tensor<4>,
cache_last_time: Tensor<4>,
unsqueeze26_out1: [i64; 1],
transpose4_out1: Tensor<3>,
concat16_out1: [i64; 4],
concat17_out1: [i64; 4],
unsqueeze19_out1: [i64; 1],
constant496_out1: Tensor<1>,
unsqueeze28_out1: Tensor<4, Bool>,
constant325_out1: f32,
expand15_out1: Tensor<4, Bool>,
) -> (
Tensor<3>,
Tensor<3>,
Tensor<3>,
Tensor<3>,
Tensor<3>,
Tensor<3>,
Tensor<3>,
Tensor<3>,
) {
let layernormalization81_out1 = {
let dtype = add101_out1.clone().dtype();
self.layernormalization81
.forward(add101_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose198_out1 = layernormalization81_out1.permute([0, 2, 1]);
let conv1d40_out1 = self.conv1d40.forward(transpose198_out1);
let split_tensors = conv1d40_out1.split_with_sizes([512, 512].into(), 1);
let [split14_out1, split14_out2] = split_tensors.try_into().unwrap();
let sigmoid54_out1 = burn::tensor::activation::sigmoid(split14_out2);
let mul111_out1 = split14_out1.mul(sigmoid54_out1);
let where42_out1 = mul111_out1
.mask_fill(unsqueeze30_out1.clone(), constant328_out1);
let pad28_out1 = where42_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let concat57_out1 = burn::tensor::Tensor::cat(
[gather28_out1, pad28_out1].into(),
2,
);
let slice60_out1 = concat57_out1.clone().slice(s![.., .., - 8..]);
let conv1d41_out1 = self.conv1d41.forward(concat57_out1);
let transpose199_out1 = conv1d41_out1.permute([0, 2, 1]);
let layernormalization82_out1 = {
let dtype = transpose199_out1.dtype();
self.layernormalization82
.forward(transpose199_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose200_out1 = layernormalization82_out1.permute([0, 2, 1]);
let sigmoid55_out1 = burn::tensor::activation::sigmoid(
transpose200_out1.clone(),
);
let mul112_out1 = transpose200_out1.mul(sigmoid55_out1);
let conv1d42_out1 = self.conv1d42.forward(mul112_out1);
let transpose201_out1 = conv1d42_out1.permute([0, 2, 1]);
let add102_out1 = add101_out1.add(transpose201_out1);
let layernormalization83_out1 = {
let dtype = add102_out1.clone().dtype();
self.layernormalization83
.forward(add102_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear126_out1 = self.linear126.forward(layernormalization83_out1);
let sigmoid56_out1 = burn::tensor::activation::sigmoid(linear126_out1.clone());
let mul113_out1 = linear126_out1.mul(sigmoid56_out1);
let linear127_out1 = self.linear127.forward(mul113_out1);
let mul114_out1 = linear127_out1
.mul((constant493_out1.clone()).unsqueeze_dims(&[0isize, 1isize]));
let add103_out1 = add102_out1.add(mul114_out1);
let layernormalization84_out1 = {
let dtype = add103_out1.dtype();
self.layernormalization84
.forward(add103_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let gather29_out1 = {
let sliced = cache_last_channel.clone().slice(s![14, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let gather30_out1 = {
let sliced = cache_last_time.clone().slice(s![14, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let layernormalization85_out1 = {
let dtype = layernormalization84_out1.clone().dtype();
self.layernormalization85
.forward(
layernormalization84_out1.clone().cast(burn::tensor::DType::F32),
)
.cast(dtype)
};
let linear128_out1 = self.linear128.forward(layernormalization85_out1);
let sigmoid57_out1 = burn::tensor::activation::sigmoid(linear128_out1.clone());
let mul115_out1 = linear128_out1.mul(sigmoid57_out1);
let linear129_out1 = self.linear129.forward(mul115_out1);
let mul116_out1 = linear129_out1
.mul((constant493_out1.clone()).unsqueeze_dims(&[0isize, 1isize]));
let add104_out1 = layernormalization84_out1.add(mul116_out1);
let layernormalization86_out1 = {
let dtype = add104_out1.clone().dtype();
self.layernormalization86
.forward(add104_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let concat58_out1 = burn::tensor::Tensor::cat(
[gather29_out1.clone(), layernormalization86_out1.clone()].into(),
1,
);
let transpose202_out1 = gather29_out1.permute([1, 0, 2]);
let slice61_out1 = transpose202_out1
.slice(s![unsqueeze26_out1[0]..9223372036854775807, .., ..]);
let transpose203_out1 = slice61_out1.permute([1, 0, 2]);
let concat59_out1 = burn::tensor::Tensor::cat(
[transpose203_out1, layernormalization86_out1.clone()].into(),
1,
);
let linear130_out1 = self.linear130.forward(layernormalization86_out1);
let reshape100_out1 = linear130_out1.reshape([1, -1, 8, 64]);
let linear131_out1 = self.linear131.forward(concat58_out1.clone());
let reshape101_out1 = linear131_out1.reshape([1, -1, 8, 64]);
let linear132_out1 = self.linear132.forward(concat58_out1);
let reshape102_out1 = linear132_out1.reshape([1, -1, 8, 64]);
let transpose204_out1 = reshape101_out1.permute([0, 2, 1, 3]);
let transpose205_out1 = reshape102_out1.permute([0, 2, 1, 3]);
let linear133_out1 = self.linear133.forward(transpose4_out1.clone());
let reshape103_out1 = linear133_out1.reshape([1, -1, 8, 64]);
let constant261_out1 = self.constant261.val();
let add105_out1 = reshape100_out1
.clone()
.add((constant261_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose206_out1 = add105_out1.permute([0, 2, 1, 3]);
let constant262_out1 = self.constant262.val();
let add106_out1 = reshape100_out1
.add((constant262_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose207_out1 = add106_out1.permute([0, 2, 1, 3]);
let transpose208_out1 = reshape103_out1.permute([0, 2, 3, 1]);
let matmul148_out1 = transpose207_out1.matmul(transpose208_out1);
let pad29_out1 = matmul148_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize), (1usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let reshape104_out1 = pad29_out1.reshape(concat16_out1);
let slice62_out1 = reshape104_out1.slice(s![.., .., 1.., ..]);
let reshape105_out1 = slice62_out1.reshape(concat17_out1);
let transpose209_out1 = reshape105_out1.permute([3, 0, 1, 2]);
let slice63_out1 = transpose209_out1
.slice(s![0..unsqueeze19_out1[0], .., .., ..]);
let transpose210_out1 = slice63_out1.permute([1, 2, 3, 0]);
let mul117_out1 = transpose210_out1
.mul((constant496_out1.clone()).unsqueeze_dims(&[0isize, 1isize, 2isize]));
let where43_out1 = mul117_out1
.mask_fill(unsqueeze28_out1.clone(), constant325_out1);
let (attention15_out1,) = {
let q = transpose206_out1;
let k = transpose204_out1;
let v = transpose205_out1;
let score_bias = where43_out1;
let hidden_rows = score_bias
.clone()
.max_dim(3)
.equal_elem(f32::NEG_INFINITY);
let attention15_out1 = burn::tensor::module::attention(
q,
k,
v,
None,
Some(score_bias),
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention15_out1 = {
let dims = attention15_out1.dims();
attention15_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention15_out1,)
};
let where44_out1 = attention15_out1
.mask_fill(expand15_out1.clone(), constant328_out1);
let transpose211_out1 = where44_out1.permute([0, 2, 1, 3]);
let reshape106_out1 = transpose211_out1.reshape([1, -1, 512]);
let linear134_out1 = self.linear134.forward(reshape106_out1);
let add107_out1 = add104_out1.add(linear134_out1);
let layernormalization87_out1 = {
let dtype = add107_out1.clone().dtype();
self.layernormalization87
.forward(add107_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose212_out1 = layernormalization87_out1.permute([0, 2, 1]);
let conv1d43_out1 = self.conv1d43.forward(transpose212_out1);
let split_tensors = conv1d43_out1.split_with_sizes([512, 512].into(), 1);
let [split15_out1, split15_out2] = split_tensors.try_into().unwrap();
let sigmoid58_out1 = burn::tensor::activation::sigmoid(split15_out2);
let mul118_out1 = split15_out1.mul(sigmoid58_out1);
let where45_out1 = mul118_out1
.mask_fill(unsqueeze30_out1.clone(), constant328_out1);
let pad30_out1 = where45_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let concat60_out1 = burn::tensor::Tensor::cat(
[gather30_out1, pad30_out1].into(),
2,
);
let slice64_out1 = concat60_out1.clone().slice(s![.., .., - 8..]);
let conv1d44_out1 = self.conv1d44.forward(concat60_out1);
let transpose213_out1 = conv1d44_out1.permute([0, 2, 1]);
let layernormalization88_out1 = {
let dtype = transpose213_out1.dtype();
self.layernormalization88
.forward(transpose213_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose214_out1 = layernormalization88_out1.permute([0, 2, 1]);
let sigmoid59_out1 = burn::tensor::activation::sigmoid(
transpose214_out1.clone(),
);
let mul119_out1 = transpose214_out1.mul(sigmoid59_out1);
let conv1d45_out1 = self.conv1d45.forward(mul119_out1);
let transpose215_out1 = conv1d45_out1.permute([0, 2, 1]);
let add108_out1 = add107_out1.add(transpose215_out1);
let layernormalization89_out1 = {
let dtype = add108_out1.clone().dtype();
self.layernormalization89
.forward(add108_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear135_out1 = self.linear135.forward(layernormalization89_out1);
let sigmoid60_out1 = burn::tensor::activation::sigmoid(linear135_out1.clone());
let mul120_out1 = linear135_out1.mul(sigmoid60_out1);
let linear136_out1 = self.linear136.forward(mul120_out1);
let mul121_out1 = linear136_out1
.mul((constant493_out1.clone()).unsqueeze_dims(&[0isize, 1isize]));
let add109_out1 = add108_out1.add(mul121_out1);
let layernormalization90_out1 = {
let dtype = add109_out1.dtype();
self.layernormalization90
.forward(add109_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let gather31_out1 = {
let sliced = cache_last_channel.clone().slice(s![15, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let gather32_out1 = {
let sliced = cache_last_time.clone().slice(s![15, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let layernormalization91_out1 = {
let dtype = layernormalization90_out1.clone().dtype();
self.layernormalization91
.forward(
layernormalization90_out1.clone().cast(burn::tensor::DType::F32),
)
.cast(dtype)
};
let linear137_out1 = self.linear137.forward(layernormalization91_out1);
let sigmoid61_out1 = burn::tensor::activation::sigmoid(linear137_out1.clone());
let mul122_out1 = linear137_out1.mul(sigmoid61_out1);
let linear138_out1 = self.linear138.forward(mul122_out1);
let mul123_out1 = linear138_out1
.mul((constant493_out1.clone()).unsqueeze_dims(&[0isize, 1isize]));
let add110_out1 = layernormalization90_out1.add(mul123_out1);
let layernormalization92_out1 = {
let dtype = add110_out1.clone().dtype();
self.layernormalization92
.forward(add110_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let concat61_out1 = burn::tensor::Tensor::cat(
[gather31_out1.clone(), layernormalization92_out1.clone()].into(),
1,
);
let transpose216_out1 = gather31_out1.permute([1, 0, 2]);
let slice65_out1 = transpose216_out1
.slice(s![unsqueeze26_out1[0]..9223372036854775807, .., ..]);
let transpose217_out1 = slice65_out1.permute([1, 0, 2]);
let concat62_out1 = burn::tensor::Tensor::cat(
[transpose217_out1, layernormalization92_out1.clone()].into(),
1,
);
let linear139_out1 = self.linear139.forward(layernormalization92_out1);
let reshape107_out1 = linear139_out1.reshape([1, -1, 8, 64]);
let linear140_out1 = self.linear140.forward(concat61_out1.clone());
let reshape108_out1 = linear140_out1.reshape([1, -1, 8, 64]);
let linear141_out1 = self.linear141.forward(concat61_out1);
let reshape109_out1 = linear141_out1.reshape([1, -1, 8, 64]);
let transpose218_out1 = reshape108_out1.permute([0, 2, 1, 3]);
let transpose219_out1 = reshape109_out1.permute([0, 2, 1, 3]);
let linear142_out1 = self.linear142.forward(transpose4_out1.clone());
let reshape110_out1 = linear142_out1.reshape([1, -1, 8, 64]);
let constant278_out1 = self.constant278.val();
let add111_out1 = reshape107_out1
.clone()
.add((constant278_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose220_out1 = add111_out1.permute([0, 2, 1, 3]);
let constant279_out1 = self.constant279.val();
let add112_out1 = reshape107_out1
.add((constant279_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose221_out1 = add112_out1.permute([0, 2, 1, 3]);
let transpose222_out1 = reshape110_out1.permute([0, 2, 3, 1]);
let matmul158_out1 = transpose221_out1.matmul(transpose222_out1);
let pad31_out1 = matmul158_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize), (1usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let reshape111_out1 = pad31_out1.reshape(concat16_out1);
let slice66_out1 = reshape111_out1.slice(s![.., .., 1.., ..]);
let reshape112_out1 = slice66_out1.reshape(concat17_out1);
let transpose223_out1 = reshape112_out1.permute([3, 0, 1, 2]);
let slice67_out1 = transpose223_out1
.slice(s![0..unsqueeze19_out1[0], .., .., ..]);
let transpose224_out1 = slice67_out1.permute([1, 2, 3, 0]);
let mul124_out1 = transpose224_out1
.mul((constant496_out1.clone()).unsqueeze_dims(&[0isize, 1isize, 2isize]));
let where46_out1 = mul124_out1
.mask_fill(unsqueeze28_out1.clone(), constant325_out1);
let (attention16_out1,) = {
let q = transpose220_out1;
let k = transpose218_out1;
let v = transpose219_out1;
let score_bias = where46_out1;
let hidden_rows = score_bias
.clone()
.max_dim(3)
.equal_elem(f32::NEG_INFINITY);
let attention16_out1 = burn::tensor::module::attention(
q,
k,
v,
None,
Some(score_bias),
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention16_out1 = {
let dims = attention16_out1.dims();
attention16_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention16_out1,)
};
let where47_out1 = attention16_out1
.mask_fill(expand15_out1.clone(), constant328_out1);
let transpose225_out1 = where47_out1.permute([0, 2, 1, 3]);
let reshape113_out1 = transpose225_out1.reshape([1, -1, 512]);
let linear143_out1 = self.linear143.forward(reshape113_out1);
let add113_out1 = add110_out1.add(linear143_out1);
let layernormalization93_out1 = {
let dtype = add113_out1.clone().dtype();
self.layernormalization93
.forward(add113_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose226_out1 = layernormalization93_out1.permute([0, 2, 1]);
let conv1d46_out1 = self.conv1d46.forward(transpose226_out1);
let split_tensors = conv1d46_out1.split_with_sizes([512, 512].into(), 1);
let [split16_out1, split16_out2] = split_tensors.try_into().unwrap();
let sigmoid62_out1 = burn::tensor::activation::sigmoid(split16_out2);
let mul125_out1 = split16_out1.mul(sigmoid62_out1);
let where48_out1 = mul125_out1.mask_fill(unsqueeze30_out1, constant328_out1);
let pad32_out1 = where48_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let concat63_out1 = burn::tensor::Tensor::cat(
[gather32_out1, pad32_out1].into(),
2,
);
let slice68_out1 = concat63_out1.clone().slice(s![.., .., - 8..]);
let conv1d47_out1 = self.conv1d47.forward(concat63_out1);
let transpose227_out1 = conv1d47_out1.permute([0, 2, 1]);
let layernormalization94_out1 = {
let dtype = transpose227_out1.dtype();
self.layernormalization94
.forward(transpose227_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose228_out1 = layernormalization94_out1.permute([0, 2, 1]);
let sigmoid63_out1 = burn::tensor::activation::sigmoid(
transpose228_out1.clone(),
);
let mul126_out1 = transpose228_out1.mul(sigmoid63_out1);
let conv1d48_out1 = self.conv1d48.forward(mul126_out1);
let transpose229_out1 = conv1d48_out1.permute([0, 2, 1]);
let add114_out1 = add113_out1.add(transpose229_out1);
let layernormalization95_out1 = {
let dtype = add114_out1.clone().dtype();
self.layernormalization95
.forward(add114_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear144_out1 = self.linear144.forward(layernormalization95_out1);
let sigmoid64_out1 = burn::tensor::activation::sigmoid(linear144_out1.clone());
let mul127_out1 = linear144_out1.mul(sigmoid64_out1);
let linear145_out1 = self.linear145.forward(mul127_out1);
let mul128_out1 = linear145_out1
.mul((constant493_out1.clone()).unsqueeze_dims(&[0isize, 1isize]));
let add115_out1 = add114_out1.add(mul128_out1);
let layernormalization96_out1 = {
let dtype = add115_out1.dtype();
self.layernormalization96
.forward(add115_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let gather33_out1 = {
let sliced = cache_last_channel.slice(s![16, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let gather34_out1 = {
let sliced = cache_last_time.slice(s![16, .., .., ..]);
sliced.squeeze_dim::<3usize>(0)
};
let layernormalization97_out1 = {
let dtype = layernormalization96_out1.clone().dtype();
self.layernormalization97
.forward(
layernormalization96_out1.clone().cast(burn::tensor::DType::F32),
)
.cast(dtype)
};
let linear146_out1 = self.linear146.forward(layernormalization97_out1);
let sigmoid65_out1 = burn::tensor::activation::sigmoid(linear146_out1.clone());
let mul129_out1 = linear146_out1.mul(sigmoid65_out1);
let linear147_out1 = self.linear147.forward(mul129_out1);
let mul130_out1 = linear147_out1
.mul((constant493_out1).unsqueeze_dims(&[0isize, 1isize]));
let add116_out1 = layernormalization96_out1.add(mul130_out1);
let layernormalization98_out1 = {
let dtype = add116_out1.clone().dtype();
self.layernormalization98
.forward(add116_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let concat64_out1 = burn::tensor::Tensor::cat(
[gather33_out1.clone(), layernormalization98_out1.clone()].into(),
1,
);
let transpose230_out1 = gather33_out1.permute([1, 0, 2]);
let slice69_out1 = transpose230_out1
.slice(s![unsqueeze26_out1[0]..9223372036854775807, .., ..]);
let transpose231_out1 = slice69_out1.permute([1, 0, 2]);
let concat65_out1 = burn::tensor::Tensor::cat(
[transpose231_out1, layernormalization98_out1.clone()].into(),
1,
);
let linear148_out1 = self.linear148.forward(layernormalization98_out1);
let reshape114_out1 = linear148_out1.reshape([1, -1, 8, 64]);
let linear149_out1 = self.linear149.forward(concat64_out1.clone());
let reshape115_out1 = linear149_out1.reshape([1, -1, 8, 64]);
let linear150_out1 = self.linear150.forward(concat64_out1);
let reshape116_out1 = linear150_out1.reshape([1, -1, 8, 64]);
let transpose232_out1 = reshape115_out1.permute([0, 2, 1, 3]);
let transpose233_out1 = reshape116_out1.permute([0, 2, 1, 3]);
let linear151_out1 = self.linear151.forward(transpose4_out1);
let reshape117_out1 = linear151_out1.reshape([1, -1, 8, 64]);
let constant295_out1 = self.constant295.val();
let add117_out1 = reshape114_out1
.clone()
.add((constant295_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose234_out1 = add117_out1.permute([0, 2, 1, 3]);
let constant296_out1 = self.constant296.val();
let add118_out1 = reshape114_out1
.add((constant296_out1).unsqueeze_dims(&[0isize, 1isize]));
let transpose235_out1 = add118_out1.permute([0, 2, 1, 3]);
let transpose236_out1 = reshape117_out1.permute([0, 2, 3, 1]);
let matmul168_out1 = transpose235_out1.matmul(transpose236_out1);
let pad33_out1 = matmul168_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize), (1usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let reshape118_out1 = pad33_out1.reshape(concat16_out1);
let slice70_out1 = reshape118_out1.slice(s![.., .., 1.., ..]);
let reshape119_out1 = slice70_out1.reshape(concat17_out1);
let transpose237_out1 = reshape119_out1.permute([3, 0, 1, 2]);
let slice71_out1 = transpose237_out1
.slice(s![0..unsqueeze19_out1[0], .., .., ..]);
let transpose238_out1 = slice71_out1.permute([1, 2, 3, 0]);
let mul131_out1 = transpose238_out1
.mul((constant496_out1).unsqueeze_dims(&[0isize, 1isize, 2isize]));
let where49_out1 = mul131_out1.mask_fill(unsqueeze28_out1, constant325_out1);
let (attention17_out1,) = {
let q = transpose234_out1;
let k = transpose232_out1;
let v = transpose233_out1;
let score_bias = where49_out1;
let hidden_rows = score_bias
.clone()
.max_dim(3)
.equal_elem(f32::NEG_INFINITY);
let attention17_out1 = burn::tensor::module::attention(
q,
k,
v,
None,
Some(score_bias),
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention17_out1 = {
let dims = attention17_out1.dims();
attention17_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention17_out1,)
};
let where50_out1 = attention17_out1.mask_fill(expand15_out1, constant328_out1);
let transpose239_out1 = where50_out1.permute([0, 2, 1, 3]);
let reshape120_out1 = transpose239_out1.reshape([1, -1, 512]);
let linear152_out1 = self.linear152.forward(reshape120_out1);
let add119_out1 = add116_out1.add(linear152_out1);
(
add119_out1,
gather34_out1,
concat59_out1,
concat62_out1,
concat65_out1,
slice60_out1,
slice64_out1,
slice68_out1,
)
}
}
#[derive(Module, Debug)]
pub struct Submodule18 {
layernormalization99: LayerNorm,
conv1d49: Conv1d,
conv1d50: Conv1d,
layernormalization100: LayerNorm,
conv1d51: Conv1d,
layernormalization101: LayerNorm,
linear153: Linear,
linear154: Linear,
layernormalization102: LayerNorm,
#[module(skip)]
device: Device,
}
impl Submodule18 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization99 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d49 = Conv1dConfig::new(512, 1024, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let conv1d50 = Conv1dConfig::new(512, 512, 9)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(512)
.with_bias(false)
.init(device);
let layernormalization100 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let conv1d51 = Conv1dConfig::new(512, 512, 1)
.with_stride(1)
.with_padding(PaddingConfig1d::Valid)
.with_dilation(1)
.with_groups(1)
.with_bias(false)
.init(device);
let layernormalization101 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear153 = LinearConfig::new(512, 2048).with_bias(false).init(device);
let linear154 = LinearConfig::new(2048, 512).with_bias(false).init(device);
let layernormalization102 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
Self {
layernormalization99,
conv1d49,
conv1d50,
layernormalization100,
conv1d51,
layernormalization101,
linear153,
linear154,
layernormalization102,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add119_out1: Tensor<3>,
unsqueeze30_out1: Tensor<3, Bool>,
constant328_out1: f32,
gather34_out1: Tensor<3>,
constant493_out1: Tensor<1>,
concat15_out1: Tensor<3>,
concat20_out1: Tensor<3>,
concat23_out1: Tensor<3>,
concat26_out1: Tensor<3>,
concat29_out1: Tensor<3>,
concat32_out1: Tensor<3>,
concat35_out1: Tensor<3>,
concat38_out1: Tensor<3>,
concat41_out1: Tensor<3>,
concat44_out1: Tensor<3>,
concat47_out1: Tensor<3>,
concat50_out1: Tensor<3>,
concat53_out1: Tensor<3>,
concat56_out1: Tensor<3>,
concat59_out1: Tensor<3>,
concat62_out1: Tensor<3>,
concat65_out1: Tensor<3>,
slice8_out1: Tensor<3>,
slice12_out1: Tensor<3>,
slice16_out1: Tensor<3>,
slice20_out1: Tensor<3>,
slice24_out1: Tensor<3>,
slice28_out1: Tensor<3>,
slice32_out1: Tensor<3>,
slice36_out1: Tensor<3>,
slice40_out1: Tensor<3>,
slice44_out1: Tensor<3>,
slice48_out1: Tensor<3>,
slice52_out1: Tensor<3>,
slice56_out1: Tensor<3>,
slice60_out1: Tensor<3>,
slice64_out1: Tensor<3>,
slice68_out1: Tensor<3>,
cache_last_channel_len: Tensor<1, Int>,
max2_out1: Tensor<1, Int>,
clip1_out1: Tensor<1, Int>,
) -> (Tensor<3>, Tensor<1, Int>, Tensor<4>, Tensor<4>, Tensor<1, Int>) {
let layernormalization99_out1 = {
let dtype = add119_out1.clone().dtype();
self.layernormalization99
.forward(add119_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose240_out1 = layernormalization99_out1.permute([0, 2, 1]);
let conv1d49_out1 = self.conv1d49.forward(transpose240_out1);
let split_tensors = conv1d49_out1.split_with_sizes([512, 512].into(), 1);
let [split17_out1, split17_out2] = split_tensors.try_into().unwrap();
let sigmoid66_out1 = burn::tensor::activation::sigmoid(split17_out2);
let mul132_out1 = split17_out1.mul(sigmoid66_out1);
let where51_out1 = mul132_out1.mask_fill(unsqueeze30_out1, constant328_out1);
let pad34_out1 = where51_out1
.pad(
[(0usize, 0usize), (0usize, 0usize), (0usize, 0usize)],
burn::tensor::ops::PadMode::Constant(0f32),
);
let concat66_out1 = burn::tensor::Tensor::cat(
[gather34_out1, pad34_out1].into(),
2,
);
let slice72_out1 = concat66_out1.clone().slice(s![.., .., - 8..]);
let conv1d50_out1 = self.conv1d50.forward(concat66_out1);
let transpose241_out1 = conv1d50_out1.permute([0, 2, 1]);
let layernormalization100_out1 = {
let dtype = transpose241_out1.dtype();
self.layernormalization100
.forward(transpose241_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose242_out1 = layernormalization100_out1.permute([0, 2, 1]);
let sigmoid67_out1 = burn::tensor::activation::sigmoid(
transpose242_out1.clone(),
);
let mul133_out1 = transpose242_out1.mul(sigmoid67_out1);
let conv1d51_out1 = self.conv1d51.forward(mul133_out1);
let transpose243_out1 = conv1d51_out1.permute([0, 2, 1]);
let add120_out1 = add119_out1.add(transpose243_out1);
let layernormalization101_out1 = {
let dtype = add120_out1.clone().dtype();
self.layernormalization101
.forward(add120_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear153_out1 = self.linear153.forward(layernormalization101_out1);
let sigmoid68_out1 = burn::tensor::activation::sigmoid(linear153_out1.clone());
let mul134_out1 = linear153_out1.mul(sigmoid68_out1);
let linear154_out1 = self.linear154.forward(mul134_out1);
let mul135_out1 = linear154_out1
.mul((constant493_out1).unsqueeze_dims(&[0isize, 1isize]));
let add121_out1 = add120_out1.add(mul135_out1);
let layernormalization102_out1 = {
let dtype = add121_out1.dtype();
self.layernormalization102
.forward(add121_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let transpose244_out1 = layernormalization102_out1.permute([0, 2, 1]);
let unsqueeze31_out1: Tensor<4> = concat15_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze32_out1: Tensor<4> = concat20_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze33_out1: Tensor<4> = concat23_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze34_out1: Tensor<4> = concat26_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze35_out1: Tensor<4> = concat29_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze36_out1: Tensor<4> = concat32_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze37_out1: Tensor<4> = concat35_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze38_out1: Tensor<4> = concat38_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze39_out1: Tensor<4> = concat41_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze40_out1: Tensor<4> = concat44_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze41_out1: Tensor<4> = concat47_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze42_out1: Tensor<4> = concat50_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze43_out1: Tensor<4> = concat53_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze44_out1: Tensor<4> = concat56_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze45_out1: Tensor<4> = concat59_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze46_out1: Tensor<4> = concat62_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze47_out1: Tensor<4> = concat65_out1.unsqueeze_dims::<4>(&[0]);
let concat67_out1 = burn::tensor::Tensor::cat(
[
unsqueeze31_out1,
unsqueeze32_out1,
unsqueeze33_out1,
unsqueeze34_out1,
unsqueeze35_out1,
unsqueeze36_out1,
unsqueeze37_out1,
unsqueeze38_out1,
unsqueeze39_out1,
unsqueeze40_out1,
unsqueeze41_out1,
unsqueeze42_out1,
unsqueeze43_out1,
unsqueeze44_out1,
unsqueeze45_out1,
unsqueeze46_out1,
unsqueeze47_out1,
]
.into(),
0,
);
let unsqueeze48_out1: Tensor<4> = slice8_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze49_out1: Tensor<4> = slice12_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze50_out1: Tensor<4> = slice16_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze51_out1: Tensor<4> = slice20_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze52_out1: Tensor<4> = slice24_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze53_out1: Tensor<4> = slice28_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze54_out1: Tensor<4> = slice32_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze55_out1: Tensor<4> = slice36_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze56_out1: Tensor<4> = slice40_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze57_out1: Tensor<4> = slice44_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze58_out1: Tensor<4> = slice48_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze59_out1: Tensor<4> = slice52_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze60_out1: Tensor<4> = slice56_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze61_out1: Tensor<4> = slice60_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze62_out1: Tensor<4> = slice64_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze63_out1: Tensor<4> = slice68_out1.unsqueeze_dims::<4>(&[0]);
let unsqueeze64_out1: Tensor<4> = slice72_out1.unsqueeze_dims::<4>(&[0]);
let concat68_out1 = burn::tensor::Tensor::cat(
[
unsqueeze48_out1,
unsqueeze49_out1,
unsqueeze50_out1,
unsqueeze51_out1,
unsqueeze52_out1,
unsqueeze53_out1,
unsqueeze54_out1,
unsqueeze55_out1,
unsqueeze56_out1,
unsqueeze57_out1,
unsqueeze58_out1,
unsqueeze59_out1,
unsqueeze60_out1,
unsqueeze61_out1,
unsqueeze62_out1,
unsqueeze63_out1,
unsqueeze64_out1,
]
.into(),
0,
);
let add122_out1 = cache_last_channel_len.add(max2_out1);
let clip2_out1 = add122_out1.clamp_max(70f64);
let slice73_out1 = concat67_out1.slice(s![.., .., - 70.., ..]);
let slice74_out1 = transpose244_out1.slice(s![.., .., 0..2]);
let clip3_out1 = clip1_out1.clamp_max(2f64);
(slice74_out1, clip3_out1, slice73_out1, concat68_out1, clip2_out1)
}
}
#[derive(Module, Debug)]
pub struct Model {
submodule1: Submodule1,
submodule2: Submodule2,
submodule3: Submodule3,
submodule4: Submodule4,
submodule5: Submodule5,
submodule6: Submodule6,
submodule7: Submodule7,
submodule8: Submodule8,
submodule9: Submodule9,
submodule10: Submodule10,
submodule11: Submodule11,
submodule12: Submodule12,
submodule13: Submodule13,
submodule14: Submodule14,
submodule15: Submodule15,
submodule16: Submodule16,
submodule17: Submodule17,
submodule18: Submodule18,
#[module(skip)]
device: Device,
}
impl Model {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let submodule1 = Submodule1::new(device);
let submodule2 = Submodule2::new(device);
let submodule3 = Submodule3::new(device);
let submodule4 = Submodule4::new(device);
let submodule5 = Submodule5::new(device);
let submodule6 = Submodule6::new(device);
let submodule7 = Submodule7::new(device);
let submodule8 = Submodule8::new(device);
let submodule9 = Submodule9::new(device);
let submodule10 = Submodule10::new(device);
let submodule11 = Submodule11::new(device);
let submodule12 = Submodule12::new(device);
let submodule13 = Submodule13::new(device);
let submodule14 = Submodule14::new(device);
let submodule15 = Submodule15::new(device);
let submodule16 = Submodule16::new(device);
let submodule17 = Submodule17::new(device);
let submodule18 = Submodule18::new(device);
Self {
submodule1,
submodule2,
submodule3,
submodule4,
submodule5,
submodule6,
submodule7,
submodule8,
submodule9,
submodule10,
submodule11,
submodule12,
submodule13,
submodule14,
submodule15,
submodule16,
submodule17,
submodule18,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
audio_signal: Tensor<3>,
length: Tensor<1, Int>,
cache_last_channel: Tensor<4>,
cache_last_time: Tensor<4>,
cache_last_channel_len: Tensor<1, Int>,
) -> (Tensor<3>, Tensor<1, Int>, Tensor<4>, Tensor<4>, Tensor<1, Int>) {
let (
sub4_out1,
max2_out1,
constant303_out1,
constant310_out1,
constant304_out1,
slice1_out1,
add6_out1,
) = self.submodule1.forward(audio_signal, length);
let (
mul17_out1,
gather1_out1,
transpose4_out1,
unsqueeze19_out1,
slice4_out1,
slice3_out1,
gather2_out1,
clip1_out1,
) = self
.submodule2
.forward(
sub4_out1,
max2_out1.clone(),
cache_last_channel_len.clone(),
constant303_out1,
constant310_out1.clone(),
constant304_out1,
cache_last_channel.clone(),
cache_last_time.clone(),
slice1_out1.clone(),
);
let (
concat18_out1,
add23_out1,
constant493_out1,
unsqueeze26_out1,
concat16_out1,
concat17_out1,
constant496_out1,
unsqueeze28_out1,
constant325_out1,
expand15_out1,
constant328_out1,
unsqueeze30_out1,
concat15_out1,
) = self
.submodule3
.forward(
mul17_out1,
slice1_out1,
gather1_out1,
max2_out1.clone(),
transpose4_out1.clone(),
constant310_out1,
add6_out1,
unsqueeze19_out1.clone(),
slice4_out1,
slice3_out1,
gather2_out1,
);
let (add29_out1, gather4_out1, concat20_out1, slice8_out1) = self
.submodule4
.forward(
concat18_out1,
add23_out1,
constant493_out1.clone(),
cache_last_channel.clone(),
cache_last_time.clone(),
unsqueeze26_out1.clone(),
transpose4_out1.clone(),
concat16_out1.clone(),
concat17_out1.clone(),
unsqueeze19_out1.clone(),
constant496_out1.clone(),
unsqueeze28_out1.clone(),
constant325_out1.clone(),
expand15_out1.clone(),
constant328_out1.clone(),
);
let (add35_out1, gather6_out1, concat23_out1, slice12_out1) = self
.submodule5
.forward(
add29_out1,
unsqueeze30_out1.clone(),
constant328_out1.clone(),
gather4_out1,
constant493_out1.clone(),
cache_last_channel.clone(),
cache_last_time.clone(),
unsqueeze26_out1.clone(),
transpose4_out1.clone(),
concat16_out1.clone(),
concat17_out1.clone(),
unsqueeze19_out1.clone(),
constant496_out1.clone(),
unsqueeze28_out1.clone(),
constant325_out1.clone(),
expand15_out1.clone(),
);
let (add41_out1, gather8_out1, concat26_out1, slice16_out1) = self
.submodule6
.forward(
add35_out1,
unsqueeze30_out1.clone(),
constant328_out1.clone(),
gather6_out1,
constant493_out1.clone(),
cache_last_channel.clone(),
cache_last_time.clone(),
unsqueeze26_out1.clone(),
transpose4_out1.clone(),
concat16_out1.clone(),
concat17_out1.clone(),
unsqueeze19_out1.clone(),
constant496_out1.clone(),
unsqueeze28_out1.clone(),
constant325_out1.clone(),
expand15_out1.clone(),
);
let (add47_out1, gather10_out1, concat29_out1, slice20_out1) = self
.submodule7
.forward(
add41_out1,
unsqueeze30_out1.clone(),
constant328_out1.clone(),
gather8_out1,
constant493_out1.clone(),
cache_last_channel.clone(),
cache_last_time.clone(),
unsqueeze26_out1.clone(),
transpose4_out1.clone(),
concat16_out1.clone(),
concat17_out1.clone(),
unsqueeze19_out1.clone(),
constant496_out1.clone(),
unsqueeze28_out1.clone(),
constant325_out1.clone(),
expand15_out1.clone(),
);
let (add53_out1, gather12_out1, concat32_out1, slice24_out1) = self
.submodule8
.forward(
add47_out1,
unsqueeze30_out1.clone(),
constant328_out1.clone(),
gather10_out1,
constant493_out1.clone(),
cache_last_channel.clone(),
cache_last_time.clone(),
unsqueeze26_out1.clone(),
transpose4_out1.clone(),
concat16_out1.clone(),
concat17_out1.clone(),
unsqueeze19_out1.clone(),
constant496_out1.clone(),
unsqueeze28_out1.clone(),
constant325_out1.clone(),
expand15_out1.clone(),
);
let (add59_out1, gather14_out1, concat35_out1, slice28_out1) = self
.submodule9
.forward(
add53_out1,
unsqueeze30_out1.clone(),
constant328_out1.clone(),
gather12_out1,
constant493_out1.clone(),
cache_last_channel.clone(),
cache_last_time.clone(),
unsqueeze26_out1.clone(),
transpose4_out1.clone(),
concat16_out1.clone(),
concat17_out1.clone(),
unsqueeze19_out1.clone(),
constant496_out1.clone(),
unsqueeze28_out1.clone(),
constant325_out1.clone(),
expand15_out1.clone(),
);
let (add65_out1, gather16_out1, concat38_out1, slice32_out1) = self
.submodule10
.forward(
add59_out1,
unsqueeze30_out1.clone(),
constant328_out1.clone(),
gather14_out1,
constant493_out1.clone(),
cache_last_channel.clone(),
cache_last_time.clone(),
unsqueeze26_out1.clone(),
transpose4_out1.clone(),
concat16_out1.clone(),
concat17_out1.clone(),
unsqueeze19_out1.clone(),
constant496_out1.clone(),
unsqueeze28_out1.clone(),
constant325_out1.clone(),
expand15_out1.clone(),
);
let (add71_out1, gather18_out1, concat41_out1, slice36_out1) = self
.submodule11
.forward(
add65_out1,
unsqueeze30_out1.clone(),
constant328_out1.clone(),
gather16_out1,
constant493_out1.clone(),
cache_last_channel.clone(),
cache_last_time.clone(),
unsqueeze26_out1.clone(),
transpose4_out1.clone(),
concat16_out1.clone(),
concat17_out1.clone(),
unsqueeze19_out1.clone(),
constant496_out1.clone(),
unsqueeze28_out1.clone(),
constant325_out1.clone(),
expand15_out1.clone(),
);
let (add77_out1, gather20_out1, concat44_out1, slice40_out1) = self
.submodule12
.forward(
add71_out1,
unsqueeze30_out1.clone(),
constant328_out1.clone(),
gather18_out1,
constant493_out1.clone(),
cache_last_channel.clone(),
cache_last_time.clone(),
unsqueeze26_out1.clone(),
transpose4_out1.clone(),
concat16_out1.clone(),
concat17_out1.clone(),
unsqueeze19_out1.clone(),
constant496_out1.clone(),
unsqueeze28_out1.clone(),
constant325_out1.clone(),
expand15_out1.clone(),
);
let (add83_out1, gather22_out1, concat47_out1, slice44_out1) = self
.submodule13
.forward(
add77_out1,
unsqueeze30_out1.clone(),
constant328_out1.clone(),
gather20_out1,
constant493_out1.clone(),
cache_last_channel.clone(),
cache_last_time.clone(),
unsqueeze26_out1.clone(),
transpose4_out1.clone(),
concat16_out1.clone(),
concat17_out1.clone(),
unsqueeze19_out1.clone(),
constant496_out1.clone(),
unsqueeze28_out1.clone(),
constant325_out1.clone(),
expand15_out1.clone(),
);
let (add89_out1, gather24_out1, concat50_out1, slice48_out1) = self
.submodule14
.forward(
add83_out1,
unsqueeze30_out1.clone(),
constant328_out1.clone(),
gather22_out1,
constant493_out1.clone(),
cache_last_channel.clone(),
cache_last_time.clone(),
unsqueeze26_out1.clone(),
transpose4_out1.clone(),
concat16_out1.clone(),
concat17_out1.clone(),
unsqueeze19_out1.clone(),
constant496_out1.clone(),
unsqueeze28_out1.clone(),
constant325_out1.clone(),
expand15_out1.clone(),
);
let (add95_out1, gather26_out1, concat53_out1, slice52_out1) = self
.submodule15
.forward(
add89_out1,
unsqueeze30_out1.clone(),
constant328_out1.clone(),
gather24_out1,
constant493_out1.clone(),
cache_last_channel.clone(),
cache_last_time.clone(),
unsqueeze26_out1.clone(),
transpose4_out1.clone(),
concat16_out1.clone(),
concat17_out1.clone(),
unsqueeze19_out1.clone(),
constant496_out1.clone(),
unsqueeze28_out1.clone(),
constant325_out1.clone(),
expand15_out1.clone(),
);
let (add101_out1, gather28_out1, concat56_out1, slice56_out1) = self
.submodule16
.forward(
add95_out1,
unsqueeze30_out1.clone(),
constant328_out1.clone(),
gather26_out1,
constant493_out1.clone(),
cache_last_channel.clone(),
cache_last_time.clone(),
unsqueeze26_out1.clone(),
transpose4_out1.clone(),
concat16_out1.clone(),
concat17_out1.clone(),
unsqueeze19_out1.clone(),
constant496_out1.clone(),
unsqueeze28_out1.clone(),
constant325_out1.clone(),
expand15_out1.clone(),
);
let (
add119_out1,
gather34_out1,
concat59_out1,
concat62_out1,
concat65_out1,
slice60_out1,
slice64_out1,
slice68_out1,
) = self
.submodule17
.forward(
add101_out1,
unsqueeze30_out1.clone(),
constant328_out1.clone(),
gather28_out1,
constant493_out1.clone(),
cache_last_channel,
cache_last_time,
unsqueeze26_out1,
transpose4_out1,
concat16_out1,
concat17_out1,
unsqueeze19_out1,
constant496_out1,
unsqueeze28_out1,
constant325_out1,
expand15_out1,
);
let (slice74_out1, clip3_out1, slice73_out1, concat68_out1, clip2_out1) = self
.submodule18
.forward(
add119_out1,
unsqueeze30_out1,
constant328_out1,
gather34_out1,
constant493_out1,
concat15_out1,
concat20_out1,
concat23_out1,
concat26_out1,
concat29_out1,
concat32_out1,
concat35_out1,
concat38_out1,
concat41_out1,
concat44_out1,
concat47_out1,
concat50_out1,
concat53_out1,
concat56_out1,
concat59_out1,
concat62_out1,
concat65_out1,
slice8_out1,
slice12_out1,
slice16_out1,
slice20_out1,
slice24_out1,
slice28_out1,
slice32_out1,
slice36_out1,
slice40_out1,
slice44_out1,
slice48_out1,
slice52_out1,
slice56_out1,
slice60_out1,
slice64_out1,
slice68_out1,
cache_last_channel_len,
max2_out1,
clip1_out1,
);
(slice74_out1, clip3_out1, slice73_out1, concat68_out1, clip2_out1)
}
}