extern crate alloc;
use burn::prelude::*;
use burn::nn::LayerNorm;
use burn::nn::LayerNormConfig;
use burn::nn::Linear;
use burn::nn::LinearConfig;
use burn::nn::PaddingConfig1d;
use burn::nn::conv::Conv1d;
use burn::nn::conv::Conv1dConfig;
use burn::nn::pool::AvgPool1d;
use burn::nn::pool::AvgPool1dConfig;
#[derive(Module, Debug)]
pub struct Submodule1 {
linear1: Linear,
constant372: burn::module::Param<Tensor<1>>,
#[module(skip)]
device: Device,
}
impl Submodule1 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let linear1 = LinearConfig::new(1024, 512).with_bias(false).init(device);
let constant372: burn::module::Param<Tensor<1>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
1,
>::from_data(
burn::tensor::TensorData::from([0f64]),
(device, burn::tensor::DType::F32),
),
device.clone(),
false,
[1].into(),
);
Self {
linear1,
constant372,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
chunk: Tensor<3>,
spkcache: Tensor<3>,
fifo: Tensor<3>,
chunk_lengths: Tensor<1, Int>,
spkcache_lengths: Tensor<1, Int>,
fifo_lengths: Tensor<1, Int>,
) -> (
i64,
[i64; 1],
[i64; 1],
Tensor<2>,
Tensor<1, Int>,
Tensor<1, Int>,
i64,
Tensor<3>,
Tensor<1, Int>,
) {
let shape1_out1: [i64; 1] = {
let axes = &chunk.clone().dims()[0..1];
let mut output = [0i64; 1];
for i in 0..1 {
output[i] = axes[i] as i64;
}
output
};
let shape2_out1: [i64; 1] = {
let axes = &chunk.clone().dims()[1..2];
let mut output = [0i64; 1];
for i in 0..1 {
output[i] = axes[i] as i64;
}
output
};
let squeeze1_out1 = shape2_out1[0] as i64;
let shape3_out1: [i64; 1] = {
let axes = &spkcache.clone().dims()[0..1];
let mut output = [0i64; 1];
for i in 0..1 {
output[i] = axes[i] as i64;
}
output
};
let squeeze2_out1 = shape3_out1[0] as i64;
let shape4_out1: [i64; 1] = {
let axes = &spkcache.clone().dims()[1..2];
let mut output = [0i64; 1];
for i in 0..1 {
output[i] = axes[i] as i64;
}
output
};
let squeeze3_out1 = shape4_out1[0] as i64;
let shape5_out1: [i64; 1] = {
let axes = &fifo.clone().dims()[1..2];
let mut output = [0i64; 1];
for i in 0..1 {
output[i] = axes[i] as i64;
}
output
};
let squeeze4_out1 = shape5_out1[0] as i64;
let constant369_out1 = 8i64;
let mod1_out1 = squeeze1_out1.rem_euclid(constant369_out1);
let constant370_out1 = -1i64;
let mul1_out1 = constant370_out1 * mod1_out1;
let add1_out1 = constant369_out1 + mul1_out1;
let mod2_out1 = add1_out1.rem_euclid(constant369_out1);
let add2_out1 = squeeze1_out1 + mod2_out1;
let div1_out1 = add2_out1 / constant369_out1;
let add3_out1 = squeeze3_out1 + squeeze4_out1;
let add4_out1 = add3_out1 + div1_out1;
let unsqueeze1_out1 = [div1_out1 as i64];
let constant371_out1: [i64; 1] = [1024i64];
let concat1_out1: [i64; 3usize] = [
&shape1_out1[..],
&unsqueeze1_out1[..],
&constant371_out1[..],
]
.concat()
.try_into()
.unwrap();
let reshape1_out1 = chunk.reshape(concat1_out1);
let linear1_out1 = self.linear1.forward(reshape1_out1);
let add5_out1 = chunk_lengths.add_scalar(constant369_out1);
let constant232_out1 = 1i64;
let sub1_out1 = add5_out1.sub_scalar(constant232_out1);
let constant231_out1 = 0i64;
let less1_out1 = sub1_out1.clone().lower_elem(constant231_out1);
let constant230_out1 = true;
let equal1_out1 = less1_out1.equal_elem(constant230_out1);
let mod3_out1 = sub1_out1.clone().remainder_scalar(constant369_out1);
let cast1_out1 = mod3_out1.bool();
let and1_out1 = equal1_out1.bool_and(cast1_out1);
let cast2_out1 = and1_out1.int().cast(burn::tensor::DType::I64);
let div2_out1 = sub1_out1.div_scalar(constant369_out1);
let sub2_out1 = div2_out1.sub(cast2_out1);
let unsqueeze2_out1: Tensor<2, Int> = spkcache_lengths
.clone()
.unsqueeze_dims::<2>(&[0]);
let unsqueeze3_out1: Tensor<2, Int> = fifo_lengths
.clone()
.unsqueeze_dims::<2>(&[0]);
let unsqueeze4_out1: Tensor<2, Int> = sub2_out1
.clone()
.unsqueeze_dims::<2>(&[0]);
let concat2_out1 = burn::tensor::Tensor::cat(
[unsqueeze2_out1, unsqueeze3_out1, unsqueeze4_out1].into(),
0,
);
let reducesum1_out1 = {
concat2_out1.sum_dim(0usize).squeeze_dims::<1usize>(&[0])
};
let mul2_out1 = squeeze2_out1 * add4_out1;
let add6_out1 = mul2_out1 + constant232_out1;
let unsqueeze5_out1 = [add6_out1 as i64];
let constant373_out1: [i64; 1] = [512i64];
let concat3_out1: [i64; 2usize] = [&unsqueeze5_out1[..], &constant373_out1[..]]
.concat()
.try_into()
.unwrap();
let constant372_out1 = self.constant372.val();
let expand1_out1 = {
let shape: [i64; 2] = concat3_out1;
constant372_out1.expand(shape)
};
let expand2_out1 = {
let input = Tensor::<
1,
Int,
>::from_data(
burn::tensor::TensorData::from([constant231_out1]),
(&self.device, burn::tensor::DType::I64),
);
input.expand(shape3_out1)
};
let range1_out1 = {
let start = 0i64;
let delta = 1i64;
assert!(delta != 0);
let n = (((squeeze2_out1 as i128 - start as i128) as f64) / delta as f64)
.ceil()
.max(0.0) as i64;
Tensor::arange(0..n, &self.device)
.cast(burn::tensor::DType::I64)
.mul_scalar(delta)
.add_scalar(start)
};
let unsqueeze6_out1: Tensor<2, Int> = range1_out1
.clone()
.unsqueeze_dims::<2>(&[1]);
let mul3_out1 = unsqueeze6_out1.mul_scalar(add4_out1);
let range2_out1 = {
let start = 0i64;
let delta = 1i64;
assert!(delta != 0);
let n = (((squeeze3_out1 as i128 - start as i128) as f64) / delta as f64)
.ceil()
.max(0.0) as i64;
Tensor::arange(0..n, &self.device)
.cast(burn::tensor::DType::I64)
.mul_scalar(delta)
.add_scalar(start)
};
let unsqueeze7_out1: Tensor<2, Int> = range2_out1.unsqueeze_dims::<2>(&[0]);
let unsqueeze8_out1: Tensor<2, Int> = spkcache_lengths
.clone()
.unsqueeze_dims::<2>(&[1]);
let less2_out1 = unsqueeze7_out1.clone().lower(unsqueeze8_out1);
let unsqueeze9_out1: Tensor<2, Int> = expand2_out1
.clone()
.unsqueeze_dims::<2>(&[1]);
let add7_out1 = mul3_out1.clone().add(unsqueeze9_out1);
let add8_out1 = add7_out1.add(unsqueeze7_out1);
let where1_out1 = Tensor::<
1,
burn::tensor::Int,
>::from_data(
burn::tensor::TensorData::from([mul2_out1 as i64]),
(&self.device, burn::tensor::DType::I64),
)
.reshape([1, 1])
.mask_where(less2_out1, add8_out1);
let reshape2_out1 = where1_out1.reshape([-1]);
let reshape3_out1 = spkcache.reshape([-1, 512]);
let unsqueeze10_out1: Tensor<2, Int> = reshape2_out1.unsqueeze_dims::<2>(&[-1]);
let scatternd1_out1 = {
let data_dims = expand1_out1.dims();
let indices_i64 = unsqueeze10_out1.cast(burn::tensor::DType::I64);
let idx_dims = indices_i64.dims();
let k = idx_dims[2 - 1];
let mut dim_sizes: alloc::vec::Vec<i64> = alloc::vec::Vec::with_capacity(k);
for i in 0..k {
dim_sizes.push(data_dims[0 + i] as i64);
}
let mut bcast_shape = [1usize; 2];
bcast_shape[2 - 1] = k;
let dims_tensor = Tensor::<
1,
Int,
>::from_data(
burn::tensor::TensorData::from(dim_sizes.as_slice()),
(&self.device, burn::tensor::DType::I64),
)
.reshape(bcast_shape);
let negative = indices_i64.clone().lower_elem(0i64);
let corrected = indices_i64.clone() + dims_tensor;
let indices_norm = indices_i64.mask_where(negative, corrected);
expand1_out1
.scatter_nd(
indices_norm,
reshape3_out1,
burn::tensor::IndexingUpdateOp::Assign,
)
};
let add9_out1 = expand2_out1.add(spkcache_lengths);
let range3_out1 = {
let start = 0i64;
let delta = 1i64;
assert!(delta != 0);
let n = (((squeeze4_out1 as i128 - start as i128) as f64) / delta as f64)
.ceil()
.max(0.0) as i64;
Tensor::arange(0..n, &self.device)
.cast(burn::tensor::DType::I64)
.mul_scalar(delta)
.add_scalar(start)
};
let unsqueeze11_out1: Tensor<2, Int> = range3_out1.unsqueeze_dims::<2>(&[0]);
let unsqueeze12_out1: Tensor<2, Int> = fifo_lengths
.clone()
.unsqueeze_dims::<2>(&[1]);
let less3_out1 = unsqueeze11_out1.clone().lower(unsqueeze12_out1);
let unsqueeze13_out1: Tensor<2, Int> = add9_out1
.clone()
.unsqueeze_dims::<2>(&[1]);
let add10_out1 = mul3_out1.clone().add(unsqueeze13_out1);
let add11_out1 = add10_out1.add(unsqueeze11_out1);
let where2_out1 = Tensor::<
1,
burn::tensor::Int,
>::from_data(
burn::tensor::TensorData::from([mul2_out1 as i64]),
(&self.device, burn::tensor::DType::I64),
)
.reshape([1, 1])
.mask_where(less3_out1, add11_out1);
let reshape4_out1 = where2_out1.reshape([-1]);
let reshape5_out1 = fifo.reshape([-1, 512]);
let shape6_out1: [i64; 1] = {
let axes = &reshape4_out1.clone().dims()[0..1];
let mut output = [0i64; 1];
for i in 0..1 {
output[i] = axes[i] as i64;
}
output
};
let concat4_out1: [i64; 2usize] = [&shape6_out1[..], &constant373_out1[..]]
.concat()
.try_into()
.unwrap();
let unsqueeze14_out1: Tensor<2, Int> = reshape4_out1.unsqueeze_dims::<2>(&[-1]);
let expand3_out1 = {
let onnx_shape: [i64; 2usize] = concat4_out1;
let input_dims = reshape5_out1.dims();
let mut shape = onnx_shape;
#[allow(clippy::needless_range_loop)]
for i in 0..2usize {
let dim_offset = i;
if shape[dim_offset] == 1 && input_dims[i] > 1 {
shape[dim_offset] = input_dims[i] as i64;
}
}
reshape5_out1.expand(shape)
};
let scatternd2_out1 = {
let data_dims = scatternd1_out1.dims();
let indices_i64 = unsqueeze14_out1.cast(burn::tensor::DType::I64);
let idx_dims = indices_i64.dims();
let k = idx_dims[2 - 1];
let mut dim_sizes: alloc::vec::Vec<i64> = alloc::vec::Vec::with_capacity(k);
for i in 0..k {
dim_sizes.push(data_dims[0 + i] as i64);
}
let mut bcast_shape = [1usize; 2];
bcast_shape[2 - 1] = k;
let dims_tensor = Tensor::<
1,
Int,
>::from_data(
burn::tensor::TensorData::from(dim_sizes.as_slice()),
(&self.device, burn::tensor::DType::I64),
)
.reshape(bcast_shape);
let negative = indices_i64.clone().lower_elem(0i64);
let corrected = indices_i64.clone() + dims_tensor;
let indices_norm = indices_i64.mask_where(negative, corrected);
scatternd1_out1
.scatter_nd(
indices_norm,
expand3_out1,
burn::tensor::IndexingUpdateOp::Assign,
)
};
let add12_out1 = add9_out1.add(fifo_lengths);
let range4_out1 = {
let start = 0i64;
let delta = 1i64;
assert!(delta != 0);
let n = (((div1_out1 as i128 - start as i128) as f64) / delta as f64)
.ceil()
.max(0.0) as i64;
Tensor::arange(0..n, &self.device)
.cast(burn::tensor::DType::I64)
.mul_scalar(delta)
.add_scalar(start)
};
let unsqueeze15_out1: Tensor<2, Int> = range4_out1.unsqueeze_dims::<2>(&[0]);
let unsqueeze16_out1: Tensor<2, Int> = sub2_out1
.clone()
.unsqueeze_dims::<2>(&[1]);
let less4_out1 = unsqueeze15_out1.clone().lower(unsqueeze16_out1);
let unsqueeze17_out1: Tensor<2, Int> = add12_out1.unsqueeze_dims::<2>(&[1]);
let add13_out1 = mul3_out1.add(unsqueeze17_out1);
let add14_out1 = add13_out1.add(unsqueeze15_out1);
let where3_out1 = Tensor::<
1,
burn::tensor::Int,
>::from_data(
burn::tensor::TensorData::from([mul2_out1 as i64]),
(&self.device, burn::tensor::DType::I64),
)
.reshape([1, 1])
.mask_where(less4_out1, add14_out1);
let reshape6_out1 = where3_out1.reshape([-1]);
let reshape7_out1 = linear1_out1.clone().reshape([-1, 512]);
let shape7_out1: [i64; 1] = {
let axes = &reshape6_out1.clone().dims()[0..1];
let mut output = [0i64; 1];
for i in 0..1 {
output[i] = axes[i] as i64;
}
output
};
let concat5_out1: [i64; 2usize] = [&shape7_out1[..], &constant373_out1[..]]
.concat()
.try_into()
.unwrap();
let unsqueeze18_out1: Tensor<2, Int> = reshape6_out1.unsqueeze_dims::<2>(&[-1]);
let expand4_out1 = {
let onnx_shape: [i64; 2usize] = concat5_out1;
let input_dims = reshape7_out1.dims();
let mut shape = onnx_shape;
#[allow(clippy::needless_range_loop)]
for i in 0..2usize {
let dim_offset = i;
if shape[dim_offset] == 1 && input_dims[i] > 1 {
shape[dim_offset] = input_dims[i] as i64;
}
}
reshape7_out1.expand(shape)
};
let scatternd3_out1 = {
let data_dims = scatternd2_out1.dims();
let indices_i64 = unsqueeze18_out1.cast(burn::tensor::DType::I64);
let idx_dims = indices_i64.dims();
let k = idx_dims[2 - 1];
let mut dim_sizes: alloc::vec::Vec<i64> = alloc::vec::Vec::with_capacity(k);
for i in 0..k {
dim_sizes.push(data_dims[0 + i] as i64);
}
let mut bcast_shape = [1usize; 2];
bcast_shape[2 - 1] = k;
let dims_tensor = Tensor::<
1,
Int,
>::from_data(
burn::tensor::TensorData::from(dim_sizes.as_slice()),
(&self.device, burn::tensor::DType::I64),
)
.reshape(bcast_shape);
let negative = indices_i64.clone().lower_elem(0i64);
let corrected = indices_i64.clone() + dims_tensor;
let indices_norm = indices_i64.mask_where(negative, corrected);
scatternd2_out1
.scatter_nd(
indices_norm,
expand4_out1,
burn::tensor::IndexingUpdateOp::Assign,
)
};
let unsqueeze19_out1 = [mul2_out1 as i64];
let slice1_out1 = scatternd3_out1.slice(s![0..unsqueeze19_out1[0], ..]);
(
add4_out1,
shape3_out1,
constant373_out1,
slice1_out1,
range1_out1,
reducesum1_out1,
constant369_out1,
linear1_out1,
sub2_out1,
)
}
}
#[derive(Module, Debug)]
pub struct Submodule2 {
layernormalization1: LayerNorm,
layernormalization2: LayerNorm,
linear2: Linear,
constant227: burn::module::Param<Tensor<2>>,
constant228: burn::module::Param<Tensor<2>>,
linear3: Linear,
layernormalization3: LayerNorm,
linear4: Linear,
linear5: Linear,
layernormalization4: LayerNorm,
linear6: Linear,
linear7: Linear,
#[module(skip)]
device: Device,
}
impl Submodule2 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization1 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let layernormalization2 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear2 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let constant227: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([5000, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[5000, 64].into(),
);
let constant228: burn::module::Param<Tensor<2>> = burn::module::Param::uninitialized(
burn::module::ParamId::new(),
move |device, _require_grad| Tensor::<
2,
>::zeros([5000, 64], (device, burn::tensor::DType::F32)),
device.clone(),
false,
[5000, 64].into(),
);
let linear3 = LinearConfig::new(512, 512).with_bias(true).init(device);
let layernormalization3 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear4 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear5 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization4 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear6 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear7 = LinearConfig::new(512, 512).with_bias(true).init(device);
Self {
layernormalization1,
layernormalization2,
linear2,
constant227,
constant228,
linear3,
layernormalization3,
linear4,
linear5,
layernormalization4,
linear6,
linear7,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add4_out1: i64,
shape3_out1: [i64; 1],
constant373_out1: [i64; 1],
slice1_out1: Tensor<2>,
range1_out1: Tensor<1, Int>,
reducesum1_out1: Tensor<1, Int>,
) -> (
Tensor<3>,
[i64; 5],
Tensor<4>,
Tensor<4>,
Tensor<4, Bool>,
[i64; 3],
[i64; 1],
[i64; 1],
Tensor<1, Int>,
[i64; 1],
) {
let unsqueeze20_out1 = [add4_out1 as i64];
let concat6_out1: [i64; 3usize] = [
&shape3_out1[..],
&unsqueeze20_out1[..],
&constant373_out1[..],
]
.concat()
.try_into()
.unwrap();
let reshape8_out1 = slice1_out1.reshape(concat6_out1);
let layernormalization1_out1 = {
let dtype = reshape8_out1.dtype();
self.layernormalization1
.forward(reshape8_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let reshape9_out1 = range1_out1.reshape([-1, 1, 1, 1]);
let range5_out1 = {
let start = 0i64;
let delta = 1i64;
assert!(delta != 0);
let n = (((add4_out1 as i128 - start as i128) as f64) / delta as f64)
.ceil()
.max(0.0) as i64;
Tensor::arange(0..n, &self.device)
.cast(burn::tensor::DType::I64)
.mul_scalar(delta)
.add_scalar(start)
};
let reshape10_out1 = range5_out1.clone().reshape([1, 1, 1, -1]);
let unsqueeze21_out1: Tensor<5, Int> = reshape9_out1.unsqueeze_dims::<5>(&[-1]);
let gathernd1_out1 = {
let data_dims = reducesum1_out1.dims();
let indices_i64 = unsqueeze21_out1.cast(burn::tensor::DType::I64);
let idx_dims = indices_i64.dims();
let k = idx_dims[5 - 1];
let mut dim_sizes: alloc::vec::Vec<i64> = alloc::vec::Vec::with_capacity(k);
for i in 0..k {
dim_sizes.push(data_dims[0 + i] as i64);
}
let mut bcast_shape = [1usize; 5];
bcast_shape[5 - 1] = k;
let dims_tensor = Tensor::<
1,
Int,
>::from_data(
burn::tensor::TensorData::from(dim_sizes.as_slice()),
(&self.device, burn::tensor::DType::I64),
)
.reshape(bcast_shape);
let negative = indices_i64.clone().lower_elem(0i64);
let corrected = indices_i64.clone() + dims_tensor;
let indices_norm = indices_i64.mask_where(negative, corrected);
let aug_indices = indices_norm;
reducesum1_out1.gather_nd(aug_indices)
};
let less5_out1 = reshape10_out1.lower(gathernd1_out1);
let constant374_out1: [i64; 1] = [1i64];
let concat7_out1: [i64; 4usize] = [
&shape3_out1[..],
&constant374_out1[..],
&unsqueeze20_out1[..],
&unsqueeze20_out1[..],
]
.concat()
.try_into()
.unwrap();
let expand5_out1 = {
let onnx_shape: [i64; 4usize] = concat7_out1;
let input_dims = less5_out1.dims();
let mut shape = onnx_shape;
#[allow(clippy::needless_range_loop)]
for i in 0..4usize {
let dim_offset = i;
if shape[dim_offset] == 1 && input_dims[i] > 1 {
shape[dim_offset] = input_dims[i] as i64;
}
}
less5_out1.expand(shape)
};
let layernormalization2_out1 = {
let dtype = layernormalization1_out1.clone().dtype();
self.layernormalization2
.forward(layernormalization1_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear2_out1 = self.linear2.forward(layernormalization2_out1);
let constant241_out1: [i64; 1] = [3i64];
let constant375_out1: [i64; 1] = [8i64];
let constant376_out1: [i64; 1] = [64i64];
let concat8_out1: [i64; 5usize] = [
&shape3_out1[..],
&unsqueeze20_out1[..],
&constant241_out1[..],
&constant375_out1[..],
&constant376_out1[..],
]
.concat()
.try_into()
.unwrap();
let reshape11_out1 = linear2_out1.reshape(concat8_out1);
let transpose1_out1 = reshape11_out1.permute([2, 0, 3, 1, 4]);
let slice2_out1 = transpose1_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze5_out1 = slice2_out1.squeeze_dims::<4>(&[0]);
let slice3_out1 = transpose1_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze6_out1 = slice3_out1.squeeze_dims::<4>(&[0]);
let slice4_out1 = transpose1_out1.slice(s![2..3, .., .., .., ..]);
let squeeze7_out1 = slice4_out1.squeeze_dims::<4>(&[0]);
let constant227_out1 = self.constant227.val();
let slice5_out1 = constant227_out1.slice(s![0..unsqueeze20_out1[0], ..]);
let reshape12_out1 = slice5_out1.reshape([1, 1, -1, 64]);
let constant228_out1 = self.constant228.val();
let slice6_out1 = constant228_out1.slice(s![0..unsqueeze20_out1[0], ..]);
let reshape13_out1 = slice6_out1.reshape([1, 1, -1, 64]);
let mul4_out1 = squeeze5_out1.clone().mul(reshape12_out1.clone());
let slice7_out1 = squeeze5_out1.clone().slice(s![.., .., .., 32..]);
let neg1_out1 = slice7_out1.neg();
let slice8_out1 = squeeze5_out1.slice(s![.., .., .., 0..32]);
let concat9_out1 = burn::tensor::Tensor::cat([neg1_out1, slice8_out1].into(), 3);
let mul5_out1 = concat9_out1.mul(reshape13_out1.clone());
let add15_out1 = mul4_out1.add(mul5_out1);
let mul6_out1 = squeeze6_out1.clone().mul(reshape12_out1.clone());
let slice9_out1 = squeeze6_out1.clone().slice(s![.., .., .., 32..]);
let neg2_out1 = slice9_out1.neg();
let slice10_out1 = squeeze6_out1.slice(s![.., .., .., 0..32]);
let concat10_out1 = burn::tensor::Tensor::cat(
[neg2_out1, slice10_out1].into(),
3,
);
let mul7_out1 = concat10_out1.mul(reshape13_out1.clone());
let add16_out1 = mul6_out1.add(mul7_out1);
let (attention1_out1,) = {
let q = add15_out1;
let k = add16_out1;
let v = squeeze7_out1;
let hide = expand5_out1.clone().bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention1_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention1_out1 = {
let dims = attention1_out1.dims();
attention1_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention1_out1,)
};
let transpose2_out1 = attention1_out1.permute([0, 2, 1, 3]);
let reshape14_out1 = transpose2_out1.reshape(concat6_out1);
let linear3_out1 = self.linear3.forward(reshape14_out1);
let add17_out1 = layernormalization1_out1.add(linear3_out1);
let layernormalization3_out1 = {
let dtype = add17_out1.clone().dtype();
self.layernormalization3
.forward(add17_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear4_out1 = self.linear4.forward(layernormalization3_out1);
let gelu1_out1 = burn::tensor::activation::gelu(linear4_out1);
let linear5_out1 = self.linear5.forward(gelu1_out1);
let add18_out1 = add17_out1.add(linear5_out1);
let layernormalization4_out1 = {
let dtype = add18_out1.clone().dtype();
self.layernormalization4
.forward(add18_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear6_out1 = self.linear6.forward(layernormalization4_out1);
let reshape15_out1 = linear6_out1.reshape(concat8_out1);
let transpose3_out1 = reshape15_out1.permute([2, 0, 3, 1, 4]);
let slice11_out1 = transpose3_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze8_out1 = slice11_out1.squeeze_dims::<4>(&[0]);
let slice12_out1 = transpose3_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze9_out1 = slice12_out1.squeeze_dims::<4>(&[0]);
let slice13_out1 = transpose3_out1.slice(s![2..3, .., .., .., ..]);
let squeeze10_out1 = slice13_out1.squeeze_dims::<4>(&[0]);
let mul8_out1 = squeeze8_out1.clone().mul(reshape12_out1.clone());
let slice14_out1 = squeeze8_out1.clone().slice(s![.., .., .., 32..]);
let neg3_out1 = slice14_out1.neg();
let slice15_out1 = squeeze8_out1.slice(s![.., .., .., 0..32]);
let concat11_out1 = burn::tensor::Tensor::cat(
[neg3_out1, slice15_out1].into(),
3,
);
let mul9_out1 = concat11_out1.mul(reshape13_out1.clone());
let add19_out1 = mul8_out1.add(mul9_out1);
let mul10_out1 = squeeze9_out1.clone().mul(reshape12_out1.clone());
let slice16_out1 = squeeze9_out1.clone().slice(s![.., .., .., 32..]);
let neg4_out1 = slice16_out1.neg();
let slice17_out1 = squeeze9_out1.slice(s![.., .., .., 0..32]);
let concat12_out1 = burn::tensor::Tensor::cat(
[neg4_out1, slice17_out1].into(),
3,
);
let mul11_out1 = concat12_out1.mul(reshape13_out1.clone());
let add20_out1 = mul10_out1.add(mul11_out1);
let (attention2_out1,) = {
let q = add19_out1;
let k = add20_out1;
let v = squeeze10_out1;
let hide = expand5_out1.clone().bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention2_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention2_out1 = {
let dims = attention2_out1.dims();
attention2_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention2_out1,)
};
let transpose4_out1 = attention2_out1.permute([0, 2, 1, 3]);
let reshape16_out1 = transpose4_out1.reshape(concat6_out1);
let linear7_out1 = self.linear7.forward(reshape16_out1);
let add21_out1 = add18_out1.add(linear7_out1);
(
add21_out1,
concat8_out1,
reshape12_out1,
reshape13_out1,
expand5_out1,
concat6_out1,
constant374_out1,
unsqueeze20_out1,
range5_out1,
constant375_out1,
)
}
}
#[derive(Module, Debug)]
pub struct Submodule3 {
layernormalization5: LayerNorm,
linear8: Linear,
linear9: Linear,
layernormalization6: LayerNorm,
linear10: Linear,
linear11: Linear,
layernormalization7: LayerNorm,
linear12: Linear,
linear13: Linear,
layernormalization8: LayerNorm,
linear14: Linear,
linear15: Linear,
#[module(skip)]
device: Device,
}
impl Submodule3 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization5 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear8 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear9 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization6 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear10 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear11 = LinearConfig::new(512, 512).with_bias(true).init(device);
let layernormalization7 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear12 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear13 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization8 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear14 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear15 = LinearConfig::new(512, 512).with_bias(true).init(device);
Self {
layernormalization5,
linear8,
linear9,
layernormalization6,
linear10,
linear11,
layernormalization7,
linear12,
linear13,
layernormalization8,
linear14,
linear15,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add21_out1: Tensor<3>,
concat8_out1: [i64; 5],
reshape12_out1: Tensor<4>,
reshape13_out1: Tensor<4>,
expand5_out1: Tensor<4, Bool>,
concat6_out1: [i64; 3],
) -> Tensor<3> {
let layernormalization5_out1 = {
let dtype = add21_out1.clone().dtype();
self.layernormalization5
.forward(add21_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear8_out1 = self.linear8.forward(layernormalization5_out1);
let gelu2_out1 = burn::tensor::activation::gelu(linear8_out1);
let linear9_out1 = self.linear9.forward(gelu2_out1);
let add22_out1 = add21_out1.add(linear9_out1);
let layernormalization6_out1 = {
let dtype = add22_out1.clone().dtype();
self.layernormalization6
.forward(add22_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear10_out1 = self.linear10.forward(layernormalization6_out1);
let reshape17_out1 = linear10_out1.reshape(concat8_out1);
let transpose5_out1 = reshape17_out1.permute([2, 0, 3, 1, 4]);
let slice18_out1 = transpose5_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze11_out1 = slice18_out1.squeeze_dims::<4>(&[0]);
let slice19_out1 = transpose5_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze12_out1 = slice19_out1.squeeze_dims::<4>(&[0]);
let slice20_out1 = transpose5_out1.slice(s![2..3, .., .., .., ..]);
let squeeze13_out1 = slice20_out1.squeeze_dims::<4>(&[0]);
let mul12_out1 = squeeze11_out1.clone().mul(reshape12_out1.clone());
let slice21_out1 = squeeze11_out1.clone().slice(s![.., .., .., 32..]);
let neg5_out1 = slice21_out1.neg();
let slice22_out1 = squeeze11_out1.slice(s![.., .., .., 0..32]);
let concat13_out1 = burn::tensor::Tensor::cat(
[neg5_out1, slice22_out1].into(),
3,
);
let mul13_out1 = concat13_out1.mul(reshape13_out1.clone());
let add23_out1 = mul12_out1.add(mul13_out1);
let mul14_out1 = squeeze12_out1.clone().mul(reshape12_out1.clone());
let slice23_out1 = squeeze12_out1.clone().slice(s![.., .., .., 32..]);
let neg6_out1 = slice23_out1.neg();
let slice24_out1 = squeeze12_out1.slice(s![.., .., .., 0..32]);
let concat14_out1 = burn::tensor::Tensor::cat(
[neg6_out1, slice24_out1].into(),
3,
);
let mul15_out1 = concat14_out1.mul(reshape13_out1.clone());
let add24_out1 = mul14_out1.add(mul15_out1);
let (attention3_out1,) = {
let q = add23_out1;
let k = add24_out1;
let v = squeeze13_out1;
let hide = expand5_out1.clone().bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention3_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention3_out1 = {
let dims = attention3_out1.dims();
attention3_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention3_out1,)
};
let transpose6_out1 = attention3_out1.permute([0, 2, 1, 3]);
let reshape18_out1 = transpose6_out1.reshape(concat6_out1);
let linear11_out1 = self.linear11.forward(reshape18_out1);
let add25_out1 = add22_out1.add(linear11_out1);
let layernormalization7_out1 = {
let dtype = add25_out1.clone().dtype();
self.layernormalization7
.forward(add25_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear12_out1 = self.linear12.forward(layernormalization7_out1);
let gelu3_out1 = burn::tensor::activation::gelu(linear12_out1);
let linear13_out1 = self.linear13.forward(gelu3_out1);
let add26_out1 = add25_out1.add(linear13_out1);
let layernormalization8_out1 = {
let dtype = add26_out1.clone().dtype();
self.layernormalization8
.forward(add26_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear14_out1 = self.linear14.forward(layernormalization8_out1);
let reshape19_out1 = linear14_out1.reshape(concat8_out1);
let transpose7_out1 = reshape19_out1.permute([2, 0, 3, 1, 4]);
let slice25_out1 = transpose7_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze14_out1 = slice25_out1.squeeze_dims::<4>(&[0]);
let slice26_out1 = transpose7_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze15_out1 = slice26_out1.squeeze_dims::<4>(&[0]);
let slice27_out1 = transpose7_out1.slice(s![2..3, .., .., .., ..]);
let squeeze16_out1 = slice27_out1.squeeze_dims::<4>(&[0]);
let mul16_out1 = squeeze14_out1.clone().mul(reshape12_out1.clone());
let slice28_out1 = squeeze14_out1.clone().slice(s![.., .., .., 32..]);
let neg7_out1 = slice28_out1.neg();
let slice29_out1 = squeeze14_out1.slice(s![.., .., .., 0..32]);
let concat15_out1 = burn::tensor::Tensor::cat(
[neg7_out1, slice29_out1].into(),
3,
);
let mul17_out1 = concat15_out1.mul(reshape13_out1.clone());
let add27_out1 = mul16_out1.add(mul17_out1);
let mul18_out1 = squeeze15_out1.clone().mul(reshape12_out1);
let slice30_out1 = squeeze15_out1.clone().slice(s![.., .., .., 32..]);
let neg8_out1 = slice30_out1.neg();
let slice31_out1 = squeeze15_out1.slice(s![.., .., .., 0..32]);
let concat16_out1 = burn::tensor::Tensor::cat(
[neg8_out1, slice31_out1].into(),
3,
);
let mul19_out1 = concat16_out1.mul(reshape13_out1);
let add28_out1 = mul18_out1.add(mul19_out1);
let (attention4_out1,) = {
let q = add27_out1;
let k = add28_out1;
let v = squeeze16_out1;
let hide = expand5_out1.bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention4_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention4_out1 = {
let dims = attention4_out1.dims();
attention4_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention4_out1,)
};
let transpose8_out1 = attention4_out1.permute([0, 2, 1, 3]);
let reshape20_out1 = transpose8_out1.reshape(concat6_out1);
let linear15_out1 = self.linear15.forward(reshape20_out1);
let add29_out1 = add26_out1.add(linear15_out1);
add29_out1
}
}
#[derive(Module, Debug)]
pub struct Submodule4 {
layernormalization9: LayerNorm,
linear16: Linear,
linear17: Linear,
layernormalization10: LayerNorm,
linear18: Linear,
linear19: Linear,
layernormalization11: LayerNorm,
linear20: Linear,
linear21: Linear,
layernormalization12: LayerNorm,
linear22: Linear,
linear23: Linear,
#[module(skip)]
device: Device,
}
impl Submodule4 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization9 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear16 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear17 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization10 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear18 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear19 = LinearConfig::new(512, 512).with_bias(true).init(device);
let layernormalization11 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear20 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear21 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization12 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear22 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear23 = LinearConfig::new(512, 512).with_bias(true).init(device);
Self {
layernormalization9,
linear16,
linear17,
layernormalization10,
linear18,
linear19,
layernormalization11,
linear20,
linear21,
layernormalization12,
linear22,
linear23,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add29_out1: Tensor<3>,
concat8_out1: [i64; 5],
reshape12_out1: Tensor<4>,
reshape13_out1: Tensor<4>,
expand5_out1: Tensor<4, Bool>,
concat6_out1: [i64; 3],
) -> Tensor<3> {
let layernormalization9_out1 = {
let dtype = add29_out1.clone().dtype();
self.layernormalization9
.forward(add29_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear16_out1 = self.linear16.forward(layernormalization9_out1);
let gelu4_out1 = burn::tensor::activation::gelu(linear16_out1);
let linear17_out1 = self.linear17.forward(gelu4_out1);
let add30_out1 = add29_out1.add(linear17_out1);
let layernormalization10_out1 = {
let dtype = add30_out1.clone().dtype();
self.layernormalization10
.forward(add30_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear18_out1 = self.linear18.forward(layernormalization10_out1);
let reshape21_out1 = linear18_out1.reshape(concat8_out1);
let transpose9_out1 = reshape21_out1.permute([2, 0, 3, 1, 4]);
let slice32_out1 = transpose9_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze17_out1 = slice32_out1.squeeze_dims::<4>(&[0]);
let slice33_out1 = transpose9_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze18_out1 = slice33_out1.squeeze_dims::<4>(&[0]);
let slice34_out1 = transpose9_out1.slice(s![2..3, .., .., .., ..]);
let squeeze19_out1 = slice34_out1.squeeze_dims::<4>(&[0]);
let mul20_out1 = squeeze17_out1.clone().mul(reshape12_out1.clone());
let slice35_out1 = squeeze17_out1.clone().slice(s![.., .., .., 32..]);
let neg9_out1 = slice35_out1.neg();
let slice36_out1 = squeeze17_out1.slice(s![.., .., .., 0..32]);
let concat17_out1 = burn::tensor::Tensor::cat(
[neg9_out1, slice36_out1].into(),
3,
);
let mul21_out1 = concat17_out1.mul(reshape13_out1.clone());
let add31_out1 = mul20_out1.add(mul21_out1);
let mul22_out1 = squeeze18_out1.clone().mul(reshape12_out1.clone());
let slice37_out1 = squeeze18_out1.clone().slice(s![.., .., .., 32..]);
let neg10_out1 = slice37_out1.neg();
let slice38_out1 = squeeze18_out1.slice(s![.., .., .., 0..32]);
let concat18_out1 = burn::tensor::Tensor::cat(
[neg10_out1, slice38_out1].into(),
3,
);
let mul23_out1 = concat18_out1.mul(reshape13_out1.clone());
let add32_out1 = mul22_out1.add(mul23_out1);
let (attention5_out1,) = {
let q = add31_out1;
let k = add32_out1;
let v = squeeze19_out1;
let hide = expand5_out1.clone().bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention5_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention5_out1 = {
let dims = attention5_out1.dims();
attention5_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention5_out1,)
};
let transpose10_out1 = attention5_out1.permute([0, 2, 1, 3]);
let reshape22_out1 = transpose10_out1.reshape(concat6_out1);
let linear19_out1 = self.linear19.forward(reshape22_out1);
let add33_out1 = add30_out1.add(linear19_out1);
let layernormalization11_out1 = {
let dtype = add33_out1.clone().dtype();
self.layernormalization11
.forward(add33_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear20_out1 = self.linear20.forward(layernormalization11_out1);
let gelu5_out1 = burn::tensor::activation::gelu(linear20_out1);
let linear21_out1 = self.linear21.forward(gelu5_out1);
let add34_out1 = add33_out1.add(linear21_out1);
let layernormalization12_out1 = {
let dtype = add34_out1.clone().dtype();
self.layernormalization12
.forward(add34_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear22_out1 = self.linear22.forward(layernormalization12_out1);
let reshape23_out1 = linear22_out1.reshape(concat8_out1);
let transpose11_out1 = reshape23_out1.permute([2, 0, 3, 1, 4]);
let slice39_out1 = transpose11_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze20_out1 = slice39_out1.squeeze_dims::<4>(&[0]);
let slice40_out1 = transpose11_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze21_out1 = slice40_out1.squeeze_dims::<4>(&[0]);
let slice41_out1 = transpose11_out1.slice(s![2..3, .., .., .., ..]);
let squeeze22_out1 = slice41_out1.squeeze_dims::<4>(&[0]);
let mul24_out1 = squeeze20_out1.clone().mul(reshape12_out1.clone());
let slice42_out1 = squeeze20_out1.clone().slice(s![.., .., .., 32..]);
let neg11_out1 = slice42_out1.neg();
let slice43_out1 = squeeze20_out1.slice(s![.., .., .., 0..32]);
let concat19_out1 = burn::tensor::Tensor::cat(
[neg11_out1, slice43_out1].into(),
3,
);
let mul25_out1 = concat19_out1.mul(reshape13_out1.clone());
let add35_out1 = mul24_out1.add(mul25_out1);
let mul26_out1 = squeeze21_out1.clone().mul(reshape12_out1);
let slice44_out1 = squeeze21_out1.clone().slice(s![.., .., .., 32..]);
let neg12_out1 = slice44_out1.neg();
let slice45_out1 = squeeze21_out1.slice(s![.., .., .., 0..32]);
let concat20_out1 = burn::tensor::Tensor::cat(
[neg12_out1, slice45_out1].into(),
3,
);
let mul27_out1 = concat20_out1.mul(reshape13_out1);
let add36_out1 = mul26_out1.add(mul27_out1);
let (attention6_out1,) = {
let q = add35_out1;
let k = add36_out1;
let v = squeeze22_out1;
let hide = expand5_out1.bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention6_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention6_out1 = {
let dims = attention6_out1.dims();
attention6_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention6_out1,)
};
let transpose12_out1 = attention6_out1.permute([0, 2, 1, 3]);
let reshape24_out1 = transpose12_out1.reshape(concat6_out1);
let linear23_out1 = self.linear23.forward(reshape24_out1);
let add37_out1 = add34_out1.add(linear23_out1);
add37_out1
}
}
#[derive(Module, Debug)]
pub struct Submodule5 {
layernormalization13: LayerNorm,
linear24: Linear,
linear25: Linear,
layernormalization14: LayerNorm,
linear26: Linear,
linear27: Linear,
layernormalization15: LayerNorm,
linear28: Linear,
linear29: Linear,
layernormalization16: LayerNorm,
linear30: Linear,
linear31: Linear,
#[module(skip)]
device: Device,
}
impl Submodule5 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization13 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear24 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear25 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization14 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear26 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear27 = LinearConfig::new(512, 512).with_bias(true).init(device);
let layernormalization15 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear28 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear29 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization16 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear30 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear31 = LinearConfig::new(512, 512).with_bias(true).init(device);
Self {
layernormalization13,
linear24,
linear25,
layernormalization14,
linear26,
linear27,
layernormalization15,
linear28,
linear29,
layernormalization16,
linear30,
linear31,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add37_out1: Tensor<3>,
concat8_out1: [i64; 5],
reshape12_out1: Tensor<4>,
reshape13_out1: Tensor<4>,
expand5_out1: Tensor<4, Bool>,
concat6_out1: [i64; 3],
) -> Tensor<3> {
let layernormalization13_out1 = {
let dtype = add37_out1.clone().dtype();
self.layernormalization13
.forward(add37_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear24_out1 = self.linear24.forward(layernormalization13_out1);
let gelu6_out1 = burn::tensor::activation::gelu(linear24_out1);
let linear25_out1 = self.linear25.forward(gelu6_out1);
let add38_out1 = add37_out1.add(linear25_out1);
let layernormalization14_out1 = {
let dtype = add38_out1.clone().dtype();
self.layernormalization14
.forward(add38_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear26_out1 = self.linear26.forward(layernormalization14_out1);
let reshape25_out1 = linear26_out1.reshape(concat8_out1);
let transpose13_out1 = reshape25_out1.permute([2, 0, 3, 1, 4]);
let slice46_out1 = transpose13_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze23_out1 = slice46_out1.squeeze_dims::<4>(&[0]);
let slice47_out1 = transpose13_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze24_out1 = slice47_out1.squeeze_dims::<4>(&[0]);
let slice48_out1 = transpose13_out1.slice(s![2..3, .., .., .., ..]);
let squeeze25_out1 = slice48_out1.squeeze_dims::<4>(&[0]);
let mul28_out1 = squeeze23_out1.clone().mul(reshape12_out1.clone());
let slice49_out1 = squeeze23_out1.clone().slice(s![.., .., .., 32..]);
let neg13_out1 = slice49_out1.neg();
let slice50_out1 = squeeze23_out1.slice(s![.., .., .., 0..32]);
let concat21_out1 = burn::tensor::Tensor::cat(
[neg13_out1, slice50_out1].into(),
3,
);
let mul29_out1 = concat21_out1.mul(reshape13_out1.clone());
let add39_out1 = mul28_out1.add(mul29_out1);
let mul30_out1 = squeeze24_out1.clone().mul(reshape12_out1.clone());
let slice51_out1 = squeeze24_out1.clone().slice(s![.., .., .., 32..]);
let neg14_out1 = slice51_out1.neg();
let slice52_out1 = squeeze24_out1.slice(s![.., .., .., 0..32]);
let concat22_out1 = burn::tensor::Tensor::cat(
[neg14_out1, slice52_out1].into(),
3,
);
let mul31_out1 = concat22_out1.mul(reshape13_out1.clone());
let add40_out1 = mul30_out1.add(mul31_out1);
let (attention7_out1,) = {
let q = add39_out1;
let k = add40_out1;
let v = squeeze25_out1;
let hide = expand5_out1.clone().bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention7_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention7_out1 = {
let dims = attention7_out1.dims();
attention7_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention7_out1,)
};
let transpose14_out1 = attention7_out1.permute([0, 2, 1, 3]);
let reshape26_out1 = transpose14_out1.reshape(concat6_out1);
let linear27_out1 = self.linear27.forward(reshape26_out1);
let add41_out1 = add38_out1.add(linear27_out1);
let layernormalization15_out1 = {
let dtype = add41_out1.clone().dtype();
self.layernormalization15
.forward(add41_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear28_out1 = self.linear28.forward(layernormalization15_out1);
let gelu7_out1 = burn::tensor::activation::gelu(linear28_out1);
let linear29_out1 = self.linear29.forward(gelu7_out1);
let add42_out1 = add41_out1.add(linear29_out1);
let layernormalization16_out1 = {
let dtype = add42_out1.clone().dtype();
self.layernormalization16
.forward(add42_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear30_out1 = self.linear30.forward(layernormalization16_out1);
let reshape27_out1 = linear30_out1.reshape(concat8_out1);
let transpose15_out1 = reshape27_out1.permute([2, 0, 3, 1, 4]);
let slice53_out1 = transpose15_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze26_out1 = slice53_out1.squeeze_dims::<4>(&[0]);
let slice54_out1 = transpose15_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze27_out1 = slice54_out1.squeeze_dims::<4>(&[0]);
let slice55_out1 = transpose15_out1.slice(s![2..3, .., .., .., ..]);
let squeeze28_out1 = slice55_out1.squeeze_dims::<4>(&[0]);
let mul32_out1 = squeeze26_out1.clone().mul(reshape12_out1.clone());
let slice56_out1 = squeeze26_out1.clone().slice(s![.., .., .., 32..]);
let neg15_out1 = slice56_out1.neg();
let slice57_out1 = squeeze26_out1.slice(s![.., .., .., 0..32]);
let concat23_out1 = burn::tensor::Tensor::cat(
[neg15_out1, slice57_out1].into(),
3,
);
let mul33_out1 = concat23_out1.mul(reshape13_out1.clone());
let add43_out1 = mul32_out1.add(mul33_out1);
let mul34_out1 = squeeze27_out1.clone().mul(reshape12_out1);
let slice58_out1 = squeeze27_out1.clone().slice(s![.., .., .., 32..]);
let neg16_out1 = slice58_out1.neg();
let slice59_out1 = squeeze27_out1.slice(s![.., .., .., 0..32]);
let concat24_out1 = burn::tensor::Tensor::cat(
[neg16_out1, slice59_out1].into(),
3,
);
let mul35_out1 = concat24_out1.mul(reshape13_out1);
let add44_out1 = mul34_out1.add(mul35_out1);
let (attention8_out1,) = {
let q = add43_out1;
let k = add44_out1;
let v = squeeze28_out1;
let hide = expand5_out1.bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention8_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention8_out1 = {
let dims = attention8_out1.dims();
attention8_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention8_out1,)
};
let transpose16_out1 = attention8_out1.permute([0, 2, 1, 3]);
let reshape28_out1 = transpose16_out1.reshape(concat6_out1);
let linear31_out1 = self.linear31.forward(reshape28_out1);
let add45_out1 = add42_out1.add(linear31_out1);
add45_out1
}
}
#[derive(Module, Debug)]
pub struct Submodule6 {
layernormalization17: LayerNorm,
linear32: Linear,
linear33: Linear,
layernormalization18: LayerNorm,
linear34: Linear,
linear35: Linear,
layernormalization19: LayerNorm,
linear36: Linear,
linear37: Linear,
layernormalization20: LayerNorm,
linear38: Linear,
linear39: Linear,
#[module(skip)]
device: Device,
}
impl Submodule6 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization17 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear32 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear33 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization18 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear34 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear35 = LinearConfig::new(512, 512).with_bias(true).init(device);
let layernormalization19 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear36 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear37 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization20 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear38 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear39 = LinearConfig::new(512, 512).with_bias(true).init(device);
Self {
layernormalization17,
linear32,
linear33,
layernormalization18,
linear34,
linear35,
layernormalization19,
linear36,
linear37,
layernormalization20,
linear38,
linear39,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add45_out1: Tensor<3>,
concat8_out1: [i64; 5],
reshape12_out1: Tensor<4>,
reshape13_out1: Tensor<4>,
expand5_out1: Tensor<4, Bool>,
concat6_out1: [i64; 3],
) -> Tensor<3> {
let layernormalization17_out1 = {
let dtype = add45_out1.clone().dtype();
self.layernormalization17
.forward(add45_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear32_out1 = self.linear32.forward(layernormalization17_out1);
let gelu8_out1 = burn::tensor::activation::gelu(linear32_out1);
let linear33_out1 = self.linear33.forward(gelu8_out1);
let add46_out1 = add45_out1.add(linear33_out1);
let layernormalization18_out1 = {
let dtype = add46_out1.clone().dtype();
self.layernormalization18
.forward(add46_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear34_out1 = self.linear34.forward(layernormalization18_out1);
let reshape29_out1 = linear34_out1.reshape(concat8_out1);
let transpose17_out1 = reshape29_out1.permute([2, 0, 3, 1, 4]);
let slice60_out1 = transpose17_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze29_out1 = slice60_out1.squeeze_dims::<4>(&[0]);
let slice61_out1 = transpose17_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze30_out1 = slice61_out1.squeeze_dims::<4>(&[0]);
let slice62_out1 = transpose17_out1.slice(s![2..3, .., .., .., ..]);
let squeeze31_out1 = slice62_out1.squeeze_dims::<4>(&[0]);
let mul36_out1 = squeeze29_out1.clone().mul(reshape12_out1.clone());
let slice63_out1 = squeeze29_out1.clone().slice(s![.., .., .., 32..]);
let neg17_out1 = slice63_out1.neg();
let slice64_out1 = squeeze29_out1.slice(s![.., .., .., 0..32]);
let concat25_out1 = burn::tensor::Tensor::cat(
[neg17_out1, slice64_out1].into(),
3,
);
let mul37_out1 = concat25_out1.mul(reshape13_out1.clone());
let add47_out1 = mul36_out1.add(mul37_out1);
let mul38_out1 = squeeze30_out1.clone().mul(reshape12_out1.clone());
let slice65_out1 = squeeze30_out1.clone().slice(s![.., .., .., 32..]);
let neg18_out1 = slice65_out1.neg();
let slice66_out1 = squeeze30_out1.slice(s![.., .., .., 0..32]);
let concat26_out1 = burn::tensor::Tensor::cat(
[neg18_out1, slice66_out1].into(),
3,
);
let mul39_out1 = concat26_out1.mul(reshape13_out1.clone());
let add48_out1 = mul38_out1.add(mul39_out1);
let (attention9_out1,) = {
let q = add47_out1;
let k = add48_out1;
let v = squeeze31_out1;
let hide = expand5_out1.clone().bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention9_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention9_out1 = {
let dims = attention9_out1.dims();
attention9_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention9_out1,)
};
let transpose18_out1 = attention9_out1.permute([0, 2, 1, 3]);
let reshape30_out1 = transpose18_out1.reshape(concat6_out1);
let linear35_out1 = self.linear35.forward(reshape30_out1);
let add49_out1 = add46_out1.add(linear35_out1);
let layernormalization19_out1 = {
let dtype = add49_out1.clone().dtype();
self.layernormalization19
.forward(add49_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear36_out1 = self.linear36.forward(layernormalization19_out1);
let gelu9_out1 = burn::tensor::activation::gelu(linear36_out1);
let linear37_out1 = self.linear37.forward(gelu9_out1);
let add50_out1 = add49_out1.add(linear37_out1);
let layernormalization20_out1 = {
let dtype = add50_out1.clone().dtype();
self.layernormalization20
.forward(add50_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear38_out1 = self.linear38.forward(layernormalization20_out1);
let reshape31_out1 = linear38_out1.reshape(concat8_out1);
let transpose19_out1 = reshape31_out1.permute([2, 0, 3, 1, 4]);
let slice67_out1 = transpose19_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze32_out1 = slice67_out1.squeeze_dims::<4>(&[0]);
let slice68_out1 = transpose19_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze33_out1 = slice68_out1.squeeze_dims::<4>(&[0]);
let slice69_out1 = transpose19_out1.slice(s![2..3, .., .., .., ..]);
let squeeze34_out1 = slice69_out1.squeeze_dims::<4>(&[0]);
let mul40_out1 = squeeze32_out1.clone().mul(reshape12_out1.clone());
let slice70_out1 = squeeze32_out1.clone().slice(s![.., .., .., 32..]);
let neg19_out1 = slice70_out1.neg();
let slice71_out1 = squeeze32_out1.slice(s![.., .., .., 0..32]);
let concat27_out1 = burn::tensor::Tensor::cat(
[neg19_out1, slice71_out1].into(),
3,
);
let mul41_out1 = concat27_out1.mul(reshape13_out1.clone());
let add51_out1 = mul40_out1.add(mul41_out1);
let mul42_out1 = squeeze33_out1.clone().mul(reshape12_out1);
let slice72_out1 = squeeze33_out1.clone().slice(s![.., .., .., 32..]);
let neg20_out1 = slice72_out1.neg();
let slice73_out1 = squeeze33_out1.slice(s![.., .., .., 0..32]);
let concat28_out1 = burn::tensor::Tensor::cat(
[neg20_out1, slice73_out1].into(),
3,
);
let mul43_out1 = concat28_out1.mul(reshape13_out1);
let add52_out1 = mul42_out1.add(mul43_out1);
let (attention10_out1,) = {
let q = add51_out1;
let k = add52_out1;
let v = squeeze34_out1;
let hide = expand5_out1.bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention10_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention10_out1 = {
let dims = attention10_out1.dims();
attention10_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention10_out1,)
};
let transpose20_out1 = attention10_out1.permute([0, 2, 1, 3]);
let reshape32_out1 = transpose20_out1.reshape(concat6_out1);
let linear39_out1 = self.linear39.forward(reshape32_out1);
let add53_out1 = add50_out1.add(linear39_out1);
add53_out1
}
}
#[derive(Module, Debug)]
pub struct Submodule7 {
layernormalization21: LayerNorm,
linear40: Linear,
linear41: Linear,
layernormalization22: LayerNorm,
linear42: Linear,
linear43: Linear,
layernormalization23: LayerNorm,
linear44: Linear,
linear45: Linear,
layernormalization24: LayerNorm,
linear46: Linear,
linear47: Linear,
#[module(skip)]
device: Device,
}
impl Submodule7 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization21 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear40 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear41 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization22 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear42 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear43 = LinearConfig::new(512, 512).with_bias(true).init(device);
let layernormalization23 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear44 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear45 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization24 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear46 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear47 = LinearConfig::new(512, 512).with_bias(true).init(device);
Self {
layernormalization21,
linear40,
linear41,
layernormalization22,
linear42,
linear43,
layernormalization23,
linear44,
linear45,
layernormalization24,
linear46,
linear47,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add53_out1: Tensor<3>,
concat8_out1: [i64; 5],
reshape12_out1: Tensor<4>,
reshape13_out1: Tensor<4>,
expand5_out1: Tensor<4, Bool>,
concat6_out1: [i64; 3],
) -> Tensor<3> {
let layernormalization21_out1 = {
let dtype = add53_out1.clone().dtype();
self.layernormalization21
.forward(add53_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear40_out1 = self.linear40.forward(layernormalization21_out1);
let gelu10_out1 = burn::tensor::activation::gelu(linear40_out1);
let linear41_out1 = self.linear41.forward(gelu10_out1);
let add54_out1 = add53_out1.add(linear41_out1);
let layernormalization22_out1 = {
let dtype = add54_out1.clone().dtype();
self.layernormalization22
.forward(add54_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear42_out1 = self.linear42.forward(layernormalization22_out1);
let reshape33_out1 = linear42_out1.reshape(concat8_out1);
let transpose21_out1 = reshape33_out1.permute([2, 0, 3, 1, 4]);
let slice74_out1 = transpose21_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze35_out1 = slice74_out1.squeeze_dims::<4>(&[0]);
let slice75_out1 = transpose21_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze36_out1 = slice75_out1.squeeze_dims::<4>(&[0]);
let slice76_out1 = transpose21_out1.slice(s![2..3, .., .., .., ..]);
let squeeze37_out1 = slice76_out1.squeeze_dims::<4>(&[0]);
let mul44_out1 = squeeze35_out1.clone().mul(reshape12_out1.clone());
let slice77_out1 = squeeze35_out1.clone().slice(s![.., .., .., 32..]);
let neg21_out1 = slice77_out1.neg();
let slice78_out1 = squeeze35_out1.slice(s![.., .., .., 0..32]);
let concat29_out1 = burn::tensor::Tensor::cat(
[neg21_out1, slice78_out1].into(),
3,
);
let mul45_out1 = concat29_out1.mul(reshape13_out1.clone());
let add55_out1 = mul44_out1.add(mul45_out1);
let mul46_out1 = squeeze36_out1.clone().mul(reshape12_out1.clone());
let slice79_out1 = squeeze36_out1.clone().slice(s![.., .., .., 32..]);
let neg22_out1 = slice79_out1.neg();
let slice80_out1 = squeeze36_out1.slice(s![.., .., .., 0..32]);
let concat30_out1 = burn::tensor::Tensor::cat(
[neg22_out1, slice80_out1].into(),
3,
);
let mul47_out1 = concat30_out1.mul(reshape13_out1.clone());
let add56_out1 = mul46_out1.add(mul47_out1);
let (attention11_out1,) = {
let q = add55_out1;
let k = add56_out1;
let v = squeeze37_out1;
let hide = expand5_out1.clone().bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention11_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention11_out1 = {
let dims = attention11_out1.dims();
attention11_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention11_out1,)
};
let transpose22_out1 = attention11_out1.permute([0, 2, 1, 3]);
let reshape34_out1 = transpose22_out1.reshape(concat6_out1);
let linear43_out1 = self.linear43.forward(reshape34_out1);
let add57_out1 = add54_out1.add(linear43_out1);
let layernormalization23_out1 = {
let dtype = add57_out1.clone().dtype();
self.layernormalization23
.forward(add57_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear44_out1 = self.linear44.forward(layernormalization23_out1);
let gelu11_out1 = burn::tensor::activation::gelu(linear44_out1);
let linear45_out1 = self.linear45.forward(gelu11_out1);
let add58_out1 = add57_out1.add(linear45_out1);
let layernormalization24_out1 = {
let dtype = add58_out1.clone().dtype();
self.layernormalization24
.forward(add58_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear46_out1 = self.linear46.forward(layernormalization24_out1);
let reshape35_out1 = linear46_out1.reshape(concat8_out1);
let transpose23_out1 = reshape35_out1.permute([2, 0, 3, 1, 4]);
let slice81_out1 = transpose23_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze38_out1 = slice81_out1.squeeze_dims::<4>(&[0]);
let slice82_out1 = transpose23_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze39_out1 = slice82_out1.squeeze_dims::<4>(&[0]);
let slice83_out1 = transpose23_out1.slice(s![2..3, .., .., .., ..]);
let squeeze40_out1 = slice83_out1.squeeze_dims::<4>(&[0]);
let mul48_out1 = squeeze38_out1.clone().mul(reshape12_out1.clone());
let slice84_out1 = squeeze38_out1.clone().slice(s![.., .., .., 32..]);
let neg23_out1 = slice84_out1.neg();
let slice85_out1 = squeeze38_out1.slice(s![.., .., .., 0..32]);
let concat31_out1 = burn::tensor::Tensor::cat(
[neg23_out1, slice85_out1].into(),
3,
);
let mul49_out1 = concat31_out1.mul(reshape13_out1.clone());
let add59_out1 = mul48_out1.add(mul49_out1);
let mul50_out1 = squeeze39_out1.clone().mul(reshape12_out1);
let slice86_out1 = squeeze39_out1.clone().slice(s![.., .., .., 32..]);
let neg24_out1 = slice86_out1.neg();
let slice87_out1 = squeeze39_out1.slice(s![.., .., .., 0..32]);
let concat32_out1 = burn::tensor::Tensor::cat(
[neg24_out1, slice87_out1].into(),
3,
);
let mul51_out1 = concat32_out1.mul(reshape13_out1);
let add60_out1 = mul50_out1.add(mul51_out1);
let (attention12_out1,) = {
let q = add59_out1;
let k = add60_out1;
let v = squeeze40_out1;
let hide = expand5_out1.bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention12_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention12_out1 = {
let dims = attention12_out1.dims();
attention12_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention12_out1,)
};
let transpose24_out1 = attention12_out1.permute([0, 2, 1, 3]);
let reshape36_out1 = transpose24_out1.reshape(concat6_out1);
let linear47_out1 = self.linear47.forward(reshape36_out1);
let add61_out1 = add58_out1.add(linear47_out1);
add61_out1
}
}
#[derive(Module, Debug)]
pub struct Submodule8 {
layernormalization25: LayerNorm,
linear48: Linear,
linear49: Linear,
layernormalization26: LayerNorm,
linear50: Linear,
linear51: Linear,
layernormalization27: LayerNorm,
linear52: Linear,
linear53: Linear,
layernormalization28: LayerNorm,
linear54: Linear,
linear55: Linear,
#[module(skip)]
device: Device,
}
impl Submodule8 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization25 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear48 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear49 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization26 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear50 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear51 = LinearConfig::new(512, 512).with_bias(true).init(device);
let layernormalization27 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear52 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear53 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization28 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear54 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear55 = LinearConfig::new(512, 512).with_bias(true).init(device);
Self {
layernormalization25,
linear48,
linear49,
layernormalization26,
linear50,
linear51,
layernormalization27,
linear52,
linear53,
layernormalization28,
linear54,
linear55,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add61_out1: Tensor<3>,
concat8_out1: [i64; 5],
reshape12_out1: Tensor<4>,
reshape13_out1: Tensor<4>,
expand5_out1: Tensor<4, Bool>,
concat6_out1: [i64; 3],
) -> Tensor<3> {
let layernormalization25_out1 = {
let dtype = add61_out1.clone().dtype();
self.layernormalization25
.forward(add61_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear48_out1 = self.linear48.forward(layernormalization25_out1);
let gelu12_out1 = burn::tensor::activation::gelu(linear48_out1);
let linear49_out1 = self.linear49.forward(gelu12_out1);
let add62_out1 = add61_out1.add(linear49_out1);
let layernormalization26_out1 = {
let dtype = add62_out1.clone().dtype();
self.layernormalization26
.forward(add62_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear50_out1 = self.linear50.forward(layernormalization26_out1);
let reshape37_out1 = linear50_out1.reshape(concat8_out1);
let transpose25_out1 = reshape37_out1.permute([2, 0, 3, 1, 4]);
let slice88_out1 = transpose25_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze41_out1 = slice88_out1.squeeze_dims::<4>(&[0]);
let slice89_out1 = transpose25_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze42_out1 = slice89_out1.squeeze_dims::<4>(&[0]);
let slice90_out1 = transpose25_out1.slice(s![2..3, .., .., .., ..]);
let squeeze43_out1 = slice90_out1.squeeze_dims::<4>(&[0]);
let mul52_out1 = squeeze41_out1.clone().mul(reshape12_out1.clone());
let slice91_out1 = squeeze41_out1.clone().slice(s![.., .., .., 32..]);
let neg25_out1 = slice91_out1.neg();
let slice92_out1 = squeeze41_out1.slice(s![.., .., .., 0..32]);
let concat33_out1 = burn::tensor::Tensor::cat(
[neg25_out1, slice92_out1].into(),
3,
);
let mul53_out1 = concat33_out1.mul(reshape13_out1.clone());
let add63_out1 = mul52_out1.add(mul53_out1);
let mul54_out1 = squeeze42_out1.clone().mul(reshape12_out1.clone());
let slice93_out1 = squeeze42_out1.clone().slice(s![.., .., .., 32..]);
let neg26_out1 = slice93_out1.neg();
let slice94_out1 = squeeze42_out1.slice(s![.., .., .., 0..32]);
let concat34_out1 = burn::tensor::Tensor::cat(
[neg26_out1, slice94_out1].into(),
3,
);
let mul55_out1 = concat34_out1.mul(reshape13_out1.clone());
let add64_out1 = mul54_out1.add(mul55_out1);
let (attention13_out1,) = {
let q = add63_out1;
let k = add64_out1;
let v = squeeze43_out1;
let hide = expand5_out1.clone().bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention13_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention13_out1 = {
let dims = attention13_out1.dims();
attention13_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention13_out1,)
};
let transpose26_out1 = attention13_out1.permute([0, 2, 1, 3]);
let reshape38_out1 = transpose26_out1.reshape(concat6_out1);
let linear51_out1 = self.linear51.forward(reshape38_out1);
let add65_out1 = add62_out1.add(linear51_out1);
let layernormalization27_out1 = {
let dtype = add65_out1.clone().dtype();
self.layernormalization27
.forward(add65_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear52_out1 = self.linear52.forward(layernormalization27_out1);
let gelu13_out1 = burn::tensor::activation::gelu(linear52_out1);
let linear53_out1 = self.linear53.forward(gelu13_out1);
let add66_out1 = add65_out1.add(linear53_out1);
let layernormalization28_out1 = {
let dtype = add66_out1.clone().dtype();
self.layernormalization28
.forward(add66_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear54_out1 = self.linear54.forward(layernormalization28_out1);
let reshape39_out1 = linear54_out1.reshape(concat8_out1);
let transpose27_out1 = reshape39_out1.permute([2, 0, 3, 1, 4]);
let slice95_out1 = transpose27_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze44_out1 = slice95_out1.squeeze_dims::<4>(&[0]);
let slice96_out1 = transpose27_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze45_out1 = slice96_out1.squeeze_dims::<4>(&[0]);
let slice97_out1 = transpose27_out1.slice(s![2..3, .., .., .., ..]);
let squeeze46_out1 = slice97_out1.squeeze_dims::<4>(&[0]);
let mul56_out1 = squeeze44_out1.clone().mul(reshape12_out1.clone());
let slice98_out1 = squeeze44_out1.clone().slice(s![.., .., .., 32..]);
let neg27_out1 = slice98_out1.neg();
let slice99_out1 = squeeze44_out1.slice(s![.., .., .., 0..32]);
let concat35_out1 = burn::tensor::Tensor::cat(
[neg27_out1, slice99_out1].into(),
3,
);
let mul57_out1 = concat35_out1.mul(reshape13_out1.clone());
let add67_out1 = mul56_out1.add(mul57_out1);
let mul58_out1 = squeeze45_out1.clone().mul(reshape12_out1);
let slice100_out1 = squeeze45_out1.clone().slice(s![.., .., .., 32..]);
let neg28_out1 = slice100_out1.neg();
let slice101_out1 = squeeze45_out1.slice(s![.., .., .., 0..32]);
let concat36_out1 = burn::tensor::Tensor::cat(
[neg28_out1, slice101_out1].into(),
3,
);
let mul59_out1 = concat36_out1.mul(reshape13_out1);
let add68_out1 = mul58_out1.add(mul59_out1);
let (attention14_out1,) = {
let q = add67_out1;
let k = add68_out1;
let v = squeeze46_out1;
let hide = expand5_out1.bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention14_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention14_out1 = {
let dims = attention14_out1.dims();
attention14_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention14_out1,)
};
let transpose28_out1 = attention14_out1.permute([0, 2, 1, 3]);
let reshape40_out1 = transpose28_out1.reshape(concat6_out1);
let linear55_out1 = self.linear55.forward(reshape40_out1);
let add69_out1 = add66_out1.add(linear55_out1);
add69_out1
}
}
#[derive(Module, Debug)]
pub struct Submodule9 {
layernormalization29: LayerNorm,
linear56: Linear,
linear57: Linear,
layernormalization30: LayerNorm,
linear58: Linear,
linear59: Linear,
layernormalization31: LayerNorm,
linear60: Linear,
linear61: Linear,
layernormalization32: LayerNorm,
linear62: Linear,
linear63: Linear,
#[module(skip)]
device: Device,
}
impl Submodule9 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization29 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear56 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear57 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization30 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear58 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear59 = LinearConfig::new(512, 512).with_bias(true).init(device);
let layernormalization31 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear60 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear61 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization32 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear62 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear63 = LinearConfig::new(512, 512).with_bias(true).init(device);
Self {
layernormalization29,
linear56,
linear57,
layernormalization30,
linear58,
linear59,
layernormalization31,
linear60,
linear61,
layernormalization32,
linear62,
linear63,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add69_out1: Tensor<3>,
concat8_out1: [i64; 5],
reshape12_out1: Tensor<4>,
reshape13_out1: Tensor<4>,
expand5_out1: Tensor<4, Bool>,
concat6_out1: [i64; 3],
) -> Tensor<3> {
let layernormalization29_out1 = {
let dtype = add69_out1.clone().dtype();
self.layernormalization29
.forward(add69_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear56_out1 = self.linear56.forward(layernormalization29_out1);
let gelu14_out1 = burn::tensor::activation::gelu(linear56_out1);
let linear57_out1 = self.linear57.forward(gelu14_out1);
let add70_out1 = add69_out1.add(linear57_out1);
let layernormalization30_out1 = {
let dtype = add70_out1.clone().dtype();
self.layernormalization30
.forward(add70_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear58_out1 = self.linear58.forward(layernormalization30_out1);
let reshape41_out1 = linear58_out1.reshape(concat8_out1);
let transpose29_out1 = reshape41_out1.permute([2, 0, 3, 1, 4]);
let slice102_out1 = transpose29_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze47_out1 = slice102_out1.squeeze_dims::<4>(&[0]);
let slice103_out1 = transpose29_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze48_out1 = slice103_out1.squeeze_dims::<4>(&[0]);
let slice104_out1 = transpose29_out1.slice(s![2..3, .., .., .., ..]);
let squeeze49_out1 = slice104_out1.squeeze_dims::<4>(&[0]);
let mul60_out1 = squeeze47_out1.clone().mul(reshape12_out1.clone());
let slice105_out1 = squeeze47_out1.clone().slice(s![.., .., .., 32..]);
let neg29_out1 = slice105_out1.neg();
let slice106_out1 = squeeze47_out1.slice(s![.., .., .., 0..32]);
let concat37_out1 = burn::tensor::Tensor::cat(
[neg29_out1, slice106_out1].into(),
3,
);
let mul61_out1 = concat37_out1.mul(reshape13_out1.clone());
let add71_out1 = mul60_out1.add(mul61_out1);
let mul62_out1 = squeeze48_out1.clone().mul(reshape12_out1.clone());
let slice107_out1 = squeeze48_out1.clone().slice(s![.., .., .., 32..]);
let neg30_out1 = slice107_out1.neg();
let slice108_out1 = squeeze48_out1.slice(s![.., .., .., 0..32]);
let concat38_out1 = burn::tensor::Tensor::cat(
[neg30_out1, slice108_out1].into(),
3,
);
let mul63_out1 = concat38_out1.mul(reshape13_out1.clone());
let add72_out1 = mul62_out1.add(mul63_out1);
let (attention15_out1,) = {
let q = add71_out1;
let k = add72_out1;
let v = squeeze49_out1;
let hide = expand5_out1.clone().bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention15_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention15_out1 = {
let dims = attention15_out1.dims();
attention15_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention15_out1,)
};
let transpose30_out1 = attention15_out1.permute([0, 2, 1, 3]);
let reshape42_out1 = transpose30_out1.reshape(concat6_out1);
let linear59_out1 = self.linear59.forward(reshape42_out1);
let add73_out1 = add70_out1.add(linear59_out1);
let layernormalization31_out1 = {
let dtype = add73_out1.clone().dtype();
self.layernormalization31
.forward(add73_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear60_out1 = self.linear60.forward(layernormalization31_out1);
let gelu15_out1 = burn::tensor::activation::gelu(linear60_out1);
let linear61_out1 = self.linear61.forward(gelu15_out1);
let add74_out1 = add73_out1.add(linear61_out1);
let layernormalization32_out1 = {
let dtype = add74_out1.clone().dtype();
self.layernormalization32
.forward(add74_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear62_out1 = self.linear62.forward(layernormalization32_out1);
let reshape43_out1 = linear62_out1.reshape(concat8_out1);
let transpose31_out1 = reshape43_out1.permute([2, 0, 3, 1, 4]);
let slice109_out1 = transpose31_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze50_out1 = slice109_out1.squeeze_dims::<4>(&[0]);
let slice110_out1 = transpose31_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze51_out1 = slice110_out1.squeeze_dims::<4>(&[0]);
let slice111_out1 = transpose31_out1.slice(s![2..3, .., .., .., ..]);
let squeeze52_out1 = slice111_out1.squeeze_dims::<4>(&[0]);
let mul64_out1 = squeeze50_out1.clone().mul(reshape12_out1.clone());
let slice112_out1 = squeeze50_out1.clone().slice(s![.., .., .., 32..]);
let neg31_out1 = slice112_out1.neg();
let slice113_out1 = squeeze50_out1.slice(s![.., .., .., 0..32]);
let concat39_out1 = burn::tensor::Tensor::cat(
[neg31_out1, slice113_out1].into(),
3,
);
let mul65_out1 = concat39_out1.mul(reshape13_out1.clone());
let add75_out1 = mul64_out1.add(mul65_out1);
let mul66_out1 = squeeze51_out1.clone().mul(reshape12_out1);
let slice114_out1 = squeeze51_out1.clone().slice(s![.., .., .., 32..]);
let neg32_out1 = slice114_out1.neg();
let slice115_out1 = squeeze51_out1.slice(s![.., .., .., 0..32]);
let concat40_out1 = burn::tensor::Tensor::cat(
[neg32_out1, slice115_out1].into(),
3,
);
let mul67_out1 = concat40_out1.mul(reshape13_out1);
let add76_out1 = mul66_out1.add(mul67_out1);
let (attention16_out1,) = {
let q = add75_out1;
let k = add76_out1;
let v = squeeze52_out1;
let hide = expand5_out1.bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention16_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention16_out1 = {
let dims = attention16_out1.dims();
attention16_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention16_out1,)
};
let transpose32_out1 = attention16_out1.permute([0, 2, 1, 3]);
let reshape44_out1 = transpose32_out1.reshape(concat6_out1);
let linear63_out1 = self.linear63.forward(reshape44_out1);
let add77_out1 = add74_out1.add(linear63_out1);
add77_out1
}
}
#[derive(Module, Debug)]
pub struct Submodule10 {
layernormalization33: LayerNorm,
linear64: Linear,
linear65: Linear,
layernormalization34: LayerNorm,
linear66: Linear,
linear67: Linear,
layernormalization35: LayerNorm,
linear68: Linear,
linear69: Linear,
layernormalization36: LayerNorm,
linear70: Linear,
linear71: Linear,
#[module(skip)]
device: Device,
}
impl Submodule10 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization33 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear64 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear65 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization34 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear66 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear67 = LinearConfig::new(512, 512).with_bias(true).init(device);
let layernormalization35 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear68 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear69 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization36 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear70 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear71 = LinearConfig::new(512, 512).with_bias(true).init(device);
Self {
layernormalization33,
linear64,
linear65,
layernormalization34,
linear66,
linear67,
layernormalization35,
linear68,
linear69,
layernormalization36,
linear70,
linear71,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add77_out1: Tensor<3>,
concat8_out1: [i64; 5],
reshape12_out1: Tensor<4>,
reshape13_out1: Tensor<4>,
expand5_out1: Tensor<4, Bool>,
concat6_out1: [i64; 3],
) -> Tensor<3> {
let layernormalization33_out1 = {
let dtype = add77_out1.clone().dtype();
self.layernormalization33
.forward(add77_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear64_out1 = self.linear64.forward(layernormalization33_out1);
let gelu16_out1 = burn::tensor::activation::gelu(linear64_out1);
let linear65_out1 = self.linear65.forward(gelu16_out1);
let add78_out1 = add77_out1.add(linear65_out1);
let layernormalization34_out1 = {
let dtype = add78_out1.clone().dtype();
self.layernormalization34
.forward(add78_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear66_out1 = self.linear66.forward(layernormalization34_out1);
let reshape45_out1 = linear66_out1.reshape(concat8_out1);
let transpose33_out1 = reshape45_out1.permute([2, 0, 3, 1, 4]);
let slice116_out1 = transpose33_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze53_out1 = slice116_out1.squeeze_dims::<4>(&[0]);
let slice117_out1 = transpose33_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze54_out1 = slice117_out1.squeeze_dims::<4>(&[0]);
let slice118_out1 = transpose33_out1.slice(s![2..3, .., .., .., ..]);
let squeeze55_out1 = slice118_out1.squeeze_dims::<4>(&[0]);
let mul68_out1 = squeeze53_out1.clone().mul(reshape12_out1.clone());
let slice119_out1 = squeeze53_out1.clone().slice(s![.., .., .., 32..]);
let neg33_out1 = slice119_out1.neg();
let slice120_out1 = squeeze53_out1.slice(s![.., .., .., 0..32]);
let concat41_out1 = burn::tensor::Tensor::cat(
[neg33_out1, slice120_out1].into(),
3,
);
let mul69_out1 = concat41_out1.mul(reshape13_out1.clone());
let add79_out1 = mul68_out1.add(mul69_out1);
let mul70_out1 = squeeze54_out1.clone().mul(reshape12_out1.clone());
let slice121_out1 = squeeze54_out1.clone().slice(s![.., .., .., 32..]);
let neg34_out1 = slice121_out1.neg();
let slice122_out1 = squeeze54_out1.slice(s![.., .., .., 0..32]);
let concat42_out1 = burn::tensor::Tensor::cat(
[neg34_out1, slice122_out1].into(),
3,
);
let mul71_out1 = concat42_out1.mul(reshape13_out1.clone());
let add80_out1 = mul70_out1.add(mul71_out1);
let (attention17_out1,) = {
let q = add79_out1;
let k = add80_out1;
let v = squeeze55_out1;
let hide = expand5_out1.clone().bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention17_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention17_out1 = {
let dims = attention17_out1.dims();
attention17_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention17_out1,)
};
let transpose34_out1 = attention17_out1.permute([0, 2, 1, 3]);
let reshape46_out1 = transpose34_out1.reshape(concat6_out1);
let linear67_out1 = self.linear67.forward(reshape46_out1);
let add81_out1 = add78_out1.add(linear67_out1);
let layernormalization35_out1 = {
let dtype = add81_out1.clone().dtype();
self.layernormalization35
.forward(add81_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear68_out1 = self.linear68.forward(layernormalization35_out1);
let gelu17_out1 = burn::tensor::activation::gelu(linear68_out1);
let linear69_out1 = self.linear69.forward(gelu17_out1);
let add82_out1 = add81_out1.add(linear69_out1);
let layernormalization36_out1 = {
let dtype = add82_out1.clone().dtype();
self.layernormalization36
.forward(add82_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear70_out1 = self.linear70.forward(layernormalization36_out1);
let reshape47_out1 = linear70_out1.reshape(concat8_out1);
let transpose35_out1 = reshape47_out1.permute([2, 0, 3, 1, 4]);
let slice123_out1 = transpose35_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze56_out1 = slice123_out1.squeeze_dims::<4>(&[0]);
let slice124_out1 = transpose35_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze57_out1 = slice124_out1.squeeze_dims::<4>(&[0]);
let slice125_out1 = transpose35_out1.slice(s![2..3, .., .., .., ..]);
let squeeze58_out1 = slice125_out1.squeeze_dims::<4>(&[0]);
let mul72_out1 = squeeze56_out1.clone().mul(reshape12_out1.clone());
let slice126_out1 = squeeze56_out1.clone().slice(s![.., .., .., 32..]);
let neg35_out1 = slice126_out1.neg();
let slice127_out1 = squeeze56_out1.slice(s![.., .., .., 0..32]);
let concat43_out1 = burn::tensor::Tensor::cat(
[neg35_out1, slice127_out1].into(),
3,
);
let mul73_out1 = concat43_out1.mul(reshape13_out1.clone());
let add83_out1 = mul72_out1.add(mul73_out1);
let mul74_out1 = squeeze57_out1.clone().mul(reshape12_out1);
let slice128_out1 = squeeze57_out1.clone().slice(s![.., .., .., 32..]);
let neg36_out1 = slice128_out1.neg();
let slice129_out1 = squeeze57_out1.slice(s![.., .., .., 0..32]);
let concat44_out1 = burn::tensor::Tensor::cat(
[neg36_out1, slice129_out1].into(),
3,
);
let mul75_out1 = concat44_out1.mul(reshape13_out1);
let add84_out1 = mul74_out1.add(mul75_out1);
let (attention18_out1,) = {
let q = add83_out1;
let k = add84_out1;
let v = squeeze58_out1;
let hide = expand5_out1.bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention18_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention18_out1 = {
let dims = attention18_out1.dims();
attention18_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention18_out1,)
};
let transpose36_out1 = attention18_out1.permute([0, 2, 1, 3]);
let reshape48_out1 = transpose36_out1.reshape(concat6_out1);
let linear71_out1 = self.linear71.forward(reshape48_out1);
let add85_out1 = add82_out1.add(linear71_out1);
add85_out1
}
}
#[derive(Module, Debug)]
pub struct Submodule11 {
layernormalization37: LayerNorm,
linear72: Linear,
linear73: Linear,
layernormalization38: LayerNorm,
linear74: Linear,
linear75: Linear,
layernormalization39: LayerNorm,
linear76: Linear,
linear77: Linear,
layernormalization40: LayerNorm,
linear78: Linear,
linear79: Linear,
#[module(skip)]
device: Device,
}
impl Submodule11 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization37 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear72 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear73 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization38 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear74 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear75 = LinearConfig::new(512, 512).with_bias(true).init(device);
let layernormalization39 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear76 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear77 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization40 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear78 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear79 = LinearConfig::new(512, 512).with_bias(true).init(device);
Self {
layernormalization37,
linear72,
linear73,
layernormalization38,
linear74,
linear75,
layernormalization39,
linear76,
linear77,
layernormalization40,
linear78,
linear79,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add85_out1: Tensor<3>,
concat8_out1: [i64; 5],
reshape12_out1: Tensor<4>,
reshape13_out1: Tensor<4>,
expand5_out1: Tensor<4, Bool>,
concat6_out1: [i64; 3],
) -> Tensor<3> {
let layernormalization37_out1 = {
let dtype = add85_out1.clone().dtype();
self.layernormalization37
.forward(add85_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear72_out1 = self.linear72.forward(layernormalization37_out1);
let gelu18_out1 = burn::tensor::activation::gelu(linear72_out1);
let linear73_out1 = self.linear73.forward(gelu18_out1);
let add86_out1 = add85_out1.add(linear73_out1);
let layernormalization38_out1 = {
let dtype = add86_out1.clone().dtype();
self.layernormalization38
.forward(add86_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear74_out1 = self.linear74.forward(layernormalization38_out1);
let reshape49_out1 = linear74_out1.reshape(concat8_out1);
let transpose37_out1 = reshape49_out1.permute([2, 0, 3, 1, 4]);
let slice130_out1 = transpose37_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze59_out1 = slice130_out1.squeeze_dims::<4>(&[0]);
let slice131_out1 = transpose37_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze60_out1 = slice131_out1.squeeze_dims::<4>(&[0]);
let slice132_out1 = transpose37_out1.slice(s![2..3, .., .., .., ..]);
let squeeze61_out1 = slice132_out1.squeeze_dims::<4>(&[0]);
let mul76_out1 = squeeze59_out1.clone().mul(reshape12_out1.clone());
let slice133_out1 = squeeze59_out1.clone().slice(s![.., .., .., 32..]);
let neg37_out1 = slice133_out1.neg();
let slice134_out1 = squeeze59_out1.slice(s![.., .., .., 0..32]);
let concat45_out1 = burn::tensor::Tensor::cat(
[neg37_out1, slice134_out1].into(),
3,
);
let mul77_out1 = concat45_out1.mul(reshape13_out1.clone());
let add87_out1 = mul76_out1.add(mul77_out1);
let mul78_out1 = squeeze60_out1.clone().mul(reshape12_out1.clone());
let slice135_out1 = squeeze60_out1.clone().slice(s![.., .., .., 32..]);
let neg38_out1 = slice135_out1.neg();
let slice136_out1 = squeeze60_out1.slice(s![.., .., .., 0..32]);
let concat46_out1 = burn::tensor::Tensor::cat(
[neg38_out1, slice136_out1].into(),
3,
);
let mul79_out1 = concat46_out1.mul(reshape13_out1.clone());
let add88_out1 = mul78_out1.add(mul79_out1);
let (attention19_out1,) = {
let q = add87_out1;
let k = add88_out1;
let v = squeeze61_out1;
let hide = expand5_out1.clone().bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention19_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention19_out1 = {
let dims = attention19_out1.dims();
attention19_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention19_out1,)
};
let transpose38_out1 = attention19_out1.permute([0, 2, 1, 3]);
let reshape50_out1 = transpose38_out1.reshape(concat6_out1);
let linear75_out1 = self.linear75.forward(reshape50_out1);
let add89_out1 = add86_out1.add(linear75_out1);
let layernormalization39_out1 = {
let dtype = add89_out1.clone().dtype();
self.layernormalization39
.forward(add89_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear76_out1 = self.linear76.forward(layernormalization39_out1);
let gelu19_out1 = burn::tensor::activation::gelu(linear76_out1);
let linear77_out1 = self.linear77.forward(gelu19_out1);
let add90_out1 = add89_out1.add(linear77_out1);
let layernormalization40_out1 = {
let dtype = add90_out1.clone().dtype();
self.layernormalization40
.forward(add90_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear78_out1 = self.linear78.forward(layernormalization40_out1);
let reshape51_out1 = linear78_out1.reshape(concat8_out1);
let transpose39_out1 = reshape51_out1.permute([2, 0, 3, 1, 4]);
let slice137_out1 = transpose39_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze62_out1 = slice137_out1.squeeze_dims::<4>(&[0]);
let slice138_out1 = transpose39_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze63_out1 = slice138_out1.squeeze_dims::<4>(&[0]);
let slice139_out1 = transpose39_out1.slice(s![2..3, .., .., .., ..]);
let squeeze64_out1 = slice139_out1.squeeze_dims::<4>(&[0]);
let mul80_out1 = squeeze62_out1.clone().mul(reshape12_out1.clone());
let slice140_out1 = squeeze62_out1.clone().slice(s![.., .., .., 32..]);
let neg39_out1 = slice140_out1.neg();
let slice141_out1 = squeeze62_out1.slice(s![.., .., .., 0..32]);
let concat47_out1 = burn::tensor::Tensor::cat(
[neg39_out1, slice141_out1].into(),
3,
);
let mul81_out1 = concat47_out1.mul(reshape13_out1.clone());
let add91_out1 = mul80_out1.add(mul81_out1);
let mul82_out1 = squeeze63_out1.clone().mul(reshape12_out1);
let slice142_out1 = squeeze63_out1.clone().slice(s![.., .., .., 32..]);
let neg40_out1 = slice142_out1.neg();
let slice143_out1 = squeeze63_out1.slice(s![.., .., .., 0..32]);
let concat48_out1 = burn::tensor::Tensor::cat(
[neg40_out1, slice143_out1].into(),
3,
);
let mul83_out1 = concat48_out1.mul(reshape13_out1);
let add92_out1 = mul82_out1.add(mul83_out1);
let (attention20_out1,) = {
let q = add91_out1;
let k = add92_out1;
let v = squeeze64_out1;
let hide = expand5_out1.bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention20_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention20_out1 = {
let dims = attention20_out1.dims();
attention20_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention20_out1,)
};
let transpose40_out1 = attention20_out1.permute([0, 2, 1, 3]);
let reshape52_out1 = transpose40_out1.reshape(concat6_out1);
let linear79_out1 = self.linear79.forward(reshape52_out1);
let add93_out1 = add90_out1.add(linear79_out1);
add93_out1
}
}
#[derive(Module, Debug)]
pub struct Submodule12 {
layernormalization41: LayerNorm,
linear80: Linear,
linear81: Linear,
layernormalization42: LayerNorm,
linear82: Linear,
linear83: Linear,
layernormalization43: LayerNorm,
linear84: Linear,
linear85: Linear,
layernormalization44: LayerNorm,
linear86: Linear,
linear87: Linear,
#[module(skip)]
device: Device,
}
impl Submodule12 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization41 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear80 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear81 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization42 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear82 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear83 = LinearConfig::new(512, 512).with_bias(true).init(device);
let layernormalization43 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear84 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear85 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization44 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear86 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear87 = LinearConfig::new(512, 512).with_bias(true).init(device);
Self {
layernormalization41,
linear80,
linear81,
layernormalization42,
linear82,
linear83,
layernormalization43,
linear84,
linear85,
layernormalization44,
linear86,
linear87,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add93_out1: Tensor<3>,
concat8_out1: [i64; 5],
reshape12_out1: Tensor<4>,
reshape13_out1: Tensor<4>,
expand5_out1: Tensor<4, Bool>,
concat6_out1: [i64; 3],
) -> Tensor<3> {
let layernormalization41_out1 = {
let dtype = add93_out1.clone().dtype();
self.layernormalization41
.forward(add93_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear80_out1 = self.linear80.forward(layernormalization41_out1);
let gelu20_out1 = burn::tensor::activation::gelu(linear80_out1);
let linear81_out1 = self.linear81.forward(gelu20_out1);
let add94_out1 = add93_out1.add(linear81_out1);
let layernormalization42_out1 = {
let dtype = add94_out1.clone().dtype();
self.layernormalization42
.forward(add94_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear82_out1 = self.linear82.forward(layernormalization42_out1);
let reshape53_out1 = linear82_out1.reshape(concat8_out1);
let transpose41_out1 = reshape53_out1.permute([2, 0, 3, 1, 4]);
let slice144_out1 = transpose41_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze65_out1 = slice144_out1.squeeze_dims::<4>(&[0]);
let slice145_out1 = transpose41_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze66_out1 = slice145_out1.squeeze_dims::<4>(&[0]);
let slice146_out1 = transpose41_out1.slice(s![2..3, .., .., .., ..]);
let squeeze67_out1 = slice146_out1.squeeze_dims::<4>(&[0]);
let mul84_out1 = squeeze65_out1.clone().mul(reshape12_out1.clone());
let slice147_out1 = squeeze65_out1.clone().slice(s![.., .., .., 32..]);
let neg41_out1 = slice147_out1.neg();
let slice148_out1 = squeeze65_out1.slice(s![.., .., .., 0..32]);
let concat49_out1 = burn::tensor::Tensor::cat(
[neg41_out1, slice148_out1].into(),
3,
);
let mul85_out1 = concat49_out1.mul(reshape13_out1.clone());
let add95_out1 = mul84_out1.add(mul85_out1);
let mul86_out1 = squeeze66_out1.clone().mul(reshape12_out1.clone());
let slice149_out1 = squeeze66_out1.clone().slice(s![.., .., .., 32..]);
let neg42_out1 = slice149_out1.neg();
let slice150_out1 = squeeze66_out1.slice(s![.., .., .., 0..32]);
let concat50_out1 = burn::tensor::Tensor::cat(
[neg42_out1, slice150_out1].into(),
3,
);
let mul87_out1 = concat50_out1.mul(reshape13_out1.clone());
let add96_out1 = mul86_out1.add(mul87_out1);
let (attention21_out1,) = {
let q = add95_out1;
let k = add96_out1;
let v = squeeze67_out1;
let hide = expand5_out1.clone().bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention21_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention21_out1 = {
let dims = attention21_out1.dims();
attention21_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention21_out1,)
};
let transpose42_out1 = attention21_out1.permute([0, 2, 1, 3]);
let reshape54_out1 = transpose42_out1.reshape(concat6_out1);
let linear83_out1 = self.linear83.forward(reshape54_out1);
let add97_out1 = add94_out1.add(linear83_out1);
let layernormalization43_out1 = {
let dtype = add97_out1.clone().dtype();
self.layernormalization43
.forward(add97_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear84_out1 = self.linear84.forward(layernormalization43_out1);
let gelu21_out1 = burn::tensor::activation::gelu(linear84_out1);
let linear85_out1 = self.linear85.forward(gelu21_out1);
let add98_out1 = add97_out1.add(linear85_out1);
let layernormalization44_out1 = {
let dtype = add98_out1.clone().dtype();
self.layernormalization44
.forward(add98_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear86_out1 = self.linear86.forward(layernormalization44_out1);
let reshape55_out1 = linear86_out1.reshape(concat8_out1);
let transpose43_out1 = reshape55_out1.permute([2, 0, 3, 1, 4]);
let slice151_out1 = transpose43_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze68_out1 = slice151_out1.squeeze_dims::<4>(&[0]);
let slice152_out1 = transpose43_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze69_out1 = slice152_out1.squeeze_dims::<4>(&[0]);
let slice153_out1 = transpose43_out1.slice(s![2..3, .., .., .., ..]);
let squeeze70_out1 = slice153_out1.squeeze_dims::<4>(&[0]);
let mul88_out1 = squeeze68_out1.clone().mul(reshape12_out1.clone());
let slice154_out1 = squeeze68_out1.clone().slice(s![.., .., .., 32..]);
let neg43_out1 = slice154_out1.neg();
let slice155_out1 = squeeze68_out1.slice(s![.., .., .., 0..32]);
let concat51_out1 = burn::tensor::Tensor::cat(
[neg43_out1, slice155_out1].into(),
3,
);
let mul89_out1 = concat51_out1.mul(reshape13_out1.clone());
let add99_out1 = mul88_out1.add(mul89_out1);
let mul90_out1 = squeeze69_out1.clone().mul(reshape12_out1);
let slice156_out1 = squeeze69_out1.clone().slice(s![.., .., .., 32..]);
let neg44_out1 = slice156_out1.neg();
let slice157_out1 = squeeze69_out1.slice(s![.., .., .., 0..32]);
let concat52_out1 = burn::tensor::Tensor::cat(
[neg44_out1, slice157_out1].into(),
3,
);
let mul91_out1 = concat52_out1.mul(reshape13_out1);
let add100_out1 = mul90_out1.add(mul91_out1);
let (attention22_out1,) = {
let q = add99_out1;
let k = add100_out1;
let v = squeeze70_out1;
let hide = expand5_out1.bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention22_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention22_out1 = {
let dims = attention22_out1.dims();
attention22_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention22_out1,)
};
let transpose44_out1 = attention22_out1.permute([0, 2, 1, 3]);
let reshape56_out1 = transpose44_out1.reshape(concat6_out1);
let linear87_out1 = self.linear87.forward(reshape56_out1);
let add101_out1 = add98_out1.add(linear87_out1);
add101_out1
}
}
#[derive(Module, Debug)]
pub struct Submodule13 {
layernormalization45: LayerNorm,
linear88: Linear,
linear89: Linear,
layernormalization46: LayerNorm,
linear90: Linear,
linear91: Linear,
layernormalization47: LayerNorm,
linear92: Linear,
linear93: Linear,
layernormalization48: LayerNorm,
linear94: Linear,
linear95: Linear,
#[module(skip)]
device: Device,
}
impl Submodule13 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization45 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear88 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear89 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization46 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear90 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear91 = LinearConfig::new(512, 512).with_bias(true).init(device);
let layernormalization47 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear92 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear93 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization48 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear94 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear95 = LinearConfig::new(512, 512).with_bias(true).init(device);
Self {
layernormalization45,
linear88,
linear89,
layernormalization46,
linear90,
linear91,
layernormalization47,
linear92,
linear93,
layernormalization48,
linear94,
linear95,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add101_out1: Tensor<3>,
concat8_out1: [i64; 5],
reshape12_out1: Tensor<4>,
reshape13_out1: Tensor<4>,
expand5_out1: Tensor<4, Bool>,
concat6_out1: [i64; 3],
) -> Tensor<3> {
let layernormalization45_out1 = {
let dtype = add101_out1.clone().dtype();
self.layernormalization45
.forward(add101_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear88_out1 = self.linear88.forward(layernormalization45_out1);
let gelu22_out1 = burn::tensor::activation::gelu(linear88_out1);
let linear89_out1 = self.linear89.forward(gelu22_out1);
let add102_out1 = add101_out1.add(linear89_out1);
let layernormalization46_out1 = {
let dtype = add102_out1.clone().dtype();
self.layernormalization46
.forward(add102_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear90_out1 = self.linear90.forward(layernormalization46_out1);
let reshape57_out1 = linear90_out1.reshape(concat8_out1);
let transpose45_out1 = reshape57_out1.permute([2, 0, 3, 1, 4]);
let slice158_out1 = transpose45_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze71_out1 = slice158_out1.squeeze_dims::<4>(&[0]);
let slice159_out1 = transpose45_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze72_out1 = slice159_out1.squeeze_dims::<4>(&[0]);
let slice160_out1 = transpose45_out1.slice(s![2..3, .., .., .., ..]);
let squeeze73_out1 = slice160_out1.squeeze_dims::<4>(&[0]);
let mul92_out1 = squeeze71_out1.clone().mul(reshape12_out1.clone());
let slice161_out1 = squeeze71_out1.clone().slice(s![.., .., .., 32..]);
let neg45_out1 = slice161_out1.neg();
let slice162_out1 = squeeze71_out1.slice(s![.., .., .., 0..32]);
let concat53_out1 = burn::tensor::Tensor::cat(
[neg45_out1, slice162_out1].into(),
3,
);
let mul93_out1 = concat53_out1.mul(reshape13_out1.clone());
let add103_out1 = mul92_out1.add(mul93_out1);
let mul94_out1 = squeeze72_out1.clone().mul(reshape12_out1.clone());
let slice163_out1 = squeeze72_out1.clone().slice(s![.., .., .., 32..]);
let neg46_out1 = slice163_out1.neg();
let slice164_out1 = squeeze72_out1.slice(s![.., .., .., 0..32]);
let concat54_out1 = burn::tensor::Tensor::cat(
[neg46_out1, slice164_out1].into(),
3,
);
let mul95_out1 = concat54_out1.mul(reshape13_out1.clone());
let add104_out1 = mul94_out1.add(mul95_out1);
let (attention23_out1,) = {
let q = add103_out1;
let k = add104_out1;
let v = squeeze73_out1;
let hide = expand5_out1.clone().bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention23_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention23_out1 = {
let dims = attention23_out1.dims();
attention23_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention23_out1,)
};
let transpose46_out1 = attention23_out1.permute([0, 2, 1, 3]);
let reshape58_out1 = transpose46_out1.reshape(concat6_out1);
let linear91_out1 = self.linear91.forward(reshape58_out1);
let add105_out1 = add102_out1.add(linear91_out1);
let layernormalization47_out1 = {
let dtype = add105_out1.clone().dtype();
self.layernormalization47
.forward(add105_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear92_out1 = self.linear92.forward(layernormalization47_out1);
let gelu23_out1 = burn::tensor::activation::gelu(linear92_out1);
let linear93_out1 = self.linear93.forward(gelu23_out1);
let add106_out1 = add105_out1.add(linear93_out1);
let layernormalization48_out1 = {
let dtype = add106_out1.clone().dtype();
self.layernormalization48
.forward(add106_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear94_out1 = self.linear94.forward(layernormalization48_out1);
let reshape59_out1 = linear94_out1.reshape(concat8_out1);
let transpose47_out1 = reshape59_out1.permute([2, 0, 3, 1, 4]);
let slice165_out1 = transpose47_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze74_out1 = slice165_out1.squeeze_dims::<4>(&[0]);
let slice166_out1 = transpose47_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze75_out1 = slice166_out1.squeeze_dims::<4>(&[0]);
let slice167_out1 = transpose47_out1.slice(s![2..3, .., .., .., ..]);
let squeeze76_out1 = slice167_out1.squeeze_dims::<4>(&[0]);
let mul96_out1 = squeeze74_out1.clone().mul(reshape12_out1.clone());
let slice168_out1 = squeeze74_out1.clone().slice(s![.., .., .., 32..]);
let neg47_out1 = slice168_out1.neg();
let slice169_out1 = squeeze74_out1.slice(s![.., .., .., 0..32]);
let concat55_out1 = burn::tensor::Tensor::cat(
[neg47_out1, slice169_out1].into(),
3,
);
let mul97_out1 = concat55_out1.mul(reshape13_out1.clone());
let add107_out1 = mul96_out1.add(mul97_out1);
let mul98_out1 = squeeze75_out1.clone().mul(reshape12_out1);
let slice170_out1 = squeeze75_out1.clone().slice(s![.., .., .., 32..]);
let neg48_out1 = slice170_out1.neg();
let slice171_out1 = squeeze75_out1.slice(s![.., .., .., 0..32]);
let concat56_out1 = burn::tensor::Tensor::cat(
[neg48_out1, slice171_out1].into(),
3,
);
let mul99_out1 = concat56_out1.mul(reshape13_out1);
let add108_out1 = mul98_out1.add(mul99_out1);
let (attention24_out1,) = {
let q = add107_out1;
let k = add108_out1;
let v = squeeze76_out1;
let hide = expand5_out1.bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention24_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention24_out1 = {
let dims = attention24_out1.dims();
attention24_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention24_out1,)
};
let transpose48_out1 = attention24_out1.permute([0, 2, 1, 3]);
let reshape60_out1 = transpose48_out1.reshape(concat6_out1);
let linear95_out1 = self.linear95.forward(reshape60_out1);
let add109_out1 = add106_out1.add(linear95_out1);
add109_out1
}
}
#[derive(Module, Debug)]
pub struct Submodule14 {
layernormalization49: LayerNorm,
linear96: Linear,
linear97: Linear,
layernormalization50: LayerNorm,
linear98: Linear,
linear99: Linear,
layernormalization51: LayerNorm,
linear100: Linear,
linear101: Linear,
layernormalization52: LayerNorm,
linear102: Linear,
linear103: Linear,
#[module(skip)]
device: Device,
}
impl Submodule14 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization49 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear96 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear97 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization50 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear98 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear99 = LinearConfig::new(512, 512).with_bias(true).init(device);
let layernormalization51 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear100 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear101 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization52 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear102 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear103 = LinearConfig::new(512, 512).with_bias(true).init(device);
Self {
layernormalization49,
linear96,
linear97,
layernormalization50,
linear98,
linear99,
layernormalization51,
linear100,
linear101,
layernormalization52,
linear102,
linear103,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add109_out1: Tensor<3>,
concat8_out1: [i64; 5],
reshape12_out1: Tensor<4>,
reshape13_out1: Tensor<4>,
expand5_out1: Tensor<4, Bool>,
concat6_out1: [i64; 3],
) -> Tensor<3> {
let layernormalization49_out1 = {
let dtype = add109_out1.clone().dtype();
self.layernormalization49
.forward(add109_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear96_out1 = self.linear96.forward(layernormalization49_out1);
let gelu24_out1 = burn::tensor::activation::gelu(linear96_out1);
let linear97_out1 = self.linear97.forward(gelu24_out1);
let add110_out1 = add109_out1.add(linear97_out1);
let layernormalization50_out1 = {
let dtype = add110_out1.clone().dtype();
self.layernormalization50
.forward(add110_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear98_out1 = self.linear98.forward(layernormalization50_out1);
let reshape61_out1 = linear98_out1.reshape(concat8_out1);
let transpose49_out1 = reshape61_out1.permute([2, 0, 3, 1, 4]);
let slice172_out1 = transpose49_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze77_out1 = slice172_out1.squeeze_dims::<4>(&[0]);
let slice173_out1 = transpose49_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze78_out1 = slice173_out1.squeeze_dims::<4>(&[0]);
let slice174_out1 = transpose49_out1.slice(s![2..3, .., .., .., ..]);
let squeeze79_out1 = slice174_out1.squeeze_dims::<4>(&[0]);
let mul100_out1 = squeeze77_out1.clone().mul(reshape12_out1.clone());
let slice175_out1 = squeeze77_out1.clone().slice(s![.., .., .., 32..]);
let neg49_out1 = slice175_out1.neg();
let slice176_out1 = squeeze77_out1.slice(s![.., .., .., 0..32]);
let concat57_out1 = burn::tensor::Tensor::cat(
[neg49_out1, slice176_out1].into(),
3,
);
let mul101_out1 = concat57_out1.mul(reshape13_out1.clone());
let add111_out1 = mul100_out1.add(mul101_out1);
let mul102_out1 = squeeze78_out1.clone().mul(reshape12_out1.clone());
let slice177_out1 = squeeze78_out1.clone().slice(s![.., .., .., 32..]);
let neg50_out1 = slice177_out1.neg();
let slice178_out1 = squeeze78_out1.slice(s![.., .., .., 0..32]);
let concat58_out1 = burn::tensor::Tensor::cat(
[neg50_out1, slice178_out1].into(),
3,
);
let mul103_out1 = concat58_out1.mul(reshape13_out1.clone());
let add112_out1 = mul102_out1.add(mul103_out1);
let (attention25_out1,) = {
let q = add111_out1;
let k = add112_out1;
let v = squeeze79_out1;
let hide = expand5_out1.clone().bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention25_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention25_out1 = {
let dims = attention25_out1.dims();
attention25_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention25_out1,)
};
let transpose50_out1 = attention25_out1.permute([0, 2, 1, 3]);
let reshape62_out1 = transpose50_out1.reshape(concat6_out1);
let linear99_out1 = self.linear99.forward(reshape62_out1);
let add113_out1 = add110_out1.add(linear99_out1);
let layernormalization51_out1 = {
let dtype = add113_out1.clone().dtype();
self.layernormalization51
.forward(add113_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear100_out1 = self.linear100.forward(layernormalization51_out1);
let gelu25_out1 = burn::tensor::activation::gelu(linear100_out1);
let linear101_out1 = self.linear101.forward(gelu25_out1);
let add114_out1 = add113_out1.add(linear101_out1);
let layernormalization52_out1 = {
let dtype = add114_out1.clone().dtype();
self.layernormalization52
.forward(add114_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear102_out1 = self.linear102.forward(layernormalization52_out1);
let reshape63_out1 = linear102_out1.reshape(concat8_out1);
let transpose51_out1 = reshape63_out1.permute([2, 0, 3, 1, 4]);
let slice179_out1 = transpose51_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze80_out1 = slice179_out1.squeeze_dims::<4>(&[0]);
let slice180_out1 = transpose51_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze81_out1 = slice180_out1.squeeze_dims::<4>(&[0]);
let slice181_out1 = transpose51_out1.slice(s![2..3, .., .., .., ..]);
let squeeze82_out1 = slice181_out1.squeeze_dims::<4>(&[0]);
let mul104_out1 = squeeze80_out1.clone().mul(reshape12_out1.clone());
let slice182_out1 = squeeze80_out1.clone().slice(s![.., .., .., 32..]);
let neg51_out1 = slice182_out1.neg();
let slice183_out1 = squeeze80_out1.slice(s![.., .., .., 0..32]);
let concat59_out1 = burn::tensor::Tensor::cat(
[neg51_out1, slice183_out1].into(),
3,
);
let mul105_out1 = concat59_out1.mul(reshape13_out1.clone());
let add115_out1 = mul104_out1.add(mul105_out1);
let mul106_out1 = squeeze81_out1.clone().mul(reshape12_out1);
let slice184_out1 = squeeze81_out1.clone().slice(s![.., .., .., 32..]);
let neg52_out1 = slice184_out1.neg();
let slice185_out1 = squeeze81_out1.slice(s![.., .., .., 0..32]);
let concat60_out1 = burn::tensor::Tensor::cat(
[neg52_out1, slice185_out1].into(),
3,
);
let mul107_out1 = concat60_out1.mul(reshape13_out1);
let add116_out1 = mul106_out1.add(mul107_out1);
let (attention26_out1,) = {
let q = add115_out1;
let k = add116_out1;
let v = squeeze82_out1;
let hide = expand5_out1.bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention26_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention26_out1 = {
let dims = attention26_out1.dims();
attention26_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention26_out1,)
};
let transpose52_out1 = attention26_out1.permute([0, 2, 1, 3]);
let reshape64_out1 = transpose52_out1.reshape(concat6_out1);
let linear103_out1 = self.linear103.forward(reshape64_out1);
let add117_out1 = add114_out1.add(linear103_out1);
add117_out1
}
}
#[derive(Module, Debug)]
pub struct Submodule15 {
layernormalization53: LayerNorm,
linear104: Linear,
linear105: Linear,
layernormalization54: LayerNorm,
linear106: Linear,
linear107: Linear,
layernormalization55: LayerNorm,
linear108: Linear,
linear109: Linear,
layernormalization56: LayerNorm,
linear110: Linear,
linear111: Linear,
#[module(skip)]
device: Device,
}
impl Submodule15 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization53 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear104 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear105 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization54 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear106 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear107 = LinearConfig::new(512, 512).with_bias(true).init(device);
let layernormalization55 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear108 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear109 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization56 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear110 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear111 = LinearConfig::new(512, 512).with_bias(true).init(device);
Self {
layernormalization53,
linear104,
linear105,
layernormalization54,
linear106,
linear107,
layernormalization55,
linear108,
linear109,
layernormalization56,
linear110,
linear111,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add117_out1: Tensor<3>,
concat8_out1: [i64; 5],
reshape12_out1: Tensor<4>,
reshape13_out1: Tensor<4>,
expand5_out1: Tensor<4, Bool>,
concat6_out1: [i64; 3],
) -> Tensor<3> {
let layernormalization53_out1 = {
let dtype = add117_out1.clone().dtype();
self.layernormalization53
.forward(add117_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear104_out1 = self.linear104.forward(layernormalization53_out1);
let gelu26_out1 = burn::tensor::activation::gelu(linear104_out1);
let linear105_out1 = self.linear105.forward(gelu26_out1);
let add118_out1 = add117_out1.add(linear105_out1);
let layernormalization54_out1 = {
let dtype = add118_out1.clone().dtype();
self.layernormalization54
.forward(add118_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear106_out1 = self.linear106.forward(layernormalization54_out1);
let reshape65_out1 = linear106_out1.reshape(concat8_out1);
let transpose53_out1 = reshape65_out1.permute([2, 0, 3, 1, 4]);
let slice186_out1 = transpose53_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze83_out1 = slice186_out1.squeeze_dims::<4>(&[0]);
let slice187_out1 = transpose53_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze84_out1 = slice187_out1.squeeze_dims::<4>(&[0]);
let slice188_out1 = transpose53_out1.slice(s![2..3, .., .., .., ..]);
let squeeze85_out1 = slice188_out1.squeeze_dims::<4>(&[0]);
let mul108_out1 = squeeze83_out1.clone().mul(reshape12_out1.clone());
let slice189_out1 = squeeze83_out1.clone().slice(s![.., .., .., 32..]);
let neg53_out1 = slice189_out1.neg();
let slice190_out1 = squeeze83_out1.slice(s![.., .., .., 0..32]);
let concat61_out1 = burn::tensor::Tensor::cat(
[neg53_out1, slice190_out1].into(),
3,
);
let mul109_out1 = concat61_out1.mul(reshape13_out1.clone());
let add119_out1 = mul108_out1.add(mul109_out1);
let mul110_out1 = squeeze84_out1.clone().mul(reshape12_out1.clone());
let slice191_out1 = squeeze84_out1.clone().slice(s![.., .., .., 32..]);
let neg54_out1 = slice191_out1.neg();
let slice192_out1 = squeeze84_out1.slice(s![.., .., .., 0..32]);
let concat62_out1 = burn::tensor::Tensor::cat(
[neg54_out1, slice192_out1].into(),
3,
);
let mul111_out1 = concat62_out1.mul(reshape13_out1.clone());
let add120_out1 = mul110_out1.add(mul111_out1);
let (attention27_out1,) = {
let q = add119_out1;
let k = add120_out1;
let v = squeeze85_out1;
let hide = expand5_out1.clone().bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention27_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention27_out1 = {
let dims = attention27_out1.dims();
attention27_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention27_out1,)
};
let transpose54_out1 = attention27_out1.permute([0, 2, 1, 3]);
let reshape66_out1 = transpose54_out1.reshape(concat6_out1);
let linear107_out1 = self.linear107.forward(reshape66_out1);
let add121_out1 = add118_out1.add(linear107_out1);
let layernormalization55_out1 = {
let dtype = add121_out1.clone().dtype();
self.layernormalization55
.forward(add121_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear108_out1 = self.linear108.forward(layernormalization55_out1);
let gelu27_out1 = burn::tensor::activation::gelu(linear108_out1);
let linear109_out1 = self.linear109.forward(gelu27_out1);
let add122_out1 = add121_out1.add(linear109_out1);
let layernormalization56_out1 = {
let dtype = add122_out1.clone().dtype();
self.layernormalization56
.forward(add122_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear110_out1 = self.linear110.forward(layernormalization56_out1);
let reshape67_out1 = linear110_out1.reshape(concat8_out1);
let transpose55_out1 = reshape67_out1.permute([2, 0, 3, 1, 4]);
let slice193_out1 = transpose55_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze86_out1 = slice193_out1.squeeze_dims::<4>(&[0]);
let slice194_out1 = transpose55_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze87_out1 = slice194_out1.squeeze_dims::<4>(&[0]);
let slice195_out1 = transpose55_out1.slice(s![2..3, .., .., .., ..]);
let squeeze88_out1 = slice195_out1.squeeze_dims::<4>(&[0]);
let mul112_out1 = squeeze86_out1.clone().mul(reshape12_out1.clone());
let slice196_out1 = squeeze86_out1.clone().slice(s![.., .., .., 32..]);
let neg55_out1 = slice196_out1.neg();
let slice197_out1 = squeeze86_out1.slice(s![.., .., .., 0..32]);
let concat63_out1 = burn::tensor::Tensor::cat(
[neg55_out1, slice197_out1].into(),
3,
);
let mul113_out1 = concat63_out1.mul(reshape13_out1.clone());
let add123_out1 = mul112_out1.add(mul113_out1);
let mul114_out1 = squeeze87_out1.clone().mul(reshape12_out1);
let slice198_out1 = squeeze87_out1.clone().slice(s![.., .., .., 32..]);
let neg56_out1 = slice198_out1.neg();
let slice199_out1 = squeeze87_out1.slice(s![.., .., .., 0..32]);
let concat64_out1 = burn::tensor::Tensor::cat(
[neg56_out1, slice199_out1].into(),
3,
);
let mul115_out1 = concat64_out1.mul(reshape13_out1);
let add124_out1 = mul114_out1.add(mul115_out1);
let (attention28_out1,) = {
let q = add123_out1;
let k = add124_out1;
let v = squeeze88_out1;
let hide = expand5_out1.bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention28_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention28_out1 = {
let dims = attention28_out1.dims();
attention28_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention28_out1,)
};
let transpose56_out1 = attention28_out1.permute([0, 2, 1, 3]);
let reshape68_out1 = transpose56_out1.reshape(concat6_out1);
let linear111_out1 = self.linear111.forward(reshape68_out1);
let add125_out1 = add122_out1.add(linear111_out1);
add125_out1
}
}
#[derive(Module, Debug)]
pub struct Submodule16 {
layernormalization57: LayerNorm,
linear112: Linear,
linear113: Linear,
layernormalization58: LayerNorm,
linear114: Linear,
linear115: Linear,
layernormalization59: LayerNorm,
linear116: Linear,
linear117: Linear,
layernormalization60: LayerNorm,
linear118: Linear,
linear119: Linear,
#[module(skip)]
device: Device,
}
impl Submodule16 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization57 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear112 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear113 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization58 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear114 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear115 = LinearConfig::new(512, 512).with_bias(true).init(device);
let layernormalization59 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear116 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear117 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization60 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear118 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear119 = LinearConfig::new(512, 512).with_bias(true).init(device);
Self {
layernormalization57,
linear112,
linear113,
layernormalization58,
linear114,
linear115,
layernormalization59,
linear116,
linear117,
layernormalization60,
linear118,
linear119,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add125_out1: Tensor<3>,
concat8_out1: [i64; 5],
reshape12_out1: Tensor<4>,
reshape13_out1: Tensor<4>,
expand5_out1: Tensor<4, Bool>,
concat6_out1: [i64; 3],
) -> Tensor<3> {
let layernormalization57_out1 = {
let dtype = add125_out1.clone().dtype();
self.layernormalization57
.forward(add125_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear112_out1 = self.linear112.forward(layernormalization57_out1);
let gelu28_out1 = burn::tensor::activation::gelu(linear112_out1);
let linear113_out1 = self.linear113.forward(gelu28_out1);
let add126_out1 = add125_out1.add(linear113_out1);
let layernormalization58_out1 = {
let dtype = add126_out1.clone().dtype();
self.layernormalization58
.forward(add126_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear114_out1 = self.linear114.forward(layernormalization58_out1);
let reshape69_out1 = linear114_out1.reshape(concat8_out1);
let transpose57_out1 = reshape69_out1.permute([2, 0, 3, 1, 4]);
let slice200_out1 = transpose57_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze89_out1 = slice200_out1.squeeze_dims::<4>(&[0]);
let slice201_out1 = transpose57_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze90_out1 = slice201_out1.squeeze_dims::<4>(&[0]);
let slice202_out1 = transpose57_out1.slice(s![2..3, .., .., .., ..]);
let squeeze91_out1 = slice202_out1.squeeze_dims::<4>(&[0]);
let mul116_out1 = squeeze89_out1.clone().mul(reshape12_out1.clone());
let slice203_out1 = squeeze89_out1.clone().slice(s![.., .., .., 32..]);
let neg57_out1 = slice203_out1.neg();
let slice204_out1 = squeeze89_out1.slice(s![.., .., .., 0..32]);
let concat65_out1 = burn::tensor::Tensor::cat(
[neg57_out1, slice204_out1].into(),
3,
);
let mul117_out1 = concat65_out1.mul(reshape13_out1.clone());
let add127_out1 = mul116_out1.add(mul117_out1);
let mul118_out1 = squeeze90_out1.clone().mul(reshape12_out1.clone());
let slice205_out1 = squeeze90_out1.clone().slice(s![.., .., .., 32..]);
let neg58_out1 = slice205_out1.neg();
let slice206_out1 = squeeze90_out1.slice(s![.., .., .., 0..32]);
let concat66_out1 = burn::tensor::Tensor::cat(
[neg58_out1, slice206_out1].into(),
3,
);
let mul119_out1 = concat66_out1.mul(reshape13_out1.clone());
let add128_out1 = mul118_out1.add(mul119_out1);
let (attention29_out1,) = {
let q = add127_out1;
let k = add128_out1;
let v = squeeze91_out1;
let hide = expand5_out1.clone().bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention29_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention29_out1 = {
let dims = attention29_out1.dims();
attention29_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention29_out1,)
};
let transpose58_out1 = attention29_out1.permute([0, 2, 1, 3]);
let reshape70_out1 = transpose58_out1.reshape(concat6_out1);
let linear115_out1 = self.linear115.forward(reshape70_out1);
let add129_out1 = add126_out1.add(linear115_out1);
let layernormalization59_out1 = {
let dtype = add129_out1.clone().dtype();
self.layernormalization59
.forward(add129_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear116_out1 = self.linear116.forward(layernormalization59_out1);
let gelu29_out1 = burn::tensor::activation::gelu(linear116_out1);
let linear117_out1 = self.linear117.forward(gelu29_out1);
let add130_out1 = add129_out1.add(linear117_out1);
let layernormalization60_out1 = {
let dtype = add130_out1.clone().dtype();
self.layernormalization60
.forward(add130_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear118_out1 = self.linear118.forward(layernormalization60_out1);
let reshape71_out1 = linear118_out1.reshape(concat8_out1);
let transpose59_out1 = reshape71_out1.permute([2, 0, 3, 1, 4]);
let slice207_out1 = transpose59_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze92_out1 = slice207_out1.squeeze_dims::<4>(&[0]);
let slice208_out1 = transpose59_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze93_out1 = slice208_out1.squeeze_dims::<4>(&[0]);
let slice209_out1 = transpose59_out1.slice(s![2..3, .., .., .., ..]);
let squeeze94_out1 = slice209_out1.squeeze_dims::<4>(&[0]);
let mul120_out1 = squeeze92_out1.clone().mul(reshape12_out1.clone());
let slice210_out1 = squeeze92_out1.clone().slice(s![.., .., .., 32..]);
let neg59_out1 = slice210_out1.neg();
let slice211_out1 = squeeze92_out1.slice(s![.., .., .., 0..32]);
let concat67_out1 = burn::tensor::Tensor::cat(
[neg59_out1, slice211_out1].into(),
3,
);
let mul121_out1 = concat67_out1.mul(reshape13_out1.clone());
let add131_out1 = mul120_out1.add(mul121_out1);
let mul122_out1 = squeeze93_out1.clone().mul(reshape12_out1);
let slice212_out1 = squeeze93_out1.clone().slice(s![.., .., .., 32..]);
let neg60_out1 = slice212_out1.neg();
let slice213_out1 = squeeze93_out1.slice(s![.., .., .., 0..32]);
let concat68_out1 = burn::tensor::Tensor::cat(
[neg60_out1, slice213_out1].into(),
3,
);
let mul123_out1 = concat68_out1.mul(reshape13_out1);
let add132_out1 = mul122_out1.add(mul123_out1);
let (attention30_out1,) = {
let q = add131_out1;
let k = add132_out1;
let v = squeeze94_out1;
let hide = expand5_out1.bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention30_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention30_out1 = {
let dims = attention30_out1.dims();
attention30_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention30_out1,)
};
let transpose60_out1 = attention30_out1.permute([0, 2, 1, 3]);
let reshape72_out1 = transpose60_out1.reshape(concat6_out1);
let linear119_out1 = self.linear119.forward(reshape72_out1);
let add133_out1 = add130_out1.add(linear119_out1);
add133_out1
}
}
#[derive(Module, Debug)]
pub struct Submodule17 {
layernormalization61: LayerNorm,
linear120: Linear,
linear121: Linear,
layernormalization62: LayerNorm,
linear122: Linear,
linear123: Linear,
layernormalization63: LayerNorm,
linear124: Linear,
linear125: Linear,
layernormalization64: LayerNorm,
linear126: Linear,
conv1d1: Conv1d,
linear127: Linear,
linear128: Linear,
averagepool1d1: AvgPool1d,
#[module(skip)]
device: Device,
}
impl Submodule17 {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let layernormalization61 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear120 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear121 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization62 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear122 = LinearConfig::new(512, 1536).with_bias(false).init(device);
let linear123 = LinearConfig::new(512, 512).with_bias(true).init(device);
let layernormalization63 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear124 = LinearConfig::new(512, 2048).with_bias(true).init(device);
let linear125 = LinearConfig::new(2048, 512).with_bias(true).init(device);
let layernormalization64 = LayerNormConfig::new(512)
.with_epsilon(0.000009999999747378752f64)
.with_bias(true)
.init(device);
let linear126 = LinearConfig::new(512, 192).with_bias(true).init(device);
let conv1d1 = Conv1dConfig::new(192, 1536, 3)
.with_stride(1)
.with_padding(PaddingConfig1d::Explicit(1, 1))
.with_dilation(1)
.with_groups(1)
.with_bias(true)
.init(device);
let linear127 = LinearConfig::new(192, 192).with_bias(true).init(device);
let linear128 = LinearConfig::new(192, 8).with_bias(true).init(device);
let averagepool1d1 = AvgPool1dConfig::new(8)
.with_stride(8)
.with_padding(PaddingConfig1d::Valid)
.with_count_include_pad(false)
.with_ceil_mode(true)
.init();
Self {
layernormalization61,
linear120,
linear121,
layernormalization62,
linear122,
linear123,
layernormalization63,
linear124,
linear125,
layernormalization64,
linear126,
conv1d1,
linear127,
linear128,
averagepool1d1,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
add133_out1: Tensor<3>,
concat8_out1: [i64; 5],
reshape12_out1: Tensor<4>,
reshape13_out1: Tensor<4>,
expand5_out1: Tensor<4, Bool>,
concat6_out1: [i64; 3],
constant374_out1: [i64; 1],
unsqueeze20_out1: [i64; 1],
range5_out1: Tensor<1, Int>,
reducesum1_out1: Tensor<1, Int>,
shape3_out1: [i64; 1],
constant375_out1: [i64; 1],
add4_out1: i64,
constant369_out1: i64,
) -> (Tensor<3>, Tensor<3>) {
let layernormalization61_out1 = {
let dtype = add133_out1.clone().dtype();
self.layernormalization61
.forward(add133_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear120_out1 = self.linear120.forward(layernormalization61_out1);
let gelu30_out1 = burn::tensor::activation::gelu(linear120_out1);
let linear121_out1 = self.linear121.forward(gelu30_out1);
let add134_out1 = add133_out1.add(linear121_out1);
let layernormalization62_out1 = {
let dtype = add134_out1.clone().dtype();
self.layernormalization62
.forward(add134_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear122_out1 = self.linear122.forward(layernormalization62_out1);
let reshape73_out1 = linear122_out1.reshape(concat8_out1);
let transpose61_out1 = reshape73_out1.permute([2, 0, 3, 1, 4]);
let slice214_out1 = transpose61_out1.clone().slice(s![0..1, .., .., .., ..]);
let squeeze95_out1 = slice214_out1.squeeze_dims::<4>(&[0]);
let slice215_out1 = transpose61_out1.clone().slice(s![1..2, .., .., .., ..]);
let squeeze96_out1 = slice215_out1.squeeze_dims::<4>(&[0]);
let slice216_out1 = transpose61_out1.slice(s![2..3, .., .., .., ..]);
let squeeze97_out1 = slice216_out1.squeeze_dims::<4>(&[0]);
let mul124_out1 = squeeze95_out1.clone().mul(reshape12_out1.clone());
let slice217_out1 = squeeze95_out1.clone().slice(s![.., .., .., 32..]);
let neg61_out1 = slice217_out1.neg();
let slice218_out1 = squeeze95_out1.slice(s![.., .., .., 0..32]);
let concat69_out1 = burn::tensor::Tensor::cat(
[neg61_out1, slice218_out1].into(),
3,
);
let mul125_out1 = concat69_out1.mul(reshape13_out1.clone());
let add135_out1 = mul124_out1.add(mul125_out1);
let mul126_out1 = squeeze96_out1.clone().mul(reshape12_out1);
let slice219_out1 = squeeze96_out1.clone().slice(s![.., .., .., 32..]);
let neg62_out1 = slice219_out1.neg();
let slice220_out1 = squeeze96_out1.slice(s![.., .., .., 0..32]);
let concat70_out1 = burn::tensor::Tensor::cat(
[neg62_out1, slice220_out1].into(),
3,
);
let mul127_out1 = concat70_out1.mul(reshape13_out1);
let add136_out1 = mul126_out1.add(mul127_out1);
let (attention31_out1,) = {
let q = add135_out1;
let k = add136_out1;
let v = squeeze97_out1;
let hide = expand5_out1.bool_not();
let hidden_rows = hide.clone().all_dim(3);
let attention31_out1 = burn::tensor::module::attention(
q,
k,
v,
Some(hide),
None,
burn::tensor::ops::AttentionModuleOptions {
scale: None,
softcap: None,
is_causal: false,
},
);
let attention31_out1 = {
let dims = attention31_out1.dims();
attention31_out1.mask_fill(hidden_rows.expand(dims), 0.0)
};
(attention31_out1,)
};
let transpose62_out1 = attention31_out1.permute([0, 2, 1, 3]);
let reshape74_out1 = transpose62_out1.reshape(concat6_out1);
let linear123_out1 = self.linear123.forward(reshape74_out1);
let add137_out1 = add134_out1.add(linear123_out1);
let layernormalization63_out1 = {
let dtype = add137_out1.clone().dtype();
self.layernormalization63
.forward(add137_out1.clone().cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear124_out1 = self.linear124.forward(layernormalization63_out1);
let gelu31_out1 = burn::tensor::activation::gelu(linear124_out1);
let linear125_out1 = self.linear125.forward(gelu31_out1);
let add138_out1 = add137_out1.add(linear125_out1);
let layernormalization64_out1 = {
let dtype = add138_out1.dtype();
self.layernormalization64
.forward(add138_out1.cast(burn::tensor::DType::F32))
.cast(dtype)
};
let linear126_out1 = self.linear126.forward(layernormalization64_out1);
let concat71_out1: [i64; 2usize] = [&constant374_out1[..], &unsqueeze20_out1[..]]
.concat()
.try_into()
.unwrap();
let expand6_out1 = {
let onnx_shape: [i64; 2usize] = concat71_out1;
let input_dims = range5_out1.dims();
let mut shape = onnx_shape;
#[allow(clippy::needless_range_loop)]
for i in 0..1usize {
let dim_offset = 1usize + i;
if shape[dim_offset] == 1 && input_dims[i] > 1 {
shape[dim_offset] = input_dims[i] as i64;
}
}
range5_out1.expand(shape)
};
let unsqueeze22_out1: Tensor<2, Int> = reducesum1_out1.unsqueeze_dims::<2>(&[1]);
let less6_out1 = expand6_out1.lower(unsqueeze22_out1);
let transpose63_out1 = linear126_out1.permute([0, 2, 1]);
let conv1d1_out1 = self.conv1d1.forward(transpose63_out1);
let transpose64_out1 = conv1d1_out1.permute([0, 2, 1]);
let constant379_out1: [i64; 1] = [192i64];
let concat72_out1: [i64; 4usize] = [
&shape3_out1[..],
&unsqueeze20_out1[..],
&constant375_out1[..],
&constant379_out1[..],
]
.concat()
.try_into()
.unwrap();
let reshape75_out1 = transpose64_out1.reshape(concat72_out1);
let mul128_out1 = add4_out1 * constant369_out1;
let unsqueeze23_out1 = [mul128_out1 as i64];
let concat73_out1: [i64; 3usize] = [
&shape3_out1[..],
&unsqueeze23_out1[..],
&constant379_out1[..],
]
.concat()
.try_into()
.unwrap();
let reshape76_out1 = reshape75_out1.reshape(concat73_out1);
let unsqueeze24_out1: Tensor<3, Bool> = less6_out1.unsqueeze_dims::<3>(&[2]);
let expand7_out1 = {
let onnx_shape: [i64; 3usize] = [1, 1, 8];
let input_dims = unsqueeze24_out1.dims();
let mut shape = onnx_shape;
#[allow(clippy::needless_range_loop)]
for i in 0..3usize {
let dim_offset = i;
if shape[dim_offset] == 1 && input_dims[i] > 1 {
shape[dim_offset] = input_dims[i] as i64;
}
}
unsqueeze24_out1.expand(shape)
};
let reshape77_out1 = expand7_out1.reshape([1, -1]);
let relu1_out1 = burn::tensor::activation::relu(reshape76_out1);
let linear127_out1 = self.linear127.forward(relu1_out1);
let relu2_out1 = burn::tensor::activation::relu(linear127_out1);
let linear128_out1 = self.linear128.forward(relu2_out1);
let sigmoid1_out1 = burn::tensor::activation::sigmoid(linear128_out1);
let unsqueeze25_out1: Tensor<3, Bool> = reshape77_out1
.unsqueeze_dims::<3>(&[-1]);
let cast3_out1 = unsqueeze25_out1.float().cast(burn::tensor::DType::F32);
let mul129_out1 = sigmoid1_out1.mul(cast3_out1);
let transpose65_out1 = mul129_out1.clone().permute([0, 2, 1]);
let averagepool1d1_out1 = self.averagepool1d1.forward(transpose65_out1);
let transpose66_out1 = averagepool1d1_out1.permute([0, 2, 1]);
(transpose66_out1, mul129_out1)
}
}
#[derive(Module, Debug)]
pub struct Model {
submodule1: Submodule1,
submodule2: Submodule2,
submodule3: Submodule3,
submodule4: Submodule4,
submodule5: Submodule5,
submodule6: Submodule6,
submodule7: Submodule7,
submodule8: Submodule8,
submodule9: Submodule9,
submodule10: Submodule10,
submodule11: Submodule11,
submodule12: Submodule12,
submodule13: Submodule13,
submodule14: Submodule14,
submodule15: Submodule15,
submodule16: Submodule16,
submodule17: Submodule17,
#[module(skip)]
device: Device,
}
impl Model {
#[allow(unused_variables)]
pub fn new(device: &Device) -> Self {
let submodule1 = Submodule1::new(device);
let submodule2 = Submodule2::new(device);
let submodule3 = Submodule3::new(device);
let submodule4 = Submodule4::new(device);
let submodule5 = Submodule5::new(device);
let submodule6 = Submodule6::new(device);
let submodule7 = Submodule7::new(device);
let submodule8 = Submodule8::new(device);
let submodule9 = Submodule9::new(device);
let submodule10 = Submodule10::new(device);
let submodule11 = Submodule11::new(device);
let submodule12 = Submodule12::new(device);
let submodule13 = Submodule13::new(device);
let submodule14 = Submodule14::new(device);
let submodule15 = Submodule15::new(device);
let submodule16 = Submodule16::new(device);
let submodule17 = Submodule17::new(device);
Self {
submodule1,
submodule2,
submodule3,
submodule4,
submodule5,
submodule6,
submodule7,
submodule8,
submodule9,
submodule10,
submodule11,
submodule12,
submodule13,
submodule14,
submodule15,
submodule16,
submodule17,
device: device.clone(),
}
}
#[allow(clippy::let_and_return, clippy::approx_constant)]
pub fn forward(
&self,
chunk: Tensor<3>,
chunk_lengths: Tensor<1, Int>,
spkcache: Tensor<3>,
spkcache_lengths: Tensor<1, Int>,
fifo: Tensor<3>,
fifo_lengths: Tensor<1, Int>,
) -> (Tensor<3>, Tensor<3>, Tensor<3>, Tensor<1, Int>) {
let (
add4_out1,
shape3_out1,
constant373_out1,
slice1_out1,
range1_out1,
reducesum1_out1,
constant369_out1,
linear1_out1,
sub2_out1,
) = self
.submodule1
.forward(
chunk,
spkcache,
fifo,
chunk_lengths,
spkcache_lengths,
fifo_lengths,
);
let (
add21_out1,
concat8_out1,
reshape12_out1,
reshape13_out1,
expand5_out1,
concat6_out1,
constant374_out1,
unsqueeze20_out1,
range5_out1,
constant375_out1,
) = self
.submodule2
.forward(
add4_out1.clone(),
shape3_out1.clone(),
constant373_out1,
slice1_out1,
range1_out1,
reducesum1_out1.clone(),
);
let add29_out1 = self
.submodule3
.forward(
add21_out1,
concat8_out1.clone(),
reshape12_out1.clone(),
reshape13_out1.clone(),
expand5_out1.clone(),
concat6_out1.clone(),
);
let add37_out1 = self
.submodule4
.forward(
add29_out1,
concat8_out1.clone(),
reshape12_out1.clone(),
reshape13_out1.clone(),
expand5_out1.clone(),
concat6_out1.clone(),
);
let add45_out1 = self
.submodule5
.forward(
add37_out1,
concat8_out1.clone(),
reshape12_out1.clone(),
reshape13_out1.clone(),
expand5_out1.clone(),
concat6_out1.clone(),
);
let add53_out1 = self
.submodule6
.forward(
add45_out1,
concat8_out1.clone(),
reshape12_out1.clone(),
reshape13_out1.clone(),
expand5_out1.clone(),
concat6_out1.clone(),
);
let add61_out1 = self
.submodule7
.forward(
add53_out1,
concat8_out1.clone(),
reshape12_out1.clone(),
reshape13_out1.clone(),
expand5_out1.clone(),
concat6_out1.clone(),
);
let add69_out1 = self
.submodule8
.forward(
add61_out1,
concat8_out1.clone(),
reshape12_out1.clone(),
reshape13_out1.clone(),
expand5_out1.clone(),
concat6_out1.clone(),
);
let add77_out1 = self
.submodule9
.forward(
add69_out1,
concat8_out1.clone(),
reshape12_out1.clone(),
reshape13_out1.clone(),
expand5_out1.clone(),
concat6_out1.clone(),
);
let add85_out1 = self
.submodule10
.forward(
add77_out1,
concat8_out1.clone(),
reshape12_out1.clone(),
reshape13_out1.clone(),
expand5_out1.clone(),
concat6_out1.clone(),
);
let add93_out1 = self
.submodule11
.forward(
add85_out1,
concat8_out1.clone(),
reshape12_out1.clone(),
reshape13_out1.clone(),
expand5_out1.clone(),
concat6_out1.clone(),
);
let add101_out1 = self
.submodule12
.forward(
add93_out1,
concat8_out1.clone(),
reshape12_out1.clone(),
reshape13_out1.clone(),
expand5_out1.clone(),
concat6_out1.clone(),
);
let add109_out1 = self
.submodule13
.forward(
add101_out1,
concat8_out1.clone(),
reshape12_out1.clone(),
reshape13_out1.clone(),
expand5_out1.clone(),
concat6_out1.clone(),
);
let add117_out1 = self
.submodule14
.forward(
add109_out1,
concat8_out1.clone(),
reshape12_out1.clone(),
reshape13_out1.clone(),
expand5_out1.clone(),
concat6_out1.clone(),
);
let add125_out1 = self
.submodule15
.forward(
add117_out1,
concat8_out1.clone(),
reshape12_out1.clone(),
reshape13_out1.clone(),
expand5_out1.clone(),
concat6_out1.clone(),
);
let add133_out1 = self
.submodule16
.forward(
add125_out1,
concat8_out1.clone(),
reshape12_out1.clone(),
reshape13_out1.clone(),
expand5_out1.clone(),
concat6_out1.clone(),
);
let (transpose66_out1, mul129_out1) = self
.submodule17
.forward(
add133_out1,
concat8_out1,
reshape12_out1,
reshape13_out1,
expand5_out1,
concat6_out1,
constant374_out1,
unsqueeze20_out1,
range5_out1,
reducesum1_out1,
shape3_out1,
constant375_out1,
add4_out1,
constant369_out1,
);
(transpose66_out1, mul129_out1, linear1_out1, sub2_out1)
}
}