use tch::{Tensor, nn::Init};
use crate::{Result, RustTorchError};
use super::{Module, ParameterPath, functional};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ConvConfig<const D: usize> {
in_channels: i64,
out_channels: i64,
kernel_size: [i64; D],
stride: [i64; D],
padding: [i64; D],
dilation: [i64; D],
groups: i64,
bias: bool,
}
impl<const D: usize> ConvConfig<D> {
pub const fn new(in_channels: i64, out_channels: i64, kernel_size: [i64; D]) -> Self {
Self {
in_channels,
out_channels,
kernel_size,
stride: [1; D],
padding: [0; D],
dilation: [1; D],
groups: 1,
bias: true,
}
}
#[must_use]
pub const fn stride(mut self, stride: [i64; D]) -> Self {
self.stride = stride;
self
}
#[must_use]
pub const fn padding(mut self, padding: [i64; D]) -> Self {
self.padding = padding;
self
}
#[must_use]
pub const fn dilation(mut self, dilation: [i64; D]) -> Self {
self.dilation = dilation;
self
}
#[must_use]
pub const fn groups(mut self, groups: i64) -> Self {
self.groups = groups;
self
}
#[must_use]
pub const fn bias(mut self, bias: bool) -> Self {
self.bias = bias;
self
}
pub fn build(self, path: &ParameterPath<'_>) -> Result<Conv<D>> {
validate_parameter_kind(path)?;
functional::validate_conv_config(
self.in_channels,
self.out_channels,
&self.kernel_size,
&self.stride,
&self.padding,
&self.dilation,
self.groups,
)?;
let fan_in = self
.kernel_size
.iter()
.try_fold(self.in_channels / self.groups, |size, &kernel| {
size.checked_mul(kernel)
})
.ok_or_else(|| invalid("kernel_size", "fan-in exceeds i64"))?;
let bound = 1.0 / (fan_in as f64).sqrt();
let init = Init::Uniform {
lo: -bound,
up: bound,
};
let mut shape = vec![self.out_channels, self.in_channels / self.groups];
shape.extend(self.kernel_size);
let weight = path.f_var("weight", &shape, init)?;
let bias = self
.bias
.then(|| path.f_var("bias", &[self.out_channels], init))
.transpose()?;
Ok(Conv {
weight,
bias,
config: self,
})
}
}
#[derive(Debug)]
pub struct Conv<const D: usize> {
weight: Tensor,
bias: Option<Tensor>,
config: ConvConfig<D>,
}
pub type Conv1d = Conv<1>;
pub type Conv2d = Conv<2>;
pub type Conv3d = Conv<3>;
impl<const D: usize> Conv<D> {
pub const fn weight(&self) -> &Tensor {
&self.weight
}
pub const fn bias(&self) -> Option<&Tensor> {
self.bias.as_ref()
}
}
impl<const D: usize> Module for Conv<D> {
fn forward(&self, input: &Tensor) -> Result<Tensor> {
functional::convolution(
input,
&self.weight,
self.bias.as_ref(),
&self.config.stride,
&self.config.padding,
&self.config.dilation,
self.config.groups,
)
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct LayerNormConfig {
normalized_shape: Vec<i64>,
eps: f64,
elementwise_affine: bool,
bias: bool,
}
impl LayerNormConfig {
pub fn new(normalized_shape: impl Into<Vec<i64>>) -> Self {
Self {
normalized_shape: normalized_shape.into(),
eps: 1e-5,
elementwise_affine: true,
bias: true,
}
}
#[must_use]
pub const fn eps(mut self, eps: f64) -> Self {
self.eps = eps;
self
}
#[must_use]
pub const fn elementwise_affine(mut self, enabled: bool) -> Self {
self.elementwise_affine = enabled;
self
}
#[must_use]
pub const fn bias(mut self, bias: bool) -> Self {
self.bias = bias;
self
}
pub fn build(self, path: &ParameterPath<'_>) -> Result<LayerNorm> {
functional::validate_layer_norm(&self.normalized_shape, self.eps)?;
if self.elementwise_affine {
validate_parameter_kind(path)?;
}
let weight = self
.elementwise_affine
.then(|| path.f_ones("weight", &self.normalized_shape))
.transpose()?;
let bias = (self.elementwise_affine && self.bias)
.then(|| path.f_zeros("bias", &self.normalized_shape))
.transpose()?;
Ok(LayerNorm {
weight,
bias,
config: self,
})
}
}
#[derive(Debug)]
pub struct LayerNorm {
weight: Option<Tensor>,
bias: Option<Tensor>,
config: LayerNormConfig,
}
impl LayerNorm {
pub const fn weight(&self) -> Option<&Tensor> {
self.weight.as_ref()
}
pub const fn bias(&self) -> Option<&Tensor> {
self.bias.as_ref()
}
}
impl Module for LayerNorm {
fn forward(&self, input: &Tensor) -> Result<Tensor> {
functional::layer_norm(
input,
&self.config.normalized_shape,
self.weight.as_ref(),
self.bias.as_ref(),
self.config.eps,
)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct EmbeddingConfig {
num_embeddings: i64,
embedding_dim: i64,
padding_idx: Option<i64>,
scale_grad_by_freq: bool,
sparse: bool,
}
impl EmbeddingConfig {
pub const fn new(num_embeddings: i64, embedding_dim: i64) -> Self {
Self {
num_embeddings,
embedding_dim,
padding_idx: None,
scale_grad_by_freq: false,
sparse: false,
}
}
#[must_use]
pub const fn padding_idx(mut self, padding_idx: i64) -> Self {
self.padding_idx = Some(padding_idx);
self
}
#[must_use]
pub const fn scale_grad_by_freq(mut self, enabled: bool) -> Self {
self.scale_grad_by_freq = enabled;
self
}
#[must_use]
pub const fn sparse(mut self, sparse: bool) -> Self {
self.sparse = sparse;
self
}
pub fn build(mut self, path: &ParameterPath<'_>) -> Result<Embedding> {
validate_parameter_kind(path)?;
if self.num_embeddings <= 0 || self.embedding_dim <= 0 {
return Err(invalid(
"embedding dimensions",
"vocabulary size and vector width must be positive",
));
}
self.padding_idx = functional::validate_embedding_options(
self.num_embeddings,
self.padding_idx,
self.scale_grad_by_freq,
self.sparse,
)?;
let weight = path.f_randn_standard("weight", &[self.num_embeddings, self.embedding_dim])?;
if let Some(index) = self.padding_idx {
let _ = tch::no_grad(|| weight.f_get(index)?.f_zero_())?;
}
Ok(Embedding {
weight,
config: self,
})
}
}
#[derive(Debug)]
pub struct Embedding {
weight: Tensor,
config: EmbeddingConfig,
}
impl Embedding {
pub const fn weight(&self) -> &Tensor {
&self.weight
}
}
impl Module for Embedding {
fn forward(&self, input: &Tensor) -> Result<Tensor> {
functional::embedding(
input,
&self.weight,
self.config.padding_idx,
self.config.scale_grad_by_freq,
self.config.sparse,
)
}
}
fn invalid(field: &'static str, reason: &str) -> RustTorchError {
RustTorchError::InvalidConfiguration {
field,
reason: reason.to_owned(),
}
}
pub(super) fn validate_parameter_kind(path: &ParameterPath<'_>) -> Result<()> {
use tch::Kind;
if matches!(
path.kind(),
Kind::Uint8
| Kind::Int8
| Kind::Int16
| Kind::Int
| Kind::Int64
| Kind::Bool
| Kind::QInt8
| Kind::QUInt8
| Kind::QInt32
) {
Err(invalid(
"parameter dtype",
"trainable parameters require a floating-point or complex dtype",
))
} else {
Ok(())
}
}