mod activations;
pub mod functional;
mod layers;
mod loss;
mod normalization;
mod sequence;
mod spatial;
pub use activations::*;
pub use normalization::*;
pub use sequence::*;
pub use spatial::*;
pub use layers::{
Conv, Conv1d, Conv2d, Conv3d, ConvConfig, Embedding, EmbeddingConfig, LayerNorm,
LayerNormConfig,
};
pub use tch::nn::VarStore;
pub use tch::nn::Path as ParameterPath;
use std::{fmt, path::Path};
use tch::{Device, Tensor};
use crate::{
DeviceSpec, Result, RustTorchError,
device::{ensure_device, resolve_device},
interop::{
LoadOptions, LoadReport, StateDictMapping, load_state_dict, load_state_dict_with_mapping,
save_state_dict,
},
};
pub trait Module: Send {
fn forward(&self, input: &Tensor) -> Result<Tensor>;
fn forward_t(&self, input: &Tensor, _training: bool) -> Result<Tensor> {
self.forward(input)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct LinearConfig {
in_features: i64,
out_features: i64,
bias: bool,
}
impl LinearConfig {
pub const fn new(in_features: i64, out_features: i64) -> Self {
Self {
in_features,
out_features,
bias: true,
}
}
#[must_use]
pub const fn bias(mut self, bias: bool) -> Self {
self.bias = bias;
self
}
pub fn build(self, path: &ParameterPath<'_>) -> Result<Linear> {
layers::validate_parameter_kind(path)?;
if self.in_features < 0 {
return Err(RustTorchError::InvalidConfiguration {
field: "in_features",
reason: "must be non-negative".to_owned(),
});
}
if self.out_features < 0 {
return Err(RustTorchError::InvalidConfiguration {
field: "out_features",
reason: "must be non-negative".to_owned(),
});
}
let bound = if self.in_features == 0 {
0.0
} else {
1.0 / (self.in_features as f64).sqrt()
};
let init = tch::nn::Init::Uniform {
lo: -bound,
up: bound,
};
let ws = path.f_var("weight", &[self.out_features, self.in_features], init)?;
let bs = self
.bias
.then(|| path.f_var("bias", &[self.out_features], init))
.transpose()?;
let inner = tch::nn::Linear { ws, bs };
Ok(Linear { inner })
}
}
#[derive(Debug)]
pub struct Linear {
inner: tch::nn::Linear,
}
impl Linear {
pub fn weight(&self) -> &Tensor {
&self.inner.ws
}
pub fn bias(&self) -> Option<&Tensor> {
self.inner.bs.as_ref()
}
pub fn forward(&self, input: &Tensor) -> Result<Tensor> {
<Self as Module>::forward(self, input)
}
}
impl Module for Linear {
fn forward(&self, input: &Tensor) -> Result<Tensor> {
functional::linear(input, &self.inner.ws, self.inner.bs.as_ref())
}
}
pub fn linear(path: &ParameterPath<'_>, in_features: i64, out_features: i64) -> Result<Linear> {
LinearConfig::new(in_features, out_features).build(path)
}
#[derive(Debug, Default)]
pub struct Identity;
impl Module for Identity {
fn forward(&self, input: &Tensor) -> Result<Tensor> {
Ok(input.shallow_clone())
}
}
#[derive(Debug, Default)]
pub struct ReLU;
impl Module for ReLU {
fn forward(&self, input: &Tensor) -> Result<Tensor> {
functional::relu(input)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[non_exhaustive]
pub enum GeluApproximation {
#[default]
None,
Tanh,
}
impl GeluApproximation {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::None => "none",
Self::Tanh => "tanh",
}
}
}
#[derive(Debug, Default)]
pub struct Gelu {
approximation: GeluApproximation,
}
impl Gelu {
pub const fn new(approximation: GeluApproximation) -> Self {
Self { approximation }
}
}
impl Module for Gelu {
fn forward(&self, input: &Tensor) -> Result<Tensor> {
functional::gelu_with_approximation(input, self.approximation)
}
}
#[derive(Debug)]
pub struct Dropout {
probability: f64,
}
impl Dropout {
pub fn new(probability: f64) -> Result<Self> {
functional::validate_dropout(probability)?;
Ok(Self { probability })
}
}
impl Module for Dropout {
fn forward(&self, input: &Tensor) -> Result<Tensor> {
self.forward_t(input, false)
}
fn forward_t(&self, input: &Tensor, training: bool) -> Result<Tensor> {
functional::dropout(input, self.probability, training)
}
}
#[derive(Debug)]
pub struct Flatten {
start_dim: i64,
end_dim: i64,
}
impl Flatten {
pub const fn new(start_dim: i64, end_dim: i64) -> Self {
Self { start_dim, end_dim }
}
}
impl Default for Flatten {
fn default() -> Self {
Self::new(1, -1)
}
}
impl Module for Flatten {
fn forward(&self, input: &Tensor) -> Result<Tensor> {
functional::flatten(input, self.start_dim, self.end_dim)
}
}
enum LayerSpec {
Custom(LayerFactory),
Conv1d(ConvConfig<1>),
Conv2d(ConvConfig<2>),
Conv3d(ConvConfig<3>),
LayerNorm(LayerNormConfig),
Embedding(EmbeddingConfig),
Linear(LinearConfig),
Identity,
ReLU,
Gelu(GeluApproximation),
Dropout(f64),
Flatten(i64, i64),
}
type LayerFactory = Box<dyn FnOnce(&ParameterPath<'_>) -> Result<Box<dyn Module>> + Send>;
#[derive(Default)]
pub struct SequentialBuilder {
layers: Vec<LayerSpec>,
}
impl SequentialBuilder {
pub fn new() -> Self {
Self::default()
}
#[must_use]
pub fn layer<M, F>(mut self, factory: F) -> Self
where
M: Module + 'static,
F: FnOnce(&ParameterPath<'_>) -> Result<M> + Send + 'static,
{
self.layers.push(LayerSpec::Custom(Box::new(move |path| {
Ok(Box::new(factory(path)?))
})));
self
}
#[must_use]
pub fn conv1d(mut self, config: ConvConfig<1>) -> Self {
self.layers.push(LayerSpec::Conv1d(config));
self
}
#[must_use]
pub fn conv2d(mut self, config: ConvConfig<2>) -> Self {
self.layers.push(LayerSpec::Conv2d(config));
self
}
#[must_use]
pub fn conv3d(mut self, config: ConvConfig<3>) -> Self {
self.layers.push(LayerSpec::Conv3d(config));
self
}
#[must_use]
pub fn layer_norm(mut self, config: LayerNormConfig) -> Self {
self.layers.push(LayerSpec::LayerNorm(config));
self
}
#[must_use]
pub fn embedding(mut self, config: EmbeddingConfig) -> Self {
self.layers.push(LayerSpec::Embedding(config));
self
}
#[must_use]
pub fn linear(mut self, in_features: i64, out_features: i64) -> Self {
self.layers.push(LayerSpec::Linear(LinearConfig::new(
in_features,
out_features,
)));
self
}
#[must_use]
pub fn linear_config(mut self, config: LinearConfig) -> Self {
self.layers.push(LayerSpec::Linear(config));
self
}
#[must_use]
pub fn identity(mut self) -> Self {
self.layers.push(LayerSpec::Identity);
self
}
#[must_use]
pub fn relu(mut self) -> Self {
self.layers.push(LayerSpec::ReLU);
self
}
#[must_use]
pub fn gelu(mut self) -> Self {
self.layers.push(LayerSpec::Gelu(GeluApproximation::None));
self
}
#[must_use]
pub fn gelu_approximate(mut self, approximation: GeluApproximation) -> Self {
self.layers.push(LayerSpec::Gelu(approximation));
self
}
#[must_use]
pub fn dropout(mut self, probability: f64) -> Self {
self.layers.push(LayerSpec::Dropout(probability));
self
}
#[must_use]
pub fn flatten(mut self, start_dim: i64, end_dim: i64) -> Self {
self.layers.push(LayerSpec::Flatten(start_dim, end_dim));
self
}
pub fn build(self, device: DeviceSpec) -> Result<Sequential> {
let device = resolve_device(device)?;
let var_store = VarStore::new(device);
let mut layers: Vec<Box<dyn Module>> = Vec::with_capacity(self.layers.len());
for (index, layer) in self.layers.into_iter().enumerate() {
let path = var_store.root() / index.to_string();
let layer: Box<dyn Module> = match layer {
LayerSpec::Custom(factory) => factory(&path)?,
LayerSpec::Conv1d(config) => Box::new(config.build(&path)?),
LayerSpec::Conv2d(config) => Box::new(config.build(&path)?),
LayerSpec::Conv3d(config) => Box::new(config.build(&path)?),
LayerSpec::LayerNorm(config) => Box::new(config.build(&path)?),
LayerSpec::Embedding(config) => Box::new(config.build(&path)?),
LayerSpec::Linear(config) => Box::new(config.build(&path)?),
LayerSpec::Identity => Box::new(Identity),
LayerSpec::ReLU => Box::new(ReLU),
LayerSpec::Gelu(approximation) => Box::new(Gelu::new(approximation)),
LayerSpec::Dropout(probability) => Box::new(Dropout::new(probability)?),
LayerSpec::Flatten(start_dim, end_dim) => {
Box::new(Flatten::new(start_dim, end_dim))
}
};
layers.push(layer);
}
Ok(Sequential {
var_store,
layers,
training: true,
})
}
}
pub struct Sequential {
var_store: VarStore,
layers: Vec<Box<dyn Module>>,
training: bool,
}
impl fmt::Debug for Sequential {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("Sequential")
.field("device", &self.device())
.field("layers", &self.layers.len())
.field("training", &self.training)
.finish()
}
}
impl Sequential {
pub fn builder() -> SequentialBuilder {
SequentialBuilder::new()
}
pub fn forward(&self, input: &Tensor) -> Result<Tensor> {
self.forward_t(input, self.training)
}
pub fn forward_t(&self, input: &Tensor, training: bool) -> Result<Tensor> {
ensure_device("Sequential input", input, self.device())?;
self.layers
.iter()
.try_fold(input.shallow_clone(), |value, layer| {
layer.forward_t(&value, training)
})
}
pub fn train(&mut self) {
self.training = true;
}
pub fn eval(&mut self) {
self.training = false;
}
pub const fn is_training(&self) -> bool {
self.training
}
pub fn device(&self) -> Device {
self.var_store.device()
}
pub const fn var_store(&self) -> &VarStore {
&self.var_store
}
pub fn to_device(&mut self, device: DeviceSpec) -> Result<()> {
self.var_store.set_device(resolve_device(device)?);
Ok(())
}
pub fn save_weights(&self, path: impl AsRef<Path>) -> Result<()> {
save_state_dict(path, &self.var_store)
}
pub fn load_weights(&self, path: impl AsRef<Path>) -> Result<LoadReport> {
load_state_dict(path, &self.var_store)
}
pub fn load_weights_with_mapping(
&self,
path: impl AsRef<Path>,
mapping: &StateDictMapping,
options: LoadOptions,
) -> Result<LoadReport> {
load_state_dict_with_mapping(path, &self.var_store, mapping, options)
}
}
impl Module for Sequential {
fn forward(&self, input: &Tensor) -> Result<Tensor> {
Sequential::forward(self, input)
}
fn forward_t(&self, input: &Tensor, training: bool) -> Result<Tensor> {
Sequential::forward_t(self, input, training)
}
}