pub mod functional;
use std::{fmt, path::Path};
use tch::{Device, Tensor, nn::VarStore, no_grad};
use crate::{
DeviceSpec, Result, RustTorchError,
device::{ensure_device, resolve_device},
interop::{
LoadOptions, LoadReport, StateDictMapping, load_state_dict, load_state_dict_with_mapping,
save_state_dict,
},
};
pub trait Module: Send {
fn forward(&self, input: &Tensor) -> Result<Tensor>;
fn forward_t(&self, input: &Tensor, _training: bool) -> Result<Tensor> {
self.forward(input)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct LinearConfig {
in_features: i64,
out_features: i64,
bias: bool,
}
impl LinearConfig {
pub const fn new(in_features: i64, out_features: i64) -> Self {
Self {
in_features,
out_features,
bias: true,
}
}
#[must_use]
pub const fn bias(mut self, bias: bool) -> Self {
self.bias = bias;
self
}
pub fn build(self, path: &tch::nn::Path<'_>) -> Result<Linear> {
if self.in_features < 0 {
return Err(RustTorchError::InvalidConfiguration {
field: "in_features",
reason: "must be non-negative".to_owned(),
});
}
if self.out_features < 0 {
return Err(RustTorchError::InvalidConfiguration {
field: "out_features",
reason: "must be non-negative".to_owned(),
});
}
let inner = tch::nn::linear(
path,
self.in_features,
self.out_features,
tch::nn::LinearConfig {
bias: self.bias,
..Default::default()
},
);
if self.in_features == 0
&& let Some(bias) = &inner.bs
{
let mut bias = bias.shallow_clone();
let _ = no_grad(|| bias.f_zero_())?;
}
Ok(Linear { inner })
}
}
#[derive(Debug)]
pub struct Linear {
inner: tch::nn::Linear,
}
impl Linear {
pub fn weight(&self) -> &Tensor {
&self.inner.ws
}
pub fn bias(&self) -> Option<&Tensor> {
self.inner.bs.as_ref()
}
pub fn forward(&self, input: &Tensor) -> Result<Tensor> {
<Self as Module>::forward(self, input)
}
}
impl Module for Linear {
fn forward(&self, input: &Tensor) -> Result<Tensor> {
functional::linear(input, &self.inner.ws, self.inner.bs.as_ref())
}
}
pub fn linear(path: &tch::nn::Path<'_>, in_features: i64, out_features: i64) -> Result<Linear> {
LinearConfig::new(in_features, out_features).build(path)
}
#[derive(Debug, Default)]
pub struct Identity;
impl Module for Identity {
fn forward(&self, input: &Tensor) -> Result<Tensor> {
Ok(input.shallow_clone())
}
}
#[derive(Debug, Default)]
pub struct ReLU;
impl Module for ReLU {
fn forward(&self, input: &Tensor) -> Result<Tensor> {
functional::relu(input)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[non_exhaustive]
pub enum GeluApproximation {
#[default]
None,
Tanh,
}
impl GeluApproximation {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::None => "none",
Self::Tanh => "tanh",
}
}
}
#[derive(Debug, Default)]
pub struct Gelu {
approximation: GeluApproximation,
}
impl Gelu {
pub const fn new(approximation: GeluApproximation) -> Self {
Self { approximation }
}
}
impl Module for Gelu {
fn forward(&self, input: &Tensor) -> Result<Tensor> {
functional::gelu_with_approximation(input, self.approximation)
}
}
#[derive(Debug)]
pub struct Dropout {
probability: f64,
}
impl Dropout {
pub fn new(probability: f64) -> Result<Self> {
functional::validate_dropout(probability)?;
Ok(Self { probability })
}
}
impl Module for Dropout {
fn forward(&self, input: &Tensor) -> Result<Tensor> {
self.forward_t(input, false)
}
fn forward_t(&self, input: &Tensor, training: bool) -> Result<Tensor> {
functional::dropout(input, self.probability, training)
}
}
#[derive(Debug)]
pub struct Flatten {
start_dim: i64,
end_dim: i64,
}
impl Flatten {
pub const fn new(start_dim: i64, end_dim: i64) -> Self {
Self { start_dim, end_dim }
}
}
impl Default for Flatten {
fn default() -> Self {
Self::new(1, -1)
}
}
impl Module for Flatten {
fn forward(&self, input: &Tensor) -> Result<Tensor> {
functional::flatten(input, self.start_dim, self.end_dim)
}
}
enum LayerSpec {
Linear(LinearConfig),
Identity,
ReLU,
Gelu(GeluApproximation),
Dropout(f64),
Flatten(i64, i64),
}
#[derive(Default)]
pub struct SequentialBuilder {
layers: Vec<LayerSpec>,
}
impl SequentialBuilder {
pub fn new() -> Self {
Self::default()
}
#[must_use]
pub fn linear(mut self, in_features: i64, out_features: i64) -> Self {
self.layers.push(LayerSpec::Linear(LinearConfig::new(
in_features,
out_features,
)));
self
}
#[must_use]
pub fn linear_config(mut self, config: LinearConfig) -> Self {
self.layers.push(LayerSpec::Linear(config));
self
}
#[must_use]
pub fn identity(mut self) -> Self {
self.layers.push(LayerSpec::Identity);
self
}
#[must_use]
pub fn relu(mut self) -> Self {
self.layers.push(LayerSpec::ReLU);
self
}
#[must_use]
pub fn gelu(mut self) -> Self {
self.layers.push(LayerSpec::Gelu(GeluApproximation::None));
self
}
#[must_use]
pub fn gelu_approximate(mut self, approximation: GeluApproximation) -> Self {
self.layers.push(LayerSpec::Gelu(approximation));
self
}
#[must_use]
pub fn dropout(mut self, probability: f64) -> Self {
self.layers.push(LayerSpec::Dropout(probability));
self
}
#[must_use]
pub fn flatten(mut self, start_dim: i64, end_dim: i64) -> Self {
self.layers.push(LayerSpec::Flatten(start_dim, end_dim));
self
}
pub fn build(self, device: DeviceSpec) -> Result<Sequential> {
let device = resolve_device(device)?;
let var_store = VarStore::new(device);
let mut layers: Vec<Box<dyn Module>> = Vec::with_capacity(self.layers.len());
for (index, layer) in self.layers.into_iter().enumerate() {
let path = var_store.root() / index.to_string();
let layer: Box<dyn Module> = match layer {
LayerSpec::Linear(config) => Box::new(config.build(&path)?),
LayerSpec::Identity => Box::new(Identity),
LayerSpec::ReLU => Box::new(ReLU),
LayerSpec::Gelu(approximation) => Box::new(Gelu::new(approximation)),
LayerSpec::Dropout(probability) => Box::new(Dropout::new(probability)?),
LayerSpec::Flatten(start_dim, end_dim) => {
Box::new(Flatten::new(start_dim, end_dim))
}
};
layers.push(layer);
}
Ok(Sequential {
var_store,
layers,
training: true,
})
}
}
pub struct Sequential {
var_store: VarStore,
layers: Vec<Box<dyn Module>>,
training: bool,
}
impl fmt::Debug for Sequential {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("Sequential")
.field("device", &self.device())
.field("layers", &self.layers.len())
.field("training", &self.training)
.finish()
}
}
impl Sequential {
pub fn builder() -> SequentialBuilder {
SequentialBuilder::new()
}
pub fn forward(&self, input: &Tensor) -> Result<Tensor> {
self.forward_t(input, self.training)
}
pub fn forward_t(&self, input: &Tensor, training: bool) -> Result<Tensor> {
ensure_device("Sequential input", input, self.device())?;
self.layers
.iter()
.try_fold(input.shallow_clone(), |value, layer| {
layer.forward_t(&value, training)
})
}
pub fn train(&mut self) {
self.training = true;
}
pub fn eval(&mut self) {
self.training = false;
}
pub const fn is_training(&self) -> bool {
self.training
}
pub fn device(&self) -> Device {
self.var_store.device()
}
pub const fn var_store(&self) -> &VarStore {
&self.var_store
}
pub fn to_device(&mut self, device: DeviceSpec) -> Result<()> {
self.var_store.set_device(resolve_device(device)?);
Ok(())
}
pub fn save_weights(&self, path: impl AsRef<Path>) -> Result<()> {
save_state_dict(path, &self.var_store)
}
pub fn load_weights(&self, path: impl AsRef<Path>) -> Result<LoadReport> {
load_state_dict(path, &self.var_store)
}
pub fn load_weights_with_mapping(
&self,
path: impl AsRef<Path>,
mapping: &StateDictMapping,
options: LoadOptions,
) -> Result<LoadReport> {
load_state_dict_with_mapping(path, &self.var_store, mapping, options)
}
}
impl Module for Sequential {
fn forward(&self, input: &Tensor) -> Result<Tensor> {
Sequential::forward(self, input)
}
fn forward_t(&self, input: &Tensor, training: bool) -> Result<Tensor> {
Sequential::forward_t(self, input, training)
}
}