use crate::nn::param::{Buffer, Param};
use crate::nn::{Dropout, ELU, GELU, Mish, ReLU, Sequential, Sigmoid, Softmax, Swish, Tanh};
use crate::shapes::error::OperationKind;
use crate::shapes::{DynShape, Shape, ShapeBuf};
use crate::tensor::dtype::DType;
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct LayerStats {
pub params: u64,
pub macs: u64,
}
impl core::ops::Add for LayerStats {
type Output = LayerStats;
fn add(self, rhs: Self) -> Self::Output {
LayerStats {
params: self.params + rhs.params,
macs: self.macs + rhs.macs,
}
}
}
impl core::ops::AddAssign for LayerStats {
fn add_assign(&mut self, rhs: Self) {
*self = *self + rhs;
}
}
impl core::iter::Sum for LayerStats {
fn sum<I: Iterator<Item = Self>>(iter: I) -> Self {
iter.fold(Self::default(), core::ops::Add::add)
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct ModelStats {
pub params: u64,
pub macs: u64,
pub flops: u64,
}
pub trait ComputeStats {
fn compute_stats(&self, batch: u64) -> LayerStats;
fn stats(&self, batch: u64) -> ModelStats {
self.compute_stats(batch).into_model_stats()
}
}
impl<
S: Shape + DynShape,
B: crate::tensor::backend::VariableBackend,
K: DType,
Train: crate::nn::param::TrainState,
> ComputeStats for Param<S, B, K, Train>
{
fn compute_stats(&self, _batch: u64) -> LayerStats {
LayerStats {
params: validated_parameter_count(&self.shape_dims()),
macs: 0,
}
}
}
impl<S: Shape + DynShape, B: crate::tensor::backend::VariableBackend, K: DType> ComputeStats
for Buffer<S, B, K>
{
fn compute_stats(&self, _batch: u64) -> LayerStats {
LayerStats::default()
}
}
pub(crate) fn validated_parameter_count(dims: &[usize]) -> u64 {
let elements = ShapeBuf::from_slice(dims)
.checked_numel(OperationKind::Storage)
.expect("parameter tensor shape crossed checked construction");
u64::try_from(elements).expect("usize parameter count must fit u64")
}
impl<T: ComputeStats> ComputeStats for Option<T> {
fn compute_stats(&self, batch: u64) -> LayerStats {
match self {
Some(v) => v.compute_stats(batch),
None => LayerStats::default(),
}
}
}
impl<L1: ComputeStats, L2: ComputeStats> ComputeStats for Sequential<L1, L2> {
fn compute_stats(&self, batch: u64) -> LayerStats {
self.0.compute_stats(batch) + self.1.compute_stats(batch)
}
}
macro_rules! impl_zero_compute_stats {
($($ty:ty),+ $(,)?) => {
$(
impl ComputeStats for $ty {
fn compute_stats(&self, _batch: u64) -> LayerStats {
LayerStats::default()
}
}
)+
};
}
impl_zero_compute_stats!(
ReLU, GELU, Swish, Mish, ELU, Sigmoid, Tanh, Softmax, Dropout
);
impl<K: typenum::Unsigned, S: typenum::Unsigned, P: typenum::Unsigned, D: typenum::Unsigned>
ComputeStats for crate::nn::max_pool2d::MaxPool2d<K, S, P, D>
{
fn compute_stats(&self, _batch: u64) -> LayerStats {
LayerStats::default()
}
}
impl<K: typenum::Unsigned, S: typenum::Unsigned, P: typenum::Unsigned, D: typenum::Unsigned>
ComputeStats for crate::nn::avg_pool2d::AvgPool2d<K, S, P, D>
{
fn compute_stats(&self, _batch: u64) -> LayerStats {
LayerStats::default()
}
}
impl<HOut: typenum::Unsigned, WOut: typenum::Unsigned> ComputeStats
for crate::nn::adaptive_avg_pool2d::AdaptiveAvgPool2d<HOut, WOut>
{
fn compute_stats(&self, _batch: u64) -> LayerStats {
LayerStats::default()
}
}
pub fn sum_stats(items: &[LayerStats]) -> LayerStats {
items.iter().copied().sum()
}
impl LayerStats {
pub fn into_model_stats(self) -> ModelStats {
ModelStats {
params: self.params,
macs: self.macs,
flops: self.macs * 2,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn layer_stats_add_sums_both_fields() {
let a = LayerStats {
params: 10,
macs: 20,
};
let b = LayerStats { params: 3, macs: 4 };
assert_eq!(
a + b,
LayerStats {
params: 13,
macs: 24
}
);
}
#[test]
fn layer_stats_sum_over_empty_iter_is_default() {
let items: Vec<LayerStats> = Vec::new();
assert_eq!(sum_stats(&items), LayerStats::default());
}
#[test]
fn into_model_stats_doubles_macs_for_flops() {
let stats = LayerStats {
params: 5,
macs: 100,
}
.into_model_stats();
assert_eq!(
stats,
ModelStats {
params: 5,
macs: 100,
flops: 200
}
);
}
}