use core::fmt::Debug;
use burn::Tensor;
pub trait LevelStatistic: Clone + Debug + Send + Sync {
fn of(frames: Tensor<3>) -> Self;
fn merge(
self,
later: Self,
) -> Self;
}
impl LevelStatistic for () {
fn of(_frames: Tensor<3>) -> Self {}
fn merge(
self,
_later: Self,
) -> Self {
}
}
#[derive(Clone, Debug)]
pub struct MaxLevel {
pub max: Tensor<1>,
}
impl LevelStatistic for MaxLevel {
fn of(frames: Tensor<3>) -> Self {
Self {
max: frames.max_dims(&[1, 2]).reshape([-1]),
}
}
fn merge(
self,
later: Self,
) -> Self {
Self {
max: self.max.max_pair(later.max),
}
}
}
#[derive(Clone, Debug)]
pub struct MomentsLevel {
pub count: usize,
pub mean: Tensor<2>,
pub m2: Tensor<2>,
}
impl MomentsLevel {
pub fn sample_variance(&self) -> Tensor<2> {
self.m2
.clone()
.div_scalar(self.count.saturating_sub(1).max(1) as f64)
}
}
impl LevelStatistic for MomentsLevel {
fn of(frames: Tensor<3>) -> Self {
let [batch, count, features] = frames.dims();
let mean = frames.clone().mean_dim(1);
let m2 = (frames - mean.clone()).powi_scalar(2).sum_dim(1);
Self {
count,
mean: mean.reshape([batch, features]),
m2: m2.reshape([batch, features]),
}
}
fn merge(
self,
later: Self,
) -> Self {
let (na, nb) = (self.count as f64, later.count as f64);
let n = na + nb;
let delta = later.mean.clone() - self.mean.clone();
Self {
count: self.count + later.count,
mean: self.mean + delta.clone().mul_scalar(nb / n),
m2: self.m2 + later.m2 + delta.powi_scalar(2).mul_scalar(na * nb / n),
}
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum LevelPolicy {
#[default]
PerWindow,
Running,
}
#[derive(Clone, Debug)]
pub struct LevelTracker<S: LevelStatistic> {
policy: LevelPolicy,
seen: Option<S>,
}
impl<S: LevelStatistic> LevelTracker<S> {
pub fn new(policy: LevelPolicy) -> Self {
Self { policy, seen: None }
}
pub fn policy(&self) -> LevelPolicy {
self.policy
}
pub fn seen(&self) -> Option<&S> {
self.seen.as_ref()
}
pub fn observe(
&mut self,
frames: Tensor<3>,
) {
if self.policy == LevelPolicy::PerWindow || frames.dims()[1] == 0 {
return;
}
let arriving = S::of(frames);
self.seen = Some(match self.seen.take() {
Some(seen) => seen.merge(arriving),
None => arriving,
});
}
pub fn reference(
&self,
window: Tensor<3>,
) -> S {
match &self.seen {
Some(seen) => seen.clone(),
None => S::of(window),
}
}
}
#[cfg(test)]
mod tests {
use burn::{
prelude::TensorData,
tensor::{
Distribution,
Tolerance,
},
};
use super::*;
use crate::support::testing::{
assert_tensor_close_to_vec,
assert_tensors_close,
cpu_device,
seeded_tensor,
};
fn frames(values: [f64; 8]) -> Tensor<3> {
Tensor::from_data(TensorData::new(values.to_vec(), [2, 2, 2]), &cpu_device())
}
#[test]
fn test_max_is_each_rows_own() {
let level = MaxLevel::of(frames([0.0, -3.0, -1.0, -20.0, 5.0, 4.0, -9.0, 1.0]));
assert_tensor_close_to_vec(&level.max, &[0.0, 5.0], Tolerance::default());
let later = MaxLevel::of(frames([7.0, -3.0, -1.0, -20.0, -5.0, -4.0, -9.0, -1.0]));
assert_tensor_close_to_vec(&level.merge(later).max, &[7.0, 5.0], Tolerance::default());
}
#[test]
fn test_moments_of_a_known_clip() {
let level = MomentsLevel::of(frames([1.0, 10.0, 3.0, 10.0, 0.0, 2.0, 4.0, 6.0]));
assert_eq!(level.count, 2);
assert_tensor_close_to_vec(&level.mean, &[2.0, 10.0, 2.0, 4.0], Tolerance::default());
assert_tensor_close_to_vec(&level.m2, &[2.0, 0.0, 8.0, 8.0], Tolerance::default());
assert_tensor_close_to_vec(
&level.sample_variance(),
&[2.0, 0.0, 8.0, 8.0],
Tolerance::default(),
);
}
#[test]
fn test_merges_match_the_whole() {
let device = cpu_device();
let clip = seeded_tensor(7, [2, 23, 5], Distribution::Normal(1.0, 3.0), &device);
let cuts = [0, 1, 8, 9, 17, 23];
let parts = |range: core::ops::Range<usize>| clip.clone().slice_dim(1, range);
let whole_max = MaxLevel::of(clip.clone());
let whole_moments = MomentsLevel::of(clip.clone());
let mut max: Option<MaxLevel> = None;
let mut moments: Option<MomentsLevel> = None;
for pair in cuts.windows(2) {
let part = parts(pair[0]..pair[1]);
let (m, s) = (MaxLevel::of(part.clone()), MomentsLevel::of(part));
max = Some(match max {
Some(max) => max.merge(m),
None => m,
});
moments = Some(match moments {
Some(moments) => moments.merge(s),
None => s,
});
}
let (max, moments) = (max.unwrap(), moments.unwrap());
assert_tensors_close(&max.max, &whole_max.max, Tolerance::default());
assert_eq!(moments.count, whole_moments.count);
assert_tensors_close(&moments.mean, &whole_moments.mean, Tolerance::default());
assert_tensors_close(
&moments.m2,
&whole_moments.m2,
Tolerance::rel_abs(1e-4, 1e-4),
);
}
#[test]
fn test_per_window_ignores_what_it_observes() {
let mut tracker = LevelTracker::<MaxLevel>::new(LevelPolicy::PerWindow);
tracker.observe(frames([99.0; 8]));
assert!(tracker.seen().is_none());
let window = frames([0.0, -3.0, -1.0, -20.0, 5.0, 4.0, -9.0, 1.0]);
assert_tensor_close_to_vec(
&tracker.reference(window).max,
&[0.0, 5.0],
Tolerance::default(),
);
}
#[test]
fn test_running_covers_everything_seen() {
let mut tracker = LevelTracker::<MaxLevel>::new(LevelPolicy::Running);
let quiet = frames([0.0, -3.0, -1.0, -20.0, 5.0, 4.0, -9.0, 1.0]);
assert_tensor_close_to_vec(
&tracker.reference(quiet.clone()).max,
&[0.0, 5.0],
Tolerance::default(),
);
tracker.observe(Tensor::zeros([2, 0, 2], &cpu_device()));
assert!(tracker.seen().is_none());
tracker.observe(frames([7.0, -3.0, -1.0, -20.0, -5.0, -4.0, -9.0, -1.0]));
tracker.observe(quiet.clone());
assert_tensor_close_to_vec(
&tracker.reference(quiet).max,
&[7.0, 5.0],
Tolerance::default(),
);
}
}