Skip to main content

kl_report/
kl_report.rs

1//! Reports what a shape autoencoder's posterior costs, checkpoint by checkpoint.
2//!
3//! The decoder is only ever trained on draws from the posterior, so the mean is
4//! off-distribution and reconstructing from it is a different, worse number.
5//! How much worse depends on how wide the posterior is, which is what the KL
6//! weight controls. This reads one or more checkpoints and reports, on the same
7//! objects and the same query points for every checkpoint:
8//!
9//! - `l1 draw` — clamped L1 of the distances decoded from a draw, which is what
10//!   training measured.
11//! - `l1 mean` — the same objects decoded from the posterior mean, which is what
12//!   a caller gets from [`ShapeVae::encode`] if they ignore the warning.
13//! - `sigma` — the posterior's mean standard deviation, and `|mean|` the mean
14//!   magnitude of its centre. Their ratio is why the two errors differ.
15//! - `active` — the fraction of latent units whose KL is above 0.01 nats, which
16//!   is the measurement a free-bits floor would be aimed at.
17//!
18//! ```bash
19//! cargo run --release --features cuda --example kl_report -- \
20//!     --tokens tokens/ --samples samples/ --cuda 0 \
21//!     kl-1e-5.safetensors kl-1e-4.safetensors kl-1e-3.safetensors
22//! ```
23
24use rand::SeedableRng;
25use rand::rngs::StdRng;
26use rusting_brain::{Corpus, Matrix, ShapeVae, losses};
27use std::path::PathBuf;
28
29struct Args {
30    tokens: PathBuf,
31    samples: PathBuf,
32    objects: usize,
33    surface: usize,
34    queries: usize,
35    clamp: f32,
36    seed: u64,
37    cuda: Option<usize>,
38    checkpoints: Vec<PathBuf>,
39}
40
41/// One object's samples, drawn once and reused for every checkpoint.
42struct Object {
43    surface: Matrix,
44    queries: Matrix,
45    distances: Vec<f32>,
46}
47
48fn main() -> Result<(), Box<dyn std::error::Error>> {
49    let args = parse()?;
50    let mut rng = StdRng::seed_from_u64(args.seed);
51
52    let mut corpus = Corpus::open(&args.tokens, &args.samples)?;
53    if corpus.is_empty() {
54        return Err("no render is paired with a mesh; check the two directories".into());
55    }
56    let objects: Vec<Object> = (0..args.objects.min(corpus.len()))
57        .map(|index| {
58            let example =
59                corpus.example(index, Some(args.surface), Some(args.queries), &mut rng)?;
60            Ok::<_, Box<dyn std::error::Error>>(Object {
61                surface: example.surface,
62                queries: example.queries,
63                distances: example.distances,
64            })
65        })
66        .collect::<Result<_, _>>()?;
67    println!(
68        "{} objects, {} surface points, {} query points, clamp {}",
69        objects.len(),
70        args.surface,
71        args.queries,
72        args.clamp
73    );
74    println!(
75        "{:<28} {:>9} {:>9} {:>7} {:>7} {:>8} {:>7}",
76        "checkpoint", "l1 draw", "l1 mean", "sigma", "|mean|", "kl/unit", "active"
77    );
78
79    for path in &args.checkpoints {
80        // A fresh generator per checkpoint, so every model is asked for the
81        // same draw of noise as well as the same objects.
82        let mut rng = StdRng::seed_from_u64(args.seed ^ 0x5a1);
83        // `mut` is only used by the device move below.
84        #[cfg_attr(not(feature = "cuda"), allow(unused_mut))]
85        let (mut model, _) = ShapeVae::load(path, &mut StdRng::seed_from_u64(args.seed))?;
86        if let Some(device) = args.cuda {
87            #[cfg(feature = "cuda")]
88            model.to_cuda(device, 0)?;
89            #[cfg(not(feature = "cuda"))]
90            return Err(format!(
91                "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
92            )
93            .into());
94        }
95
96        let (mut draw_l1, mut mean_l1, mut sigma, mut magnitude, mut kl, mut active) =
97            (0.0, 0.0, 0.0, 0.0, 0.0, 0.0);
98        for object in &objects {
99            let (mean, log_variance) = model.encode(&object.surface)?;
100            let (latent, _) = ShapeVae::sample(&mean, &log_variance, &mut rng);
101
102            let from_draw = model.decode(&latent, &object.queries, 4096)?;
103            let from_mean = model.decode(&mean, &object.queries, 4096)?;
104            draw_l1 += losses::clamped_l1(&from_draw, &object.distances, args.clamp).0;
105            mean_l1 += losses::clamped_l1(&from_mean, &object.distances, args.clamp).0;
106
107            let units = mean.data.len() as f32;
108            magnitude += mean.data.iter().map(|value| value.abs()).sum::<f32>() / units;
109            sigma += log_variance
110                .data
111                .iter()
112                .map(|value| (0.5 * value).exp())
113                .sum::<f32>()
114                / units;
115            // The per-unit KL against a unit Gaussian, which is what a
116            // free-bits floor would be applied to one unit at a time.
117            let mut per_unit = 0.0;
118            let mut above = 0.0;
119            for (mean, log_variance) in mean.data.iter().zip(&log_variance.data) {
120                let nats = 0.5 * (mean * mean + log_variance.exp() - 1.0 - log_variance);
121                per_unit += nats;
122                if nats > 0.01 {
123                    above += 1.0;
124                }
125            }
126            kl += per_unit / units;
127            active += above / units;
128        }
129
130        let count = objects.len() as f32;
131        println!(
132            "{:<28} {:>9.5} {:>9.5} {:>7.3} {:>7.3} {:>8.4} {:>6.1}%",
133            path.file_name()
134                .unwrap_or(path.as_os_str())
135                .to_string_lossy(),
136            draw_l1 / count,
137            mean_l1 / count,
138            sigma / count,
139            magnitude / count,
140            kl / count,
141            100.0 * active / count
142        );
143    }
144    Ok(())
145}
146
147fn parse() -> Result<Args, Box<dyn std::error::Error>> {
148    let mut rest = std::env::args().skip(1);
149    let mut args = Args {
150        tokens: PathBuf::from("tokens"),
151        samples: PathBuf::from("samples"),
152        objects: 16,
153        surface: 2048,
154        queries: 4096,
155        clamp: 0.1,
156        seed: 0,
157        cuda: None,
158        checkpoints: Vec::new(),
159    };
160    while let Some(flag) = rest.next() {
161        let mut value = || rest.next().ok_or(format!("{flag} needs a value"));
162        match flag.as_str() {
163            "--tokens" => args.tokens = value()?.into(),
164            "--samples" => args.samples = value()?.into(),
165            "--objects" => args.objects = value()?.parse()?,
166            "--surface" => args.surface = value()?.parse()?,
167            "--queries" => args.queries = value()?.parse()?,
168            "--clamp" => args.clamp = value()?.parse()?,
169            "--seed" => args.seed = value()?.parse()?,
170            "--cuda" => args.cuda = Some(value()?.parse()?),
171            other if other.starts_with("--") => {
172                return Err(format!("unknown flag {other}\n{}", usage()).into());
173            }
174            path => args.checkpoints.push(path.into()),
175        }
176    }
177    if args.checkpoints.is_empty() || args.objects == 0 {
178        return Err(usage().into());
179    }
180    Ok(args)
181}
182
183fn usage() -> String {
184    "usage: kl_report [--tokens <dir>] [--samples <dir>] [--objects n] [--surface n] \
185     [--queries n] [--clamp f] [--seed n] [--cuda device] <checkpoint>..."
186        .to_string()
187}