1use rand::SeedableRng;
25use rand::rngs::StdRng;
26use rusting_brain::{Corpus, Matrix, ShapeVae, losses};
27use std::path::PathBuf;
28
29struct Args {
30 tokens: PathBuf,
31 samples: PathBuf,
32 objects: usize,
33 surface: usize,
34 queries: usize,
35 clamp: f32,
36 seed: u64,
37 cuda: Option<usize>,
38 checkpoints: Vec<PathBuf>,
39}
40
41struct Object {
43 surface: Matrix,
44 queries: Matrix,
45 distances: Vec<f32>,
46}
47
48fn main() -> Result<(), Box<dyn std::error::Error>> {
49 let args = parse()?;
50 let mut rng = StdRng::seed_from_u64(args.seed);
51
52 let mut corpus = Corpus::open(&args.tokens, &args.samples)?;
53 if corpus.is_empty() {
54 return Err("no render is paired with a mesh; check the two directories".into());
55 }
56 let objects: Vec<Object> = (0..args.objects.min(corpus.len()))
57 .map(|index| {
58 let example =
59 corpus.example(index, Some(args.surface), Some(args.queries), &mut rng)?;
60 Ok::<_, Box<dyn std::error::Error>>(Object {
61 surface: example.surface,
62 queries: example.queries,
63 distances: example.distances,
64 })
65 })
66 .collect::<Result<_, _>>()?;
67 println!(
68 "{} objects, {} surface points, {} query points, clamp {}",
69 objects.len(),
70 args.surface,
71 args.queries,
72 args.clamp
73 );
74 println!(
75 "{:<28} {:>9} {:>9} {:>7} {:>7} {:>8} {:>7}",
76 "checkpoint", "l1 draw", "l1 mean", "sigma", "|mean|", "kl/unit", "active"
77 );
78
79 for path in &args.checkpoints {
80 let mut rng = StdRng::seed_from_u64(args.seed ^ 0x5a1);
83 #[cfg_attr(not(feature = "cuda"), allow(unused_mut))]
85 let (mut model, _) = ShapeVae::load(path, &mut StdRng::seed_from_u64(args.seed))?;
86 if let Some(device) = args.cuda {
87 #[cfg(feature = "cuda")]
88 model.to_cuda(device, 0)?;
89 #[cfg(not(feature = "cuda"))]
90 return Err(format!(
91 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
92 )
93 .into());
94 }
95
96 let (mut draw_l1, mut mean_l1, mut sigma, mut magnitude, mut kl, mut active) =
97 (0.0, 0.0, 0.0, 0.0, 0.0, 0.0);
98 for object in &objects {
99 let (mean, log_variance) = model.encode(&object.surface)?;
100 let (latent, _) = ShapeVae::sample(&mean, &log_variance, &mut rng);
101
102 let from_draw = model.decode(&latent, &object.queries, 4096)?;
103 let from_mean = model.decode(&mean, &object.queries, 4096)?;
104 draw_l1 += losses::clamped_l1(&from_draw, &object.distances, args.clamp).0;
105 mean_l1 += losses::clamped_l1(&from_mean, &object.distances, args.clamp).0;
106
107 let units = mean.data.len() as f32;
108 magnitude += mean.data.iter().map(|value| value.abs()).sum::<f32>() / units;
109 sigma += log_variance
110 .data
111 .iter()
112 .map(|value| (0.5 * value).exp())
113 .sum::<f32>()
114 / units;
115 let mut per_unit = 0.0;
118 let mut above = 0.0;
119 for (mean, log_variance) in mean.data.iter().zip(&log_variance.data) {
120 let nats = 0.5 * (mean * mean + log_variance.exp() - 1.0 - log_variance);
121 per_unit += nats;
122 if nats > 0.01 {
123 above += 1.0;
124 }
125 }
126 kl += per_unit / units;
127 active += above / units;
128 }
129
130 let count = objects.len() as f32;
131 println!(
132 "{:<28} {:>9.5} {:>9.5} {:>7.3} {:>7.3} {:>8.4} {:>6.1}%",
133 path.file_name()
134 .unwrap_or(path.as_os_str())
135 .to_string_lossy(),
136 draw_l1 / count,
137 mean_l1 / count,
138 sigma / count,
139 magnitude / count,
140 kl / count,
141 100.0 * active / count
142 );
143 }
144 Ok(())
145}
146
147fn parse() -> Result<Args, Box<dyn std::error::Error>> {
148 let mut rest = std::env::args().skip(1);
149 let mut args = Args {
150 tokens: PathBuf::from("tokens"),
151 samples: PathBuf::from("samples"),
152 objects: 16,
153 surface: 2048,
154 queries: 4096,
155 clamp: 0.1,
156 seed: 0,
157 cuda: None,
158 checkpoints: Vec::new(),
159 };
160 while let Some(flag) = rest.next() {
161 let mut value = || rest.next().ok_or(format!("{flag} needs a value"));
162 match flag.as_str() {
163 "--tokens" => args.tokens = value()?.into(),
164 "--samples" => args.samples = value()?.into(),
165 "--objects" => args.objects = value()?.parse()?,
166 "--surface" => args.surface = value()?.parse()?,
167 "--queries" => args.queries = value()?.parse()?,
168 "--clamp" => args.clamp = value()?.parse()?,
169 "--seed" => args.seed = value()?.parse()?,
170 "--cuda" => args.cuda = Some(value()?.parse()?),
171 other if other.starts_with("--") => {
172 return Err(format!("unknown flag {other}\n{}", usage()).into());
173 }
174 path => args.checkpoints.push(path.into()),
175 }
176 }
177 if args.checkpoints.is_empty() || args.objects == 0 {
178 return Err(usage().into());
179 }
180 Ok(args)
181}
182
183fn usage() -> String {
184 "usage: kl_report [--tokens <dir>] [--samples <dir>] [--objects n] [--surface n] \
185 [--queries n] [--clamp f] [--seed n] [--cuda device] <checkpoint>..."
186 .to_string()
187}