pub struct ShapeVae { /* private fields */ }Expand description
Surface points in, a signed distance field out.
Implementations§
Source§impl ShapeVae
impl ShapeVae
Sourcepub fn new(
config: ShapeVaeConfig,
rng: &mut StdRng,
) -> Result<Self, NetworkError>
pub fn new( config: ShapeVaeConfig, rng: &mut StdRng, ) -> Result<Self, NetworkError>
Examples found in repository?
126fn train_vae(
127 args: &Args,
128 stream: BatchStream,
129 rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131 let (mut model, mut step) = match &args.resume {
132 Some(path) => {
133 let (model, metadata) = ShapeVae::load(path, rng)?;
134 (model, started_at(&metadata))
135 }
136 None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137 };
138 if let Some(device) = args.cuda {
139 #[cfg(feature = "cuda")]
140 {
141 // 0 as the budget: the caller picked the card, and the weights are
142 // a small part of what a run of this size needs.
143 model.to_cuda(device, 0)?;
144 println!("shape autoencoder: training on CUDA device {device}");
145 }
146 #[cfg(not(feature = "cuda"))]
147 return Err(format!(
148 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149 )
150 .into());
151 }
152 let optimizer = adam(args.learning_rate);
153 println!(
154 "shape autoencoder: {} parameters, from step {step}",
155 model.num_parameters()
156 );
157
158 let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159 for batch in stream {
160 let batch = batch?;
161 let count = batch.len();
162 let points = batch.surface.rows / count;
163 let queries = batch.queries.rows / count;
164
165 // Packing helps the ordinary 4K-query training shape, but at 16K
166 // queries the larger attention workspace is slower and batch eight can
167 // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168 let cuda_batch = model.on_device() && queries <= 4096;
169 #[cfg(feature = "cuda")]
170 if cuda_batch {
171 let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172 // The packed losses are means across the batch; the reporting
173 // accumulator below historically stores a sum over examples.
174 reconstruction += loss * count as f32;
175 divergence += kl * count as f32;
176 tint += color * count as f32;
177 }
178
179 if !cuda_batch {
180 for index in 0..count {
181 let surface = slice(&batch.surface, index, points);
182 let (mean, log_variance, cache) = model.encode_train(&surface)?;
183 let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185 let (prediction, decoded) =
186 model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187 let target = &batch.distances[index * queries..(index + 1) * queries];
188 let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189 let (kl, grad_mean, grad_log_variance) =
190 losses::kl_divergence(&mean.data, &log_variance.data);
191 reconstruction += loss;
192 divergence += kl;
193
194 let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196 // The colour targets sit on the surface points, not the distance
197 // queries, so the colour head trains on a second short decode.
198 // The loader's points are already in random order, so the first
199 // `--color-points` of them are a random draw.
200 if let Some(colors) = &batch.colors
201 && args.color > 0.0
202 && args.color_points > 0
203 {
204 let wanted = args.color_points.min(points);
205 let positions = positions(&surface, wanted);
206 let targets = head_rows(&slice(colors, index, points), wanted);
207 let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209 let predicted = model.colors(&decoded)?;
210 let mut grad = predicted.clone();
211 let scale = args.color / predicted.data.len() as f32;
212 for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213 let error = *slot - target;
214 tint += error * error / predicted.data.len() as f32;
215 *slot = 2.0 * scale * error;
216 }
217
218 let from_colors = model.decode_backward_colored(
219 &decoded,
220 &vec![0.0; positions.rows],
221 Some(&grad),
222 )?;
223 for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224 *slot += value;
225 }
226 }
227
228 let (mut into_mean, mut into_log_variance) =
229 ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230 for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231 *slot += args.kl * value;
232 }
233 for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234 *slot += args.kl * value;
235 }
236 model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237 }
238 }
239
240 step += 1;
241 // The gradients were summed over the batch, so the step is scaled by
242 // its size rather than the loop dividing every gradient it makes.
243 optimizers::step_clipped(
244 &mut model.params_mut(),
245 &optimizer,
246 step,
247 1.0 / count as f32,
248 1.0,
249 )?;
250 optimizers::zero_grad(&mut model.params_mut());
251
252 if step % args.report == 0 {
253 let scale = (args.report * count) as f32;
254 println!(
255 "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256 reconstruction / scale,
257 divergence / scale,
258 tint / scale
259 );
260 reconstruction = 0.0;
261 divergence = 0.0;
262 tint = 0.0;
263 }
264 if step % args.checkpoint_every == 0 || step == args.steps {
265 model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266 println!("wrote {} at step {step}", args.out.display());
267 }
268 if step >= args.steps {
269 break;
270 }
271 }
272 Ok(())
273}Sourcepub fn config(&self) -> &ShapeVaeConfig
pub fn config(&self) -> &ShapeVaeConfig
Examples found in repository?
340fn train_flow(
341 args: &Args,
342 (cond_tokens, cond_dim): (usize, usize),
343 stream: BatchStream,
344 rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346 // Frozen: the flow transformer is trained against a latent space that has
347 // already stopped moving, so only `encode` is ever called here.
348 let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349 let shape = *vae.config();
350
351 let (mut model, mut step) = match &args.resume {
352 Some(path) => {
353 let (model, metadata) = FlowTransformer::load(path, rng)?;
354 (model, started_at(&metadata))
355 }
356 None => {
357 // The latent shape is the autoencoder's and the condition's shape
358 // is the corpus's, whatever a supplied configuration says.
359 let config = FlowConfig {
360 latents: shape.latents,
361 latent_dim: shape.latent_dim,
362 cond_tokens,
363 cond_dim,
364 ..read_config(args, FlowConfig::default())?
365 };
366 (FlowTransformer::new(config, rng)?, 0)
367 }
368 };
369 if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370 return Err(format!(
371 "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372 model.config().cond_tokens,
373 model.config().cond_dim
374 )
375 .into());
376 }
377 model.set_optimizer(adam(args.learning_rate));
378 if let Some(device) = args.cuda {
379 #[cfg(feature = "cuda")]
380 {
381 // 0 as the budget: the caller picked the card, and the blocks'
382 // weights are a small part of what a run of this size needs.
383 model.to_cuda(device, 0)?;
384 println!("flow transformer: training on CUDA device {device}");
385 }
386 #[cfg(not(feature = "cuda"))]
387 return Err(format!(
388 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389 )
390 .into());
391 }
392 println!(
393 "flow transformer: {} parameters, from step {step}",
394 model.num_parameters()
395 );
396
397 let mut running = 0.0;
398 for batch in stream {
399 let batch = batch?;
400 let count = batch.len();
401 let points = batch.surface.rows / count;
402
403 let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404 for index in 0..count {
405 // A draw from the posterior, not its mean. Every autoencoder step
406 // decoded a draw, so a draw is the only kind of latent the frozen
407 // decoder has ever answered for. The posterior is wide enough for
408 // that to matter — sigma about twice the magnitude of the mean —
409 // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410 // a draw, so a flow trained towards the mean would be aiming at the
411 // one latent the decoder is worst at.
412 let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413 let width = shape.latents * shape.latent_dim;
414 for (slot, value) in clean.data[index * width..(index + 1) * width]
415 .iter_mut()
416 .zip(&latent.data)
417 {
418 *slot = value * args.latent_scale;
419 }
420 }
421 if step == 0 {
422 println!("latent spread after scaling: {:.3}", spread(&clean));
423 }
424
425 running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426 step += 1;
427 model.step_clipped(1.0, 1.0)?;
428 model.zero_grad();
429
430 if step % args.report == 0 {
431 println!(
432 "step {step}: velocity mse {:.5}",
433 running / args.report as f32
434 );
435 running = 0.0;
436 }
437 if step % args.checkpoint_every == 0 || step == args.steps {
438 model.save(
439 &args.out,
440 &BTreeMap::from([
441 step_at(step),
442 ("latent_scale".to_string(), args.latent_scale.to_string()),
443 ]),
444 )?;
445 println!("wrote {} at step {step}", args.out.display());
446 }
447 if step >= args.steps {
448 break;
449 }
450 }
451 Ok(())
452}Sourcepub fn encode(&self, surface: &Matrix) -> Result<(Matrix, Matrix), NetworkError>
pub fn encode(&self, surface: &Matrix) -> Result<(Matrix, Matrix), NetworkError>
The mean and the log-variance of the latent distribution, each
[latents, latent_dim].
surface is [points, 6]: a position and its outward normal.
§The mean is not what the decoder was trained on
Every training step decodes a draw from this distribution, never its mean, so the decoder has only ever seen latents with the posterior’s noise on them. The mean sits in the middle of that cloud, in a spot no training step ever visited, and the reconstruction from it is visibly worse: on the reference model at 24,000 steps the posterior is wide — sigma around 0.53 against a mean magnitude of 0.23 — and reconstructing from the mean gives a clamped L1 of 0.029 where a draw gives 0.0025, an order of magnitude.
So reconstruction goes through ShapeVae::encode_sample. Reach for
the mean when the question is about the distribution itself — measuring
how wide it is, feeding a flow model that learns to produce means, or
checking a KL term — not when the answer is a mesh.
Examples found in repository?
48fn main() -> Result<(), Box<dyn std::error::Error>> {
49 let args = parse()?;
50 let mut rng = StdRng::seed_from_u64(args.seed);
51
52 let mut corpus = Corpus::open(&args.tokens, &args.samples)?;
53 if corpus.is_empty() {
54 return Err("no render is paired with a mesh; check the two directories".into());
55 }
56 let objects: Vec<Object> = (0..args.objects.min(corpus.len()))
57 .map(|index| {
58 let example =
59 corpus.example(index, Some(args.surface), Some(args.queries), &mut rng)?;
60 Ok::<_, Box<dyn std::error::Error>>(Object {
61 surface: example.surface,
62 queries: example.queries,
63 distances: example.distances,
64 })
65 })
66 .collect::<Result<_, _>>()?;
67 println!(
68 "{} objects, {} surface points, {} query points, clamp {}",
69 objects.len(),
70 args.surface,
71 args.queries,
72 args.clamp
73 );
74 println!(
75 "{:<28} {:>9} {:>9} {:>7} {:>7} {:>8} {:>7}",
76 "checkpoint", "l1 draw", "l1 mean", "sigma", "|mean|", "kl/unit", "active"
77 );
78
79 for path in &args.checkpoints {
80 // A fresh generator per checkpoint, so every model is asked for the
81 // same draw of noise as well as the same objects.
82 let mut rng = StdRng::seed_from_u64(args.seed ^ 0x5a1);
83 // `mut` is only used by the device move below.
84 #[cfg_attr(not(feature = "cuda"), allow(unused_mut))]
85 let (mut model, _) = ShapeVae::load(path, &mut StdRng::seed_from_u64(args.seed))?;
86 if let Some(device) = args.cuda {
87 #[cfg(feature = "cuda")]
88 model.to_cuda(device, 0)?;
89 #[cfg(not(feature = "cuda"))]
90 return Err(format!(
91 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
92 )
93 .into());
94 }
95
96 let (mut draw_l1, mut mean_l1, mut sigma, mut magnitude, mut kl, mut active) =
97 (0.0, 0.0, 0.0, 0.0, 0.0, 0.0);
98 for object in &objects {
99 let (mean, log_variance) = model.encode(&object.surface)?;
100 let (latent, _) = ShapeVae::sample(&mean, &log_variance, &mut rng);
101
102 let from_draw = model.decode(&latent, &object.queries, 4096)?;
103 let from_mean = model.decode(&mean, &object.queries, 4096)?;
104 draw_l1 += losses::clamped_l1(&from_draw, &object.distances, args.clamp).0;
105 mean_l1 += losses::clamped_l1(&from_mean, &object.distances, args.clamp).0;
106
107 let units = mean.data.len() as f32;
108 magnitude += mean.data.iter().map(|value| value.abs()).sum::<f32>() / units;
109 sigma += log_variance
110 .data
111 .iter()
112 .map(|value| (0.5 * value).exp())
113 .sum::<f32>()
114 / units;
115 // The per-unit KL against a unit Gaussian, which is what a
116 // free-bits floor would be applied to one unit at a time.
117 let mut per_unit = 0.0;
118 let mut above = 0.0;
119 for (mean, log_variance) in mean.data.iter().zip(&log_variance.data) {
120 let nats = 0.5 * (mean * mean + log_variance.exp() - 1.0 - log_variance);
121 per_unit += nats;
122 if nats > 0.01 {
123 above += 1.0;
124 }
125 }
126 kl += per_unit / units;
127 active += above / units;
128 }
129
130 let count = objects.len() as f32;
131 println!(
132 "{:<28} {:>9.5} {:>9.5} {:>7.3} {:>7.3} {:>8.4} {:>6.1}%",
133 path.file_name()
134 .unwrap_or(path.as_os_str())
135 .to_string_lossy(),
136 draw_l1 / count,
137 mean_l1 / count,
138 sigma / count,
139 magnitude / count,
140 kl / count,
141 100.0 * active / count
142 );
143 }
144 Ok(())
145}Sourcepub fn encode_sample(
&self,
surface: &Matrix,
rng: &mut StdRng,
) -> Result<Matrix, NetworkError>
pub fn encode_sample( &self, surface: &Matrix, rng: &mut StdRng, ) -> Result<Matrix, NetworkError>
One latent drawn from the posterior, which is what the decoder was trained to read.
ShapeVae::encode followed by ShapeVae::sample, in the order a
training step does it. This is the call for reconstructing a shape: see
the warning on encode for what the mean costs instead.
Examples found in repository?
340fn train_flow(
341 args: &Args,
342 (cond_tokens, cond_dim): (usize, usize),
343 stream: BatchStream,
344 rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346 // Frozen: the flow transformer is trained against a latent space that has
347 // already stopped moving, so only `encode` is ever called here.
348 let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349 let shape = *vae.config();
350
351 let (mut model, mut step) = match &args.resume {
352 Some(path) => {
353 let (model, metadata) = FlowTransformer::load(path, rng)?;
354 (model, started_at(&metadata))
355 }
356 None => {
357 // The latent shape is the autoencoder's and the condition's shape
358 // is the corpus's, whatever a supplied configuration says.
359 let config = FlowConfig {
360 latents: shape.latents,
361 latent_dim: shape.latent_dim,
362 cond_tokens,
363 cond_dim,
364 ..read_config(args, FlowConfig::default())?
365 };
366 (FlowTransformer::new(config, rng)?, 0)
367 }
368 };
369 if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370 return Err(format!(
371 "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372 model.config().cond_tokens,
373 model.config().cond_dim
374 )
375 .into());
376 }
377 model.set_optimizer(adam(args.learning_rate));
378 if let Some(device) = args.cuda {
379 #[cfg(feature = "cuda")]
380 {
381 // 0 as the budget: the caller picked the card, and the blocks'
382 // weights are a small part of what a run of this size needs.
383 model.to_cuda(device, 0)?;
384 println!("flow transformer: training on CUDA device {device}");
385 }
386 #[cfg(not(feature = "cuda"))]
387 return Err(format!(
388 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389 )
390 .into());
391 }
392 println!(
393 "flow transformer: {} parameters, from step {step}",
394 model.num_parameters()
395 );
396
397 let mut running = 0.0;
398 for batch in stream {
399 let batch = batch?;
400 let count = batch.len();
401 let points = batch.surface.rows / count;
402
403 let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404 for index in 0..count {
405 // A draw from the posterior, not its mean. Every autoencoder step
406 // decoded a draw, so a draw is the only kind of latent the frozen
407 // decoder has ever answered for. The posterior is wide enough for
408 // that to matter — sigma about twice the magnitude of the mean —
409 // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410 // a draw, so a flow trained towards the mean would be aiming at the
411 // one latent the decoder is worst at.
412 let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413 let width = shape.latents * shape.latent_dim;
414 for (slot, value) in clean.data[index * width..(index + 1) * width]
415 .iter_mut()
416 .zip(&latent.data)
417 {
418 *slot = value * args.latent_scale;
419 }
420 }
421 if step == 0 {
422 println!("latent spread after scaling: {:.3}", spread(&clean));
423 }
424
425 running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426 step += 1;
427 model.step_clipped(1.0, 1.0)?;
428 model.zero_grad();
429
430 if step % args.report == 0 {
431 println!(
432 "step {step}: velocity mse {:.5}",
433 running / args.report as f32
434 );
435 running = 0.0;
436 }
437 if step % args.checkpoint_every == 0 || step == args.steps {
438 model.save(
439 &args.out,
440 &BTreeMap::from([
441 step_at(step),
442 ("latent_scale".to_string(), args.latent_scale.to_string()),
443 ]),
444 )?;
445 println!("wrote {} at step {step}", args.out.display());
446 }
447 if step >= args.steps {
448 break;
449 }
450 }
451 Ok(())
452}Sourcepub fn encode_train(
&self,
surface: &Matrix,
) -> Result<(Matrix, Matrix, EncoderCache), NetworkError>
pub fn encode_train( &self, surface: &Matrix, ) -> Result<(Matrix, Matrix, EncoderCache), NetworkError>
The same, keeping what the backward pass needs.
Examples found in repository?
126fn train_vae(
127 args: &Args,
128 stream: BatchStream,
129 rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131 let (mut model, mut step) = match &args.resume {
132 Some(path) => {
133 let (model, metadata) = ShapeVae::load(path, rng)?;
134 (model, started_at(&metadata))
135 }
136 None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137 };
138 if let Some(device) = args.cuda {
139 #[cfg(feature = "cuda")]
140 {
141 // 0 as the budget: the caller picked the card, and the weights are
142 // a small part of what a run of this size needs.
143 model.to_cuda(device, 0)?;
144 println!("shape autoencoder: training on CUDA device {device}");
145 }
146 #[cfg(not(feature = "cuda"))]
147 return Err(format!(
148 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149 )
150 .into());
151 }
152 let optimizer = adam(args.learning_rate);
153 println!(
154 "shape autoencoder: {} parameters, from step {step}",
155 model.num_parameters()
156 );
157
158 let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159 for batch in stream {
160 let batch = batch?;
161 let count = batch.len();
162 let points = batch.surface.rows / count;
163 let queries = batch.queries.rows / count;
164
165 // Packing helps the ordinary 4K-query training shape, but at 16K
166 // queries the larger attention workspace is slower and batch eight can
167 // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168 let cuda_batch = model.on_device() && queries <= 4096;
169 #[cfg(feature = "cuda")]
170 if cuda_batch {
171 let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172 // The packed losses are means across the batch; the reporting
173 // accumulator below historically stores a sum over examples.
174 reconstruction += loss * count as f32;
175 divergence += kl * count as f32;
176 tint += color * count as f32;
177 }
178
179 if !cuda_batch {
180 for index in 0..count {
181 let surface = slice(&batch.surface, index, points);
182 let (mean, log_variance, cache) = model.encode_train(&surface)?;
183 let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185 let (prediction, decoded) =
186 model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187 let target = &batch.distances[index * queries..(index + 1) * queries];
188 let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189 let (kl, grad_mean, grad_log_variance) =
190 losses::kl_divergence(&mean.data, &log_variance.data);
191 reconstruction += loss;
192 divergence += kl;
193
194 let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196 // The colour targets sit on the surface points, not the distance
197 // queries, so the colour head trains on a second short decode.
198 // The loader's points are already in random order, so the first
199 // `--color-points` of them are a random draw.
200 if let Some(colors) = &batch.colors
201 && args.color > 0.0
202 && args.color_points > 0
203 {
204 let wanted = args.color_points.min(points);
205 let positions = positions(&surface, wanted);
206 let targets = head_rows(&slice(colors, index, points), wanted);
207 let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209 let predicted = model.colors(&decoded)?;
210 let mut grad = predicted.clone();
211 let scale = args.color / predicted.data.len() as f32;
212 for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213 let error = *slot - target;
214 tint += error * error / predicted.data.len() as f32;
215 *slot = 2.0 * scale * error;
216 }
217
218 let from_colors = model.decode_backward_colored(
219 &decoded,
220 &vec![0.0; positions.rows],
221 Some(&grad),
222 )?;
223 for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224 *slot += value;
225 }
226 }
227
228 let (mut into_mean, mut into_log_variance) =
229 ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230 for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231 *slot += args.kl * value;
232 }
233 for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234 *slot += args.kl * value;
235 }
236 model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237 }
238 }
239
240 step += 1;
241 // The gradients were summed over the batch, so the step is scaled by
242 // its size rather than the loop dividing every gradient it makes.
243 optimizers::step_clipped(
244 &mut model.params_mut(),
245 &optimizer,
246 step,
247 1.0 / count as f32,
248 1.0,
249 )?;
250 optimizers::zero_grad(&mut model.params_mut());
251
252 if step % args.report == 0 {
253 let scale = (args.report * count) as f32;
254 println!(
255 "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256 reconstruction / scale,
257 divergence / scale,
258 tint / scale
259 );
260 reconstruction = 0.0;
261 divergence = 0.0;
262 tint = 0.0;
263 }
264 if step % args.checkpoint_every == 0 || step == args.steps {
265 model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266 println!("wrote {} at step {step}", args.out.display());
267 }
268 if step >= args.steps {
269 break;
270 }
271 }
272 Ok(())
273}Sourcepub fn encode_backward(
&mut self,
cache: &EncoderCache,
grad_mean: &Matrix,
grad_log_variance: &Matrix,
) -> Result<(), NetworkError>
pub fn encode_backward( &mut self, cache: &EncoderCache, grad_mean: &Matrix, grad_log_variance: &Matrix, ) -> Result<(), NetworkError>
Accumulates the encoder’s weight gradients.
Nothing is returned: the surface points are data, not a layer below.
Examples found in repository?
126fn train_vae(
127 args: &Args,
128 stream: BatchStream,
129 rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131 let (mut model, mut step) = match &args.resume {
132 Some(path) => {
133 let (model, metadata) = ShapeVae::load(path, rng)?;
134 (model, started_at(&metadata))
135 }
136 None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137 };
138 if let Some(device) = args.cuda {
139 #[cfg(feature = "cuda")]
140 {
141 // 0 as the budget: the caller picked the card, and the weights are
142 // a small part of what a run of this size needs.
143 model.to_cuda(device, 0)?;
144 println!("shape autoencoder: training on CUDA device {device}");
145 }
146 #[cfg(not(feature = "cuda"))]
147 return Err(format!(
148 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149 )
150 .into());
151 }
152 let optimizer = adam(args.learning_rate);
153 println!(
154 "shape autoencoder: {} parameters, from step {step}",
155 model.num_parameters()
156 );
157
158 let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159 for batch in stream {
160 let batch = batch?;
161 let count = batch.len();
162 let points = batch.surface.rows / count;
163 let queries = batch.queries.rows / count;
164
165 // Packing helps the ordinary 4K-query training shape, but at 16K
166 // queries the larger attention workspace is slower and batch eight can
167 // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168 let cuda_batch = model.on_device() && queries <= 4096;
169 #[cfg(feature = "cuda")]
170 if cuda_batch {
171 let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172 // The packed losses are means across the batch; the reporting
173 // accumulator below historically stores a sum over examples.
174 reconstruction += loss * count as f32;
175 divergence += kl * count as f32;
176 tint += color * count as f32;
177 }
178
179 if !cuda_batch {
180 for index in 0..count {
181 let surface = slice(&batch.surface, index, points);
182 let (mean, log_variance, cache) = model.encode_train(&surface)?;
183 let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185 let (prediction, decoded) =
186 model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187 let target = &batch.distances[index * queries..(index + 1) * queries];
188 let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189 let (kl, grad_mean, grad_log_variance) =
190 losses::kl_divergence(&mean.data, &log_variance.data);
191 reconstruction += loss;
192 divergence += kl;
193
194 let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196 // The colour targets sit on the surface points, not the distance
197 // queries, so the colour head trains on a second short decode.
198 // The loader's points are already in random order, so the first
199 // `--color-points` of them are a random draw.
200 if let Some(colors) = &batch.colors
201 && args.color > 0.0
202 && args.color_points > 0
203 {
204 let wanted = args.color_points.min(points);
205 let positions = positions(&surface, wanted);
206 let targets = head_rows(&slice(colors, index, points), wanted);
207 let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209 let predicted = model.colors(&decoded)?;
210 let mut grad = predicted.clone();
211 let scale = args.color / predicted.data.len() as f32;
212 for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213 let error = *slot - target;
214 tint += error * error / predicted.data.len() as f32;
215 *slot = 2.0 * scale * error;
216 }
217
218 let from_colors = model.decode_backward_colored(
219 &decoded,
220 &vec![0.0; positions.rows],
221 Some(&grad),
222 )?;
223 for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224 *slot += value;
225 }
226 }
227
228 let (mut into_mean, mut into_log_variance) =
229 ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230 for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231 *slot += args.kl * value;
232 }
233 for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234 *slot += args.kl * value;
235 }
236 model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237 }
238 }
239
240 step += 1;
241 // The gradients were summed over the batch, so the step is scaled by
242 // its size rather than the loop dividing every gradient it makes.
243 optimizers::step_clipped(
244 &mut model.params_mut(),
245 &optimizer,
246 step,
247 1.0 / count as f32,
248 1.0,
249 )?;
250 optimizers::zero_grad(&mut model.params_mut());
251
252 if step % args.report == 0 {
253 let scale = (args.report * count) as f32;
254 println!(
255 "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256 reconstruction / scale,
257 divergence / scale,
258 tint / scale
259 );
260 reconstruction = 0.0;
261 divergence = 0.0;
262 tint = 0.0;
263 }
264 if step % args.checkpoint_every == 0 || step == args.steps {
265 model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266 println!("wrote {} at step {step}", args.out.display());
267 }
268 if step >= args.steps {
269 break;
270 }
271 }
272 Ok(())
273}Sourcepub fn sample<R: Rng>(
mean: &Matrix,
log_variance: &Matrix,
rng: &mut R,
) -> (Matrix, Matrix)
pub fn sample<R: Rng>( mean: &Matrix, log_variance: &Matrix, rng: &mut R, ) -> (Matrix, Matrix)
Draws a latent from the distribution the encoder described.
z = mean + exp(log_variance / 2) * noise. The noise is returned
because the backward pass needs it, and because a caller that wants a
deterministic reconstruction passes the mean straight to
decode and never calls this at all.
Examples found in repository?
48fn main() -> Result<(), Box<dyn std::error::Error>> {
49 let args = parse()?;
50 let mut rng = StdRng::seed_from_u64(args.seed);
51
52 let mut corpus = Corpus::open(&args.tokens, &args.samples)?;
53 if corpus.is_empty() {
54 return Err("no render is paired with a mesh; check the two directories".into());
55 }
56 let objects: Vec<Object> = (0..args.objects.min(corpus.len()))
57 .map(|index| {
58 let example =
59 corpus.example(index, Some(args.surface), Some(args.queries), &mut rng)?;
60 Ok::<_, Box<dyn std::error::Error>>(Object {
61 surface: example.surface,
62 queries: example.queries,
63 distances: example.distances,
64 })
65 })
66 .collect::<Result<_, _>>()?;
67 println!(
68 "{} objects, {} surface points, {} query points, clamp {}",
69 objects.len(),
70 args.surface,
71 args.queries,
72 args.clamp
73 );
74 println!(
75 "{:<28} {:>9} {:>9} {:>7} {:>7} {:>8} {:>7}",
76 "checkpoint", "l1 draw", "l1 mean", "sigma", "|mean|", "kl/unit", "active"
77 );
78
79 for path in &args.checkpoints {
80 // A fresh generator per checkpoint, so every model is asked for the
81 // same draw of noise as well as the same objects.
82 let mut rng = StdRng::seed_from_u64(args.seed ^ 0x5a1);
83 // `mut` is only used by the device move below.
84 #[cfg_attr(not(feature = "cuda"), allow(unused_mut))]
85 let (mut model, _) = ShapeVae::load(path, &mut StdRng::seed_from_u64(args.seed))?;
86 if let Some(device) = args.cuda {
87 #[cfg(feature = "cuda")]
88 model.to_cuda(device, 0)?;
89 #[cfg(not(feature = "cuda"))]
90 return Err(format!(
91 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
92 )
93 .into());
94 }
95
96 let (mut draw_l1, mut mean_l1, mut sigma, mut magnitude, mut kl, mut active) =
97 (0.0, 0.0, 0.0, 0.0, 0.0, 0.0);
98 for object in &objects {
99 let (mean, log_variance) = model.encode(&object.surface)?;
100 let (latent, _) = ShapeVae::sample(&mean, &log_variance, &mut rng);
101
102 let from_draw = model.decode(&latent, &object.queries, 4096)?;
103 let from_mean = model.decode(&mean, &object.queries, 4096)?;
104 draw_l1 += losses::clamped_l1(&from_draw, &object.distances, args.clamp).0;
105 mean_l1 += losses::clamped_l1(&from_mean, &object.distances, args.clamp).0;
106
107 let units = mean.data.len() as f32;
108 magnitude += mean.data.iter().map(|value| value.abs()).sum::<f32>() / units;
109 sigma += log_variance
110 .data
111 .iter()
112 .map(|value| (0.5 * value).exp())
113 .sum::<f32>()
114 / units;
115 // The per-unit KL against a unit Gaussian, which is what a
116 // free-bits floor would be applied to one unit at a time.
117 let mut per_unit = 0.0;
118 let mut above = 0.0;
119 for (mean, log_variance) in mean.data.iter().zip(&log_variance.data) {
120 let nats = 0.5 * (mean * mean + log_variance.exp() - 1.0 - log_variance);
121 per_unit += nats;
122 if nats > 0.01 {
123 above += 1.0;
124 }
125 }
126 kl += per_unit / units;
127 active += above / units;
128 }
129
130 let count = objects.len() as f32;
131 println!(
132 "{:<28} {:>9.5} {:>9.5} {:>7.3} {:>7.3} {:>8.4} {:>6.1}%",
133 path.file_name()
134 .unwrap_or(path.as_os_str())
135 .to_string_lossy(),
136 draw_l1 / count,
137 mean_l1 / count,
138 sigma / count,
139 magnitude / count,
140 kl / count,
141 100.0 * active / count
142 );
143 }
144 Ok(())
145}More examples
126fn train_vae(
127 args: &Args,
128 stream: BatchStream,
129 rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131 let (mut model, mut step) = match &args.resume {
132 Some(path) => {
133 let (model, metadata) = ShapeVae::load(path, rng)?;
134 (model, started_at(&metadata))
135 }
136 None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137 };
138 if let Some(device) = args.cuda {
139 #[cfg(feature = "cuda")]
140 {
141 // 0 as the budget: the caller picked the card, and the weights are
142 // a small part of what a run of this size needs.
143 model.to_cuda(device, 0)?;
144 println!("shape autoencoder: training on CUDA device {device}");
145 }
146 #[cfg(not(feature = "cuda"))]
147 return Err(format!(
148 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149 )
150 .into());
151 }
152 let optimizer = adam(args.learning_rate);
153 println!(
154 "shape autoencoder: {} parameters, from step {step}",
155 model.num_parameters()
156 );
157
158 let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159 for batch in stream {
160 let batch = batch?;
161 let count = batch.len();
162 let points = batch.surface.rows / count;
163 let queries = batch.queries.rows / count;
164
165 // Packing helps the ordinary 4K-query training shape, but at 16K
166 // queries the larger attention workspace is slower and batch eight can
167 // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168 let cuda_batch = model.on_device() && queries <= 4096;
169 #[cfg(feature = "cuda")]
170 if cuda_batch {
171 let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172 // The packed losses are means across the batch; the reporting
173 // accumulator below historically stores a sum over examples.
174 reconstruction += loss * count as f32;
175 divergence += kl * count as f32;
176 tint += color * count as f32;
177 }
178
179 if !cuda_batch {
180 for index in 0..count {
181 let surface = slice(&batch.surface, index, points);
182 let (mean, log_variance, cache) = model.encode_train(&surface)?;
183 let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185 let (prediction, decoded) =
186 model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187 let target = &batch.distances[index * queries..(index + 1) * queries];
188 let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189 let (kl, grad_mean, grad_log_variance) =
190 losses::kl_divergence(&mean.data, &log_variance.data);
191 reconstruction += loss;
192 divergence += kl;
193
194 let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196 // The colour targets sit on the surface points, not the distance
197 // queries, so the colour head trains on a second short decode.
198 // The loader's points are already in random order, so the first
199 // `--color-points` of them are a random draw.
200 if let Some(colors) = &batch.colors
201 && args.color > 0.0
202 && args.color_points > 0
203 {
204 let wanted = args.color_points.min(points);
205 let positions = positions(&surface, wanted);
206 let targets = head_rows(&slice(colors, index, points), wanted);
207 let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209 let predicted = model.colors(&decoded)?;
210 let mut grad = predicted.clone();
211 let scale = args.color / predicted.data.len() as f32;
212 for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213 let error = *slot - target;
214 tint += error * error / predicted.data.len() as f32;
215 *slot = 2.0 * scale * error;
216 }
217
218 let from_colors = model.decode_backward_colored(
219 &decoded,
220 &vec![0.0; positions.rows],
221 Some(&grad),
222 )?;
223 for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224 *slot += value;
225 }
226 }
227
228 let (mut into_mean, mut into_log_variance) =
229 ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230 for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231 *slot += args.kl * value;
232 }
233 for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234 *slot += args.kl * value;
235 }
236 model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237 }
238 }
239
240 step += 1;
241 // The gradients were summed over the batch, so the step is scaled by
242 // its size rather than the loop dividing every gradient it makes.
243 optimizers::step_clipped(
244 &mut model.params_mut(),
245 &optimizer,
246 step,
247 1.0 / count as f32,
248 1.0,
249 )?;
250 optimizers::zero_grad(&mut model.params_mut());
251
252 if step % args.report == 0 {
253 let scale = (args.report * count) as f32;
254 println!(
255 "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256 reconstruction / scale,
257 divergence / scale,
258 tint / scale
259 );
260 reconstruction = 0.0;
261 divergence = 0.0;
262 tint = 0.0;
263 }
264 if step % args.checkpoint_every == 0 || step == args.steps {
265 model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266 println!("wrote {} at step {step}", args.out.display());
267 }
268 if step >= args.steps {
269 break;
270 }
271 }
272 Ok(())
273}Sourcepub fn sample_backward(
grad_latent: &Matrix,
log_variance: &Matrix,
noise: &Matrix,
) -> (Matrix, Matrix)
pub fn sample_backward( grad_latent: &Matrix, log_variance: &Matrix, noise: &Matrix, ) -> (Matrix, Matrix)
Splits dL/dlatent back into dL/dmean and dL/dlog_variance.
The mean is on a straight path so it takes the gradient unchanged; the
log-variance only reaches the latent through the scaled noise, which is
exactly latent - mean.
Examples found in repository?
126fn train_vae(
127 args: &Args,
128 stream: BatchStream,
129 rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131 let (mut model, mut step) = match &args.resume {
132 Some(path) => {
133 let (model, metadata) = ShapeVae::load(path, rng)?;
134 (model, started_at(&metadata))
135 }
136 None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137 };
138 if let Some(device) = args.cuda {
139 #[cfg(feature = "cuda")]
140 {
141 // 0 as the budget: the caller picked the card, and the weights are
142 // a small part of what a run of this size needs.
143 model.to_cuda(device, 0)?;
144 println!("shape autoencoder: training on CUDA device {device}");
145 }
146 #[cfg(not(feature = "cuda"))]
147 return Err(format!(
148 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149 )
150 .into());
151 }
152 let optimizer = adam(args.learning_rate);
153 println!(
154 "shape autoencoder: {} parameters, from step {step}",
155 model.num_parameters()
156 );
157
158 let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159 for batch in stream {
160 let batch = batch?;
161 let count = batch.len();
162 let points = batch.surface.rows / count;
163 let queries = batch.queries.rows / count;
164
165 // Packing helps the ordinary 4K-query training shape, but at 16K
166 // queries the larger attention workspace is slower and batch eight can
167 // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168 let cuda_batch = model.on_device() && queries <= 4096;
169 #[cfg(feature = "cuda")]
170 if cuda_batch {
171 let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172 // The packed losses are means across the batch; the reporting
173 // accumulator below historically stores a sum over examples.
174 reconstruction += loss * count as f32;
175 divergence += kl * count as f32;
176 tint += color * count as f32;
177 }
178
179 if !cuda_batch {
180 for index in 0..count {
181 let surface = slice(&batch.surface, index, points);
182 let (mean, log_variance, cache) = model.encode_train(&surface)?;
183 let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185 let (prediction, decoded) =
186 model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187 let target = &batch.distances[index * queries..(index + 1) * queries];
188 let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189 let (kl, grad_mean, grad_log_variance) =
190 losses::kl_divergence(&mean.data, &log_variance.data);
191 reconstruction += loss;
192 divergence += kl;
193
194 let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196 // The colour targets sit on the surface points, not the distance
197 // queries, so the colour head trains on a second short decode.
198 // The loader's points are already in random order, so the first
199 // `--color-points` of them are a random draw.
200 if let Some(colors) = &batch.colors
201 && args.color > 0.0
202 && args.color_points > 0
203 {
204 let wanted = args.color_points.min(points);
205 let positions = positions(&surface, wanted);
206 let targets = head_rows(&slice(colors, index, points), wanted);
207 let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209 let predicted = model.colors(&decoded)?;
210 let mut grad = predicted.clone();
211 let scale = args.color / predicted.data.len() as f32;
212 for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213 let error = *slot - target;
214 tint += error * error / predicted.data.len() as f32;
215 *slot = 2.0 * scale * error;
216 }
217
218 let from_colors = model.decode_backward_colored(
219 &decoded,
220 &vec![0.0; positions.rows],
221 Some(&grad),
222 )?;
223 for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224 *slot += value;
225 }
226 }
227
228 let (mut into_mean, mut into_log_variance) =
229 ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230 for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231 *slot += args.kl * value;
232 }
233 for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234 *slot += args.kl * value;
235 }
236 model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237 }
238 }
239
240 step += 1;
241 // The gradients were summed over the batch, so the step is scaled by
242 // its size rather than the loop dividing every gradient it makes.
243 optimizers::step_clipped(
244 &mut model.params_mut(),
245 &optimizer,
246 step,
247 1.0 / count as f32,
248 1.0,
249 )?;
250 optimizers::zero_grad(&mut model.params_mut());
251
252 if step % args.report == 0 {
253 let scale = (args.report * count) as f32;
254 println!(
255 "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256 reconstruction / scale,
257 divergence / scale,
258 tint / scale
259 );
260 reconstruction = 0.0;
261 divergence = 0.0;
262 tint = 0.0;
263 }
264 if step % args.checkpoint_every == 0 || step == args.steps {
265 model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266 println!("wrote {} at step {step}", args.out.display());
267 }
268 if step >= args.steps {
269 break;
270 }
271 }
272 Ok(())
273}Sourcepub fn decode(
&self,
latent: &Matrix,
queries: &Matrix,
chunk: usize,
) -> Result<Vec<f32>, NetworkError>
pub fn decode( &self, latent: &Matrix, queries: &Matrix, chunk: usize, ) -> Result<Vec<f32>, NetworkError>
One signed distance per query point, decoded chunk points at a time.
queries is [n, 3]. The chunk size is what bounds the memory: the
attention scores are [chunk, latents], and a 128³ marching grid asks
about 2.1 million points that will not fit at once.
Examples found in repository?
48fn main() -> Result<(), Box<dyn std::error::Error>> {
49 let args = parse()?;
50 let mut rng = StdRng::seed_from_u64(args.seed);
51
52 let mut corpus = Corpus::open(&args.tokens, &args.samples)?;
53 if corpus.is_empty() {
54 return Err("no render is paired with a mesh; check the two directories".into());
55 }
56 let objects: Vec<Object> = (0..args.objects.min(corpus.len()))
57 .map(|index| {
58 let example =
59 corpus.example(index, Some(args.surface), Some(args.queries), &mut rng)?;
60 Ok::<_, Box<dyn std::error::Error>>(Object {
61 surface: example.surface,
62 queries: example.queries,
63 distances: example.distances,
64 })
65 })
66 .collect::<Result<_, _>>()?;
67 println!(
68 "{} objects, {} surface points, {} query points, clamp {}",
69 objects.len(),
70 args.surface,
71 args.queries,
72 args.clamp
73 );
74 println!(
75 "{:<28} {:>9} {:>9} {:>7} {:>7} {:>8} {:>7}",
76 "checkpoint", "l1 draw", "l1 mean", "sigma", "|mean|", "kl/unit", "active"
77 );
78
79 for path in &args.checkpoints {
80 // A fresh generator per checkpoint, so every model is asked for the
81 // same draw of noise as well as the same objects.
82 let mut rng = StdRng::seed_from_u64(args.seed ^ 0x5a1);
83 // `mut` is only used by the device move below.
84 #[cfg_attr(not(feature = "cuda"), allow(unused_mut))]
85 let (mut model, _) = ShapeVae::load(path, &mut StdRng::seed_from_u64(args.seed))?;
86 if let Some(device) = args.cuda {
87 #[cfg(feature = "cuda")]
88 model.to_cuda(device, 0)?;
89 #[cfg(not(feature = "cuda"))]
90 return Err(format!(
91 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
92 )
93 .into());
94 }
95
96 let (mut draw_l1, mut mean_l1, mut sigma, mut magnitude, mut kl, mut active) =
97 (0.0, 0.0, 0.0, 0.0, 0.0, 0.0);
98 for object in &objects {
99 let (mean, log_variance) = model.encode(&object.surface)?;
100 let (latent, _) = ShapeVae::sample(&mean, &log_variance, &mut rng);
101
102 let from_draw = model.decode(&latent, &object.queries, 4096)?;
103 let from_mean = model.decode(&mean, &object.queries, 4096)?;
104 draw_l1 += losses::clamped_l1(&from_draw, &object.distances, args.clamp).0;
105 mean_l1 += losses::clamped_l1(&from_mean, &object.distances, args.clamp).0;
106
107 let units = mean.data.len() as f32;
108 magnitude += mean.data.iter().map(|value| value.abs()).sum::<f32>() / units;
109 sigma += log_variance
110 .data
111 .iter()
112 .map(|value| (0.5 * value).exp())
113 .sum::<f32>()
114 / units;
115 // The per-unit KL against a unit Gaussian, which is what a
116 // free-bits floor would be applied to one unit at a time.
117 let mut per_unit = 0.0;
118 let mut above = 0.0;
119 for (mean, log_variance) in mean.data.iter().zip(&log_variance.data) {
120 let nats = 0.5 * (mean * mean + log_variance.exp() - 1.0 - log_variance);
121 per_unit += nats;
122 if nats > 0.01 {
123 above += 1.0;
124 }
125 }
126 kl += per_unit / units;
127 active += above / units;
128 }
129
130 let count = objects.len() as f32;
131 println!(
132 "{:<28} {:>9.5} {:>9.5} {:>7.3} {:>7.3} {:>8.4} {:>6.1}%",
133 path.file_name()
134 .unwrap_or(path.as_os_str())
135 .to_string_lossy(),
136 draw_l1 / count,
137 mean_l1 / count,
138 sigma / count,
139 magnitude / count,
140 kl / count,
141 100.0 * active / count
142 );
143 }
144 Ok(())
145}Sourcepub fn decode_train(
&self,
latent: &Matrix,
queries: &Matrix,
) -> Result<(Vec<f32>, DecoderCache), NetworkError>
pub fn decode_train( &self, latent: &Matrix, queries: &Matrix, ) -> Result<(Vec<f32>, DecoderCache), NetworkError>
One chunk of query points, keeping what the backward pass needs.
Training decodes a subsample of a mesh’s query points per step, which is one chunk, so it calls this directly.
Examples found in repository?
126fn train_vae(
127 args: &Args,
128 stream: BatchStream,
129 rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131 let (mut model, mut step) = match &args.resume {
132 Some(path) => {
133 let (model, metadata) = ShapeVae::load(path, rng)?;
134 (model, started_at(&metadata))
135 }
136 None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137 };
138 if let Some(device) = args.cuda {
139 #[cfg(feature = "cuda")]
140 {
141 // 0 as the budget: the caller picked the card, and the weights are
142 // a small part of what a run of this size needs.
143 model.to_cuda(device, 0)?;
144 println!("shape autoencoder: training on CUDA device {device}");
145 }
146 #[cfg(not(feature = "cuda"))]
147 return Err(format!(
148 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149 )
150 .into());
151 }
152 let optimizer = adam(args.learning_rate);
153 println!(
154 "shape autoencoder: {} parameters, from step {step}",
155 model.num_parameters()
156 );
157
158 let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159 for batch in stream {
160 let batch = batch?;
161 let count = batch.len();
162 let points = batch.surface.rows / count;
163 let queries = batch.queries.rows / count;
164
165 // Packing helps the ordinary 4K-query training shape, but at 16K
166 // queries the larger attention workspace is slower and batch eight can
167 // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168 let cuda_batch = model.on_device() && queries <= 4096;
169 #[cfg(feature = "cuda")]
170 if cuda_batch {
171 let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172 // The packed losses are means across the batch; the reporting
173 // accumulator below historically stores a sum over examples.
174 reconstruction += loss * count as f32;
175 divergence += kl * count as f32;
176 tint += color * count as f32;
177 }
178
179 if !cuda_batch {
180 for index in 0..count {
181 let surface = slice(&batch.surface, index, points);
182 let (mean, log_variance, cache) = model.encode_train(&surface)?;
183 let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185 let (prediction, decoded) =
186 model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187 let target = &batch.distances[index * queries..(index + 1) * queries];
188 let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189 let (kl, grad_mean, grad_log_variance) =
190 losses::kl_divergence(&mean.data, &log_variance.data);
191 reconstruction += loss;
192 divergence += kl;
193
194 let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196 // The colour targets sit on the surface points, not the distance
197 // queries, so the colour head trains on a second short decode.
198 // The loader's points are already in random order, so the first
199 // `--color-points` of them are a random draw.
200 if let Some(colors) = &batch.colors
201 && args.color > 0.0
202 && args.color_points > 0
203 {
204 let wanted = args.color_points.min(points);
205 let positions = positions(&surface, wanted);
206 let targets = head_rows(&slice(colors, index, points), wanted);
207 let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209 let predicted = model.colors(&decoded)?;
210 let mut grad = predicted.clone();
211 let scale = args.color / predicted.data.len() as f32;
212 for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213 let error = *slot - target;
214 tint += error * error / predicted.data.len() as f32;
215 *slot = 2.0 * scale * error;
216 }
217
218 let from_colors = model.decode_backward_colored(
219 &decoded,
220 &vec![0.0; positions.rows],
221 Some(&grad),
222 )?;
223 for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224 *slot += value;
225 }
226 }
227
228 let (mut into_mean, mut into_log_variance) =
229 ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230 for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231 *slot += args.kl * value;
232 }
233 for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234 *slot += args.kl * value;
235 }
236 model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237 }
238 }
239
240 step += 1;
241 // The gradients were summed over the batch, so the step is scaled by
242 // its size rather than the loop dividing every gradient it makes.
243 optimizers::step_clipped(
244 &mut model.params_mut(),
245 &optimizer,
246 step,
247 1.0 / count as f32,
248 1.0,
249 )?;
250 optimizers::zero_grad(&mut model.params_mut());
251
252 if step % args.report == 0 {
253 let scale = (args.report * count) as f32;
254 println!(
255 "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256 reconstruction / scale,
257 divergence / scale,
258 tint / scale
259 );
260 reconstruction = 0.0;
261 divergence = 0.0;
262 tint = 0.0;
263 }
264 if step % args.checkpoint_every == 0 || step == args.steps {
265 model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266 println!("wrote {} at step {step}", args.out.display());
267 }
268 if step >= args.steps {
269 break;
270 }
271 }
272 Ok(())
273}Sourcepub fn colors(&self, cache: &DecoderCache) -> Result<Matrix, NetworkError>
pub fn colors(&self, cache: &DecoderCache) -> Result<Matrix, NetworkError>
The colour at each point of a decoded chunk, [n, 3].
The distance and the colour share everything but the last projection,
so this reads a cache decode_train already
built rather than decoding again.
Examples found in repository?
126fn train_vae(
127 args: &Args,
128 stream: BatchStream,
129 rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131 let (mut model, mut step) = match &args.resume {
132 Some(path) => {
133 let (model, metadata) = ShapeVae::load(path, rng)?;
134 (model, started_at(&metadata))
135 }
136 None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137 };
138 if let Some(device) = args.cuda {
139 #[cfg(feature = "cuda")]
140 {
141 // 0 as the budget: the caller picked the card, and the weights are
142 // a small part of what a run of this size needs.
143 model.to_cuda(device, 0)?;
144 println!("shape autoencoder: training on CUDA device {device}");
145 }
146 #[cfg(not(feature = "cuda"))]
147 return Err(format!(
148 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149 )
150 .into());
151 }
152 let optimizer = adam(args.learning_rate);
153 println!(
154 "shape autoencoder: {} parameters, from step {step}",
155 model.num_parameters()
156 );
157
158 let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159 for batch in stream {
160 let batch = batch?;
161 let count = batch.len();
162 let points = batch.surface.rows / count;
163 let queries = batch.queries.rows / count;
164
165 // Packing helps the ordinary 4K-query training shape, but at 16K
166 // queries the larger attention workspace is slower and batch eight can
167 // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168 let cuda_batch = model.on_device() && queries <= 4096;
169 #[cfg(feature = "cuda")]
170 if cuda_batch {
171 let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172 // The packed losses are means across the batch; the reporting
173 // accumulator below historically stores a sum over examples.
174 reconstruction += loss * count as f32;
175 divergence += kl * count as f32;
176 tint += color * count as f32;
177 }
178
179 if !cuda_batch {
180 for index in 0..count {
181 let surface = slice(&batch.surface, index, points);
182 let (mean, log_variance, cache) = model.encode_train(&surface)?;
183 let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185 let (prediction, decoded) =
186 model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187 let target = &batch.distances[index * queries..(index + 1) * queries];
188 let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189 let (kl, grad_mean, grad_log_variance) =
190 losses::kl_divergence(&mean.data, &log_variance.data);
191 reconstruction += loss;
192 divergence += kl;
193
194 let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196 // The colour targets sit on the surface points, not the distance
197 // queries, so the colour head trains on a second short decode.
198 // The loader's points are already in random order, so the first
199 // `--color-points` of them are a random draw.
200 if let Some(colors) = &batch.colors
201 && args.color > 0.0
202 && args.color_points > 0
203 {
204 let wanted = args.color_points.min(points);
205 let positions = positions(&surface, wanted);
206 let targets = head_rows(&slice(colors, index, points), wanted);
207 let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209 let predicted = model.colors(&decoded)?;
210 let mut grad = predicted.clone();
211 let scale = args.color / predicted.data.len() as f32;
212 for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213 let error = *slot - target;
214 tint += error * error / predicted.data.len() as f32;
215 *slot = 2.0 * scale * error;
216 }
217
218 let from_colors = model.decode_backward_colored(
219 &decoded,
220 &vec![0.0; positions.rows],
221 Some(&grad),
222 )?;
223 for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224 *slot += value;
225 }
226 }
227
228 let (mut into_mean, mut into_log_variance) =
229 ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230 for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231 *slot += args.kl * value;
232 }
233 for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234 *slot += args.kl * value;
235 }
236 model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237 }
238 }
239
240 step += 1;
241 // The gradients were summed over the batch, so the step is scaled by
242 // its size rather than the loop dividing every gradient it makes.
243 optimizers::step_clipped(
244 &mut model.params_mut(),
245 &optimizer,
246 step,
247 1.0 / count as f32,
248 1.0,
249 )?;
250 optimizers::zero_grad(&mut model.params_mut());
251
252 if step % args.report == 0 {
253 let scale = (args.report * count) as f32;
254 println!(
255 "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256 reconstruction / scale,
257 divergence / scale,
258 tint / scale
259 );
260 reconstruction = 0.0;
261 divergence = 0.0;
262 tint = 0.0;
263 }
264 if step % args.checkpoint_every == 0 || step == args.steps {
265 model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266 println!("wrote {} at step {step}", args.out.display());
267 }
268 if step >= args.steps {
269 break;
270 }
271 }
272 Ok(())
273}Sourcepub fn decode_colors(
&self,
latent: &Matrix,
points: &Matrix,
chunk: usize,
) -> Result<Vec<[f32; 3]>, NetworkError>
pub fn decode_colors( &self, latent: &Matrix, points: &Matrix, chunk: usize, ) -> Result<Vec<[f32; 3]>, NetworkError>
One colour per point, chunked like decode.
Sourcepub fn decode_backward(
&mut self,
cache: &DecoderCache,
grad_distances: &[f32],
) -> Result<Matrix, NetworkError>
pub fn decode_backward( &mut self, cache: &DecoderCache, grad_distances: &[f32], ) -> Result<Matrix, NetworkError>
Accumulates the decoder’s weight gradients and returns dL/dlatent.
Examples found in repository?
126fn train_vae(
127 args: &Args,
128 stream: BatchStream,
129 rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131 let (mut model, mut step) = match &args.resume {
132 Some(path) => {
133 let (model, metadata) = ShapeVae::load(path, rng)?;
134 (model, started_at(&metadata))
135 }
136 None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137 };
138 if let Some(device) = args.cuda {
139 #[cfg(feature = "cuda")]
140 {
141 // 0 as the budget: the caller picked the card, and the weights are
142 // a small part of what a run of this size needs.
143 model.to_cuda(device, 0)?;
144 println!("shape autoencoder: training on CUDA device {device}");
145 }
146 #[cfg(not(feature = "cuda"))]
147 return Err(format!(
148 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149 )
150 .into());
151 }
152 let optimizer = adam(args.learning_rate);
153 println!(
154 "shape autoencoder: {} parameters, from step {step}",
155 model.num_parameters()
156 );
157
158 let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159 for batch in stream {
160 let batch = batch?;
161 let count = batch.len();
162 let points = batch.surface.rows / count;
163 let queries = batch.queries.rows / count;
164
165 // Packing helps the ordinary 4K-query training shape, but at 16K
166 // queries the larger attention workspace is slower and batch eight can
167 // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168 let cuda_batch = model.on_device() && queries <= 4096;
169 #[cfg(feature = "cuda")]
170 if cuda_batch {
171 let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172 // The packed losses are means across the batch; the reporting
173 // accumulator below historically stores a sum over examples.
174 reconstruction += loss * count as f32;
175 divergence += kl * count as f32;
176 tint += color * count as f32;
177 }
178
179 if !cuda_batch {
180 for index in 0..count {
181 let surface = slice(&batch.surface, index, points);
182 let (mean, log_variance, cache) = model.encode_train(&surface)?;
183 let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185 let (prediction, decoded) =
186 model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187 let target = &batch.distances[index * queries..(index + 1) * queries];
188 let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189 let (kl, grad_mean, grad_log_variance) =
190 losses::kl_divergence(&mean.data, &log_variance.data);
191 reconstruction += loss;
192 divergence += kl;
193
194 let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196 // The colour targets sit on the surface points, not the distance
197 // queries, so the colour head trains on a second short decode.
198 // The loader's points are already in random order, so the first
199 // `--color-points` of them are a random draw.
200 if let Some(colors) = &batch.colors
201 && args.color > 0.0
202 && args.color_points > 0
203 {
204 let wanted = args.color_points.min(points);
205 let positions = positions(&surface, wanted);
206 let targets = head_rows(&slice(colors, index, points), wanted);
207 let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209 let predicted = model.colors(&decoded)?;
210 let mut grad = predicted.clone();
211 let scale = args.color / predicted.data.len() as f32;
212 for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213 let error = *slot - target;
214 tint += error * error / predicted.data.len() as f32;
215 *slot = 2.0 * scale * error;
216 }
217
218 let from_colors = model.decode_backward_colored(
219 &decoded,
220 &vec![0.0; positions.rows],
221 Some(&grad),
222 )?;
223 for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224 *slot += value;
225 }
226 }
227
228 let (mut into_mean, mut into_log_variance) =
229 ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230 for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231 *slot += args.kl * value;
232 }
233 for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234 *slot += args.kl * value;
235 }
236 model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237 }
238 }
239
240 step += 1;
241 // The gradients were summed over the batch, so the step is scaled by
242 // its size rather than the loop dividing every gradient it makes.
243 optimizers::step_clipped(
244 &mut model.params_mut(),
245 &optimizer,
246 step,
247 1.0 / count as f32,
248 1.0,
249 )?;
250 optimizers::zero_grad(&mut model.params_mut());
251
252 if step % args.report == 0 {
253 let scale = (args.report * count) as f32;
254 println!(
255 "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256 reconstruction / scale,
257 divergence / scale,
258 tint / scale
259 );
260 reconstruction = 0.0;
261 divergence = 0.0;
262 tint = 0.0;
263 }
264 if step % args.checkpoint_every == 0 || step == args.steps {
265 model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266 println!("wrote {} at step {step}", args.out.display());
267 }
268 if step >= args.steps {
269 break;
270 }
271 }
272 Ok(())
273}Sourcepub fn decode_backward_colored(
&mut self,
cache: &DecoderCache,
grad_distances: &[f32],
grad_colors: Option<&Matrix>,
) -> Result<Matrix, NetworkError>
pub fn decode_backward_colored( &mut self, cache: &DecoderCache, grad_distances: &[f32], grad_colors: Option<&Matrix>, ) -> Result<Matrix, NetworkError>
The same, with the colour head’s gradients as well.
Examples found in repository?
126fn train_vae(
127 args: &Args,
128 stream: BatchStream,
129 rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131 let (mut model, mut step) = match &args.resume {
132 Some(path) => {
133 let (model, metadata) = ShapeVae::load(path, rng)?;
134 (model, started_at(&metadata))
135 }
136 None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137 };
138 if let Some(device) = args.cuda {
139 #[cfg(feature = "cuda")]
140 {
141 // 0 as the budget: the caller picked the card, and the weights are
142 // a small part of what a run of this size needs.
143 model.to_cuda(device, 0)?;
144 println!("shape autoencoder: training on CUDA device {device}");
145 }
146 #[cfg(not(feature = "cuda"))]
147 return Err(format!(
148 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149 )
150 .into());
151 }
152 let optimizer = adam(args.learning_rate);
153 println!(
154 "shape autoencoder: {} parameters, from step {step}",
155 model.num_parameters()
156 );
157
158 let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159 for batch in stream {
160 let batch = batch?;
161 let count = batch.len();
162 let points = batch.surface.rows / count;
163 let queries = batch.queries.rows / count;
164
165 // Packing helps the ordinary 4K-query training shape, but at 16K
166 // queries the larger attention workspace is slower and batch eight can
167 // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168 let cuda_batch = model.on_device() && queries <= 4096;
169 #[cfg(feature = "cuda")]
170 if cuda_batch {
171 let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172 // The packed losses are means across the batch; the reporting
173 // accumulator below historically stores a sum over examples.
174 reconstruction += loss * count as f32;
175 divergence += kl * count as f32;
176 tint += color * count as f32;
177 }
178
179 if !cuda_batch {
180 for index in 0..count {
181 let surface = slice(&batch.surface, index, points);
182 let (mean, log_variance, cache) = model.encode_train(&surface)?;
183 let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185 let (prediction, decoded) =
186 model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187 let target = &batch.distances[index * queries..(index + 1) * queries];
188 let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189 let (kl, grad_mean, grad_log_variance) =
190 losses::kl_divergence(&mean.data, &log_variance.data);
191 reconstruction += loss;
192 divergence += kl;
193
194 let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196 // The colour targets sit on the surface points, not the distance
197 // queries, so the colour head trains on a second short decode.
198 // The loader's points are already in random order, so the first
199 // `--color-points` of them are a random draw.
200 if let Some(colors) = &batch.colors
201 && args.color > 0.0
202 && args.color_points > 0
203 {
204 let wanted = args.color_points.min(points);
205 let positions = positions(&surface, wanted);
206 let targets = head_rows(&slice(colors, index, points), wanted);
207 let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209 let predicted = model.colors(&decoded)?;
210 let mut grad = predicted.clone();
211 let scale = args.color / predicted.data.len() as f32;
212 for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213 let error = *slot - target;
214 tint += error * error / predicted.data.len() as f32;
215 *slot = 2.0 * scale * error;
216 }
217
218 let from_colors = model.decode_backward_colored(
219 &decoded,
220 &vec![0.0; positions.rows],
221 Some(&grad),
222 )?;
223 for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224 *slot += value;
225 }
226 }
227
228 let (mut into_mean, mut into_log_variance) =
229 ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230 for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231 *slot += args.kl * value;
232 }
233 for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234 *slot += args.kl * value;
235 }
236 model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237 }
238 }
239
240 step += 1;
241 // The gradients were summed over the batch, so the step is scaled by
242 // its size rather than the loop dividing every gradient it makes.
243 optimizers::step_clipped(
244 &mut model.params_mut(),
245 &optimizer,
246 step,
247 1.0 / count as f32,
248 1.0,
249 )?;
250 optimizers::zero_grad(&mut model.params_mut());
251
252 if step % args.report == 0 {
253 let scale = (args.report * count) as f32;
254 println!(
255 "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256 reconstruction / scale,
257 divergence / scale,
258 tint / scale
259 );
260 reconstruction = 0.0;
261 divergence = 0.0;
262 tint = 0.0;
263 }
264 if step % args.checkpoint_every == 0 || step == args.steps {
265 model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266 println!("wrote {} at step {step}", args.out.display());
267 }
268 if step >= args.steps {
269 break;
270 }
271 }
272 Ok(())
273}Sourcepub fn field<'a>(
&'a self,
latent: &'a Matrix,
chunk: usize,
) -> impl Fn(&Matrix) -> Result<Vec<f32>, NetworkError> + Sync + 'a
pub fn field<'a>( &'a self, latent: &'a Matrix, chunk: usize, ) -> impl Fn(&Matrix) -> Result<Vec<f32>, NetworkError> + Sync + 'a
A closure over a fixed latent, shaped for
crate::mesh::marching_tetrahedra, which hands it one grid slice at a
time.
Sourcepub fn on_device(&self) -> bool
pub fn on_device(&self) -> bool
Whether the projections currently live on a device.
Examples found in repository?
126fn train_vae(
127 args: &Args,
128 stream: BatchStream,
129 rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131 let (mut model, mut step) = match &args.resume {
132 Some(path) => {
133 let (model, metadata) = ShapeVae::load(path, rng)?;
134 (model, started_at(&metadata))
135 }
136 None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137 };
138 if let Some(device) = args.cuda {
139 #[cfg(feature = "cuda")]
140 {
141 // 0 as the budget: the caller picked the card, and the weights are
142 // a small part of what a run of this size needs.
143 model.to_cuda(device, 0)?;
144 println!("shape autoencoder: training on CUDA device {device}");
145 }
146 #[cfg(not(feature = "cuda"))]
147 return Err(format!(
148 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149 )
150 .into());
151 }
152 let optimizer = adam(args.learning_rate);
153 println!(
154 "shape autoencoder: {} parameters, from step {step}",
155 model.num_parameters()
156 );
157
158 let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159 for batch in stream {
160 let batch = batch?;
161 let count = batch.len();
162 let points = batch.surface.rows / count;
163 let queries = batch.queries.rows / count;
164
165 // Packing helps the ordinary 4K-query training shape, but at 16K
166 // queries the larger attention workspace is slower and batch eight can
167 // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168 let cuda_batch = model.on_device() && queries <= 4096;
169 #[cfg(feature = "cuda")]
170 if cuda_batch {
171 let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172 // The packed losses are means across the batch; the reporting
173 // accumulator below historically stores a sum over examples.
174 reconstruction += loss * count as f32;
175 divergence += kl * count as f32;
176 tint += color * count as f32;
177 }
178
179 if !cuda_batch {
180 for index in 0..count {
181 let surface = slice(&batch.surface, index, points);
182 let (mean, log_variance, cache) = model.encode_train(&surface)?;
183 let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185 let (prediction, decoded) =
186 model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187 let target = &batch.distances[index * queries..(index + 1) * queries];
188 let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189 let (kl, grad_mean, grad_log_variance) =
190 losses::kl_divergence(&mean.data, &log_variance.data);
191 reconstruction += loss;
192 divergence += kl;
193
194 let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196 // The colour targets sit on the surface points, not the distance
197 // queries, so the colour head trains on a second short decode.
198 // The loader's points are already in random order, so the first
199 // `--color-points` of them are a random draw.
200 if let Some(colors) = &batch.colors
201 && args.color > 0.0
202 && args.color_points > 0
203 {
204 let wanted = args.color_points.min(points);
205 let positions = positions(&surface, wanted);
206 let targets = head_rows(&slice(colors, index, points), wanted);
207 let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209 let predicted = model.colors(&decoded)?;
210 let mut grad = predicted.clone();
211 let scale = args.color / predicted.data.len() as f32;
212 for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213 let error = *slot - target;
214 tint += error * error / predicted.data.len() as f32;
215 *slot = 2.0 * scale * error;
216 }
217
218 let from_colors = model.decode_backward_colored(
219 &decoded,
220 &vec![0.0; positions.rows],
221 Some(&grad),
222 )?;
223 for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224 *slot += value;
225 }
226 }
227
228 let (mut into_mean, mut into_log_variance) =
229 ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230 for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231 *slot += args.kl * value;
232 }
233 for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234 *slot += args.kl * value;
235 }
236 model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237 }
238 }
239
240 step += 1;
241 // The gradients were summed over the batch, so the step is scaled by
242 // its size rather than the loop dividing every gradient it makes.
243 optimizers::step_clipped(
244 &mut model.params_mut(),
245 &optimizer,
246 step,
247 1.0 / count as f32,
248 1.0,
249 )?;
250 optimizers::zero_grad(&mut model.params_mut());
251
252 if step % args.report == 0 {
253 let scale = (args.report * count) as f32;
254 println!(
255 "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256 reconstruction / scale,
257 divergence / scale,
258 tint / scale
259 );
260 reconstruction = 0.0;
261 divergence = 0.0;
262 tint = 0.0;
263 }
264 if step % args.checkpoint_every == 0 || step == args.steps {
265 model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266 println!("wrote {} at step {step}", args.out.display());
267 }
268 if step >= args.steps {
269 break;
270 }
271 }
272 Ok(())
273}Sourcepub fn params_mut(&mut self) -> Vec<&mut Param>
pub fn params_mut(&mut self) -> Vec<&mut Param>
Examples found in repository?
126fn train_vae(
127 args: &Args,
128 stream: BatchStream,
129 rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131 let (mut model, mut step) = match &args.resume {
132 Some(path) => {
133 let (model, metadata) = ShapeVae::load(path, rng)?;
134 (model, started_at(&metadata))
135 }
136 None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137 };
138 if let Some(device) = args.cuda {
139 #[cfg(feature = "cuda")]
140 {
141 // 0 as the budget: the caller picked the card, and the weights are
142 // a small part of what a run of this size needs.
143 model.to_cuda(device, 0)?;
144 println!("shape autoencoder: training on CUDA device {device}");
145 }
146 #[cfg(not(feature = "cuda"))]
147 return Err(format!(
148 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149 )
150 .into());
151 }
152 let optimizer = adam(args.learning_rate);
153 println!(
154 "shape autoencoder: {} parameters, from step {step}",
155 model.num_parameters()
156 );
157
158 let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159 for batch in stream {
160 let batch = batch?;
161 let count = batch.len();
162 let points = batch.surface.rows / count;
163 let queries = batch.queries.rows / count;
164
165 // Packing helps the ordinary 4K-query training shape, but at 16K
166 // queries the larger attention workspace is slower and batch eight can
167 // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168 let cuda_batch = model.on_device() && queries <= 4096;
169 #[cfg(feature = "cuda")]
170 if cuda_batch {
171 let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172 // The packed losses are means across the batch; the reporting
173 // accumulator below historically stores a sum over examples.
174 reconstruction += loss * count as f32;
175 divergence += kl * count as f32;
176 tint += color * count as f32;
177 }
178
179 if !cuda_batch {
180 for index in 0..count {
181 let surface = slice(&batch.surface, index, points);
182 let (mean, log_variance, cache) = model.encode_train(&surface)?;
183 let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185 let (prediction, decoded) =
186 model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187 let target = &batch.distances[index * queries..(index + 1) * queries];
188 let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189 let (kl, grad_mean, grad_log_variance) =
190 losses::kl_divergence(&mean.data, &log_variance.data);
191 reconstruction += loss;
192 divergence += kl;
193
194 let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196 // The colour targets sit on the surface points, not the distance
197 // queries, so the colour head trains on a second short decode.
198 // The loader's points are already in random order, so the first
199 // `--color-points` of them are a random draw.
200 if let Some(colors) = &batch.colors
201 && args.color > 0.0
202 && args.color_points > 0
203 {
204 let wanted = args.color_points.min(points);
205 let positions = positions(&surface, wanted);
206 let targets = head_rows(&slice(colors, index, points), wanted);
207 let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209 let predicted = model.colors(&decoded)?;
210 let mut grad = predicted.clone();
211 let scale = args.color / predicted.data.len() as f32;
212 for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213 let error = *slot - target;
214 tint += error * error / predicted.data.len() as f32;
215 *slot = 2.0 * scale * error;
216 }
217
218 let from_colors = model.decode_backward_colored(
219 &decoded,
220 &vec![0.0; positions.rows],
221 Some(&grad),
222 )?;
223 for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224 *slot += value;
225 }
226 }
227
228 let (mut into_mean, mut into_log_variance) =
229 ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230 for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231 *slot += args.kl * value;
232 }
233 for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234 *slot += args.kl * value;
235 }
236 model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237 }
238 }
239
240 step += 1;
241 // The gradients were summed over the batch, so the step is scaled by
242 // its size rather than the loop dividing every gradient it makes.
243 optimizers::step_clipped(
244 &mut model.params_mut(),
245 &optimizer,
246 step,
247 1.0 / count as f32,
248 1.0,
249 )?;
250 optimizers::zero_grad(&mut model.params_mut());
251
252 if step % args.report == 0 {
253 let scale = (args.report * count) as f32;
254 println!(
255 "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256 reconstruction / scale,
257 divergence / scale,
258 tint / scale
259 );
260 reconstruction = 0.0;
261 divergence = 0.0;
262 tint = 0.0;
263 }
264 if step % args.checkpoint_every == 0 || step == args.steps {
265 model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266 println!("wrote {} at step {step}", args.out.display());
267 }
268 if step >= args.steps {
269 break;
270 }
271 }
272 Ok(())
273}Sourcepub fn num_parameters(&mut self) -> usize
pub fn num_parameters(&mut self) -> usize
Examples found in repository?
126fn train_vae(
127 args: &Args,
128 stream: BatchStream,
129 rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131 let (mut model, mut step) = match &args.resume {
132 Some(path) => {
133 let (model, metadata) = ShapeVae::load(path, rng)?;
134 (model, started_at(&metadata))
135 }
136 None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137 };
138 if let Some(device) = args.cuda {
139 #[cfg(feature = "cuda")]
140 {
141 // 0 as the budget: the caller picked the card, and the weights are
142 // a small part of what a run of this size needs.
143 model.to_cuda(device, 0)?;
144 println!("shape autoencoder: training on CUDA device {device}");
145 }
146 #[cfg(not(feature = "cuda"))]
147 return Err(format!(
148 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149 )
150 .into());
151 }
152 let optimizer = adam(args.learning_rate);
153 println!(
154 "shape autoencoder: {} parameters, from step {step}",
155 model.num_parameters()
156 );
157
158 let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159 for batch in stream {
160 let batch = batch?;
161 let count = batch.len();
162 let points = batch.surface.rows / count;
163 let queries = batch.queries.rows / count;
164
165 // Packing helps the ordinary 4K-query training shape, but at 16K
166 // queries the larger attention workspace is slower and batch eight can
167 // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168 let cuda_batch = model.on_device() && queries <= 4096;
169 #[cfg(feature = "cuda")]
170 if cuda_batch {
171 let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172 // The packed losses are means across the batch; the reporting
173 // accumulator below historically stores a sum over examples.
174 reconstruction += loss * count as f32;
175 divergence += kl * count as f32;
176 tint += color * count as f32;
177 }
178
179 if !cuda_batch {
180 for index in 0..count {
181 let surface = slice(&batch.surface, index, points);
182 let (mean, log_variance, cache) = model.encode_train(&surface)?;
183 let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185 let (prediction, decoded) =
186 model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187 let target = &batch.distances[index * queries..(index + 1) * queries];
188 let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189 let (kl, grad_mean, grad_log_variance) =
190 losses::kl_divergence(&mean.data, &log_variance.data);
191 reconstruction += loss;
192 divergence += kl;
193
194 let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196 // The colour targets sit on the surface points, not the distance
197 // queries, so the colour head trains on a second short decode.
198 // The loader's points are already in random order, so the first
199 // `--color-points` of them are a random draw.
200 if let Some(colors) = &batch.colors
201 && args.color > 0.0
202 && args.color_points > 0
203 {
204 let wanted = args.color_points.min(points);
205 let positions = positions(&surface, wanted);
206 let targets = head_rows(&slice(colors, index, points), wanted);
207 let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209 let predicted = model.colors(&decoded)?;
210 let mut grad = predicted.clone();
211 let scale = args.color / predicted.data.len() as f32;
212 for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213 let error = *slot - target;
214 tint += error * error / predicted.data.len() as f32;
215 *slot = 2.0 * scale * error;
216 }
217
218 let from_colors = model.decode_backward_colored(
219 &decoded,
220 &vec![0.0; positions.rows],
221 Some(&grad),
222 )?;
223 for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224 *slot += value;
225 }
226 }
227
228 let (mut into_mean, mut into_log_variance) =
229 ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230 for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231 *slot += args.kl * value;
232 }
233 for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234 *slot += args.kl * value;
235 }
236 model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237 }
238 }
239
240 step += 1;
241 // The gradients were summed over the batch, so the step is scaled by
242 // its size rather than the loop dividing every gradient it makes.
243 optimizers::step_clipped(
244 &mut model.params_mut(),
245 &optimizer,
246 step,
247 1.0 / count as f32,
248 1.0,
249 )?;
250 optimizers::zero_grad(&mut model.params_mut());
251
252 if step % args.report == 0 {
253 let scale = (args.report * count) as f32;
254 println!(
255 "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256 reconstruction / scale,
257 divergence / scale,
258 tint / scale
259 );
260 reconstruction = 0.0;
261 divergence = 0.0;
262 tint = 0.0;
263 }
264 if step % args.checkpoint_every == 0 || step == args.steps {
265 model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266 println!("wrote {} at step {step}", args.out.display());
267 }
268 if step >= args.steps {
269 break;
270 }
271 }
272 Ok(())
273}Sourcepub fn save<P: AsRef<Path>>(
&mut self,
path: P,
metadata: &BTreeMap<String, String>,
) -> Result<(), NetworkError>
pub fn save<P: AsRef<Path>>( &mut self, path: P, metadata: &BTreeMap<String, String>, ) -> Result<(), NetworkError>
Writes the weights, the Adam moments and the configuration.
The caller’s metadata is written beside the configuration, which is
where a training loop puts its step counter. The key config belongs
to the model and is overwritten.
Examples found in repository?
126fn train_vae(
127 args: &Args,
128 stream: BatchStream,
129 rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131 let (mut model, mut step) = match &args.resume {
132 Some(path) => {
133 let (model, metadata) = ShapeVae::load(path, rng)?;
134 (model, started_at(&metadata))
135 }
136 None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137 };
138 if let Some(device) = args.cuda {
139 #[cfg(feature = "cuda")]
140 {
141 // 0 as the budget: the caller picked the card, and the weights are
142 // a small part of what a run of this size needs.
143 model.to_cuda(device, 0)?;
144 println!("shape autoencoder: training on CUDA device {device}");
145 }
146 #[cfg(not(feature = "cuda"))]
147 return Err(format!(
148 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149 )
150 .into());
151 }
152 let optimizer = adam(args.learning_rate);
153 println!(
154 "shape autoencoder: {} parameters, from step {step}",
155 model.num_parameters()
156 );
157
158 let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159 for batch in stream {
160 let batch = batch?;
161 let count = batch.len();
162 let points = batch.surface.rows / count;
163 let queries = batch.queries.rows / count;
164
165 // Packing helps the ordinary 4K-query training shape, but at 16K
166 // queries the larger attention workspace is slower and batch eight can
167 // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168 let cuda_batch = model.on_device() && queries <= 4096;
169 #[cfg(feature = "cuda")]
170 if cuda_batch {
171 let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172 // The packed losses are means across the batch; the reporting
173 // accumulator below historically stores a sum over examples.
174 reconstruction += loss * count as f32;
175 divergence += kl * count as f32;
176 tint += color * count as f32;
177 }
178
179 if !cuda_batch {
180 for index in 0..count {
181 let surface = slice(&batch.surface, index, points);
182 let (mean, log_variance, cache) = model.encode_train(&surface)?;
183 let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185 let (prediction, decoded) =
186 model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187 let target = &batch.distances[index * queries..(index + 1) * queries];
188 let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189 let (kl, grad_mean, grad_log_variance) =
190 losses::kl_divergence(&mean.data, &log_variance.data);
191 reconstruction += loss;
192 divergence += kl;
193
194 let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196 // The colour targets sit on the surface points, not the distance
197 // queries, so the colour head trains on a second short decode.
198 // The loader's points are already in random order, so the first
199 // `--color-points` of them are a random draw.
200 if let Some(colors) = &batch.colors
201 && args.color > 0.0
202 && args.color_points > 0
203 {
204 let wanted = args.color_points.min(points);
205 let positions = positions(&surface, wanted);
206 let targets = head_rows(&slice(colors, index, points), wanted);
207 let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209 let predicted = model.colors(&decoded)?;
210 let mut grad = predicted.clone();
211 let scale = args.color / predicted.data.len() as f32;
212 for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213 let error = *slot - target;
214 tint += error * error / predicted.data.len() as f32;
215 *slot = 2.0 * scale * error;
216 }
217
218 let from_colors = model.decode_backward_colored(
219 &decoded,
220 &vec![0.0; positions.rows],
221 Some(&grad),
222 )?;
223 for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224 *slot += value;
225 }
226 }
227
228 let (mut into_mean, mut into_log_variance) =
229 ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230 for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231 *slot += args.kl * value;
232 }
233 for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234 *slot += args.kl * value;
235 }
236 model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237 }
238 }
239
240 step += 1;
241 // The gradients were summed over the batch, so the step is scaled by
242 // its size rather than the loop dividing every gradient it makes.
243 optimizers::step_clipped(
244 &mut model.params_mut(),
245 &optimizer,
246 step,
247 1.0 / count as f32,
248 1.0,
249 )?;
250 optimizers::zero_grad(&mut model.params_mut());
251
252 if step % args.report == 0 {
253 let scale = (args.report * count) as f32;
254 println!(
255 "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256 reconstruction / scale,
257 divergence / scale,
258 tint / scale
259 );
260 reconstruction = 0.0;
261 divergence = 0.0;
262 tint = 0.0;
263 }
264 if step % args.checkpoint_every == 0 || step == args.steps {
265 model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266 println!("wrote {} at step {step}", args.out.display());
267 }
268 if step >= args.steps {
269 break;
270 }
271 }
272 Ok(())
273}Sourcepub fn load<P: AsRef<Path>>(
path: P,
rng: &mut StdRng,
) -> Result<(Self, BTreeMap<String, String>), NetworkError>
pub fn load<P: AsRef<Path>>( path: P, rng: &mut StdRng, ) -> Result<(Self, BTreeMap<String, String>), NetworkError>
Rebuilds a model from what save wrote, and returns the
rest of the metadata.
The shape comes out of the file, so a caller does not have to remember
which configuration a checkpoint was trained with; rng only fills
weights that the file then overwrites.
Examples found in repository?
48fn main() -> Result<(), Box<dyn std::error::Error>> {
49 let args = parse()?;
50 let mut rng = StdRng::seed_from_u64(args.seed);
51
52 let mut corpus = Corpus::open(&args.tokens, &args.samples)?;
53 if corpus.is_empty() {
54 return Err("no render is paired with a mesh; check the two directories".into());
55 }
56 let objects: Vec<Object> = (0..args.objects.min(corpus.len()))
57 .map(|index| {
58 let example =
59 corpus.example(index, Some(args.surface), Some(args.queries), &mut rng)?;
60 Ok::<_, Box<dyn std::error::Error>>(Object {
61 surface: example.surface,
62 queries: example.queries,
63 distances: example.distances,
64 })
65 })
66 .collect::<Result<_, _>>()?;
67 println!(
68 "{} objects, {} surface points, {} query points, clamp {}",
69 objects.len(),
70 args.surface,
71 args.queries,
72 args.clamp
73 );
74 println!(
75 "{:<28} {:>9} {:>9} {:>7} {:>7} {:>8} {:>7}",
76 "checkpoint", "l1 draw", "l1 mean", "sigma", "|mean|", "kl/unit", "active"
77 );
78
79 for path in &args.checkpoints {
80 // A fresh generator per checkpoint, so every model is asked for the
81 // same draw of noise as well as the same objects.
82 let mut rng = StdRng::seed_from_u64(args.seed ^ 0x5a1);
83 // `mut` is only used by the device move below.
84 #[cfg_attr(not(feature = "cuda"), allow(unused_mut))]
85 let (mut model, _) = ShapeVae::load(path, &mut StdRng::seed_from_u64(args.seed))?;
86 if let Some(device) = args.cuda {
87 #[cfg(feature = "cuda")]
88 model.to_cuda(device, 0)?;
89 #[cfg(not(feature = "cuda"))]
90 return Err(format!(
91 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
92 )
93 .into());
94 }
95
96 let (mut draw_l1, mut mean_l1, mut sigma, mut magnitude, mut kl, mut active) =
97 (0.0, 0.0, 0.0, 0.0, 0.0, 0.0);
98 for object in &objects {
99 let (mean, log_variance) = model.encode(&object.surface)?;
100 let (latent, _) = ShapeVae::sample(&mean, &log_variance, &mut rng);
101
102 let from_draw = model.decode(&latent, &object.queries, 4096)?;
103 let from_mean = model.decode(&mean, &object.queries, 4096)?;
104 draw_l1 += losses::clamped_l1(&from_draw, &object.distances, args.clamp).0;
105 mean_l1 += losses::clamped_l1(&from_mean, &object.distances, args.clamp).0;
106
107 let units = mean.data.len() as f32;
108 magnitude += mean.data.iter().map(|value| value.abs()).sum::<f32>() / units;
109 sigma += log_variance
110 .data
111 .iter()
112 .map(|value| (0.5 * value).exp())
113 .sum::<f32>()
114 / units;
115 // The per-unit KL against a unit Gaussian, which is what a
116 // free-bits floor would be applied to one unit at a time.
117 let mut per_unit = 0.0;
118 let mut above = 0.0;
119 for (mean, log_variance) in mean.data.iter().zip(&log_variance.data) {
120 let nats = 0.5 * (mean * mean + log_variance.exp() - 1.0 - log_variance);
121 per_unit += nats;
122 if nats > 0.01 {
123 above += 1.0;
124 }
125 }
126 kl += per_unit / units;
127 active += above / units;
128 }
129
130 let count = objects.len() as f32;
131 println!(
132 "{:<28} {:>9.5} {:>9.5} {:>7.3} {:>7.3} {:>8.4} {:>6.1}%",
133 path.file_name()
134 .unwrap_or(path.as_os_str())
135 .to_string_lossy(),
136 draw_l1 / count,
137 mean_l1 / count,
138 sigma / count,
139 magnitude / count,
140 kl / count,
141 100.0 * active / count
142 );
143 }
144 Ok(())
145}More examples
126fn train_vae(
127 args: &Args,
128 stream: BatchStream,
129 rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131 let (mut model, mut step) = match &args.resume {
132 Some(path) => {
133 let (model, metadata) = ShapeVae::load(path, rng)?;
134 (model, started_at(&metadata))
135 }
136 None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137 };
138 if let Some(device) = args.cuda {
139 #[cfg(feature = "cuda")]
140 {
141 // 0 as the budget: the caller picked the card, and the weights are
142 // a small part of what a run of this size needs.
143 model.to_cuda(device, 0)?;
144 println!("shape autoencoder: training on CUDA device {device}");
145 }
146 #[cfg(not(feature = "cuda"))]
147 return Err(format!(
148 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149 )
150 .into());
151 }
152 let optimizer = adam(args.learning_rate);
153 println!(
154 "shape autoencoder: {} parameters, from step {step}",
155 model.num_parameters()
156 );
157
158 let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159 for batch in stream {
160 let batch = batch?;
161 let count = batch.len();
162 let points = batch.surface.rows / count;
163 let queries = batch.queries.rows / count;
164
165 // Packing helps the ordinary 4K-query training shape, but at 16K
166 // queries the larger attention workspace is slower and batch eight can
167 // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168 let cuda_batch = model.on_device() && queries <= 4096;
169 #[cfg(feature = "cuda")]
170 if cuda_batch {
171 let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172 // The packed losses are means across the batch; the reporting
173 // accumulator below historically stores a sum over examples.
174 reconstruction += loss * count as f32;
175 divergence += kl * count as f32;
176 tint += color * count as f32;
177 }
178
179 if !cuda_batch {
180 for index in 0..count {
181 let surface = slice(&batch.surface, index, points);
182 let (mean, log_variance, cache) = model.encode_train(&surface)?;
183 let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185 let (prediction, decoded) =
186 model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187 let target = &batch.distances[index * queries..(index + 1) * queries];
188 let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189 let (kl, grad_mean, grad_log_variance) =
190 losses::kl_divergence(&mean.data, &log_variance.data);
191 reconstruction += loss;
192 divergence += kl;
193
194 let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196 // The colour targets sit on the surface points, not the distance
197 // queries, so the colour head trains on a second short decode.
198 // The loader's points are already in random order, so the first
199 // `--color-points` of them are a random draw.
200 if let Some(colors) = &batch.colors
201 && args.color > 0.0
202 && args.color_points > 0
203 {
204 let wanted = args.color_points.min(points);
205 let positions = positions(&surface, wanted);
206 let targets = head_rows(&slice(colors, index, points), wanted);
207 let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209 let predicted = model.colors(&decoded)?;
210 let mut grad = predicted.clone();
211 let scale = args.color / predicted.data.len() as f32;
212 for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213 let error = *slot - target;
214 tint += error * error / predicted.data.len() as f32;
215 *slot = 2.0 * scale * error;
216 }
217
218 let from_colors = model.decode_backward_colored(
219 &decoded,
220 &vec![0.0; positions.rows],
221 Some(&grad),
222 )?;
223 for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224 *slot += value;
225 }
226 }
227
228 let (mut into_mean, mut into_log_variance) =
229 ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230 for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231 *slot += args.kl * value;
232 }
233 for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234 *slot += args.kl * value;
235 }
236 model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237 }
238 }
239
240 step += 1;
241 // The gradients were summed over the batch, so the step is scaled by
242 // its size rather than the loop dividing every gradient it makes.
243 optimizers::step_clipped(
244 &mut model.params_mut(),
245 &optimizer,
246 step,
247 1.0 / count as f32,
248 1.0,
249 )?;
250 optimizers::zero_grad(&mut model.params_mut());
251
252 if step % args.report == 0 {
253 let scale = (args.report * count) as f32;
254 println!(
255 "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256 reconstruction / scale,
257 divergence / scale,
258 tint / scale
259 );
260 reconstruction = 0.0;
261 divergence = 0.0;
262 tint = 0.0;
263 }
264 if step % args.checkpoint_every == 0 || step == args.steps {
265 model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266 println!("wrote {} at step {step}", args.out.display());
267 }
268 if step >= args.steps {
269 break;
270 }
271 }
272 Ok(())
273}
274
275/// One packed device step. Packing is important here: the old loop launched
276/// and synchronized the whole encoder and decoder once per shape.
277#[cfg(feature = "cuda")]
278fn train_vae_cuda_batch(
279 args: &Args,
280 model: &mut ShapeVae,
281 batch: &Batch,
282 rng: &mut StdRng,
283) -> Result<(f32, f32, f32), Box<dyn std::error::Error>> {
284 let count = batch.len();
285 let points = batch.surface.rows / count;
286 let (mean, log_variance, encoder) = model.encode_train_batch(&batch.surface, count)?;
287 let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
288
289 let (prediction, decoded) = model.decode_train_batch(&latent, &batch.queries, count)?;
290 let (reconstruction, mut grad) = losses::clamped_l1(&prediction, &batch.distances, args.clamp);
291 let (divergence, mut grad_mean, mut grad_log_variance) =
292 losses::kl_divergence(&mean.data, &log_variance.data);
293 // The shape-at-a-time path accumulates one mean-loss gradient per example
294 // and lets the optimizer divide by the batch. Re-expand this packed mean
295 // so gradient clipping and the optimizer see the same summed gradient.
296 let batch_scale = count as f32;
297 grad.iter_mut().for_each(|value| *value *= batch_scale);
298 grad_mean.iter_mut().for_each(|value| *value *= batch_scale);
299 grad_log_variance
300 .iter_mut()
301 .for_each(|value| *value *= batch_scale);
302 let mut grad_latent = model.decode_backward(&decoded, &grad)?;
303
304 let mut tint = 0.0;
305 if let Some(colors) = &batch.colors
306 && args.color > 0.0
307 && args.color_points > 0
308 {
309 let wanted = args.color_points.min(points);
310 let positions = packed_prefix_rows(&batch.surface, count, points, wanted, 3);
311 let targets = packed_prefix_rows(colors, count, points, wanted, 3);
312 let (_, decoded) = model.decode_train_batch(&latent, &positions, count)?;
313 let predicted = model.colors(&decoded)?;
314 let mut grad = predicted.clone();
315 let scale = args.color * batch_scale / predicted.data.len() as f32;
316 for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
317 let error = *slot - target;
318 tint += error * error / predicted.data.len() as f32;
319 *slot = 2.0 * scale * error;
320 }
321 let from_colors =
322 model.decode_backward_colored(&decoded, &vec![0.0; positions.rows], Some(&grad))?;
323 for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
324 *slot += value;
325 }
326 }
327
328 let (mut into_mean, mut into_log_variance) =
329 ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
330 for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
331 *slot += args.kl * value;
332 }
333 for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
334 *slot += args.kl * value;
335 }
336 model.encode_backward(&encoder, &into_mean, &into_log_variance)?;
337 Ok((reconstruction, divergence, tint))
338}
339
340fn train_flow(
341 args: &Args,
342 (cond_tokens, cond_dim): (usize, usize),
343 stream: BatchStream,
344 rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346 // Frozen: the flow transformer is trained against a latent space that has
347 // already stopped moving, so only `encode` is ever called here.
348 let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349 let shape = *vae.config();
350
351 let (mut model, mut step) = match &args.resume {
352 Some(path) => {
353 let (model, metadata) = FlowTransformer::load(path, rng)?;
354 (model, started_at(&metadata))
355 }
356 None => {
357 // The latent shape is the autoencoder's and the condition's shape
358 // is the corpus's, whatever a supplied configuration says.
359 let config = FlowConfig {
360 latents: shape.latents,
361 latent_dim: shape.latent_dim,
362 cond_tokens,
363 cond_dim,
364 ..read_config(args, FlowConfig::default())?
365 };
366 (FlowTransformer::new(config, rng)?, 0)
367 }
368 };
369 if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370 return Err(format!(
371 "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372 model.config().cond_tokens,
373 model.config().cond_dim
374 )
375 .into());
376 }
377 model.set_optimizer(adam(args.learning_rate));
378 if let Some(device) = args.cuda {
379 #[cfg(feature = "cuda")]
380 {
381 // 0 as the budget: the caller picked the card, and the blocks'
382 // weights are a small part of what a run of this size needs.
383 model.to_cuda(device, 0)?;
384 println!("flow transformer: training on CUDA device {device}");
385 }
386 #[cfg(not(feature = "cuda"))]
387 return Err(format!(
388 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389 )
390 .into());
391 }
392 println!(
393 "flow transformer: {} parameters, from step {step}",
394 model.num_parameters()
395 );
396
397 let mut running = 0.0;
398 for batch in stream {
399 let batch = batch?;
400 let count = batch.len();
401 let points = batch.surface.rows / count;
402
403 let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404 for index in 0..count {
405 // A draw from the posterior, not its mean. Every autoencoder step
406 // decoded a draw, so a draw is the only kind of latent the frozen
407 // decoder has ever answered for. The posterior is wide enough for
408 // that to matter — sigma about twice the magnitude of the mean —
409 // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410 // a draw, so a flow trained towards the mean would be aiming at the
411 // one latent the decoder is worst at.
412 let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413 let width = shape.latents * shape.latent_dim;
414 for (slot, value) in clean.data[index * width..(index + 1) * width]
415 .iter_mut()
416 .zip(&latent.data)
417 {
418 *slot = value * args.latent_scale;
419 }
420 }
421 if step == 0 {
422 println!("latent spread after scaling: {:.3}", spread(&clean));
423 }
424
425 running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426 step += 1;
427 model.step_clipped(1.0, 1.0)?;
428 model.zero_grad();
429
430 if step % args.report == 0 {
431 println!(
432 "step {step}: velocity mse {:.5}",
433 running / args.report as f32
434 );
435 running = 0.0;
436 }
437 if step % args.checkpoint_every == 0 || step == args.steps {
438 model.save(
439 &args.out,
440 &BTreeMap::from([
441 step_at(step),
442 ("latent_scale".to_string(), args.latent_scale.to_string()),
443 ]),
444 )?;
445 println!("wrote {} at step {step}", args.out.display());
446 }
447 if step >= args.steps {
448 break;
449 }
450 }
451 Ok(())
452}Trait Implementations§
Source§impl<'de> Deserialize<'de> for ShapeVae
impl<'de> Deserialize<'de> for ShapeVae
Source§fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where
__D: Deserializer<'de>,
fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where
__D: Deserializer<'de>,
Auto Trait Implementations§
impl Forget for ShapeVae
impl Freeze for ShapeVae
impl Move for ShapeVae
impl RefUnwindSafe for ShapeVae
impl Send for ShapeVae
impl Sync for ShapeVae
impl Unpin for ShapeVae
impl UnsafeUnpin for ShapeVae
impl UnwindSafe for ShapeVae
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Source§impl<T> CloneToUninit for Twhere
T: Clone,
impl<T> CloneToUninit for Twhere
T: Clone,
impl<T> DeserializeOwned for Twhere
T: for<'de> Deserialize<'de>,
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more