1use rand::SeedableRng;
46use rand::rngs::StdRng;
47#[cfg(feature = "cuda")]
48use rusting_brain::Batch;
49use rusting_brain::{
50 BatchConfig, BatchStream, Corpus, FlowConfig, FlowTransformer, Matrix, Optimizer, ShapeVae,
51 ShapeVaeConfig, losses, optimizers,
52};
53use std::collections::BTreeMap;
54use std::path::PathBuf;
55
56#[derive(Clone, Copy, PartialEq)]
57enum Stage {
58 Vae,
59 Flow,
60}
61
62struct Args {
63 stage: Stage,
64 tokens: PathBuf,
65 samples: PathBuf,
66 vae: PathBuf,
67 out: PathBuf,
68 resume: Option<PathBuf>,
69 config: Option<PathBuf>,
70 steps: usize,
71 batch: usize,
72 surface: usize,
73 queries: usize,
74 learning_rate: f32,
75 kl: f32,
76 clamp: f32,
77 color: f32,
79 color_points: usize,
82 dropout: f32,
83 latent_scale: f32,
84 report: usize,
85 checkpoint_every: usize,
86 seed: u64,
87 cuda: Option<usize>,
89}
90
91fn main() -> Result<(), Box<dyn std::error::Error>> {
92 let args = parse()?;
93 let mut rng = StdRng::seed_from_u64(args.seed);
94
95 let mut corpus = Corpus::open(&args.tokens, &args.samples)?;
96 if corpus.is_empty() {
97 return Err("no render is paired with a mesh; check the two directories".into());
98 }
99 println!("{} paired examples", corpus.len());
100
101 let condition = match args.stage {
104 Stage::Vae => None,
105 Stage::Flow => {
106 let example = corpus.example(0, Some(1), Some(1), &mut rng)?;
107 Some((example.tokens.rows, example.tokens.cols))
108 }
109 };
110
111 let stream = corpus.stream(BatchConfig {
112 batch: args.batch,
113 surface_points: Some(args.surface),
114 queries: Some(args.queries),
115 epochs: None,
116 shuffle: true,
117 seed: args.seed,
118 })?;
119
120 match condition {
121 None => train_vae(&args, stream, &mut rng),
122 Some(condition) => train_flow(&args, condition, stream, &mut rng),
123 }
124}
125
126fn train_vae(
127 args: &Args,
128 stream: BatchStream,
129 rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131 let (mut model, mut step) = match &args.resume {
132 Some(path) => {
133 let (model, metadata) = ShapeVae::load(path, rng)?;
134 (model, started_at(&metadata))
135 }
136 None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137 };
138 if let Some(device) = args.cuda {
139 #[cfg(feature = "cuda")]
140 {
141 model.to_cuda(device, 0)?;
144 println!("shape autoencoder: training on CUDA device {device}");
145 }
146 #[cfg(not(feature = "cuda"))]
147 return Err(format!(
148 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149 )
150 .into());
151 }
152 let optimizer = adam(args.learning_rate);
153 println!(
154 "shape autoencoder: {} parameters, from step {step}",
155 model.num_parameters()
156 );
157
158 let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159 for batch in stream {
160 let batch = batch?;
161 let count = batch.len();
162 let points = batch.surface.rows / count;
163 let queries = batch.queries.rows / count;
164
165 let cuda_batch = model.on_device() && queries <= 4096;
169 #[cfg(feature = "cuda")]
170 if cuda_batch {
171 let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172 reconstruction += loss * count as f32;
175 divergence += kl * count as f32;
176 tint += color * count as f32;
177 }
178
179 if !cuda_batch {
180 for index in 0..count {
181 let surface = slice(&batch.surface, index, points);
182 let (mean, log_variance, cache) = model.encode_train(&surface)?;
183 let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185 let (prediction, decoded) =
186 model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187 let target = &batch.distances[index * queries..(index + 1) * queries];
188 let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189 let (kl, grad_mean, grad_log_variance) =
190 losses::kl_divergence(&mean.data, &log_variance.data);
191 reconstruction += loss;
192 divergence += kl;
193
194 let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196 if let Some(colors) = &batch.colors
201 && args.color > 0.0
202 && args.color_points > 0
203 {
204 let wanted = args.color_points.min(points);
205 let positions = positions(&surface, wanted);
206 let targets = head_rows(&slice(colors, index, points), wanted);
207 let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209 let predicted = model.colors(&decoded)?;
210 let mut grad = predicted.clone();
211 let scale = args.color / predicted.data.len() as f32;
212 for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213 let error = *slot - target;
214 tint += error * error / predicted.data.len() as f32;
215 *slot = 2.0 * scale * error;
216 }
217
218 let from_colors = model.decode_backward_colored(
219 &decoded,
220 &vec![0.0; positions.rows],
221 Some(&grad),
222 )?;
223 for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224 *slot += value;
225 }
226 }
227
228 let (mut into_mean, mut into_log_variance) =
229 ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230 for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231 *slot += args.kl * value;
232 }
233 for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234 *slot += args.kl * value;
235 }
236 model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237 }
238 }
239
240 step += 1;
241 optimizers::step_clipped(
244 &mut model.params_mut(),
245 &optimizer,
246 step,
247 1.0 / count as f32,
248 1.0,
249 )?;
250 optimizers::zero_grad(&mut model.params_mut());
251
252 if step % args.report == 0 {
253 let scale = (args.report * count) as f32;
254 println!(
255 "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256 reconstruction / scale,
257 divergence / scale,
258 tint / scale
259 );
260 reconstruction = 0.0;
261 divergence = 0.0;
262 tint = 0.0;
263 }
264 if step % args.checkpoint_every == 0 || step == args.steps {
265 model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266 println!("wrote {} at step {step}", args.out.display());
267 }
268 if step >= args.steps {
269 break;
270 }
271 }
272 Ok(())
273}
274
275#[cfg(feature = "cuda")]
278fn train_vae_cuda_batch(
279 args: &Args,
280 model: &mut ShapeVae,
281 batch: &Batch,
282 rng: &mut StdRng,
283) -> Result<(f32, f32, f32), Box<dyn std::error::Error>> {
284 let count = batch.len();
285 let points = batch.surface.rows / count;
286 let (mean, log_variance, encoder) = model.encode_train_batch(&batch.surface, count)?;
287 let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
288
289 let (prediction, decoded) = model.decode_train_batch(&latent, &batch.queries, count)?;
290 let (reconstruction, mut grad) = losses::clamped_l1(&prediction, &batch.distances, args.clamp);
291 let (divergence, mut grad_mean, mut grad_log_variance) =
292 losses::kl_divergence(&mean.data, &log_variance.data);
293 let batch_scale = count as f32;
297 grad.iter_mut().for_each(|value| *value *= batch_scale);
298 grad_mean.iter_mut().for_each(|value| *value *= batch_scale);
299 grad_log_variance
300 .iter_mut()
301 .for_each(|value| *value *= batch_scale);
302 let mut grad_latent = model.decode_backward(&decoded, &grad)?;
303
304 let mut tint = 0.0;
305 if let Some(colors) = &batch.colors
306 && args.color > 0.0
307 && args.color_points > 0
308 {
309 let wanted = args.color_points.min(points);
310 let positions = packed_prefix_rows(&batch.surface, count, points, wanted, 3);
311 let targets = packed_prefix_rows(colors, count, points, wanted, 3);
312 let (_, decoded) = model.decode_train_batch(&latent, &positions, count)?;
313 let predicted = model.colors(&decoded)?;
314 let mut grad = predicted.clone();
315 let scale = args.color * batch_scale / predicted.data.len() as f32;
316 for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
317 let error = *slot - target;
318 tint += error * error / predicted.data.len() as f32;
319 *slot = 2.0 * scale * error;
320 }
321 let from_colors =
322 model.decode_backward_colored(&decoded, &vec![0.0; positions.rows], Some(&grad))?;
323 for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
324 *slot += value;
325 }
326 }
327
328 let (mut into_mean, mut into_log_variance) =
329 ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
330 for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
331 *slot += args.kl * value;
332 }
333 for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
334 *slot += args.kl * value;
335 }
336 model.encode_backward(&encoder, &into_mean, &into_log_variance)?;
337 Ok((reconstruction, divergence, tint))
338}
339
340fn train_flow(
341 args: &Args,
342 (cond_tokens, cond_dim): (usize, usize),
343 stream: BatchStream,
344 rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346 let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349 let shape = *vae.config();
350
351 let (mut model, mut step) = match &args.resume {
352 Some(path) => {
353 let (model, metadata) = FlowTransformer::load(path, rng)?;
354 (model, started_at(&metadata))
355 }
356 None => {
357 let config = FlowConfig {
360 latents: shape.latents,
361 latent_dim: shape.latent_dim,
362 cond_tokens,
363 cond_dim,
364 ..read_config(args, FlowConfig::default())?
365 };
366 (FlowTransformer::new(config, rng)?, 0)
367 }
368 };
369 if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370 return Err(format!(
371 "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372 model.config().cond_tokens,
373 model.config().cond_dim
374 )
375 .into());
376 }
377 model.set_optimizer(adam(args.learning_rate));
378 if let Some(device) = args.cuda {
379 #[cfg(feature = "cuda")]
380 {
381 model.to_cuda(device, 0)?;
384 println!("flow transformer: training on CUDA device {device}");
385 }
386 #[cfg(not(feature = "cuda"))]
387 return Err(format!(
388 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389 )
390 .into());
391 }
392 println!(
393 "flow transformer: {} parameters, from step {step}",
394 model.num_parameters()
395 );
396
397 let mut running = 0.0;
398 for batch in stream {
399 let batch = batch?;
400 let count = batch.len();
401 let points = batch.surface.rows / count;
402
403 let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404 for index in 0..count {
405 let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413 let width = shape.latents * shape.latent_dim;
414 for (slot, value) in clean.data[index * width..(index + 1) * width]
415 .iter_mut()
416 .zip(&latent.data)
417 {
418 *slot = value * args.latent_scale;
419 }
420 }
421 if step == 0 {
422 println!("latent spread after scaling: {:.3}", spread(&clean));
423 }
424
425 running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426 step += 1;
427 model.step_clipped(1.0, 1.0)?;
428 model.zero_grad();
429
430 if step % args.report == 0 {
431 println!(
432 "step {step}: velocity mse {:.5}",
433 running / args.report as f32
434 );
435 running = 0.0;
436 }
437 if step % args.checkpoint_every == 0 || step == args.steps {
438 model.save(
439 &args.out,
440 &BTreeMap::from([
441 step_at(step),
442 ("latent_scale".to_string(), args.latent_scale.to_string()),
443 ]),
444 )?;
445 println!("wrote {} at step {step}", args.out.display());
446 }
447 if step >= args.steps {
448 break;
449 }
450 }
451 Ok(())
452}
453
454fn shape_config(args: &Args) -> Result<ShapeVaeConfig, Box<dyn std::error::Error>> {
456 read_config(args, ShapeVaeConfig::default())
457}
458
459fn read_config<T: serde::de::DeserializeOwned>(
465 args: &Args,
466 fallback: T,
467) -> Result<T, Box<dyn std::error::Error>> {
468 match &args.config {
469 Some(path) => Ok(serde_json::from_str(&std::fs::read_to_string(path)?)?),
470 None => Ok(fallback),
471 }
472}
473
474fn head_rows(matrix: &Matrix, rows: usize) -> Matrix {
477 Matrix::from_vec(
478 rows,
479 matrix.cols,
480 matrix.data[..rows * matrix.cols].to_vec(),
481 )
482}
483
484#[cfg(feature = "cuda")]
487fn packed_prefix_rows(
488 matrix: &Matrix,
489 examples: usize,
490 rows_per_example: usize,
491 wanted: usize,
492 cols: usize,
493) -> Matrix {
494 let mut data = Vec::with_capacity(examples * wanted * cols);
495 for example in 0..examples {
496 let first = example * rows_per_example;
497 for row in first..first + wanted {
498 data.extend_from_slice(&matrix.row(row)[..cols]);
499 }
500 }
501 Matrix::from_vec(examples * wanted, cols, data)
502}
503
504fn positions(surface: &Matrix, rows: usize) -> Matrix {
506 Matrix::from_vec(
507 rows,
508 3,
509 surface
510 .data
511 .chunks_exact(surface.cols)
512 .take(rows)
513 .flat_map(|point| point[..3].iter().copied())
514 .collect(),
515 )
516}
517
518fn slice(stacked: &Matrix, index: usize, rows: usize) -> Matrix {
519 let width = rows * stacked.cols;
520 Matrix::from_vec(
521 rows,
522 stacked.cols,
523 stacked.data[index * width..(index + 1) * width].to_vec(),
524 )
525}
526
527fn spread(values: &Matrix) -> f32 {
529 let count = values.data.len().max(1) as f32;
530 let mean = values.data.iter().sum::<f32>() / count;
531 (values
532 .data
533 .iter()
534 .map(|value| (value - mean) * (value - mean))
535 .sum::<f32>()
536 / count)
537 .sqrt()
538}
539
540fn adam(learning_rate: f32) -> Optimizer {
541 Optimizer::Adam {
542 learning_rate,
543 beta1: 0.9,
544 beta2: 0.95,
545 epsilon: 1e-8,
546 weight_decay: 0.0,
547 }
548}
549
550fn step_at(step: usize) -> (String, String) {
551 ("step".to_string(), step.to_string())
552}
553
554fn started_at(metadata: &BTreeMap<String, String>) -> usize {
555 metadata
556 .get("step")
557 .and_then(|step| step.parse().ok())
558 .unwrap_or(0)
559}
560
561fn parse() -> Result<Args, Box<dyn std::error::Error>> {
562 let mut rest = std::env::args().skip(1);
563 let stage = match rest.next().as_deref() {
564 Some("vae") => Stage::Vae,
565 Some("flow") => Stage::Flow,
566 _ => return Err(usage().into()),
567 };
568 let mut args = Args {
569 stage,
570 tokens: PathBuf::from("tokens"),
571 samples: PathBuf::from("samples"),
572 vae: PathBuf::from("vae.safetensors"),
573 out: PathBuf::from(match stage {
574 Stage::Vae => "vae.safetensors",
575 Stage::Flow => "flow.safetensors",
576 }),
577 resume: None,
578 config: None,
579 steps: 10_000,
580 batch: 4,
581 surface: 2048,
582 queries: 4096,
583 learning_rate: 1e-4,
584 kl: 1e-4,
585 clamp: 0.1,
586 color: 0.1,
587 color_points: 512,
588 dropout: 0.1,
589 latent_scale: 1.0,
590 report: 50,
591 checkpoint_every: 1000,
592 seed: 0,
593 cuda: None,
594 };
595 while let Some(flag) = rest.next() {
596 let mut value = || rest.next().ok_or(format!("{flag} needs a value"));
597 match flag.as_str() {
598 "--tokens" => args.tokens = value()?.into(),
599 "--samples" => args.samples = value()?.into(),
600 "--vae" => args.vae = value()?.into(),
601 "--out" => args.out = value()?.into(),
602 "--resume" => args.resume = Some(value()?.into()),
603 "--config" => args.config = Some(value()?.into()),
604 "--steps" => args.steps = value()?.parse()?,
605 "--batch" => args.batch = value()?.parse()?,
606 "--surface" => args.surface = value()?.parse()?,
607 "--queries" => args.queries = value()?.parse()?,
608 "--learning-rate" => args.learning_rate = value()?.parse()?,
609 "--kl" => args.kl = value()?.parse()?,
610 "--clamp" => args.clamp = value()?.parse()?,
611 "--color" => args.color = value()?.parse()?,
612 "--color-points" => args.color_points = value()?.parse()?,
613 "--dropout" => args.dropout = value()?.parse()?,
614 "--latent-scale" => args.latent_scale = value()?.parse()?,
615 "--report" => args.report = value()?.parse()?,
616 "--checkpoint-every" => args.checkpoint_every = value()?.parse()?,
617 "--seed" => args.seed = value()?.parse()?,
618 "--cuda" => args.cuda = Some(value()?.parse()?),
619 other => return Err(format!("unknown flag {other}\n{}", usage()).into()),
620 }
621 }
622 if args.batch == 0 || args.steps == 0 {
623 return Err("--batch and --steps both need to be at least one".into());
624 }
625 if args.report == 0 || args.checkpoint_every == 0 {
626 return Err("--report and --checkpoint-every are counted in steps, so at least one".into());
627 }
628 if !(0.0..=1.0).contains(&args.dropout) {
629 return Err("--dropout is the chance of dropping a condition, so 0 to 1".into());
630 }
631 Ok(args)
632}
633
634fn usage() -> String {
635 "usage: train_shape <vae|flow> [--tokens <dir>] [--samples <dir>] [--vae <file>] \
636 [--out <file>] [--resume <file>] [--config <file.json>] [--steps n] [--batch n] [--surface n] [--queries n] \
637 [--learning-rate f] [--kl f] [--clamp f] [--color f] [--color-points n] [--dropout f] \
638 [--latent-scale f] [--report n] \
639 [--checkpoint-every n] [--seed n] [--cuda device]"
640 .to_string()
641}