Skip to main content

train_shape/
train_shape.rs

1//! Trains the two halves of the image-to-3D pipeline, one stage at a time.
2//!
3//! Both stages read the same corpus: the image tokens `vit_tokens` wrote and
4//! the surface and query samples `mesh_samples` wrote, paired by name.
5//!
6//! ```bash
7//! # Stage one: the shape autoencoder, which never looks at the images.
8//! cargo run --release --example train_shape -- vae \
9//!     --tokens tokens/ --samples samples/ --out vae.safetensors --steps 20000
10//!
11//! # Stage two: the flow transformer, conditioned on the tokens, writing into
12//! # the latent space stage one just built.
13//! cargo run --release --example train_shape -- flow \
14//!     --tokens tokens/ --samples samples/ --vae vae.safetensors \
15//!     --out flow.safetensors --steps 40000
16//! ```
17//!
18//! `--cuda <device>` moves both stages onto a CUDA device, with the crate
19//! built `--features cuda`: stage one's projections and attentions through
20//! [`rusting_brain::gpu_shape`], stage two's transformer blocks through
21//! `gpu_flow`. Stage one packs short equal-sized point clouds so a batch shares
22//! projection, attention and synchronization launches; large query sets stay
23//! shape-at-a-time to bound the attention workspace.
24//!
25//! The autoencoder is frozen during stage two: it is only asked to encode, and
26//! its gradients are never touched. That is what makes the latent space a
27//! fixed target rather than one that moves under the flow transformer.
28//!
29//! Stage one also trains the colour head, on the meshes whose samples carry
30//! vertex colours: `--color` weights its error against the distance error and
31//! `--color-points 0` turns it off. A corpus with no colours anywhere trains
32//! exactly as it did before the head existed, since a batch without colour
33//! targets skips it.
34//!
35//! `--latent-scale` multiplies the encoder's latent before the flow sees it.
36//! Flow matching mixes the latent with unit-variance noise, so a latent whose
37//! own spread is far from one trains badly. The first batch prints the spread
38//! it measured, and the scale is written into the checkpoint so
39//! `image_to_3d` divides by the same number.
40//!
41//! The CPU path remains one shape at a time inside a batch. CUDA batches are
42//! uniform because the corpus sampler already selects the same point counts
43//! for every shape, so no padding or mask is needed.
44
45use rand::SeedableRng;
46use rand::rngs::StdRng;
47#[cfg(feature = "cuda")]
48use rusting_brain::Batch;
49use rusting_brain::{
50    BatchConfig, BatchStream, Corpus, FlowConfig, FlowTransformer, Matrix, Optimizer, ShapeVae,
51    ShapeVaeConfig, losses, optimizers,
52};
53use std::collections::BTreeMap;
54use std::path::PathBuf;
55
56#[derive(Clone, Copy, PartialEq)]
57enum Stage {
58    Vae,
59    Flow,
60}
61
62struct Args {
63    stage: Stage,
64    tokens: PathBuf,
65    samples: PathBuf,
66    vae: PathBuf,
67    out: PathBuf,
68    resume: Option<PathBuf>,
69    config: Option<PathBuf>,
70    steps: usize,
71    batch: usize,
72    surface: usize,
73    queries: usize,
74    learning_rate: f32,
75    kl: f32,
76    clamp: f32,
77    /// How heavily the colour head's error counts against the distance error.
78    color: f32,
79    /// Surface points per example the colour head is trained on. 0 turns the
80    /// colour head off.
81    color_points: usize,
82    dropout: f32,
83    latent_scale: f32,
84    report: usize,
85    checkpoint_every: usize,
86    seed: u64,
87    /// Which CUDA device the flow transformer's blocks train on, if any.
88    cuda: Option<usize>,
89}
90
91fn main() -> Result<(), Box<dyn std::error::Error>> {
92    let args = parse()?;
93    let mut rng = StdRng::seed_from_u64(args.seed);
94
95    let mut corpus = Corpus::open(&args.tokens, &args.samples)?;
96    if corpus.is_empty() {
97        return Err("no render is paired with a mesh; check the two directories".into());
98    }
99    println!("{} paired examples", corpus.len());
100
101    // The token shape is the condition's shape, and only the file knows it.
102    // One example with a single point sampled is the cheapest way to ask.
103    let condition = match args.stage {
104        Stage::Vae => None,
105        Stage::Flow => {
106            let example = corpus.example(0, Some(1), Some(1), &mut rng)?;
107            Some((example.tokens.rows, example.tokens.cols))
108        }
109    };
110
111    let stream = corpus.stream(BatchConfig {
112        batch: args.batch,
113        surface_points: Some(args.surface),
114        queries: Some(args.queries),
115        epochs: None,
116        shuffle: true,
117        seed: args.seed,
118    })?;
119
120    match condition {
121        None => train_vae(&args, stream, &mut rng),
122        Some(condition) => train_flow(&args, condition, stream, &mut rng),
123    }
124}
125
126fn train_vae(
127    args: &Args,
128    stream: BatchStream,
129    rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131    let (mut model, mut step) = match &args.resume {
132        Some(path) => {
133            let (model, metadata) = ShapeVae::load(path, rng)?;
134            (model, started_at(&metadata))
135        }
136        None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137    };
138    if let Some(device) = args.cuda {
139        #[cfg(feature = "cuda")]
140        {
141            // 0 as the budget: the caller picked the card, and the weights are
142            // a small part of what a run of this size needs.
143            model.to_cuda(device, 0)?;
144            println!("shape autoencoder: training on CUDA device {device}");
145        }
146        #[cfg(not(feature = "cuda"))]
147        return Err(format!(
148            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149        )
150        .into());
151    }
152    let optimizer = adam(args.learning_rate);
153    println!(
154        "shape autoencoder: {} parameters, from step {step}",
155        model.num_parameters()
156    );
157
158    let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159    for batch in stream {
160        let batch = batch?;
161        let count = batch.len();
162        let points = batch.surface.rows / count;
163        let queries = batch.queries.rows / count;
164
165        // Packing helps the ordinary 4K-query training shape, but at 16K
166        // queries the larger attention workspace is slower and batch eight can
167        // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168        let cuda_batch = model.on_device() && queries <= 4096;
169        #[cfg(feature = "cuda")]
170        if cuda_batch {
171            let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172            // The packed losses are means across the batch; the reporting
173            // accumulator below historically stores a sum over examples.
174            reconstruction += loss * count as f32;
175            divergence += kl * count as f32;
176            tint += color * count as f32;
177        }
178
179        if !cuda_batch {
180            for index in 0..count {
181                let surface = slice(&batch.surface, index, points);
182                let (mean, log_variance, cache) = model.encode_train(&surface)?;
183                let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185                let (prediction, decoded) =
186                    model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187                let target = &batch.distances[index * queries..(index + 1) * queries];
188                let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189                let (kl, grad_mean, grad_log_variance) =
190                    losses::kl_divergence(&mean.data, &log_variance.data);
191                reconstruction += loss;
192                divergence += kl;
193
194                let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196                // The colour targets sit on the surface points, not the distance
197                // queries, so the colour head trains on a second short decode.
198                // The loader's points are already in random order, so the first
199                // `--color-points` of them are a random draw.
200                if let Some(colors) = &batch.colors
201                    && args.color > 0.0
202                    && args.color_points > 0
203                {
204                    let wanted = args.color_points.min(points);
205                    let positions = positions(&surface, wanted);
206                    let targets = head_rows(&slice(colors, index, points), wanted);
207                    let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209                    let predicted = model.colors(&decoded)?;
210                    let mut grad = predicted.clone();
211                    let scale = args.color / predicted.data.len() as f32;
212                    for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213                        let error = *slot - target;
214                        tint += error * error / predicted.data.len() as f32;
215                        *slot = 2.0 * scale * error;
216                    }
217
218                    let from_colors = model.decode_backward_colored(
219                        &decoded,
220                        &vec![0.0; positions.rows],
221                        Some(&grad),
222                    )?;
223                    for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224                        *slot += value;
225                    }
226                }
227
228                let (mut into_mean, mut into_log_variance) =
229                    ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230                for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231                    *slot += args.kl * value;
232                }
233                for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234                    *slot += args.kl * value;
235                }
236                model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237            }
238        }
239
240        step += 1;
241        // The gradients were summed over the batch, so the step is scaled by
242        // its size rather than the loop dividing every gradient it makes.
243        optimizers::step_clipped(
244            &mut model.params_mut(),
245            &optimizer,
246            step,
247            1.0 / count as f32,
248            1.0,
249        )?;
250        optimizers::zero_grad(&mut model.params_mut());
251
252        if step % args.report == 0 {
253            let scale = (args.report * count) as f32;
254            println!(
255                "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256                reconstruction / scale,
257                divergence / scale,
258                tint / scale
259            );
260            reconstruction = 0.0;
261            divergence = 0.0;
262            tint = 0.0;
263        }
264        if step % args.checkpoint_every == 0 || step == args.steps {
265            model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266            println!("wrote {} at step {step}", args.out.display());
267        }
268        if step >= args.steps {
269            break;
270        }
271    }
272    Ok(())
273}
274
275/// One packed device step. Packing is important here: the old loop launched
276/// and synchronized the whole encoder and decoder once per shape.
277#[cfg(feature = "cuda")]
278fn train_vae_cuda_batch(
279    args: &Args,
280    model: &mut ShapeVae,
281    batch: &Batch,
282    rng: &mut StdRng,
283) -> Result<(f32, f32, f32), Box<dyn std::error::Error>> {
284    let count = batch.len();
285    let points = batch.surface.rows / count;
286    let (mean, log_variance, encoder) = model.encode_train_batch(&batch.surface, count)?;
287    let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
288
289    let (prediction, decoded) = model.decode_train_batch(&latent, &batch.queries, count)?;
290    let (reconstruction, mut grad) = losses::clamped_l1(&prediction, &batch.distances, args.clamp);
291    let (divergence, mut grad_mean, mut grad_log_variance) =
292        losses::kl_divergence(&mean.data, &log_variance.data);
293    // The shape-at-a-time path accumulates one mean-loss gradient per example
294    // and lets the optimizer divide by the batch. Re-expand this packed mean
295    // so gradient clipping and the optimizer see the same summed gradient.
296    let batch_scale = count as f32;
297    grad.iter_mut().for_each(|value| *value *= batch_scale);
298    grad_mean.iter_mut().for_each(|value| *value *= batch_scale);
299    grad_log_variance
300        .iter_mut()
301        .for_each(|value| *value *= batch_scale);
302    let mut grad_latent = model.decode_backward(&decoded, &grad)?;
303
304    let mut tint = 0.0;
305    if let Some(colors) = &batch.colors
306        && args.color > 0.0
307        && args.color_points > 0
308    {
309        let wanted = args.color_points.min(points);
310        let positions = packed_prefix_rows(&batch.surface, count, points, wanted, 3);
311        let targets = packed_prefix_rows(colors, count, points, wanted, 3);
312        let (_, decoded) = model.decode_train_batch(&latent, &positions, count)?;
313        let predicted = model.colors(&decoded)?;
314        let mut grad = predicted.clone();
315        let scale = args.color * batch_scale / predicted.data.len() as f32;
316        for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
317            let error = *slot - target;
318            tint += error * error / predicted.data.len() as f32;
319            *slot = 2.0 * scale * error;
320        }
321        let from_colors =
322            model.decode_backward_colored(&decoded, &vec![0.0; positions.rows], Some(&grad))?;
323        for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
324            *slot += value;
325        }
326    }
327
328    let (mut into_mean, mut into_log_variance) =
329        ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
330    for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
331        *slot += args.kl * value;
332    }
333    for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
334        *slot += args.kl * value;
335    }
336    model.encode_backward(&encoder, &into_mean, &into_log_variance)?;
337    Ok((reconstruction, divergence, tint))
338}
339
340fn train_flow(
341    args: &Args,
342    (cond_tokens, cond_dim): (usize, usize),
343    stream: BatchStream,
344    rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346    // Frozen: the flow transformer is trained against a latent space that has
347    // already stopped moving, so only `encode` is ever called here.
348    let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349    let shape = *vae.config();
350
351    let (mut model, mut step) = match &args.resume {
352        Some(path) => {
353            let (model, metadata) = FlowTransformer::load(path, rng)?;
354            (model, started_at(&metadata))
355        }
356        None => {
357            // The latent shape is the autoencoder's and the condition's shape
358            // is the corpus's, whatever a supplied configuration says.
359            let config = FlowConfig {
360                latents: shape.latents,
361                latent_dim: shape.latent_dim,
362                cond_tokens,
363                cond_dim,
364                ..read_config(args, FlowConfig::default())?
365            };
366            (FlowTransformer::new(config, rng)?, 0)
367        }
368    };
369    if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370        return Err(format!(
371            "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372            model.config().cond_tokens,
373            model.config().cond_dim
374        )
375        .into());
376    }
377    model.set_optimizer(adam(args.learning_rate));
378    if let Some(device) = args.cuda {
379        #[cfg(feature = "cuda")]
380        {
381            // 0 as the budget: the caller picked the card, and the blocks'
382            // weights are a small part of what a run of this size needs.
383            model.to_cuda(device, 0)?;
384            println!("flow transformer: training on CUDA device {device}");
385        }
386        #[cfg(not(feature = "cuda"))]
387        return Err(format!(
388            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389        )
390        .into());
391    }
392    println!(
393        "flow transformer: {} parameters, from step {step}",
394        model.num_parameters()
395    );
396
397    let mut running = 0.0;
398    for batch in stream {
399        let batch = batch?;
400        let count = batch.len();
401        let points = batch.surface.rows / count;
402
403        let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404        for index in 0..count {
405            // A draw from the posterior, not its mean. Every autoencoder step
406            // decoded a draw, so a draw is the only kind of latent the frozen
407            // decoder has ever answered for. The posterior is wide enough for
408            // that to matter — sigma about twice the magnitude of the mean —
409            // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410            // a draw, so a flow trained towards the mean would be aiming at the
411            // one latent the decoder is worst at.
412            let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413            let width = shape.latents * shape.latent_dim;
414            for (slot, value) in clean.data[index * width..(index + 1) * width]
415                .iter_mut()
416                .zip(&latent.data)
417            {
418                *slot = value * args.latent_scale;
419            }
420        }
421        if step == 0 {
422            println!("latent spread after scaling: {:.3}", spread(&clean));
423        }
424
425        running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426        step += 1;
427        model.step_clipped(1.0, 1.0)?;
428        model.zero_grad();
429
430        if step % args.report == 0 {
431            println!(
432                "step {step}: velocity mse {:.5}",
433                running / args.report as f32
434            );
435            running = 0.0;
436        }
437        if step % args.checkpoint_every == 0 || step == args.steps {
438            model.save(
439                &args.out,
440                &BTreeMap::from([
441                    step_at(step),
442                    ("latent_scale".to_string(), args.latent_scale.to_string()),
443                ]),
444            )?;
445            println!("wrote {} at step {step}", args.out.display());
446        }
447        if step >= args.steps {
448            break;
449        }
450    }
451    Ok(())
452}
453
454/// The autoencoder's shape, from `--config` or the plan's default.
455fn shape_config(args: &Args) -> Result<ShapeVaeConfig, Box<dyn std::error::Error>> {
456    read_config(args, ShapeVaeConfig::default())
457}
458
459/// A configuration read from `--config`, or the default when there is none.
460///
461/// The configurations are `serde` types already, so a JSON file is the whole
462/// of the knob: no per-field flag, and a run's shape is a file it can keep
463/// beside its checkpoint.
464fn read_config<T: serde::de::DeserializeOwned>(
465    args: &Args,
466    fallback: T,
467) -> Result<T, Box<dyn std::error::Error>> {
468    match &args.config {
469        Some(path) => Ok(serde_json::from_str(&std::fs::read_to_string(path)?)?),
470        None => Ok(fallback),
471    }
472}
473
474/// One example's rows out of a batch's stacked matrix.
475/// The first `rows` rows of a matrix.
476fn head_rows(matrix: &Matrix, rows: usize) -> Matrix {
477    Matrix::from_vec(
478        rows,
479        matrix.cols,
480        matrix.data[..rows * matrix.cols].to_vec(),
481    )
482}
483
484/// The first `wanted` rows from every equally sized example in a packed batch.
485/// `cols` may select a prefix, which drops normals from surface rows.
486#[cfg(feature = "cuda")]
487fn packed_prefix_rows(
488    matrix: &Matrix,
489    examples: usize,
490    rows_per_example: usize,
491    wanted: usize,
492    cols: usize,
493) -> Matrix {
494    let mut data = Vec::with_capacity(examples * wanted * cols);
495    for example in 0..examples {
496        let first = example * rows_per_example;
497        for row in first..first + wanted {
498            data.extend_from_slice(&matrix.row(row)[..cols]);
499        }
500    }
501    Matrix::from_vec(examples * wanted, cols, data)
502}
503
504/// The positions of the first `rows` surface points, dropping the normals.
505fn positions(surface: &Matrix, rows: usize) -> Matrix {
506    Matrix::from_vec(
507        rows,
508        3,
509        surface
510            .data
511            .chunks_exact(surface.cols)
512            .take(rows)
513            .flat_map(|point| point[..3].iter().copied())
514            .collect(),
515    )
516}
517
518fn slice(stacked: &Matrix, index: usize, rows: usize) -> Matrix {
519    let width = rows * stacked.cols;
520    Matrix::from_vec(
521        rows,
522        stacked.cols,
523        stacked.data[index * width..(index + 1) * width].to_vec(),
524    )
525}
526
527/// The standard deviation of a matrix, which is what `--latent-scale` tunes.
528fn spread(values: &Matrix) -> f32 {
529    let count = values.data.len().max(1) as f32;
530    let mean = values.data.iter().sum::<f32>() / count;
531    (values
532        .data
533        .iter()
534        .map(|value| (value - mean) * (value - mean))
535        .sum::<f32>()
536        / count)
537        .sqrt()
538}
539
540fn adam(learning_rate: f32) -> Optimizer {
541    Optimizer::Adam {
542        learning_rate,
543        beta1: 0.9,
544        beta2: 0.95,
545        epsilon: 1e-8,
546        weight_decay: 0.0,
547    }
548}
549
550fn step_at(step: usize) -> (String, String) {
551    ("step".to_string(), step.to_string())
552}
553
554fn started_at(metadata: &BTreeMap<String, String>) -> usize {
555    metadata
556        .get("step")
557        .and_then(|step| step.parse().ok())
558        .unwrap_or(0)
559}
560
561fn parse() -> Result<Args, Box<dyn std::error::Error>> {
562    let mut rest = std::env::args().skip(1);
563    let stage = match rest.next().as_deref() {
564        Some("vae") => Stage::Vae,
565        Some("flow") => Stage::Flow,
566        _ => return Err(usage().into()),
567    };
568    let mut args = Args {
569        stage,
570        tokens: PathBuf::from("tokens"),
571        samples: PathBuf::from("samples"),
572        vae: PathBuf::from("vae.safetensors"),
573        out: PathBuf::from(match stage {
574            Stage::Vae => "vae.safetensors",
575            Stage::Flow => "flow.safetensors",
576        }),
577        resume: None,
578        config: None,
579        steps: 10_000,
580        batch: 4,
581        surface: 2048,
582        queries: 4096,
583        learning_rate: 1e-4,
584        kl: 1e-4,
585        clamp: 0.1,
586        color: 0.1,
587        color_points: 512,
588        dropout: 0.1,
589        latent_scale: 1.0,
590        report: 50,
591        checkpoint_every: 1000,
592        seed: 0,
593        cuda: None,
594    };
595    while let Some(flag) = rest.next() {
596        let mut value = || rest.next().ok_or(format!("{flag} needs a value"));
597        match flag.as_str() {
598            "--tokens" => args.tokens = value()?.into(),
599            "--samples" => args.samples = value()?.into(),
600            "--vae" => args.vae = value()?.into(),
601            "--out" => args.out = value()?.into(),
602            "--resume" => args.resume = Some(value()?.into()),
603            "--config" => args.config = Some(value()?.into()),
604            "--steps" => args.steps = value()?.parse()?,
605            "--batch" => args.batch = value()?.parse()?,
606            "--surface" => args.surface = value()?.parse()?,
607            "--queries" => args.queries = value()?.parse()?,
608            "--learning-rate" => args.learning_rate = value()?.parse()?,
609            "--kl" => args.kl = value()?.parse()?,
610            "--clamp" => args.clamp = value()?.parse()?,
611            "--color" => args.color = value()?.parse()?,
612            "--color-points" => args.color_points = value()?.parse()?,
613            "--dropout" => args.dropout = value()?.parse()?,
614            "--latent-scale" => args.latent_scale = value()?.parse()?,
615            "--report" => args.report = value()?.parse()?,
616            "--checkpoint-every" => args.checkpoint_every = value()?.parse()?,
617            "--seed" => args.seed = value()?.parse()?,
618            "--cuda" => args.cuda = Some(value()?.parse()?),
619            other => return Err(format!("unknown flag {other}\n{}", usage()).into()),
620        }
621    }
622    if args.batch == 0 || args.steps == 0 {
623        return Err("--batch and --steps both need to be at least one".into());
624    }
625    if args.report == 0 || args.checkpoint_every == 0 {
626        return Err("--report and --checkpoint-every are counted in steps, so at least one".into());
627    }
628    if !(0.0..=1.0).contains(&args.dropout) {
629        return Err("--dropout is the chance of dropping a condition, so 0 to 1".into());
630    }
631    Ok(args)
632}
633
634fn usage() -> String {
635    "usage: train_shape <vae|flow> [--tokens <dir>] [--samples <dir>] [--vae <file>] \
636     [--out <file>] [--resume <file>] [--config <file.json>] [--steps n] [--batch n] [--surface n] [--queries n] \
637     [--learning-rate f] [--kl f] [--clamp f] [--color f] [--color-points n] [--dropout f] \
638     [--latent-scale f] [--report n] \
639     [--checkpoint-every n] [--seed n] [--cuda device]"
640        .to_string()
641}