Skip to main content

ShapeVae

Struct ShapeVae 

Source
pub struct ShapeVae { /* private fields */ }
Expand description

Surface points in, a signed distance field out.

Implementations§

Source§

impl ShapeVae

Source

pub fn new( config: ShapeVaeConfig, rng: &mut StdRng, ) -> Result<Self, NetworkError>

Examples found in repository?
examples/train_shape.rs (line 136)
126fn train_vae(
127    args: &Args,
128    stream: BatchStream,
129    rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131    let (mut model, mut step) = match &args.resume {
132        Some(path) => {
133            let (model, metadata) = ShapeVae::load(path, rng)?;
134            (model, started_at(&metadata))
135        }
136        None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137    };
138    if let Some(device) = args.cuda {
139        #[cfg(feature = "cuda")]
140        {
141            // 0 as the budget: the caller picked the card, and the weights are
142            // a small part of what a run of this size needs.
143            model.to_cuda(device, 0)?;
144            println!("shape autoencoder: training on CUDA device {device}");
145        }
146        #[cfg(not(feature = "cuda"))]
147        return Err(format!(
148            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149        )
150        .into());
151    }
152    let optimizer = adam(args.learning_rate);
153    println!(
154        "shape autoencoder: {} parameters, from step {step}",
155        model.num_parameters()
156    );
157
158    let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159    for batch in stream {
160        let batch = batch?;
161        let count = batch.len();
162        let points = batch.surface.rows / count;
163        let queries = batch.queries.rows / count;
164
165        // Packing helps the ordinary 4K-query training shape, but at 16K
166        // queries the larger attention workspace is slower and batch eight can
167        // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168        let cuda_batch = model.on_device() && queries <= 4096;
169        #[cfg(feature = "cuda")]
170        if cuda_batch {
171            let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172            // The packed losses are means across the batch; the reporting
173            // accumulator below historically stores a sum over examples.
174            reconstruction += loss * count as f32;
175            divergence += kl * count as f32;
176            tint += color * count as f32;
177        }
178
179        if !cuda_batch {
180            for index in 0..count {
181                let surface = slice(&batch.surface, index, points);
182                let (mean, log_variance, cache) = model.encode_train(&surface)?;
183                let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185                let (prediction, decoded) =
186                    model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187                let target = &batch.distances[index * queries..(index + 1) * queries];
188                let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189                let (kl, grad_mean, grad_log_variance) =
190                    losses::kl_divergence(&mean.data, &log_variance.data);
191                reconstruction += loss;
192                divergence += kl;
193
194                let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196                // The colour targets sit on the surface points, not the distance
197                // queries, so the colour head trains on a second short decode.
198                // The loader's points are already in random order, so the first
199                // `--color-points` of them are a random draw.
200                if let Some(colors) = &batch.colors
201                    && args.color > 0.0
202                    && args.color_points > 0
203                {
204                    let wanted = args.color_points.min(points);
205                    let positions = positions(&surface, wanted);
206                    let targets = head_rows(&slice(colors, index, points), wanted);
207                    let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209                    let predicted = model.colors(&decoded)?;
210                    let mut grad = predicted.clone();
211                    let scale = args.color / predicted.data.len() as f32;
212                    for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213                        let error = *slot - target;
214                        tint += error * error / predicted.data.len() as f32;
215                        *slot = 2.0 * scale * error;
216                    }
217
218                    let from_colors = model.decode_backward_colored(
219                        &decoded,
220                        &vec![0.0; positions.rows],
221                        Some(&grad),
222                    )?;
223                    for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224                        *slot += value;
225                    }
226                }
227
228                let (mut into_mean, mut into_log_variance) =
229                    ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230                for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231                    *slot += args.kl * value;
232                }
233                for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234                    *slot += args.kl * value;
235                }
236                model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237            }
238        }
239
240        step += 1;
241        // The gradients were summed over the batch, so the step is scaled by
242        // its size rather than the loop dividing every gradient it makes.
243        optimizers::step_clipped(
244            &mut model.params_mut(),
245            &optimizer,
246            step,
247            1.0 / count as f32,
248            1.0,
249        )?;
250        optimizers::zero_grad(&mut model.params_mut());
251
252        if step % args.report == 0 {
253            let scale = (args.report * count) as f32;
254            println!(
255                "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256                reconstruction / scale,
257                divergence / scale,
258                tint / scale
259            );
260            reconstruction = 0.0;
261            divergence = 0.0;
262            tint = 0.0;
263        }
264        if step % args.checkpoint_every == 0 || step == args.steps {
265            model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266            println!("wrote {} at step {step}", args.out.display());
267        }
268        if step >= args.steps {
269            break;
270        }
271    }
272    Ok(())
273}
Source

pub fn config(&self) -> &ShapeVaeConfig

Examples found in repository?
examples/train_shape.rs (line 349)
340fn train_flow(
341    args: &Args,
342    (cond_tokens, cond_dim): (usize, usize),
343    stream: BatchStream,
344    rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346    // Frozen: the flow transformer is trained against a latent space that has
347    // already stopped moving, so only `encode` is ever called here.
348    let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349    let shape = *vae.config();
350
351    let (mut model, mut step) = match &args.resume {
352        Some(path) => {
353            let (model, metadata) = FlowTransformer::load(path, rng)?;
354            (model, started_at(&metadata))
355        }
356        None => {
357            // The latent shape is the autoencoder's and the condition's shape
358            // is the corpus's, whatever a supplied configuration says.
359            let config = FlowConfig {
360                latents: shape.latents,
361                latent_dim: shape.latent_dim,
362                cond_tokens,
363                cond_dim,
364                ..read_config(args, FlowConfig::default())?
365            };
366            (FlowTransformer::new(config, rng)?, 0)
367        }
368    };
369    if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370        return Err(format!(
371            "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372            model.config().cond_tokens,
373            model.config().cond_dim
374        )
375        .into());
376    }
377    model.set_optimizer(adam(args.learning_rate));
378    if let Some(device) = args.cuda {
379        #[cfg(feature = "cuda")]
380        {
381            // 0 as the budget: the caller picked the card, and the blocks'
382            // weights are a small part of what a run of this size needs.
383            model.to_cuda(device, 0)?;
384            println!("flow transformer: training on CUDA device {device}");
385        }
386        #[cfg(not(feature = "cuda"))]
387        return Err(format!(
388            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389        )
390        .into());
391    }
392    println!(
393        "flow transformer: {} parameters, from step {step}",
394        model.num_parameters()
395    );
396
397    let mut running = 0.0;
398    for batch in stream {
399        let batch = batch?;
400        let count = batch.len();
401        let points = batch.surface.rows / count;
402
403        let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404        for index in 0..count {
405            // A draw from the posterior, not its mean. Every autoencoder step
406            // decoded a draw, so a draw is the only kind of latent the frozen
407            // decoder has ever answered for. The posterior is wide enough for
408            // that to matter — sigma about twice the magnitude of the mean —
409            // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410            // a draw, so a flow trained towards the mean would be aiming at the
411            // one latent the decoder is worst at.
412            let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413            let width = shape.latents * shape.latent_dim;
414            for (slot, value) in clean.data[index * width..(index + 1) * width]
415                .iter_mut()
416                .zip(&latent.data)
417            {
418                *slot = value * args.latent_scale;
419            }
420        }
421        if step == 0 {
422            println!("latent spread after scaling: {:.3}", spread(&clean));
423        }
424
425        running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426        step += 1;
427        model.step_clipped(1.0, 1.0)?;
428        model.zero_grad();
429
430        if step % args.report == 0 {
431            println!(
432                "step {step}: velocity mse {:.5}",
433                running / args.report as f32
434            );
435            running = 0.0;
436        }
437        if step % args.checkpoint_every == 0 || step == args.steps {
438            model.save(
439                &args.out,
440                &BTreeMap::from([
441                    step_at(step),
442                    ("latent_scale".to_string(), args.latent_scale.to_string()),
443                ]),
444            )?;
445            println!("wrote {} at step {step}", args.out.display());
446        }
447        if step >= args.steps {
448            break;
449        }
450    }
451    Ok(())
452}
Source

pub fn encode(&self, surface: &Matrix) -> Result<(Matrix, Matrix), NetworkError>

The mean and the log-variance of the latent distribution, each [latents, latent_dim].

surface is [points, 6]: a position and its outward normal.

§The mean is not what the decoder was trained on

Every training step decodes a draw from this distribution, never its mean, so the decoder has only ever seen latents with the posterior’s noise on them. The mean sits in the middle of that cloud, in a spot no training step ever visited, and the reconstruction from it is visibly worse: on the reference model at 24,000 steps the posterior is wide — sigma around 0.53 against a mean magnitude of 0.23 — and reconstructing from the mean gives a clamped L1 of 0.029 where a draw gives 0.0025, an order of magnitude.

So reconstruction goes through ShapeVae::encode_sample. Reach for the mean when the question is about the distribution itself — measuring how wide it is, feeding a flow model that learns to produce means, or checking a KL term — not when the answer is a mesh.

Examples found in repository?
examples/kl_report.rs (line 99)
48fn main() -> Result<(), Box<dyn std::error::Error>> {
49    let args = parse()?;
50    let mut rng = StdRng::seed_from_u64(args.seed);
51
52    let mut corpus = Corpus::open(&args.tokens, &args.samples)?;
53    if corpus.is_empty() {
54        return Err("no render is paired with a mesh; check the two directories".into());
55    }
56    let objects: Vec<Object> = (0..args.objects.min(corpus.len()))
57        .map(|index| {
58            let example =
59                corpus.example(index, Some(args.surface), Some(args.queries), &mut rng)?;
60            Ok::<_, Box<dyn std::error::Error>>(Object {
61                surface: example.surface,
62                queries: example.queries,
63                distances: example.distances,
64            })
65        })
66        .collect::<Result<_, _>>()?;
67    println!(
68        "{} objects, {} surface points, {} query points, clamp {}",
69        objects.len(),
70        args.surface,
71        args.queries,
72        args.clamp
73    );
74    println!(
75        "{:<28} {:>9} {:>9} {:>7} {:>7} {:>8} {:>7}",
76        "checkpoint", "l1 draw", "l1 mean", "sigma", "|mean|", "kl/unit", "active"
77    );
78
79    for path in &args.checkpoints {
80        // A fresh generator per checkpoint, so every model is asked for the
81        // same draw of noise as well as the same objects.
82        let mut rng = StdRng::seed_from_u64(args.seed ^ 0x5a1);
83        // `mut` is only used by the device move below.
84        #[cfg_attr(not(feature = "cuda"), allow(unused_mut))]
85        let (mut model, _) = ShapeVae::load(path, &mut StdRng::seed_from_u64(args.seed))?;
86        if let Some(device) = args.cuda {
87            #[cfg(feature = "cuda")]
88            model.to_cuda(device, 0)?;
89            #[cfg(not(feature = "cuda"))]
90            return Err(format!(
91                "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
92            )
93            .into());
94        }
95
96        let (mut draw_l1, mut mean_l1, mut sigma, mut magnitude, mut kl, mut active) =
97            (0.0, 0.0, 0.0, 0.0, 0.0, 0.0);
98        for object in &objects {
99            let (mean, log_variance) = model.encode(&object.surface)?;
100            let (latent, _) = ShapeVae::sample(&mean, &log_variance, &mut rng);
101
102            let from_draw = model.decode(&latent, &object.queries, 4096)?;
103            let from_mean = model.decode(&mean, &object.queries, 4096)?;
104            draw_l1 += losses::clamped_l1(&from_draw, &object.distances, args.clamp).0;
105            mean_l1 += losses::clamped_l1(&from_mean, &object.distances, args.clamp).0;
106
107            let units = mean.data.len() as f32;
108            magnitude += mean.data.iter().map(|value| value.abs()).sum::<f32>() / units;
109            sigma += log_variance
110                .data
111                .iter()
112                .map(|value| (0.5 * value).exp())
113                .sum::<f32>()
114                / units;
115            // The per-unit KL against a unit Gaussian, which is what a
116            // free-bits floor would be applied to one unit at a time.
117            let mut per_unit = 0.0;
118            let mut above = 0.0;
119            for (mean, log_variance) in mean.data.iter().zip(&log_variance.data) {
120                let nats = 0.5 * (mean * mean + log_variance.exp() - 1.0 - log_variance);
121                per_unit += nats;
122                if nats > 0.01 {
123                    above += 1.0;
124                }
125            }
126            kl += per_unit / units;
127            active += above / units;
128        }
129
130        let count = objects.len() as f32;
131        println!(
132            "{:<28} {:>9.5} {:>9.5} {:>7.3} {:>7.3} {:>8.4} {:>6.1}%",
133            path.file_name()
134                .unwrap_or(path.as_os_str())
135                .to_string_lossy(),
136            draw_l1 / count,
137            mean_l1 / count,
138            sigma / count,
139            magnitude / count,
140            kl / count,
141            100.0 * active / count
142        );
143    }
144    Ok(())
145}
Source

pub fn encode_sample( &self, surface: &Matrix, rng: &mut StdRng, ) -> Result<Matrix, NetworkError>

One latent drawn from the posterior, which is what the decoder was trained to read.

ShapeVae::encode followed by ShapeVae::sample, in the order a training step does it. This is the call for reconstructing a shape: see the warning on encode for what the mean costs instead.

Examples found in repository?
examples/train_shape.rs (line 412)
340fn train_flow(
341    args: &Args,
342    (cond_tokens, cond_dim): (usize, usize),
343    stream: BatchStream,
344    rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346    // Frozen: the flow transformer is trained against a latent space that has
347    // already stopped moving, so only `encode` is ever called here.
348    let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349    let shape = *vae.config();
350
351    let (mut model, mut step) = match &args.resume {
352        Some(path) => {
353            let (model, metadata) = FlowTransformer::load(path, rng)?;
354            (model, started_at(&metadata))
355        }
356        None => {
357            // The latent shape is the autoencoder's and the condition's shape
358            // is the corpus's, whatever a supplied configuration says.
359            let config = FlowConfig {
360                latents: shape.latents,
361                latent_dim: shape.latent_dim,
362                cond_tokens,
363                cond_dim,
364                ..read_config(args, FlowConfig::default())?
365            };
366            (FlowTransformer::new(config, rng)?, 0)
367        }
368    };
369    if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370        return Err(format!(
371            "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372            model.config().cond_tokens,
373            model.config().cond_dim
374        )
375        .into());
376    }
377    model.set_optimizer(adam(args.learning_rate));
378    if let Some(device) = args.cuda {
379        #[cfg(feature = "cuda")]
380        {
381            // 0 as the budget: the caller picked the card, and the blocks'
382            // weights are a small part of what a run of this size needs.
383            model.to_cuda(device, 0)?;
384            println!("flow transformer: training on CUDA device {device}");
385        }
386        #[cfg(not(feature = "cuda"))]
387        return Err(format!(
388            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389        )
390        .into());
391    }
392    println!(
393        "flow transformer: {} parameters, from step {step}",
394        model.num_parameters()
395    );
396
397    let mut running = 0.0;
398    for batch in stream {
399        let batch = batch?;
400        let count = batch.len();
401        let points = batch.surface.rows / count;
402
403        let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404        for index in 0..count {
405            // A draw from the posterior, not its mean. Every autoencoder step
406            // decoded a draw, so a draw is the only kind of latent the frozen
407            // decoder has ever answered for. The posterior is wide enough for
408            // that to matter — sigma about twice the magnitude of the mean —
409            // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410            // a draw, so a flow trained towards the mean would be aiming at the
411            // one latent the decoder is worst at.
412            let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413            let width = shape.latents * shape.latent_dim;
414            for (slot, value) in clean.data[index * width..(index + 1) * width]
415                .iter_mut()
416                .zip(&latent.data)
417            {
418                *slot = value * args.latent_scale;
419            }
420        }
421        if step == 0 {
422            println!("latent spread after scaling: {:.3}", spread(&clean));
423        }
424
425        running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426        step += 1;
427        model.step_clipped(1.0, 1.0)?;
428        model.zero_grad();
429
430        if step % args.report == 0 {
431            println!(
432                "step {step}: velocity mse {:.5}",
433                running / args.report as f32
434            );
435            running = 0.0;
436        }
437        if step % args.checkpoint_every == 0 || step == args.steps {
438            model.save(
439                &args.out,
440                &BTreeMap::from([
441                    step_at(step),
442                    ("latent_scale".to_string(), args.latent_scale.to_string()),
443                ]),
444            )?;
445            println!("wrote {} at step {step}", args.out.display());
446        }
447        if step >= args.steps {
448            break;
449        }
450    }
451    Ok(())
452}
Source

pub fn encode_train( &self, surface: &Matrix, ) -> Result<(Matrix, Matrix, EncoderCache), NetworkError>

The same, keeping what the backward pass needs.

Examples found in repository?
examples/train_shape.rs (line 182)
126fn train_vae(
127    args: &Args,
128    stream: BatchStream,
129    rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131    let (mut model, mut step) = match &args.resume {
132        Some(path) => {
133            let (model, metadata) = ShapeVae::load(path, rng)?;
134            (model, started_at(&metadata))
135        }
136        None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137    };
138    if let Some(device) = args.cuda {
139        #[cfg(feature = "cuda")]
140        {
141            // 0 as the budget: the caller picked the card, and the weights are
142            // a small part of what a run of this size needs.
143            model.to_cuda(device, 0)?;
144            println!("shape autoencoder: training on CUDA device {device}");
145        }
146        #[cfg(not(feature = "cuda"))]
147        return Err(format!(
148            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149        )
150        .into());
151    }
152    let optimizer = adam(args.learning_rate);
153    println!(
154        "shape autoencoder: {} parameters, from step {step}",
155        model.num_parameters()
156    );
157
158    let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159    for batch in stream {
160        let batch = batch?;
161        let count = batch.len();
162        let points = batch.surface.rows / count;
163        let queries = batch.queries.rows / count;
164
165        // Packing helps the ordinary 4K-query training shape, but at 16K
166        // queries the larger attention workspace is slower and batch eight can
167        // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168        let cuda_batch = model.on_device() && queries <= 4096;
169        #[cfg(feature = "cuda")]
170        if cuda_batch {
171            let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172            // The packed losses are means across the batch; the reporting
173            // accumulator below historically stores a sum over examples.
174            reconstruction += loss * count as f32;
175            divergence += kl * count as f32;
176            tint += color * count as f32;
177        }
178
179        if !cuda_batch {
180            for index in 0..count {
181                let surface = slice(&batch.surface, index, points);
182                let (mean, log_variance, cache) = model.encode_train(&surface)?;
183                let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185                let (prediction, decoded) =
186                    model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187                let target = &batch.distances[index * queries..(index + 1) * queries];
188                let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189                let (kl, grad_mean, grad_log_variance) =
190                    losses::kl_divergence(&mean.data, &log_variance.data);
191                reconstruction += loss;
192                divergence += kl;
193
194                let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196                // The colour targets sit on the surface points, not the distance
197                // queries, so the colour head trains on a second short decode.
198                // The loader's points are already in random order, so the first
199                // `--color-points` of them are a random draw.
200                if let Some(colors) = &batch.colors
201                    && args.color > 0.0
202                    && args.color_points > 0
203                {
204                    let wanted = args.color_points.min(points);
205                    let positions = positions(&surface, wanted);
206                    let targets = head_rows(&slice(colors, index, points), wanted);
207                    let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209                    let predicted = model.colors(&decoded)?;
210                    let mut grad = predicted.clone();
211                    let scale = args.color / predicted.data.len() as f32;
212                    for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213                        let error = *slot - target;
214                        tint += error * error / predicted.data.len() as f32;
215                        *slot = 2.0 * scale * error;
216                    }
217
218                    let from_colors = model.decode_backward_colored(
219                        &decoded,
220                        &vec![0.0; positions.rows],
221                        Some(&grad),
222                    )?;
223                    for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224                        *slot += value;
225                    }
226                }
227
228                let (mut into_mean, mut into_log_variance) =
229                    ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230                for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231                    *slot += args.kl * value;
232                }
233                for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234                    *slot += args.kl * value;
235                }
236                model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237            }
238        }
239
240        step += 1;
241        // The gradients were summed over the batch, so the step is scaled by
242        // its size rather than the loop dividing every gradient it makes.
243        optimizers::step_clipped(
244            &mut model.params_mut(),
245            &optimizer,
246            step,
247            1.0 / count as f32,
248            1.0,
249        )?;
250        optimizers::zero_grad(&mut model.params_mut());
251
252        if step % args.report == 0 {
253            let scale = (args.report * count) as f32;
254            println!(
255                "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256                reconstruction / scale,
257                divergence / scale,
258                tint / scale
259            );
260            reconstruction = 0.0;
261            divergence = 0.0;
262            tint = 0.0;
263        }
264        if step % args.checkpoint_every == 0 || step == args.steps {
265            model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266            println!("wrote {} at step {step}", args.out.display());
267        }
268        if step >= args.steps {
269            break;
270        }
271    }
272    Ok(())
273}
Source

pub fn encode_backward( &mut self, cache: &EncoderCache, grad_mean: &Matrix, grad_log_variance: &Matrix, ) -> Result<(), NetworkError>

Accumulates the encoder’s weight gradients.

Nothing is returned: the surface points are data, not a layer below.

Examples found in repository?
examples/train_shape.rs (line 236)
126fn train_vae(
127    args: &Args,
128    stream: BatchStream,
129    rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131    let (mut model, mut step) = match &args.resume {
132        Some(path) => {
133            let (model, metadata) = ShapeVae::load(path, rng)?;
134            (model, started_at(&metadata))
135        }
136        None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137    };
138    if let Some(device) = args.cuda {
139        #[cfg(feature = "cuda")]
140        {
141            // 0 as the budget: the caller picked the card, and the weights are
142            // a small part of what a run of this size needs.
143            model.to_cuda(device, 0)?;
144            println!("shape autoencoder: training on CUDA device {device}");
145        }
146        #[cfg(not(feature = "cuda"))]
147        return Err(format!(
148            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149        )
150        .into());
151    }
152    let optimizer = adam(args.learning_rate);
153    println!(
154        "shape autoencoder: {} parameters, from step {step}",
155        model.num_parameters()
156    );
157
158    let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159    for batch in stream {
160        let batch = batch?;
161        let count = batch.len();
162        let points = batch.surface.rows / count;
163        let queries = batch.queries.rows / count;
164
165        // Packing helps the ordinary 4K-query training shape, but at 16K
166        // queries the larger attention workspace is slower and batch eight can
167        // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168        let cuda_batch = model.on_device() && queries <= 4096;
169        #[cfg(feature = "cuda")]
170        if cuda_batch {
171            let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172            // The packed losses are means across the batch; the reporting
173            // accumulator below historically stores a sum over examples.
174            reconstruction += loss * count as f32;
175            divergence += kl * count as f32;
176            tint += color * count as f32;
177        }
178
179        if !cuda_batch {
180            for index in 0..count {
181                let surface = slice(&batch.surface, index, points);
182                let (mean, log_variance, cache) = model.encode_train(&surface)?;
183                let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185                let (prediction, decoded) =
186                    model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187                let target = &batch.distances[index * queries..(index + 1) * queries];
188                let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189                let (kl, grad_mean, grad_log_variance) =
190                    losses::kl_divergence(&mean.data, &log_variance.data);
191                reconstruction += loss;
192                divergence += kl;
193
194                let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196                // The colour targets sit on the surface points, not the distance
197                // queries, so the colour head trains on a second short decode.
198                // The loader's points are already in random order, so the first
199                // `--color-points` of them are a random draw.
200                if let Some(colors) = &batch.colors
201                    && args.color > 0.0
202                    && args.color_points > 0
203                {
204                    let wanted = args.color_points.min(points);
205                    let positions = positions(&surface, wanted);
206                    let targets = head_rows(&slice(colors, index, points), wanted);
207                    let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209                    let predicted = model.colors(&decoded)?;
210                    let mut grad = predicted.clone();
211                    let scale = args.color / predicted.data.len() as f32;
212                    for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213                        let error = *slot - target;
214                        tint += error * error / predicted.data.len() as f32;
215                        *slot = 2.0 * scale * error;
216                    }
217
218                    let from_colors = model.decode_backward_colored(
219                        &decoded,
220                        &vec![0.0; positions.rows],
221                        Some(&grad),
222                    )?;
223                    for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224                        *slot += value;
225                    }
226                }
227
228                let (mut into_mean, mut into_log_variance) =
229                    ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230                for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231                    *slot += args.kl * value;
232                }
233                for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234                    *slot += args.kl * value;
235                }
236                model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237            }
238        }
239
240        step += 1;
241        // The gradients were summed over the batch, so the step is scaled by
242        // its size rather than the loop dividing every gradient it makes.
243        optimizers::step_clipped(
244            &mut model.params_mut(),
245            &optimizer,
246            step,
247            1.0 / count as f32,
248            1.0,
249        )?;
250        optimizers::zero_grad(&mut model.params_mut());
251
252        if step % args.report == 0 {
253            let scale = (args.report * count) as f32;
254            println!(
255                "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256                reconstruction / scale,
257                divergence / scale,
258                tint / scale
259            );
260            reconstruction = 0.0;
261            divergence = 0.0;
262            tint = 0.0;
263        }
264        if step % args.checkpoint_every == 0 || step == args.steps {
265            model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266            println!("wrote {} at step {step}", args.out.display());
267        }
268        if step >= args.steps {
269            break;
270        }
271    }
272    Ok(())
273}
Source

pub fn sample<R: Rng>( mean: &Matrix, log_variance: &Matrix, rng: &mut R, ) -> (Matrix, Matrix)

Draws a latent from the distribution the encoder described.

z = mean + exp(log_variance / 2) * noise. The noise is returned because the backward pass needs it, and because a caller that wants a deterministic reconstruction passes the mean straight to decode and never calls this at all.

Examples found in repository?
examples/kl_report.rs (line 100)
48fn main() -> Result<(), Box<dyn std::error::Error>> {
49    let args = parse()?;
50    let mut rng = StdRng::seed_from_u64(args.seed);
51
52    let mut corpus = Corpus::open(&args.tokens, &args.samples)?;
53    if corpus.is_empty() {
54        return Err("no render is paired with a mesh; check the two directories".into());
55    }
56    let objects: Vec<Object> = (0..args.objects.min(corpus.len()))
57        .map(|index| {
58            let example =
59                corpus.example(index, Some(args.surface), Some(args.queries), &mut rng)?;
60            Ok::<_, Box<dyn std::error::Error>>(Object {
61                surface: example.surface,
62                queries: example.queries,
63                distances: example.distances,
64            })
65        })
66        .collect::<Result<_, _>>()?;
67    println!(
68        "{} objects, {} surface points, {} query points, clamp {}",
69        objects.len(),
70        args.surface,
71        args.queries,
72        args.clamp
73    );
74    println!(
75        "{:<28} {:>9} {:>9} {:>7} {:>7} {:>8} {:>7}",
76        "checkpoint", "l1 draw", "l1 mean", "sigma", "|mean|", "kl/unit", "active"
77    );
78
79    for path in &args.checkpoints {
80        // A fresh generator per checkpoint, so every model is asked for the
81        // same draw of noise as well as the same objects.
82        let mut rng = StdRng::seed_from_u64(args.seed ^ 0x5a1);
83        // `mut` is only used by the device move below.
84        #[cfg_attr(not(feature = "cuda"), allow(unused_mut))]
85        let (mut model, _) = ShapeVae::load(path, &mut StdRng::seed_from_u64(args.seed))?;
86        if let Some(device) = args.cuda {
87            #[cfg(feature = "cuda")]
88            model.to_cuda(device, 0)?;
89            #[cfg(not(feature = "cuda"))]
90            return Err(format!(
91                "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
92            )
93            .into());
94        }
95
96        let (mut draw_l1, mut mean_l1, mut sigma, mut magnitude, mut kl, mut active) =
97            (0.0, 0.0, 0.0, 0.0, 0.0, 0.0);
98        for object in &objects {
99            let (mean, log_variance) = model.encode(&object.surface)?;
100            let (latent, _) = ShapeVae::sample(&mean, &log_variance, &mut rng);
101
102            let from_draw = model.decode(&latent, &object.queries, 4096)?;
103            let from_mean = model.decode(&mean, &object.queries, 4096)?;
104            draw_l1 += losses::clamped_l1(&from_draw, &object.distances, args.clamp).0;
105            mean_l1 += losses::clamped_l1(&from_mean, &object.distances, args.clamp).0;
106
107            let units = mean.data.len() as f32;
108            magnitude += mean.data.iter().map(|value| value.abs()).sum::<f32>() / units;
109            sigma += log_variance
110                .data
111                .iter()
112                .map(|value| (0.5 * value).exp())
113                .sum::<f32>()
114                / units;
115            // The per-unit KL against a unit Gaussian, which is what a
116            // free-bits floor would be applied to one unit at a time.
117            let mut per_unit = 0.0;
118            let mut above = 0.0;
119            for (mean, log_variance) in mean.data.iter().zip(&log_variance.data) {
120                let nats = 0.5 * (mean * mean + log_variance.exp() - 1.0 - log_variance);
121                per_unit += nats;
122                if nats > 0.01 {
123                    above += 1.0;
124                }
125            }
126            kl += per_unit / units;
127            active += above / units;
128        }
129
130        let count = objects.len() as f32;
131        println!(
132            "{:<28} {:>9.5} {:>9.5} {:>7.3} {:>7.3} {:>8.4} {:>6.1}%",
133            path.file_name()
134                .unwrap_or(path.as_os_str())
135                .to_string_lossy(),
136            draw_l1 / count,
137            mean_l1 / count,
138            sigma / count,
139            magnitude / count,
140            kl / count,
141            100.0 * active / count
142        );
143    }
144    Ok(())
145}
More examples
Hide additional examples
examples/train_shape.rs (line 183)
126fn train_vae(
127    args: &Args,
128    stream: BatchStream,
129    rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131    let (mut model, mut step) = match &args.resume {
132        Some(path) => {
133            let (model, metadata) = ShapeVae::load(path, rng)?;
134            (model, started_at(&metadata))
135        }
136        None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137    };
138    if let Some(device) = args.cuda {
139        #[cfg(feature = "cuda")]
140        {
141            // 0 as the budget: the caller picked the card, and the weights are
142            // a small part of what a run of this size needs.
143            model.to_cuda(device, 0)?;
144            println!("shape autoencoder: training on CUDA device {device}");
145        }
146        #[cfg(not(feature = "cuda"))]
147        return Err(format!(
148            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149        )
150        .into());
151    }
152    let optimizer = adam(args.learning_rate);
153    println!(
154        "shape autoencoder: {} parameters, from step {step}",
155        model.num_parameters()
156    );
157
158    let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159    for batch in stream {
160        let batch = batch?;
161        let count = batch.len();
162        let points = batch.surface.rows / count;
163        let queries = batch.queries.rows / count;
164
165        // Packing helps the ordinary 4K-query training shape, but at 16K
166        // queries the larger attention workspace is slower and batch eight can
167        // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168        let cuda_batch = model.on_device() && queries <= 4096;
169        #[cfg(feature = "cuda")]
170        if cuda_batch {
171            let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172            // The packed losses are means across the batch; the reporting
173            // accumulator below historically stores a sum over examples.
174            reconstruction += loss * count as f32;
175            divergence += kl * count as f32;
176            tint += color * count as f32;
177        }
178
179        if !cuda_batch {
180            for index in 0..count {
181                let surface = slice(&batch.surface, index, points);
182                let (mean, log_variance, cache) = model.encode_train(&surface)?;
183                let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185                let (prediction, decoded) =
186                    model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187                let target = &batch.distances[index * queries..(index + 1) * queries];
188                let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189                let (kl, grad_mean, grad_log_variance) =
190                    losses::kl_divergence(&mean.data, &log_variance.data);
191                reconstruction += loss;
192                divergence += kl;
193
194                let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196                // The colour targets sit on the surface points, not the distance
197                // queries, so the colour head trains on a second short decode.
198                // The loader's points are already in random order, so the first
199                // `--color-points` of them are a random draw.
200                if let Some(colors) = &batch.colors
201                    && args.color > 0.0
202                    && args.color_points > 0
203                {
204                    let wanted = args.color_points.min(points);
205                    let positions = positions(&surface, wanted);
206                    let targets = head_rows(&slice(colors, index, points), wanted);
207                    let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209                    let predicted = model.colors(&decoded)?;
210                    let mut grad = predicted.clone();
211                    let scale = args.color / predicted.data.len() as f32;
212                    for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213                        let error = *slot - target;
214                        tint += error * error / predicted.data.len() as f32;
215                        *slot = 2.0 * scale * error;
216                    }
217
218                    let from_colors = model.decode_backward_colored(
219                        &decoded,
220                        &vec![0.0; positions.rows],
221                        Some(&grad),
222                    )?;
223                    for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224                        *slot += value;
225                    }
226                }
227
228                let (mut into_mean, mut into_log_variance) =
229                    ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230                for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231                    *slot += args.kl * value;
232                }
233                for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234                    *slot += args.kl * value;
235                }
236                model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237            }
238        }
239
240        step += 1;
241        // The gradients were summed over the batch, so the step is scaled by
242        // its size rather than the loop dividing every gradient it makes.
243        optimizers::step_clipped(
244            &mut model.params_mut(),
245            &optimizer,
246            step,
247            1.0 / count as f32,
248            1.0,
249        )?;
250        optimizers::zero_grad(&mut model.params_mut());
251
252        if step % args.report == 0 {
253            let scale = (args.report * count) as f32;
254            println!(
255                "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256                reconstruction / scale,
257                divergence / scale,
258                tint / scale
259            );
260            reconstruction = 0.0;
261            divergence = 0.0;
262            tint = 0.0;
263        }
264        if step % args.checkpoint_every == 0 || step == args.steps {
265            model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266            println!("wrote {} at step {step}", args.out.display());
267        }
268        if step >= args.steps {
269            break;
270        }
271    }
272    Ok(())
273}
Source

pub fn sample_backward( grad_latent: &Matrix, log_variance: &Matrix, noise: &Matrix, ) -> (Matrix, Matrix)

Splits dL/dlatent back into dL/dmean and dL/dlog_variance.

The mean is on a straight path so it takes the gradient unchanged; the log-variance only reaches the latent through the scaled noise, which is exactly latent - mean.

Examples found in repository?
examples/train_shape.rs (line 229)
126fn train_vae(
127    args: &Args,
128    stream: BatchStream,
129    rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131    let (mut model, mut step) = match &args.resume {
132        Some(path) => {
133            let (model, metadata) = ShapeVae::load(path, rng)?;
134            (model, started_at(&metadata))
135        }
136        None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137    };
138    if let Some(device) = args.cuda {
139        #[cfg(feature = "cuda")]
140        {
141            // 0 as the budget: the caller picked the card, and the weights are
142            // a small part of what a run of this size needs.
143            model.to_cuda(device, 0)?;
144            println!("shape autoencoder: training on CUDA device {device}");
145        }
146        #[cfg(not(feature = "cuda"))]
147        return Err(format!(
148            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149        )
150        .into());
151    }
152    let optimizer = adam(args.learning_rate);
153    println!(
154        "shape autoencoder: {} parameters, from step {step}",
155        model.num_parameters()
156    );
157
158    let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159    for batch in stream {
160        let batch = batch?;
161        let count = batch.len();
162        let points = batch.surface.rows / count;
163        let queries = batch.queries.rows / count;
164
165        // Packing helps the ordinary 4K-query training shape, but at 16K
166        // queries the larger attention workspace is slower and batch eight can
167        // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168        let cuda_batch = model.on_device() && queries <= 4096;
169        #[cfg(feature = "cuda")]
170        if cuda_batch {
171            let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172            // The packed losses are means across the batch; the reporting
173            // accumulator below historically stores a sum over examples.
174            reconstruction += loss * count as f32;
175            divergence += kl * count as f32;
176            tint += color * count as f32;
177        }
178
179        if !cuda_batch {
180            for index in 0..count {
181                let surface = slice(&batch.surface, index, points);
182                let (mean, log_variance, cache) = model.encode_train(&surface)?;
183                let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185                let (prediction, decoded) =
186                    model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187                let target = &batch.distances[index * queries..(index + 1) * queries];
188                let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189                let (kl, grad_mean, grad_log_variance) =
190                    losses::kl_divergence(&mean.data, &log_variance.data);
191                reconstruction += loss;
192                divergence += kl;
193
194                let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196                // The colour targets sit on the surface points, not the distance
197                // queries, so the colour head trains on a second short decode.
198                // The loader's points are already in random order, so the first
199                // `--color-points` of them are a random draw.
200                if let Some(colors) = &batch.colors
201                    && args.color > 0.0
202                    && args.color_points > 0
203                {
204                    let wanted = args.color_points.min(points);
205                    let positions = positions(&surface, wanted);
206                    let targets = head_rows(&slice(colors, index, points), wanted);
207                    let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209                    let predicted = model.colors(&decoded)?;
210                    let mut grad = predicted.clone();
211                    let scale = args.color / predicted.data.len() as f32;
212                    for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213                        let error = *slot - target;
214                        tint += error * error / predicted.data.len() as f32;
215                        *slot = 2.0 * scale * error;
216                    }
217
218                    let from_colors = model.decode_backward_colored(
219                        &decoded,
220                        &vec![0.0; positions.rows],
221                        Some(&grad),
222                    )?;
223                    for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224                        *slot += value;
225                    }
226                }
227
228                let (mut into_mean, mut into_log_variance) =
229                    ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230                for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231                    *slot += args.kl * value;
232                }
233                for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234                    *slot += args.kl * value;
235                }
236                model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237            }
238        }
239
240        step += 1;
241        // The gradients were summed over the batch, so the step is scaled by
242        // its size rather than the loop dividing every gradient it makes.
243        optimizers::step_clipped(
244            &mut model.params_mut(),
245            &optimizer,
246            step,
247            1.0 / count as f32,
248            1.0,
249        )?;
250        optimizers::zero_grad(&mut model.params_mut());
251
252        if step % args.report == 0 {
253            let scale = (args.report * count) as f32;
254            println!(
255                "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256                reconstruction / scale,
257                divergence / scale,
258                tint / scale
259            );
260            reconstruction = 0.0;
261            divergence = 0.0;
262            tint = 0.0;
263        }
264        if step % args.checkpoint_every == 0 || step == args.steps {
265            model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266            println!("wrote {} at step {step}", args.out.display());
267        }
268        if step >= args.steps {
269            break;
270        }
271    }
272    Ok(())
273}
Source

pub fn decode( &self, latent: &Matrix, queries: &Matrix, chunk: usize, ) -> Result<Vec<f32>, NetworkError>

One signed distance per query point, decoded chunk points at a time.

queries is [n, 3]. The chunk size is what bounds the memory: the attention scores are [chunk, latents], and a 128³ marching grid asks about 2.1 million points that will not fit at once.

Examples found in repository?
examples/kl_report.rs (line 102)
48fn main() -> Result<(), Box<dyn std::error::Error>> {
49    let args = parse()?;
50    let mut rng = StdRng::seed_from_u64(args.seed);
51
52    let mut corpus = Corpus::open(&args.tokens, &args.samples)?;
53    if corpus.is_empty() {
54        return Err("no render is paired with a mesh; check the two directories".into());
55    }
56    let objects: Vec<Object> = (0..args.objects.min(corpus.len()))
57        .map(|index| {
58            let example =
59                corpus.example(index, Some(args.surface), Some(args.queries), &mut rng)?;
60            Ok::<_, Box<dyn std::error::Error>>(Object {
61                surface: example.surface,
62                queries: example.queries,
63                distances: example.distances,
64            })
65        })
66        .collect::<Result<_, _>>()?;
67    println!(
68        "{} objects, {} surface points, {} query points, clamp {}",
69        objects.len(),
70        args.surface,
71        args.queries,
72        args.clamp
73    );
74    println!(
75        "{:<28} {:>9} {:>9} {:>7} {:>7} {:>8} {:>7}",
76        "checkpoint", "l1 draw", "l1 mean", "sigma", "|mean|", "kl/unit", "active"
77    );
78
79    for path in &args.checkpoints {
80        // A fresh generator per checkpoint, so every model is asked for the
81        // same draw of noise as well as the same objects.
82        let mut rng = StdRng::seed_from_u64(args.seed ^ 0x5a1);
83        // `mut` is only used by the device move below.
84        #[cfg_attr(not(feature = "cuda"), allow(unused_mut))]
85        let (mut model, _) = ShapeVae::load(path, &mut StdRng::seed_from_u64(args.seed))?;
86        if let Some(device) = args.cuda {
87            #[cfg(feature = "cuda")]
88            model.to_cuda(device, 0)?;
89            #[cfg(not(feature = "cuda"))]
90            return Err(format!(
91                "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
92            )
93            .into());
94        }
95
96        let (mut draw_l1, mut mean_l1, mut sigma, mut magnitude, mut kl, mut active) =
97            (0.0, 0.0, 0.0, 0.0, 0.0, 0.0);
98        for object in &objects {
99            let (mean, log_variance) = model.encode(&object.surface)?;
100            let (latent, _) = ShapeVae::sample(&mean, &log_variance, &mut rng);
101
102            let from_draw = model.decode(&latent, &object.queries, 4096)?;
103            let from_mean = model.decode(&mean, &object.queries, 4096)?;
104            draw_l1 += losses::clamped_l1(&from_draw, &object.distances, args.clamp).0;
105            mean_l1 += losses::clamped_l1(&from_mean, &object.distances, args.clamp).0;
106
107            let units = mean.data.len() as f32;
108            magnitude += mean.data.iter().map(|value| value.abs()).sum::<f32>() / units;
109            sigma += log_variance
110                .data
111                .iter()
112                .map(|value| (0.5 * value).exp())
113                .sum::<f32>()
114                / units;
115            // The per-unit KL against a unit Gaussian, which is what a
116            // free-bits floor would be applied to one unit at a time.
117            let mut per_unit = 0.0;
118            let mut above = 0.0;
119            for (mean, log_variance) in mean.data.iter().zip(&log_variance.data) {
120                let nats = 0.5 * (mean * mean + log_variance.exp() - 1.0 - log_variance);
121                per_unit += nats;
122                if nats > 0.01 {
123                    above += 1.0;
124                }
125            }
126            kl += per_unit / units;
127            active += above / units;
128        }
129
130        let count = objects.len() as f32;
131        println!(
132            "{:<28} {:>9.5} {:>9.5} {:>7.3} {:>7.3} {:>8.4} {:>6.1}%",
133            path.file_name()
134                .unwrap_or(path.as_os_str())
135                .to_string_lossy(),
136            draw_l1 / count,
137            mean_l1 / count,
138            sigma / count,
139            magnitude / count,
140            kl / count,
141            100.0 * active / count
142        );
143    }
144    Ok(())
145}
Source

pub fn decode_train( &self, latent: &Matrix, queries: &Matrix, ) -> Result<(Vec<f32>, DecoderCache), NetworkError>

One chunk of query points, keeping what the backward pass needs.

Training decodes a subsample of a mesh’s query points per step, which is one chunk, so it calls this directly.

Examples found in repository?
examples/train_shape.rs (line 186)
126fn train_vae(
127    args: &Args,
128    stream: BatchStream,
129    rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131    let (mut model, mut step) = match &args.resume {
132        Some(path) => {
133            let (model, metadata) = ShapeVae::load(path, rng)?;
134            (model, started_at(&metadata))
135        }
136        None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137    };
138    if let Some(device) = args.cuda {
139        #[cfg(feature = "cuda")]
140        {
141            // 0 as the budget: the caller picked the card, and the weights are
142            // a small part of what a run of this size needs.
143            model.to_cuda(device, 0)?;
144            println!("shape autoencoder: training on CUDA device {device}");
145        }
146        #[cfg(not(feature = "cuda"))]
147        return Err(format!(
148            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149        )
150        .into());
151    }
152    let optimizer = adam(args.learning_rate);
153    println!(
154        "shape autoencoder: {} parameters, from step {step}",
155        model.num_parameters()
156    );
157
158    let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159    for batch in stream {
160        let batch = batch?;
161        let count = batch.len();
162        let points = batch.surface.rows / count;
163        let queries = batch.queries.rows / count;
164
165        // Packing helps the ordinary 4K-query training shape, but at 16K
166        // queries the larger attention workspace is slower and batch eight can
167        // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168        let cuda_batch = model.on_device() && queries <= 4096;
169        #[cfg(feature = "cuda")]
170        if cuda_batch {
171            let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172            // The packed losses are means across the batch; the reporting
173            // accumulator below historically stores a sum over examples.
174            reconstruction += loss * count as f32;
175            divergence += kl * count as f32;
176            tint += color * count as f32;
177        }
178
179        if !cuda_batch {
180            for index in 0..count {
181                let surface = slice(&batch.surface, index, points);
182                let (mean, log_variance, cache) = model.encode_train(&surface)?;
183                let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185                let (prediction, decoded) =
186                    model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187                let target = &batch.distances[index * queries..(index + 1) * queries];
188                let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189                let (kl, grad_mean, grad_log_variance) =
190                    losses::kl_divergence(&mean.data, &log_variance.data);
191                reconstruction += loss;
192                divergence += kl;
193
194                let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196                // The colour targets sit on the surface points, not the distance
197                // queries, so the colour head trains on a second short decode.
198                // The loader's points are already in random order, so the first
199                // `--color-points` of them are a random draw.
200                if let Some(colors) = &batch.colors
201                    && args.color > 0.0
202                    && args.color_points > 0
203                {
204                    let wanted = args.color_points.min(points);
205                    let positions = positions(&surface, wanted);
206                    let targets = head_rows(&slice(colors, index, points), wanted);
207                    let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209                    let predicted = model.colors(&decoded)?;
210                    let mut grad = predicted.clone();
211                    let scale = args.color / predicted.data.len() as f32;
212                    for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213                        let error = *slot - target;
214                        tint += error * error / predicted.data.len() as f32;
215                        *slot = 2.0 * scale * error;
216                    }
217
218                    let from_colors = model.decode_backward_colored(
219                        &decoded,
220                        &vec![0.0; positions.rows],
221                        Some(&grad),
222                    )?;
223                    for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224                        *slot += value;
225                    }
226                }
227
228                let (mut into_mean, mut into_log_variance) =
229                    ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230                for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231                    *slot += args.kl * value;
232                }
233                for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234                    *slot += args.kl * value;
235                }
236                model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237            }
238        }
239
240        step += 1;
241        // The gradients were summed over the batch, so the step is scaled by
242        // its size rather than the loop dividing every gradient it makes.
243        optimizers::step_clipped(
244            &mut model.params_mut(),
245            &optimizer,
246            step,
247            1.0 / count as f32,
248            1.0,
249        )?;
250        optimizers::zero_grad(&mut model.params_mut());
251
252        if step % args.report == 0 {
253            let scale = (args.report * count) as f32;
254            println!(
255                "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256                reconstruction / scale,
257                divergence / scale,
258                tint / scale
259            );
260            reconstruction = 0.0;
261            divergence = 0.0;
262            tint = 0.0;
263        }
264        if step % args.checkpoint_every == 0 || step == args.steps {
265            model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266            println!("wrote {} at step {step}", args.out.display());
267        }
268        if step >= args.steps {
269            break;
270        }
271    }
272    Ok(())
273}
Source

pub fn colors(&self, cache: &DecoderCache) -> Result<Matrix, NetworkError>

The colour at each point of a decoded chunk, [n, 3].

The distance and the colour share everything but the last projection, so this reads a cache decode_train already built rather than decoding again.

Examples found in repository?
examples/train_shape.rs (line 209)
126fn train_vae(
127    args: &Args,
128    stream: BatchStream,
129    rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131    let (mut model, mut step) = match &args.resume {
132        Some(path) => {
133            let (model, metadata) = ShapeVae::load(path, rng)?;
134            (model, started_at(&metadata))
135        }
136        None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137    };
138    if let Some(device) = args.cuda {
139        #[cfg(feature = "cuda")]
140        {
141            // 0 as the budget: the caller picked the card, and the weights are
142            // a small part of what a run of this size needs.
143            model.to_cuda(device, 0)?;
144            println!("shape autoencoder: training on CUDA device {device}");
145        }
146        #[cfg(not(feature = "cuda"))]
147        return Err(format!(
148            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149        )
150        .into());
151    }
152    let optimizer = adam(args.learning_rate);
153    println!(
154        "shape autoencoder: {} parameters, from step {step}",
155        model.num_parameters()
156    );
157
158    let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159    for batch in stream {
160        let batch = batch?;
161        let count = batch.len();
162        let points = batch.surface.rows / count;
163        let queries = batch.queries.rows / count;
164
165        // Packing helps the ordinary 4K-query training shape, but at 16K
166        // queries the larger attention workspace is slower and batch eight can
167        // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168        let cuda_batch = model.on_device() && queries <= 4096;
169        #[cfg(feature = "cuda")]
170        if cuda_batch {
171            let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172            // The packed losses are means across the batch; the reporting
173            // accumulator below historically stores a sum over examples.
174            reconstruction += loss * count as f32;
175            divergence += kl * count as f32;
176            tint += color * count as f32;
177        }
178
179        if !cuda_batch {
180            for index in 0..count {
181                let surface = slice(&batch.surface, index, points);
182                let (mean, log_variance, cache) = model.encode_train(&surface)?;
183                let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185                let (prediction, decoded) =
186                    model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187                let target = &batch.distances[index * queries..(index + 1) * queries];
188                let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189                let (kl, grad_mean, grad_log_variance) =
190                    losses::kl_divergence(&mean.data, &log_variance.data);
191                reconstruction += loss;
192                divergence += kl;
193
194                let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196                // The colour targets sit on the surface points, not the distance
197                // queries, so the colour head trains on a second short decode.
198                // The loader's points are already in random order, so the first
199                // `--color-points` of them are a random draw.
200                if let Some(colors) = &batch.colors
201                    && args.color > 0.0
202                    && args.color_points > 0
203                {
204                    let wanted = args.color_points.min(points);
205                    let positions = positions(&surface, wanted);
206                    let targets = head_rows(&slice(colors, index, points), wanted);
207                    let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209                    let predicted = model.colors(&decoded)?;
210                    let mut grad = predicted.clone();
211                    let scale = args.color / predicted.data.len() as f32;
212                    for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213                        let error = *slot - target;
214                        tint += error * error / predicted.data.len() as f32;
215                        *slot = 2.0 * scale * error;
216                    }
217
218                    let from_colors = model.decode_backward_colored(
219                        &decoded,
220                        &vec![0.0; positions.rows],
221                        Some(&grad),
222                    )?;
223                    for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224                        *slot += value;
225                    }
226                }
227
228                let (mut into_mean, mut into_log_variance) =
229                    ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230                for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231                    *slot += args.kl * value;
232                }
233                for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234                    *slot += args.kl * value;
235                }
236                model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237            }
238        }
239
240        step += 1;
241        // The gradients were summed over the batch, so the step is scaled by
242        // its size rather than the loop dividing every gradient it makes.
243        optimizers::step_clipped(
244            &mut model.params_mut(),
245            &optimizer,
246            step,
247            1.0 / count as f32,
248            1.0,
249        )?;
250        optimizers::zero_grad(&mut model.params_mut());
251
252        if step % args.report == 0 {
253            let scale = (args.report * count) as f32;
254            println!(
255                "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256                reconstruction / scale,
257                divergence / scale,
258                tint / scale
259            );
260            reconstruction = 0.0;
261            divergence = 0.0;
262            tint = 0.0;
263        }
264        if step % args.checkpoint_every == 0 || step == args.steps {
265            model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266            println!("wrote {} at step {step}", args.out.display());
267        }
268        if step >= args.steps {
269            break;
270        }
271    }
272    Ok(())
273}
Source

pub fn decode_colors( &self, latent: &Matrix, points: &Matrix, chunk: usize, ) -> Result<Vec<[f32; 3]>, NetworkError>

One colour per point, chunked like decode.

Source

pub fn decode_backward( &mut self, cache: &DecoderCache, grad_distances: &[f32], ) -> Result<Matrix, NetworkError>

Accumulates the decoder’s weight gradients and returns dL/dlatent.

Examples found in repository?
examples/train_shape.rs (line 194)
126fn train_vae(
127    args: &Args,
128    stream: BatchStream,
129    rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131    let (mut model, mut step) = match &args.resume {
132        Some(path) => {
133            let (model, metadata) = ShapeVae::load(path, rng)?;
134            (model, started_at(&metadata))
135        }
136        None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137    };
138    if let Some(device) = args.cuda {
139        #[cfg(feature = "cuda")]
140        {
141            // 0 as the budget: the caller picked the card, and the weights are
142            // a small part of what a run of this size needs.
143            model.to_cuda(device, 0)?;
144            println!("shape autoencoder: training on CUDA device {device}");
145        }
146        #[cfg(not(feature = "cuda"))]
147        return Err(format!(
148            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149        )
150        .into());
151    }
152    let optimizer = adam(args.learning_rate);
153    println!(
154        "shape autoencoder: {} parameters, from step {step}",
155        model.num_parameters()
156    );
157
158    let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159    for batch in stream {
160        let batch = batch?;
161        let count = batch.len();
162        let points = batch.surface.rows / count;
163        let queries = batch.queries.rows / count;
164
165        // Packing helps the ordinary 4K-query training shape, but at 16K
166        // queries the larger attention workspace is slower and batch eight can
167        // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168        let cuda_batch = model.on_device() && queries <= 4096;
169        #[cfg(feature = "cuda")]
170        if cuda_batch {
171            let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172            // The packed losses are means across the batch; the reporting
173            // accumulator below historically stores a sum over examples.
174            reconstruction += loss * count as f32;
175            divergence += kl * count as f32;
176            tint += color * count as f32;
177        }
178
179        if !cuda_batch {
180            for index in 0..count {
181                let surface = slice(&batch.surface, index, points);
182                let (mean, log_variance, cache) = model.encode_train(&surface)?;
183                let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185                let (prediction, decoded) =
186                    model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187                let target = &batch.distances[index * queries..(index + 1) * queries];
188                let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189                let (kl, grad_mean, grad_log_variance) =
190                    losses::kl_divergence(&mean.data, &log_variance.data);
191                reconstruction += loss;
192                divergence += kl;
193
194                let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196                // The colour targets sit on the surface points, not the distance
197                // queries, so the colour head trains on a second short decode.
198                // The loader's points are already in random order, so the first
199                // `--color-points` of them are a random draw.
200                if let Some(colors) = &batch.colors
201                    && args.color > 0.0
202                    && args.color_points > 0
203                {
204                    let wanted = args.color_points.min(points);
205                    let positions = positions(&surface, wanted);
206                    let targets = head_rows(&slice(colors, index, points), wanted);
207                    let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209                    let predicted = model.colors(&decoded)?;
210                    let mut grad = predicted.clone();
211                    let scale = args.color / predicted.data.len() as f32;
212                    for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213                        let error = *slot - target;
214                        tint += error * error / predicted.data.len() as f32;
215                        *slot = 2.0 * scale * error;
216                    }
217
218                    let from_colors = model.decode_backward_colored(
219                        &decoded,
220                        &vec![0.0; positions.rows],
221                        Some(&grad),
222                    )?;
223                    for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224                        *slot += value;
225                    }
226                }
227
228                let (mut into_mean, mut into_log_variance) =
229                    ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230                for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231                    *slot += args.kl * value;
232                }
233                for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234                    *slot += args.kl * value;
235                }
236                model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237            }
238        }
239
240        step += 1;
241        // The gradients were summed over the batch, so the step is scaled by
242        // its size rather than the loop dividing every gradient it makes.
243        optimizers::step_clipped(
244            &mut model.params_mut(),
245            &optimizer,
246            step,
247            1.0 / count as f32,
248            1.0,
249        )?;
250        optimizers::zero_grad(&mut model.params_mut());
251
252        if step % args.report == 0 {
253            let scale = (args.report * count) as f32;
254            println!(
255                "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256                reconstruction / scale,
257                divergence / scale,
258                tint / scale
259            );
260            reconstruction = 0.0;
261            divergence = 0.0;
262            tint = 0.0;
263        }
264        if step % args.checkpoint_every == 0 || step == args.steps {
265            model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266            println!("wrote {} at step {step}", args.out.display());
267        }
268        if step >= args.steps {
269            break;
270        }
271    }
272    Ok(())
273}
Source

pub fn decode_backward_colored( &mut self, cache: &DecoderCache, grad_distances: &[f32], grad_colors: Option<&Matrix>, ) -> Result<Matrix, NetworkError>

The same, with the colour head’s gradients as well.

Examples found in repository?
examples/train_shape.rs (lines 218-222)
126fn train_vae(
127    args: &Args,
128    stream: BatchStream,
129    rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131    let (mut model, mut step) = match &args.resume {
132        Some(path) => {
133            let (model, metadata) = ShapeVae::load(path, rng)?;
134            (model, started_at(&metadata))
135        }
136        None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137    };
138    if let Some(device) = args.cuda {
139        #[cfg(feature = "cuda")]
140        {
141            // 0 as the budget: the caller picked the card, and the weights are
142            // a small part of what a run of this size needs.
143            model.to_cuda(device, 0)?;
144            println!("shape autoencoder: training on CUDA device {device}");
145        }
146        #[cfg(not(feature = "cuda"))]
147        return Err(format!(
148            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149        )
150        .into());
151    }
152    let optimizer = adam(args.learning_rate);
153    println!(
154        "shape autoencoder: {} parameters, from step {step}",
155        model.num_parameters()
156    );
157
158    let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159    for batch in stream {
160        let batch = batch?;
161        let count = batch.len();
162        let points = batch.surface.rows / count;
163        let queries = batch.queries.rows / count;
164
165        // Packing helps the ordinary 4K-query training shape, but at 16K
166        // queries the larger attention workspace is slower and batch eight can
167        // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168        let cuda_batch = model.on_device() && queries <= 4096;
169        #[cfg(feature = "cuda")]
170        if cuda_batch {
171            let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172            // The packed losses are means across the batch; the reporting
173            // accumulator below historically stores a sum over examples.
174            reconstruction += loss * count as f32;
175            divergence += kl * count as f32;
176            tint += color * count as f32;
177        }
178
179        if !cuda_batch {
180            for index in 0..count {
181                let surface = slice(&batch.surface, index, points);
182                let (mean, log_variance, cache) = model.encode_train(&surface)?;
183                let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185                let (prediction, decoded) =
186                    model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187                let target = &batch.distances[index * queries..(index + 1) * queries];
188                let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189                let (kl, grad_mean, grad_log_variance) =
190                    losses::kl_divergence(&mean.data, &log_variance.data);
191                reconstruction += loss;
192                divergence += kl;
193
194                let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196                // The colour targets sit on the surface points, not the distance
197                // queries, so the colour head trains on a second short decode.
198                // The loader's points are already in random order, so the first
199                // `--color-points` of them are a random draw.
200                if let Some(colors) = &batch.colors
201                    && args.color > 0.0
202                    && args.color_points > 0
203                {
204                    let wanted = args.color_points.min(points);
205                    let positions = positions(&surface, wanted);
206                    let targets = head_rows(&slice(colors, index, points), wanted);
207                    let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209                    let predicted = model.colors(&decoded)?;
210                    let mut grad = predicted.clone();
211                    let scale = args.color / predicted.data.len() as f32;
212                    for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213                        let error = *slot - target;
214                        tint += error * error / predicted.data.len() as f32;
215                        *slot = 2.0 * scale * error;
216                    }
217
218                    let from_colors = model.decode_backward_colored(
219                        &decoded,
220                        &vec![0.0; positions.rows],
221                        Some(&grad),
222                    )?;
223                    for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224                        *slot += value;
225                    }
226                }
227
228                let (mut into_mean, mut into_log_variance) =
229                    ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230                for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231                    *slot += args.kl * value;
232                }
233                for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234                    *slot += args.kl * value;
235                }
236                model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237            }
238        }
239
240        step += 1;
241        // The gradients were summed over the batch, so the step is scaled by
242        // its size rather than the loop dividing every gradient it makes.
243        optimizers::step_clipped(
244            &mut model.params_mut(),
245            &optimizer,
246            step,
247            1.0 / count as f32,
248            1.0,
249        )?;
250        optimizers::zero_grad(&mut model.params_mut());
251
252        if step % args.report == 0 {
253            let scale = (args.report * count) as f32;
254            println!(
255                "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256                reconstruction / scale,
257                divergence / scale,
258                tint / scale
259            );
260            reconstruction = 0.0;
261            divergence = 0.0;
262            tint = 0.0;
263        }
264        if step % args.checkpoint_every == 0 || step == args.steps {
265            model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266            println!("wrote {} at step {step}", args.out.display());
267        }
268        if step >= args.steps {
269            break;
270        }
271    }
272    Ok(())
273}
Source

pub fn field<'a>( &'a self, latent: &'a Matrix, chunk: usize, ) -> impl Fn(&Matrix) -> Result<Vec<f32>, NetworkError> + Sync + 'a

A closure over a fixed latent, shaped for crate::mesh::marching_tetrahedra, which hands it one grid slice at a time.

Source

pub fn on_device(&self) -> bool

Whether the projections currently live on a device.

Examples found in repository?
examples/train_shape.rs (line 168)
126fn train_vae(
127    args: &Args,
128    stream: BatchStream,
129    rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131    let (mut model, mut step) = match &args.resume {
132        Some(path) => {
133            let (model, metadata) = ShapeVae::load(path, rng)?;
134            (model, started_at(&metadata))
135        }
136        None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137    };
138    if let Some(device) = args.cuda {
139        #[cfg(feature = "cuda")]
140        {
141            // 0 as the budget: the caller picked the card, and the weights are
142            // a small part of what a run of this size needs.
143            model.to_cuda(device, 0)?;
144            println!("shape autoencoder: training on CUDA device {device}");
145        }
146        #[cfg(not(feature = "cuda"))]
147        return Err(format!(
148            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149        )
150        .into());
151    }
152    let optimizer = adam(args.learning_rate);
153    println!(
154        "shape autoencoder: {} parameters, from step {step}",
155        model.num_parameters()
156    );
157
158    let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159    for batch in stream {
160        let batch = batch?;
161        let count = batch.len();
162        let points = batch.surface.rows / count;
163        let queries = batch.queries.rows / count;
164
165        // Packing helps the ordinary 4K-query training shape, but at 16K
166        // queries the larger attention workspace is slower and batch eight can
167        // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168        let cuda_batch = model.on_device() && queries <= 4096;
169        #[cfg(feature = "cuda")]
170        if cuda_batch {
171            let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172            // The packed losses are means across the batch; the reporting
173            // accumulator below historically stores a sum over examples.
174            reconstruction += loss * count as f32;
175            divergence += kl * count as f32;
176            tint += color * count as f32;
177        }
178
179        if !cuda_batch {
180            for index in 0..count {
181                let surface = slice(&batch.surface, index, points);
182                let (mean, log_variance, cache) = model.encode_train(&surface)?;
183                let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185                let (prediction, decoded) =
186                    model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187                let target = &batch.distances[index * queries..(index + 1) * queries];
188                let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189                let (kl, grad_mean, grad_log_variance) =
190                    losses::kl_divergence(&mean.data, &log_variance.data);
191                reconstruction += loss;
192                divergence += kl;
193
194                let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196                // The colour targets sit on the surface points, not the distance
197                // queries, so the colour head trains on a second short decode.
198                // The loader's points are already in random order, so the first
199                // `--color-points` of them are a random draw.
200                if let Some(colors) = &batch.colors
201                    && args.color > 0.0
202                    && args.color_points > 0
203                {
204                    let wanted = args.color_points.min(points);
205                    let positions = positions(&surface, wanted);
206                    let targets = head_rows(&slice(colors, index, points), wanted);
207                    let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209                    let predicted = model.colors(&decoded)?;
210                    let mut grad = predicted.clone();
211                    let scale = args.color / predicted.data.len() as f32;
212                    for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213                        let error = *slot - target;
214                        tint += error * error / predicted.data.len() as f32;
215                        *slot = 2.0 * scale * error;
216                    }
217
218                    let from_colors = model.decode_backward_colored(
219                        &decoded,
220                        &vec![0.0; positions.rows],
221                        Some(&grad),
222                    )?;
223                    for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224                        *slot += value;
225                    }
226                }
227
228                let (mut into_mean, mut into_log_variance) =
229                    ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230                for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231                    *slot += args.kl * value;
232                }
233                for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234                    *slot += args.kl * value;
235                }
236                model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237            }
238        }
239
240        step += 1;
241        // The gradients were summed over the batch, so the step is scaled by
242        // its size rather than the loop dividing every gradient it makes.
243        optimizers::step_clipped(
244            &mut model.params_mut(),
245            &optimizer,
246            step,
247            1.0 / count as f32,
248            1.0,
249        )?;
250        optimizers::zero_grad(&mut model.params_mut());
251
252        if step % args.report == 0 {
253            let scale = (args.report * count) as f32;
254            println!(
255                "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256                reconstruction / scale,
257                divergence / scale,
258                tint / scale
259            );
260            reconstruction = 0.0;
261            divergence = 0.0;
262            tint = 0.0;
263        }
264        if step % args.checkpoint_every == 0 || step == args.steps {
265            model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266            println!("wrote {} at step {step}", args.out.display());
267        }
268        if step >= args.steps {
269            break;
270        }
271    }
272    Ok(())
273}
Source

pub fn params_mut(&mut self) -> Vec<&mut Param>

Examples found in repository?
examples/train_shape.rs (line 244)
126fn train_vae(
127    args: &Args,
128    stream: BatchStream,
129    rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131    let (mut model, mut step) = match &args.resume {
132        Some(path) => {
133            let (model, metadata) = ShapeVae::load(path, rng)?;
134            (model, started_at(&metadata))
135        }
136        None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137    };
138    if let Some(device) = args.cuda {
139        #[cfg(feature = "cuda")]
140        {
141            // 0 as the budget: the caller picked the card, and the weights are
142            // a small part of what a run of this size needs.
143            model.to_cuda(device, 0)?;
144            println!("shape autoencoder: training on CUDA device {device}");
145        }
146        #[cfg(not(feature = "cuda"))]
147        return Err(format!(
148            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149        )
150        .into());
151    }
152    let optimizer = adam(args.learning_rate);
153    println!(
154        "shape autoencoder: {} parameters, from step {step}",
155        model.num_parameters()
156    );
157
158    let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159    for batch in stream {
160        let batch = batch?;
161        let count = batch.len();
162        let points = batch.surface.rows / count;
163        let queries = batch.queries.rows / count;
164
165        // Packing helps the ordinary 4K-query training shape, but at 16K
166        // queries the larger attention workspace is slower and batch eight can
167        // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168        let cuda_batch = model.on_device() && queries <= 4096;
169        #[cfg(feature = "cuda")]
170        if cuda_batch {
171            let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172            // The packed losses are means across the batch; the reporting
173            // accumulator below historically stores a sum over examples.
174            reconstruction += loss * count as f32;
175            divergence += kl * count as f32;
176            tint += color * count as f32;
177        }
178
179        if !cuda_batch {
180            for index in 0..count {
181                let surface = slice(&batch.surface, index, points);
182                let (mean, log_variance, cache) = model.encode_train(&surface)?;
183                let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185                let (prediction, decoded) =
186                    model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187                let target = &batch.distances[index * queries..(index + 1) * queries];
188                let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189                let (kl, grad_mean, grad_log_variance) =
190                    losses::kl_divergence(&mean.data, &log_variance.data);
191                reconstruction += loss;
192                divergence += kl;
193
194                let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196                // The colour targets sit on the surface points, not the distance
197                // queries, so the colour head trains on a second short decode.
198                // The loader's points are already in random order, so the first
199                // `--color-points` of them are a random draw.
200                if let Some(colors) = &batch.colors
201                    && args.color > 0.0
202                    && args.color_points > 0
203                {
204                    let wanted = args.color_points.min(points);
205                    let positions = positions(&surface, wanted);
206                    let targets = head_rows(&slice(colors, index, points), wanted);
207                    let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209                    let predicted = model.colors(&decoded)?;
210                    let mut grad = predicted.clone();
211                    let scale = args.color / predicted.data.len() as f32;
212                    for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213                        let error = *slot - target;
214                        tint += error * error / predicted.data.len() as f32;
215                        *slot = 2.0 * scale * error;
216                    }
217
218                    let from_colors = model.decode_backward_colored(
219                        &decoded,
220                        &vec![0.0; positions.rows],
221                        Some(&grad),
222                    )?;
223                    for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224                        *slot += value;
225                    }
226                }
227
228                let (mut into_mean, mut into_log_variance) =
229                    ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230                for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231                    *slot += args.kl * value;
232                }
233                for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234                    *slot += args.kl * value;
235                }
236                model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237            }
238        }
239
240        step += 1;
241        // The gradients were summed over the batch, so the step is scaled by
242        // its size rather than the loop dividing every gradient it makes.
243        optimizers::step_clipped(
244            &mut model.params_mut(),
245            &optimizer,
246            step,
247            1.0 / count as f32,
248            1.0,
249        )?;
250        optimizers::zero_grad(&mut model.params_mut());
251
252        if step % args.report == 0 {
253            let scale = (args.report * count) as f32;
254            println!(
255                "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256                reconstruction / scale,
257                divergence / scale,
258                tint / scale
259            );
260            reconstruction = 0.0;
261            divergence = 0.0;
262            tint = 0.0;
263        }
264        if step % args.checkpoint_every == 0 || step == args.steps {
265            model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266            println!("wrote {} at step {step}", args.out.display());
267        }
268        if step >= args.steps {
269            break;
270        }
271    }
272    Ok(())
273}
Source

pub fn num_parameters(&mut self) -> usize

Examples found in repository?
examples/train_shape.rs (line 155)
126fn train_vae(
127    args: &Args,
128    stream: BatchStream,
129    rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131    let (mut model, mut step) = match &args.resume {
132        Some(path) => {
133            let (model, metadata) = ShapeVae::load(path, rng)?;
134            (model, started_at(&metadata))
135        }
136        None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137    };
138    if let Some(device) = args.cuda {
139        #[cfg(feature = "cuda")]
140        {
141            // 0 as the budget: the caller picked the card, and the weights are
142            // a small part of what a run of this size needs.
143            model.to_cuda(device, 0)?;
144            println!("shape autoencoder: training on CUDA device {device}");
145        }
146        #[cfg(not(feature = "cuda"))]
147        return Err(format!(
148            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149        )
150        .into());
151    }
152    let optimizer = adam(args.learning_rate);
153    println!(
154        "shape autoencoder: {} parameters, from step {step}",
155        model.num_parameters()
156    );
157
158    let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159    for batch in stream {
160        let batch = batch?;
161        let count = batch.len();
162        let points = batch.surface.rows / count;
163        let queries = batch.queries.rows / count;
164
165        // Packing helps the ordinary 4K-query training shape, but at 16K
166        // queries the larger attention workspace is slower and batch eight can
167        // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168        let cuda_batch = model.on_device() && queries <= 4096;
169        #[cfg(feature = "cuda")]
170        if cuda_batch {
171            let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172            // The packed losses are means across the batch; the reporting
173            // accumulator below historically stores a sum over examples.
174            reconstruction += loss * count as f32;
175            divergence += kl * count as f32;
176            tint += color * count as f32;
177        }
178
179        if !cuda_batch {
180            for index in 0..count {
181                let surface = slice(&batch.surface, index, points);
182                let (mean, log_variance, cache) = model.encode_train(&surface)?;
183                let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185                let (prediction, decoded) =
186                    model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187                let target = &batch.distances[index * queries..(index + 1) * queries];
188                let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189                let (kl, grad_mean, grad_log_variance) =
190                    losses::kl_divergence(&mean.data, &log_variance.data);
191                reconstruction += loss;
192                divergence += kl;
193
194                let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196                // The colour targets sit on the surface points, not the distance
197                // queries, so the colour head trains on a second short decode.
198                // The loader's points are already in random order, so the first
199                // `--color-points` of them are a random draw.
200                if let Some(colors) = &batch.colors
201                    && args.color > 0.0
202                    && args.color_points > 0
203                {
204                    let wanted = args.color_points.min(points);
205                    let positions = positions(&surface, wanted);
206                    let targets = head_rows(&slice(colors, index, points), wanted);
207                    let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209                    let predicted = model.colors(&decoded)?;
210                    let mut grad = predicted.clone();
211                    let scale = args.color / predicted.data.len() as f32;
212                    for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213                        let error = *slot - target;
214                        tint += error * error / predicted.data.len() as f32;
215                        *slot = 2.0 * scale * error;
216                    }
217
218                    let from_colors = model.decode_backward_colored(
219                        &decoded,
220                        &vec![0.0; positions.rows],
221                        Some(&grad),
222                    )?;
223                    for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224                        *slot += value;
225                    }
226                }
227
228                let (mut into_mean, mut into_log_variance) =
229                    ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230                for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231                    *slot += args.kl * value;
232                }
233                for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234                    *slot += args.kl * value;
235                }
236                model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237            }
238        }
239
240        step += 1;
241        // The gradients were summed over the batch, so the step is scaled by
242        // its size rather than the loop dividing every gradient it makes.
243        optimizers::step_clipped(
244            &mut model.params_mut(),
245            &optimizer,
246            step,
247            1.0 / count as f32,
248            1.0,
249        )?;
250        optimizers::zero_grad(&mut model.params_mut());
251
252        if step % args.report == 0 {
253            let scale = (args.report * count) as f32;
254            println!(
255                "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256                reconstruction / scale,
257                divergence / scale,
258                tint / scale
259            );
260            reconstruction = 0.0;
261            divergence = 0.0;
262            tint = 0.0;
263        }
264        if step % args.checkpoint_every == 0 || step == args.steps {
265            model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266            println!("wrote {} at step {step}", args.out.display());
267        }
268        if step >= args.steps {
269            break;
270        }
271    }
272    Ok(())
273}
Source

pub fn save<P: AsRef<Path>>( &mut self, path: P, metadata: &BTreeMap<String, String>, ) -> Result<(), NetworkError>

Writes the weights, the Adam moments and the configuration.

The caller’s metadata is written beside the configuration, which is where a training loop puts its step counter. The key config belongs to the model and is overwritten.

Examples found in repository?
examples/train_shape.rs (line 265)
126fn train_vae(
127    args: &Args,
128    stream: BatchStream,
129    rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131    let (mut model, mut step) = match &args.resume {
132        Some(path) => {
133            let (model, metadata) = ShapeVae::load(path, rng)?;
134            (model, started_at(&metadata))
135        }
136        None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137    };
138    if let Some(device) = args.cuda {
139        #[cfg(feature = "cuda")]
140        {
141            // 0 as the budget: the caller picked the card, and the weights are
142            // a small part of what a run of this size needs.
143            model.to_cuda(device, 0)?;
144            println!("shape autoencoder: training on CUDA device {device}");
145        }
146        #[cfg(not(feature = "cuda"))]
147        return Err(format!(
148            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149        )
150        .into());
151    }
152    let optimizer = adam(args.learning_rate);
153    println!(
154        "shape autoencoder: {} parameters, from step {step}",
155        model.num_parameters()
156    );
157
158    let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159    for batch in stream {
160        let batch = batch?;
161        let count = batch.len();
162        let points = batch.surface.rows / count;
163        let queries = batch.queries.rows / count;
164
165        // Packing helps the ordinary 4K-query training shape, but at 16K
166        // queries the larger attention workspace is slower and batch eight can
167        // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168        let cuda_batch = model.on_device() && queries <= 4096;
169        #[cfg(feature = "cuda")]
170        if cuda_batch {
171            let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172            // The packed losses are means across the batch; the reporting
173            // accumulator below historically stores a sum over examples.
174            reconstruction += loss * count as f32;
175            divergence += kl * count as f32;
176            tint += color * count as f32;
177        }
178
179        if !cuda_batch {
180            for index in 0..count {
181                let surface = slice(&batch.surface, index, points);
182                let (mean, log_variance, cache) = model.encode_train(&surface)?;
183                let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185                let (prediction, decoded) =
186                    model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187                let target = &batch.distances[index * queries..(index + 1) * queries];
188                let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189                let (kl, grad_mean, grad_log_variance) =
190                    losses::kl_divergence(&mean.data, &log_variance.data);
191                reconstruction += loss;
192                divergence += kl;
193
194                let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196                // The colour targets sit on the surface points, not the distance
197                // queries, so the colour head trains on a second short decode.
198                // The loader's points are already in random order, so the first
199                // `--color-points` of them are a random draw.
200                if let Some(colors) = &batch.colors
201                    && args.color > 0.0
202                    && args.color_points > 0
203                {
204                    let wanted = args.color_points.min(points);
205                    let positions = positions(&surface, wanted);
206                    let targets = head_rows(&slice(colors, index, points), wanted);
207                    let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209                    let predicted = model.colors(&decoded)?;
210                    let mut grad = predicted.clone();
211                    let scale = args.color / predicted.data.len() as f32;
212                    for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213                        let error = *slot - target;
214                        tint += error * error / predicted.data.len() as f32;
215                        *slot = 2.0 * scale * error;
216                    }
217
218                    let from_colors = model.decode_backward_colored(
219                        &decoded,
220                        &vec![0.0; positions.rows],
221                        Some(&grad),
222                    )?;
223                    for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224                        *slot += value;
225                    }
226                }
227
228                let (mut into_mean, mut into_log_variance) =
229                    ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230                for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231                    *slot += args.kl * value;
232                }
233                for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234                    *slot += args.kl * value;
235                }
236                model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237            }
238        }
239
240        step += 1;
241        // The gradients were summed over the batch, so the step is scaled by
242        // its size rather than the loop dividing every gradient it makes.
243        optimizers::step_clipped(
244            &mut model.params_mut(),
245            &optimizer,
246            step,
247            1.0 / count as f32,
248            1.0,
249        )?;
250        optimizers::zero_grad(&mut model.params_mut());
251
252        if step % args.report == 0 {
253            let scale = (args.report * count) as f32;
254            println!(
255                "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256                reconstruction / scale,
257                divergence / scale,
258                tint / scale
259            );
260            reconstruction = 0.0;
261            divergence = 0.0;
262            tint = 0.0;
263        }
264        if step % args.checkpoint_every == 0 || step == args.steps {
265            model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266            println!("wrote {} at step {step}", args.out.display());
267        }
268        if step >= args.steps {
269            break;
270        }
271    }
272    Ok(())
273}
Source

pub fn load<P: AsRef<Path>>( path: P, rng: &mut StdRng, ) -> Result<(Self, BTreeMap<String, String>), NetworkError>

Rebuilds a model from what save wrote, and returns the rest of the metadata.

The shape comes out of the file, so a caller does not have to remember which configuration a checkpoint was trained with; rng only fills weights that the file then overwrites.

Examples found in repository?
examples/kl_report.rs (line 85)
48fn main() -> Result<(), Box<dyn std::error::Error>> {
49    let args = parse()?;
50    let mut rng = StdRng::seed_from_u64(args.seed);
51
52    let mut corpus = Corpus::open(&args.tokens, &args.samples)?;
53    if corpus.is_empty() {
54        return Err("no render is paired with a mesh; check the two directories".into());
55    }
56    let objects: Vec<Object> = (0..args.objects.min(corpus.len()))
57        .map(|index| {
58            let example =
59                corpus.example(index, Some(args.surface), Some(args.queries), &mut rng)?;
60            Ok::<_, Box<dyn std::error::Error>>(Object {
61                surface: example.surface,
62                queries: example.queries,
63                distances: example.distances,
64            })
65        })
66        .collect::<Result<_, _>>()?;
67    println!(
68        "{} objects, {} surface points, {} query points, clamp {}",
69        objects.len(),
70        args.surface,
71        args.queries,
72        args.clamp
73    );
74    println!(
75        "{:<28} {:>9} {:>9} {:>7} {:>7} {:>8} {:>7}",
76        "checkpoint", "l1 draw", "l1 mean", "sigma", "|mean|", "kl/unit", "active"
77    );
78
79    for path in &args.checkpoints {
80        // A fresh generator per checkpoint, so every model is asked for the
81        // same draw of noise as well as the same objects.
82        let mut rng = StdRng::seed_from_u64(args.seed ^ 0x5a1);
83        // `mut` is only used by the device move below.
84        #[cfg_attr(not(feature = "cuda"), allow(unused_mut))]
85        let (mut model, _) = ShapeVae::load(path, &mut StdRng::seed_from_u64(args.seed))?;
86        if let Some(device) = args.cuda {
87            #[cfg(feature = "cuda")]
88            model.to_cuda(device, 0)?;
89            #[cfg(not(feature = "cuda"))]
90            return Err(format!(
91                "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
92            )
93            .into());
94        }
95
96        let (mut draw_l1, mut mean_l1, mut sigma, mut magnitude, mut kl, mut active) =
97            (0.0, 0.0, 0.0, 0.0, 0.0, 0.0);
98        for object in &objects {
99            let (mean, log_variance) = model.encode(&object.surface)?;
100            let (latent, _) = ShapeVae::sample(&mean, &log_variance, &mut rng);
101
102            let from_draw = model.decode(&latent, &object.queries, 4096)?;
103            let from_mean = model.decode(&mean, &object.queries, 4096)?;
104            draw_l1 += losses::clamped_l1(&from_draw, &object.distances, args.clamp).0;
105            mean_l1 += losses::clamped_l1(&from_mean, &object.distances, args.clamp).0;
106
107            let units = mean.data.len() as f32;
108            magnitude += mean.data.iter().map(|value| value.abs()).sum::<f32>() / units;
109            sigma += log_variance
110                .data
111                .iter()
112                .map(|value| (0.5 * value).exp())
113                .sum::<f32>()
114                / units;
115            // The per-unit KL against a unit Gaussian, which is what a
116            // free-bits floor would be applied to one unit at a time.
117            let mut per_unit = 0.0;
118            let mut above = 0.0;
119            for (mean, log_variance) in mean.data.iter().zip(&log_variance.data) {
120                let nats = 0.5 * (mean * mean + log_variance.exp() - 1.0 - log_variance);
121                per_unit += nats;
122                if nats > 0.01 {
123                    above += 1.0;
124                }
125            }
126            kl += per_unit / units;
127            active += above / units;
128        }
129
130        let count = objects.len() as f32;
131        println!(
132            "{:<28} {:>9.5} {:>9.5} {:>7.3} {:>7.3} {:>8.4} {:>6.1}%",
133            path.file_name()
134                .unwrap_or(path.as_os_str())
135                .to_string_lossy(),
136            draw_l1 / count,
137            mean_l1 / count,
138            sigma / count,
139            magnitude / count,
140            kl / count,
141            100.0 * active / count
142        );
143    }
144    Ok(())
145}
More examples
Hide additional examples
examples/train_shape.rs (line 133)
126fn train_vae(
127    args: &Args,
128    stream: BatchStream,
129    rng: &mut StdRng,
130) -> Result<(), Box<dyn std::error::Error>> {
131    let (mut model, mut step) = match &args.resume {
132        Some(path) => {
133            let (model, metadata) = ShapeVae::load(path, rng)?;
134            (model, started_at(&metadata))
135        }
136        None => (ShapeVae::new(shape_config(args)?, rng)?, 0),
137    };
138    if let Some(device) = args.cuda {
139        #[cfg(feature = "cuda")]
140        {
141            // 0 as the budget: the caller picked the card, and the weights are
142            // a small part of what a run of this size needs.
143            model.to_cuda(device, 0)?;
144            println!("shape autoencoder: training on CUDA device {device}");
145        }
146        #[cfg(not(feature = "cuda"))]
147        return Err(format!(
148            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
149        )
150        .into());
151    }
152    let optimizer = adam(args.learning_rate);
153    println!(
154        "shape autoencoder: {} parameters, from step {step}",
155        model.num_parameters()
156    );
157
158    let (mut reconstruction, mut divergence, mut tint) = (0.0, 0.0, 0.0);
159    for batch in stream {
160        let batch = batch?;
161        let count = batch.len();
162        let points = batch.surface.rows / count;
163        let queries = batch.queries.rows / count;
164
165        // Packing helps the ordinary 4K-query training shape, but at 16K
166        // queries the larger attention workspace is slower and batch eight can
167        // exceed a 12 GiB card. Keep that workload shape-at-a-time.
168        let cuda_batch = model.on_device() && queries <= 4096;
169        #[cfg(feature = "cuda")]
170        if cuda_batch {
171            let (loss, kl, color) = train_vae_cuda_batch(args, &mut model, &batch, rng)?;
172            // The packed losses are means across the batch; the reporting
173            // accumulator below historically stores a sum over examples.
174            reconstruction += loss * count as f32;
175            divergence += kl * count as f32;
176            tint += color * count as f32;
177        }
178
179        if !cuda_batch {
180            for index in 0..count {
181                let surface = slice(&batch.surface, index, points);
182                let (mean, log_variance, cache) = model.encode_train(&surface)?;
183                let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
184
185                let (prediction, decoded) =
186                    model.decode_train(&latent, &slice(&batch.queries, index, queries))?;
187                let target = &batch.distances[index * queries..(index + 1) * queries];
188                let (loss, grad) = losses::clamped_l1(&prediction, target, args.clamp);
189                let (kl, grad_mean, grad_log_variance) =
190                    losses::kl_divergence(&mean.data, &log_variance.data);
191                reconstruction += loss;
192                divergence += kl;
193
194                let mut grad_latent = model.decode_backward(&decoded, &grad)?;
195
196                // The colour targets sit on the surface points, not the distance
197                // queries, so the colour head trains on a second short decode.
198                // The loader's points are already in random order, so the first
199                // `--color-points` of them are a random draw.
200                if let Some(colors) = &batch.colors
201                    && args.color > 0.0
202                    && args.color_points > 0
203                {
204                    let wanted = args.color_points.min(points);
205                    let positions = positions(&surface, wanted);
206                    let targets = head_rows(&slice(colors, index, points), wanted);
207                    let (_, decoded) = model.decode_train(&latent, &positions)?;
208
209                    let predicted = model.colors(&decoded)?;
210                    let mut grad = predicted.clone();
211                    let scale = args.color / predicted.data.len() as f32;
212                    for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
213                        let error = *slot - target;
214                        tint += error * error / predicted.data.len() as f32;
215                        *slot = 2.0 * scale * error;
216                    }
217
218                    let from_colors = model.decode_backward_colored(
219                        &decoded,
220                        &vec![0.0; positions.rows],
221                        Some(&grad),
222                    )?;
223                    for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
224                        *slot += value;
225                    }
226                }
227
228                let (mut into_mean, mut into_log_variance) =
229                    ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
230                for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
231                    *slot += args.kl * value;
232                }
233                for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
234                    *slot += args.kl * value;
235                }
236                model.encode_backward(&cache, &into_mean, &into_log_variance)?;
237            }
238        }
239
240        step += 1;
241        // The gradients were summed over the batch, so the step is scaled by
242        // its size rather than the loop dividing every gradient it makes.
243        optimizers::step_clipped(
244            &mut model.params_mut(),
245            &optimizer,
246            step,
247            1.0 / count as f32,
248            1.0,
249        )?;
250        optimizers::zero_grad(&mut model.params_mut());
251
252        if step % args.report == 0 {
253            let scale = (args.report * count) as f32;
254            println!(
255                "step {step}: l1 {:.5}, kl {:.5}, color {:.5}",
256                reconstruction / scale,
257                divergence / scale,
258                tint / scale
259            );
260            reconstruction = 0.0;
261            divergence = 0.0;
262            tint = 0.0;
263        }
264        if step % args.checkpoint_every == 0 || step == args.steps {
265            model.save(&args.out, &BTreeMap::from([step_at(step)]))?;
266            println!("wrote {} at step {step}", args.out.display());
267        }
268        if step >= args.steps {
269            break;
270        }
271    }
272    Ok(())
273}
274
275/// One packed device step. Packing is important here: the old loop launched
276/// and synchronized the whole encoder and decoder once per shape.
277#[cfg(feature = "cuda")]
278fn train_vae_cuda_batch(
279    args: &Args,
280    model: &mut ShapeVae,
281    batch: &Batch,
282    rng: &mut StdRng,
283) -> Result<(f32, f32, f32), Box<dyn std::error::Error>> {
284    let count = batch.len();
285    let points = batch.surface.rows / count;
286    let (mean, log_variance, encoder) = model.encode_train_batch(&batch.surface, count)?;
287    let (latent, noise) = ShapeVae::sample(&mean, &log_variance, rng);
288
289    let (prediction, decoded) = model.decode_train_batch(&latent, &batch.queries, count)?;
290    let (reconstruction, mut grad) = losses::clamped_l1(&prediction, &batch.distances, args.clamp);
291    let (divergence, mut grad_mean, mut grad_log_variance) =
292        losses::kl_divergence(&mean.data, &log_variance.data);
293    // The shape-at-a-time path accumulates one mean-loss gradient per example
294    // and lets the optimizer divide by the batch. Re-expand this packed mean
295    // so gradient clipping and the optimizer see the same summed gradient.
296    let batch_scale = count as f32;
297    grad.iter_mut().for_each(|value| *value *= batch_scale);
298    grad_mean.iter_mut().for_each(|value| *value *= batch_scale);
299    grad_log_variance
300        .iter_mut()
301        .for_each(|value| *value *= batch_scale);
302    let mut grad_latent = model.decode_backward(&decoded, &grad)?;
303
304    let mut tint = 0.0;
305    if let Some(colors) = &batch.colors
306        && args.color > 0.0
307        && args.color_points > 0
308    {
309        let wanted = args.color_points.min(points);
310        let positions = packed_prefix_rows(&batch.surface, count, points, wanted, 3);
311        let targets = packed_prefix_rows(colors, count, points, wanted, 3);
312        let (_, decoded) = model.decode_train_batch(&latent, &positions, count)?;
313        let predicted = model.colors(&decoded)?;
314        let mut grad = predicted.clone();
315        let scale = args.color * batch_scale / predicted.data.len() as f32;
316        for (slot, target) in grad.data.iter_mut().zip(&targets.data) {
317            let error = *slot - target;
318            tint += error * error / predicted.data.len() as f32;
319            *slot = 2.0 * scale * error;
320        }
321        let from_colors =
322            model.decode_backward_colored(&decoded, &vec![0.0; positions.rows], Some(&grad))?;
323        for (slot, value) in grad_latent.data.iter_mut().zip(&from_colors.data) {
324            *slot += value;
325        }
326    }
327
328    let (mut into_mean, mut into_log_variance) =
329        ShapeVae::sample_backward(&grad_latent, &log_variance, &noise);
330    for (slot, value) in into_mean.data.iter_mut().zip(&grad_mean) {
331        *slot += args.kl * value;
332    }
333    for (slot, value) in into_log_variance.data.iter_mut().zip(&grad_log_variance) {
334        *slot += args.kl * value;
335    }
336    model.encode_backward(&encoder, &into_mean, &into_log_variance)?;
337    Ok((reconstruction, divergence, tint))
338}
339
340fn train_flow(
341    args: &Args,
342    (cond_tokens, cond_dim): (usize, usize),
343    stream: BatchStream,
344    rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346    // Frozen: the flow transformer is trained against a latent space that has
347    // already stopped moving, so only `encode` is ever called here.
348    let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349    let shape = *vae.config();
350
351    let (mut model, mut step) = match &args.resume {
352        Some(path) => {
353            let (model, metadata) = FlowTransformer::load(path, rng)?;
354            (model, started_at(&metadata))
355        }
356        None => {
357            // The latent shape is the autoencoder's and the condition's shape
358            // is the corpus's, whatever a supplied configuration says.
359            let config = FlowConfig {
360                latents: shape.latents,
361                latent_dim: shape.latent_dim,
362                cond_tokens,
363                cond_dim,
364                ..read_config(args, FlowConfig::default())?
365            };
366            (FlowTransformer::new(config, rng)?, 0)
367        }
368    };
369    if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370        return Err(format!(
371            "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372            model.config().cond_tokens,
373            model.config().cond_dim
374        )
375        .into());
376    }
377    model.set_optimizer(adam(args.learning_rate));
378    if let Some(device) = args.cuda {
379        #[cfg(feature = "cuda")]
380        {
381            // 0 as the budget: the caller picked the card, and the blocks'
382            // weights are a small part of what a run of this size needs.
383            model.to_cuda(device, 0)?;
384            println!("flow transformer: training on CUDA device {device}");
385        }
386        #[cfg(not(feature = "cuda"))]
387        return Err(format!(
388            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389        )
390        .into());
391    }
392    println!(
393        "flow transformer: {} parameters, from step {step}",
394        model.num_parameters()
395    );
396
397    let mut running = 0.0;
398    for batch in stream {
399        let batch = batch?;
400        let count = batch.len();
401        let points = batch.surface.rows / count;
402
403        let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404        for index in 0..count {
405            // A draw from the posterior, not its mean. Every autoencoder step
406            // decoded a draw, so a draw is the only kind of latent the frozen
407            // decoder has ever answered for. The posterior is wide enough for
408            // that to matter — sigma about twice the magnitude of the mean —
409            // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410            // a draw, so a flow trained towards the mean would be aiming at the
411            // one latent the decoder is worst at.
412            let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413            let width = shape.latents * shape.latent_dim;
414            for (slot, value) in clean.data[index * width..(index + 1) * width]
415                .iter_mut()
416                .zip(&latent.data)
417            {
418                *slot = value * args.latent_scale;
419            }
420        }
421        if step == 0 {
422            println!("latent spread after scaling: {:.3}", spread(&clean));
423        }
424
425        running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426        step += 1;
427        model.step_clipped(1.0, 1.0)?;
428        model.zero_grad();
429
430        if step % args.report == 0 {
431            println!(
432                "step {step}: velocity mse {:.5}",
433                running / args.report as f32
434            );
435            running = 0.0;
436        }
437        if step % args.checkpoint_every == 0 || step == args.steps {
438            model.save(
439                &args.out,
440                &BTreeMap::from([
441                    step_at(step),
442                    ("latent_scale".to_string(), args.latent_scale.to_string()),
443                ]),
444            )?;
445            println!("wrote {} at step {step}", args.out.display());
446        }
447        if step >= args.steps {
448            break;
449        }
450    }
451    Ok(())
452}

Trait Implementations§

Source§

impl Clone for ShapeVae

Source§

fn clone(&self) -> Self

Returns a duplicate of the value. Read more
1.0.0 (const: unstable) · Source§

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source. Read more
Source§

impl Debug for ShapeVae

Source§

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter. Read more
Source§

impl<'de> Deserialize<'de> for ShapeVae

Source§

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>
where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer. Read more
Source§

impl PartialEq for ShapeVae

Compares the model, not where it happens to be running: two models with the same weights are equal whether or not one of them holds a device context.

Source§

fn eq(&self, other: &Self) -> bool

Equality operator ==. Read more
1.0.0 (const: unstable) · Source§

fn ne(&self, other: &Rhs) -> bool

Inequality operator !=. Read more
Source§

impl Serialize for ShapeVae

Source§

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>
where __S: Serializer,

Serialize this value into the given Serde serializer. Read more

Auto Trait Implementations§

Blanket Implementations§

Source§

impl<T> Any for T
where T: 'static + ?Sized,

Source§

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more
Source§

impl<T> Borrow<T> for T
where T: ?Sized,

Source§

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more
Source§

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source§

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more
Source§

impl<T> CloneToUninit for T
where T: Clone,

Source§

unsafe fn clone_to_uninit(&self, dest: *mut u8)

🔬This is a nightly-only experimental API. (clone_to_uninit)
Performs copy-assignment from self to dest. Read more
Source§

impl<T> DeserializeOwned for T
where T: for<'de> Deserialize<'de>,

Source§

impl<T> From<T> for T

Source§

fn from(t: T) -> T

Returns the argument unchanged.

Source§

impl<T, U> Into<U> for T
where U: From<T>,

Source§

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source§

impl<T> IntoEither for T

Source§

fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ

Converts self into a Left variant of Either<Self, Self> if into_left is true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more
Source§

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
where F: FnOnce(&Self) -> bool,

Converts self into a Left variant of Either<Self, Self> if into_left(&self) returns true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more
Source§

impl<T> Pointable for T

Source§

const ALIGN: usize

The alignment of pointer.
Source§

type Init = T

The type for initializers.
Source§

unsafe fn init(init: <T as Pointable>::Init) -> usize

Initializes a with the given initializer. Read more
Source§

unsafe fn deref<'a>(ptr: usize) -> &'a T

Dereferences the given pointer. Read more
Source§

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

Mutably dereferences the given pointer. Read more
Source§

unsafe fn drop(ptr: usize)

Drops the object pointed to by the given pointer. Read more
Source§

impl<T> ToOwned for T
where T: Clone,

Source§

type Owned = T

The resulting type after obtaining ownership.
Source§

fn to_owned(&self) -> T

Creates owned data from borrowed data, usually by cloning. Read more
Source§

fn clone_into(&self, target: &mut T)

Uses borrowed data to replace owned data, usually by cloning. Read more
Source§

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source§

type Error = !

The type returned in the event of a conversion error.
Source§

fn try_from(value: U) -> Result<T, !>

Performs the conversion.
Source§

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source§

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.
Source§

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.
Source§

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

Source§

fn vzip(self) -> V