Skip to main content

FlowTransformer

Struct FlowTransformer 

Source
pub struct FlowTransformer { /* private fields */ }
Expand description

Noise and a picture in, a shape latent out.

Implementations§

Source§

impl FlowTransformer

Source

pub fn new(config: FlowConfig, rng: &mut StdRng) -> Result<Self, NetworkError>

Examples found in repository?
examples/train_shape.rs (line 366)
340fn train_flow(
341    args: &Args,
342    (cond_tokens, cond_dim): (usize, usize),
343    stream: BatchStream,
344    rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346    // Frozen: the flow transformer is trained against a latent space that has
347    // already stopped moving, so only `encode` is ever called here.
348    let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349    let shape = *vae.config();
350
351    let (mut model, mut step) = match &args.resume {
352        Some(path) => {
353            let (model, metadata) = FlowTransformer::load(path, rng)?;
354            (model, started_at(&metadata))
355        }
356        None => {
357            // The latent shape is the autoencoder's and the condition's shape
358            // is the corpus's, whatever a supplied configuration says.
359            let config = FlowConfig {
360                latents: shape.latents,
361                latent_dim: shape.latent_dim,
362                cond_tokens,
363                cond_dim,
364                ..read_config(args, FlowConfig::default())?
365            };
366            (FlowTransformer::new(config, rng)?, 0)
367        }
368    };
369    if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370        return Err(format!(
371            "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372            model.config().cond_tokens,
373            model.config().cond_dim
374        )
375        .into());
376    }
377    model.set_optimizer(adam(args.learning_rate));
378    if let Some(device) = args.cuda {
379        #[cfg(feature = "cuda")]
380        {
381            // 0 as the budget: the caller picked the card, and the blocks'
382            // weights are a small part of what a run of this size needs.
383            model.to_cuda(device, 0)?;
384            println!("flow transformer: training on CUDA device {device}");
385        }
386        #[cfg(not(feature = "cuda"))]
387        return Err(format!(
388            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389        )
390        .into());
391    }
392    println!(
393        "flow transformer: {} parameters, from step {step}",
394        model.num_parameters()
395    );
396
397    let mut running = 0.0;
398    for batch in stream {
399        let batch = batch?;
400        let count = batch.len();
401        let points = batch.surface.rows / count;
402
403        let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404        for index in 0..count {
405            // A draw from the posterior, not its mean. Every autoencoder step
406            // decoded a draw, so a draw is the only kind of latent the frozen
407            // decoder has ever answered for. The posterior is wide enough for
408            // that to matter — sigma about twice the magnitude of the mean —
409            // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410            // a draw, so a flow trained towards the mean would be aiming at the
411            // one latent the decoder is worst at.
412            let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413            let width = shape.latents * shape.latent_dim;
414            for (slot, value) in clean.data[index * width..(index + 1) * width]
415                .iter_mut()
416                .zip(&latent.data)
417            {
418                *slot = value * args.latent_scale;
419            }
420        }
421        if step == 0 {
422            println!("latent spread after scaling: {:.3}", spread(&clean));
423        }
424
425        running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426        step += 1;
427        model.step_clipped(1.0, 1.0)?;
428        model.zero_grad();
429
430        if step % args.report == 0 {
431            println!(
432                "step {step}: velocity mse {:.5}",
433                running / args.report as f32
434            );
435            running = 0.0;
436        }
437        if step % args.checkpoint_every == 0 || step == args.steps {
438            model.save(
439                &args.out,
440                &BTreeMap::from([
441                    step_at(step),
442                    ("latent_scale".to_string(), args.latent_scale.to_string()),
443                ]),
444            )?;
445            println!("wrote {} at step {step}", args.out.display());
446        }
447        if step >= args.steps {
448            break;
449        }
450    }
451    Ok(())
452}
Source

pub fn config(&self) -> &FlowConfig

Examples found in repository?
examples/train_shape.rs (line 369)
340fn train_flow(
341    args: &Args,
342    (cond_tokens, cond_dim): (usize, usize),
343    stream: BatchStream,
344    rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346    // Frozen: the flow transformer is trained against a latent space that has
347    // already stopped moving, so only `encode` is ever called here.
348    let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349    let shape = *vae.config();
350
351    let (mut model, mut step) = match &args.resume {
352        Some(path) => {
353            let (model, metadata) = FlowTransformer::load(path, rng)?;
354            (model, started_at(&metadata))
355        }
356        None => {
357            // The latent shape is the autoencoder's and the condition's shape
358            // is the corpus's, whatever a supplied configuration says.
359            let config = FlowConfig {
360                latents: shape.latents,
361                latent_dim: shape.latent_dim,
362                cond_tokens,
363                cond_dim,
364                ..read_config(args, FlowConfig::default())?
365            };
366            (FlowTransformer::new(config, rng)?, 0)
367        }
368    };
369    if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370        return Err(format!(
371            "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372            model.config().cond_tokens,
373            model.config().cond_dim
374        )
375        .into());
376    }
377    model.set_optimizer(adam(args.learning_rate));
378    if let Some(device) = args.cuda {
379        #[cfg(feature = "cuda")]
380        {
381            // 0 as the budget: the caller picked the card, and the blocks'
382            // weights are a small part of what a run of this size needs.
383            model.to_cuda(device, 0)?;
384            println!("flow transformer: training on CUDA device {device}");
385        }
386        #[cfg(not(feature = "cuda"))]
387        return Err(format!(
388            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389        )
390        .into());
391    }
392    println!(
393        "flow transformer: {} parameters, from step {step}",
394        model.num_parameters()
395    );
396
397    let mut running = 0.0;
398    for batch in stream {
399        let batch = batch?;
400        let count = batch.len();
401        let points = batch.surface.rows / count;
402
403        let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404        for index in 0..count {
405            // A draw from the posterior, not its mean. Every autoencoder step
406            // decoded a draw, so a draw is the only kind of latent the frozen
407            // decoder has ever answered for. The posterior is wide enough for
408            // that to matter — sigma about twice the magnitude of the mean —
409            // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410            // a draw, so a flow trained towards the mean would be aiming at the
411            // one latent the decoder is worst at.
412            let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413            let width = shape.latents * shape.latent_dim;
414            for (slot, value) in clean.data[index * width..(index + 1) * width]
415                .iter_mut()
416                .zip(&latent.data)
417            {
418                *slot = value * args.latent_scale;
419            }
420        }
421        if step == 0 {
422            println!("latent spread after scaling: {:.3}", spread(&clean));
423        }
424
425        running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426        step += 1;
427        model.step_clipped(1.0, 1.0)?;
428        model.zero_grad();
429
430        if step % args.report == 0 {
431            println!(
432                "step {step}: velocity mse {:.5}",
433                running / args.report as f32
434            );
435            running = 0.0;
436        }
437        if step % args.checkpoint_every == 0 || step == args.steps {
438            model.save(
439                &args.out,
440                &BTreeMap::from([
441                    step_at(step),
442                    ("latent_scale".to_string(), args.latent_scale.to_string()),
443                ]),
444            )?;
445            println!("wrote {} at step {step}", args.out.display());
446        }
447        if step >= args.steps {
448            break;
449        }
450    }
451    Ok(())
452}
Source

pub fn set_optimizer(&mut self, optimizer: Optimizer)

Examples found in repository?
examples/train_shape.rs (line 377)
340fn train_flow(
341    args: &Args,
342    (cond_tokens, cond_dim): (usize, usize),
343    stream: BatchStream,
344    rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346    // Frozen: the flow transformer is trained against a latent space that has
347    // already stopped moving, so only `encode` is ever called here.
348    let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349    let shape = *vae.config();
350
351    let (mut model, mut step) = match &args.resume {
352        Some(path) => {
353            let (model, metadata) = FlowTransformer::load(path, rng)?;
354            (model, started_at(&metadata))
355        }
356        None => {
357            // The latent shape is the autoencoder's and the condition's shape
358            // is the corpus's, whatever a supplied configuration says.
359            let config = FlowConfig {
360                latents: shape.latents,
361                latent_dim: shape.latent_dim,
362                cond_tokens,
363                cond_dim,
364                ..read_config(args, FlowConfig::default())?
365            };
366            (FlowTransformer::new(config, rng)?, 0)
367        }
368    };
369    if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370        return Err(format!(
371            "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372            model.config().cond_tokens,
373            model.config().cond_dim
374        )
375        .into());
376    }
377    model.set_optimizer(adam(args.learning_rate));
378    if let Some(device) = args.cuda {
379        #[cfg(feature = "cuda")]
380        {
381            // 0 as the budget: the caller picked the card, and the blocks'
382            // weights are a small part of what a run of this size needs.
383            model.to_cuda(device, 0)?;
384            println!("flow transformer: training on CUDA device {device}");
385        }
386        #[cfg(not(feature = "cuda"))]
387        return Err(format!(
388            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389        )
390        .into());
391    }
392    println!(
393        "flow transformer: {} parameters, from step {step}",
394        model.num_parameters()
395    );
396
397    let mut running = 0.0;
398    for batch in stream {
399        let batch = batch?;
400        let count = batch.len();
401        let points = batch.surface.rows / count;
402
403        let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404        for index in 0..count {
405            // A draw from the posterior, not its mean. Every autoencoder step
406            // decoded a draw, so a draw is the only kind of latent the frozen
407            // decoder has ever answered for. The posterior is wide enough for
408            // that to matter — sigma about twice the magnitude of the mean —
409            // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410            // a draw, so a flow trained towards the mean would be aiming at the
411            // one latent the decoder is worst at.
412            let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413            let width = shape.latents * shape.latent_dim;
414            for (slot, value) in clean.data[index * width..(index + 1) * width]
415                .iter_mut()
416                .zip(&latent.data)
417            {
418                *slot = value * args.latent_scale;
419            }
420        }
421        if step == 0 {
422            println!("latent spread after scaling: {:.3}", spread(&clean));
423        }
424
425        running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426        step += 1;
427        model.step_clipped(1.0, 1.0)?;
428        model.zero_grad();
429
430        if step % args.report == 0 {
431            println!(
432                "step {step}: velocity mse {:.5}",
433                running / args.report as f32
434            );
435            running = 0.0;
436        }
437        if step % args.checkpoint_every == 0 || step == args.steps {
438            model.save(
439                &args.out,
440                &BTreeMap::from([
441                    step_at(step),
442                    ("latent_scale".to_string(), args.latent_scale.to_string()),
443                ]),
444            )?;
445            println!("wrote {} at step {step}", args.out.display());
446        }
447        if step >= args.steps {
448            break;
449        }
450    }
451    Ok(())
452}
Source

pub fn null_tokens(&self) -> &Matrix

The learned stand-in for a dropped condition, [cond_tokens, cond_dim].

Source

pub fn forward( &self, latent: &Matrix, tokens: &Matrix, times: &[f32], ) -> Result<Matrix, NetworkError>

The predicted velocity at time, [sequences * latents, latent_dim].

time runs from 1.0 at pure noise down to 0.0 at a clean latent, which is the same quantity Scheduler::sigmas produces.

Source

pub fn forward_train( &self, latent: &Matrix, tokens: &Matrix, times: &[f32], ) -> Result<(Matrix, FlowCache), NetworkError>

The same, keeping what the backward pass needs.

Source

pub fn backward( &mut self, cache: &FlowCache, grad_velocity: &Matrix, ) -> Result<Matrix, NetworkError>

Accumulates every weight gradient and returns dL/dtokens.

The image tower is frozen, so the caller normally drops what comes back; it is returned rather than swallowed because the gradient is what a later stage would need to fine-tune the tower.

Source

pub fn train_step<R: Rng>( &mut self, clean: &Matrix, tokens: &Matrix, dropout: f32, rng: &mut R, ) -> Result<f32, NetworkError>

One flow-matching step: mix, predict, score, and accumulate gradients.

clean is the shape autoencoder’s latent for the target mesh and tokens the frozen image tokens that describe it. dropout is the probability that a sequence’s condition is replaced by null_tokens, which is what makes classifier-free guidance possible at sampling time; 0.1 is the usual value and 0.0 turns it off.

Returns the mean squared error against the velocity that actually joins the two endpoints.

Examples found in repository?
examples/train_shape.rs (line 425)
340fn train_flow(
341    args: &Args,
342    (cond_tokens, cond_dim): (usize, usize),
343    stream: BatchStream,
344    rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346    // Frozen: the flow transformer is trained against a latent space that has
347    // already stopped moving, so only `encode` is ever called here.
348    let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349    let shape = *vae.config();
350
351    let (mut model, mut step) = match &args.resume {
352        Some(path) => {
353            let (model, metadata) = FlowTransformer::load(path, rng)?;
354            (model, started_at(&metadata))
355        }
356        None => {
357            // The latent shape is the autoencoder's and the condition's shape
358            // is the corpus's, whatever a supplied configuration says.
359            let config = FlowConfig {
360                latents: shape.latents,
361                latent_dim: shape.latent_dim,
362                cond_tokens,
363                cond_dim,
364                ..read_config(args, FlowConfig::default())?
365            };
366            (FlowTransformer::new(config, rng)?, 0)
367        }
368    };
369    if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370        return Err(format!(
371            "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372            model.config().cond_tokens,
373            model.config().cond_dim
374        )
375        .into());
376    }
377    model.set_optimizer(adam(args.learning_rate));
378    if let Some(device) = args.cuda {
379        #[cfg(feature = "cuda")]
380        {
381            // 0 as the budget: the caller picked the card, and the blocks'
382            // weights are a small part of what a run of this size needs.
383            model.to_cuda(device, 0)?;
384            println!("flow transformer: training on CUDA device {device}");
385        }
386        #[cfg(not(feature = "cuda"))]
387        return Err(format!(
388            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389        )
390        .into());
391    }
392    println!(
393        "flow transformer: {} parameters, from step {step}",
394        model.num_parameters()
395    );
396
397    let mut running = 0.0;
398    for batch in stream {
399        let batch = batch?;
400        let count = batch.len();
401        let points = batch.surface.rows / count;
402
403        let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404        for index in 0..count {
405            // A draw from the posterior, not its mean. Every autoencoder step
406            // decoded a draw, so a draw is the only kind of latent the frozen
407            // decoder has ever answered for. The posterior is wide enough for
408            // that to matter — sigma about twice the magnitude of the mean —
409            // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410            // a draw, so a flow trained towards the mean would be aiming at the
411            // one latent the decoder is worst at.
412            let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413            let width = shape.latents * shape.latent_dim;
414            for (slot, value) in clean.data[index * width..(index + 1) * width]
415                .iter_mut()
416                .zip(&latent.data)
417            {
418                *slot = value * args.latent_scale;
419            }
420        }
421        if step == 0 {
422            println!("latent spread after scaling: {:.3}", spread(&clean));
423        }
424
425        running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426        step += 1;
427        model.step_clipped(1.0, 1.0)?;
428        model.zero_grad();
429
430        if step % args.report == 0 {
431            println!(
432                "step {step}: velocity mse {:.5}",
433                running / args.report as f32
434            );
435            running = 0.0;
436        }
437        if step % args.checkpoint_every == 0 || step == args.steps {
438            model.save(
439                &args.out,
440                &BTreeMap::from([
441                    step_at(step),
442                    ("latent_scale".to_string(), args.latent_scale.to_string()),
443                ]),
444            )?;
445            println!("wrote {} at step {step}", args.out.display());
446        }
447        if step >= args.steps {
448            break;
449        }
450    }
451    Ok(())
452}
Source

pub fn drop_condition<R: Rng>( &self, tokens: &Matrix, dropout: f32, rng: &mut R, ) -> Result<(Matrix, Vec<bool>), NetworkError>

Replaces each sequence’s conditioning tokens with the learned null embedding, independently, with probability dropout.

The flags say which sequences were replaced, which is what lets the backward pass send those sequences’ token gradient to the null embedding instead of throwing it away.

Source

pub fn step_clipped( &mut self, scale: f32, max_norm: f32, ) -> Result<f32, NetworkError>

Applies the optimizer with the gradients clipped to a global norm, and returns the pre-clip norm.

Examples found in repository?
examples/train_shape.rs (line 427)
340fn train_flow(
341    args: &Args,
342    (cond_tokens, cond_dim): (usize, usize),
343    stream: BatchStream,
344    rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346    // Frozen: the flow transformer is trained against a latent space that has
347    // already stopped moving, so only `encode` is ever called here.
348    let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349    let shape = *vae.config();
350
351    let (mut model, mut step) = match &args.resume {
352        Some(path) => {
353            let (model, metadata) = FlowTransformer::load(path, rng)?;
354            (model, started_at(&metadata))
355        }
356        None => {
357            // The latent shape is the autoencoder's and the condition's shape
358            // is the corpus's, whatever a supplied configuration says.
359            let config = FlowConfig {
360                latents: shape.latents,
361                latent_dim: shape.latent_dim,
362                cond_tokens,
363                cond_dim,
364                ..read_config(args, FlowConfig::default())?
365            };
366            (FlowTransformer::new(config, rng)?, 0)
367        }
368    };
369    if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370        return Err(format!(
371            "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372            model.config().cond_tokens,
373            model.config().cond_dim
374        )
375        .into());
376    }
377    model.set_optimizer(adam(args.learning_rate));
378    if let Some(device) = args.cuda {
379        #[cfg(feature = "cuda")]
380        {
381            // 0 as the budget: the caller picked the card, and the blocks'
382            // weights are a small part of what a run of this size needs.
383            model.to_cuda(device, 0)?;
384            println!("flow transformer: training on CUDA device {device}");
385        }
386        #[cfg(not(feature = "cuda"))]
387        return Err(format!(
388            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389        )
390        .into());
391    }
392    println!(
393        "flow transformer: {} parameters, from step {step}",
394        model.num_parameters()
395    );
396
397    let mut running = 0.0;
398    for batch in stream {
399        let batch = batch?;
400        let count = batch.len();
401        let points = batch.surface.rows / count;
402
403        let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404        for index in 0..count {
405            // A draw from the posterior, not its mean. Every autoencoder step
406            // decoded a draw, so a draw is the only kind of latent the frozen
407            // decoder has ever answered for. The posterior is wide enough for
408            // that to matter — sigma about twice the magnitude of the mean —
409            // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410            // a draw, so a flow trained towards the mean would be aiming at the
411            // one latent the decoder is worst at.
412            let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413            let width = shape.latents * shape.latent_dim;
414            for (slot, value) in clean.data[index * width..(index + 1) * width]
415                .iter_mut()
416                .zip(&latent.data)
417            {
418                *slot = value * args.latent_scale;
419            }
420        }
421        if step == 0 {
422            println!("latent spread after scaling: {:.3}", spread(&clean));
423        }
424
425        running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426        step += 1;
427        model.step_clipped(1.0, 1.0)?;
428        model.zero_grad();
429
430        if step % args.report == 0 {
431            println!(
432                "step {step}: velocity mse {:.5}",
433                running / args.report as f32
434            );
435            running = 0.0;
436        }
437        if step % args.checkpoint_every == 0 || step == args.steps {
438            model.save(
439                &args.out,
440                &BTreeMap::from([
441                    step_at(step),
442                    ("latent_scale".to_string(), args.latent_scale.to_string()),
443                ]),
444            )?;
445            println!("wrote {} at step {step}", args.out.display());
446        }
447        if step >= args.steps {
448            break;
449        }
450    }
451    Ok(())
452}
Source

pub fn zero_grad(&mut self)

Examples found in repository?
examples/train_shape.rs (line 428)
340fn train_flow(
341    args: &Args,
342    (cond_tokens, cond_dim): (usize, usize),
343    stream: BatchStream,
344    rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346    // Frozen: the flow transformer is trained against a latent space that has
347    // already stopped moving, so only `encode` is ever called here.
348    let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349    let shape = *vae.config();
350
351    let (mut model, mut step) = match &args.resume {
352        Some(path) => {
353            let (model, metadata) = FlowTransformer::load(path, rng)?;
354            (model, started_at(&metadata))
355        }
356        None => {
357            // The latent shape is the autoencoder's and the condition's shape
358            // is the corpus's, whatever a supplied configuration says.
359            let config = FlowConfig {
360                latents: shape.latents,
361                latent_dim: shape.latent_dim,
362                cond_tokens,
363                cond_dim,
364                ..read_config(args, FlowConfig::default())?
365            };
366            (FlowTransformer::new(config, rng)?, 0)
367        }
368    };
369    if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370        return Err(format!(
371            "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372            model.config().cond_tokens,
373            model.config().cond_dim
374        )
375        .into());
376    }
377    model.set_optimizer(adam(args.learning_rate));
378    if let Some(device) = args.cuda {
379        #[cfg(feature = "cuda")]
380        {
381            // 0 as the budget: the caller picked the card, and the blocks'
382            // weights are a small part of what a run of this size needs.
383            model.to_cuda(device, 0)?;
384            println!("flow transformer: training on CUDA device {device}");
385        }
386        #[cfg(not(feature = "cuda"))]
387        return Err(format!(
388            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389        )
390        .into());
391    }
392    println!(
393        "flow transformer: {} parameters, from step {step}",
394        model.num_parameters()
395    );
396
397    let mut running = 0.0;
398    for batch in stream {
399        let batch = batch?;
400        let count = batch.len();
401        let points = batch.surface.rows / count;
402
403        let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404        for index in 0..count {
405            // A draw from the posterior, not its mean. Every autoencoder step
406            // decoded a draw, so a draw is the only kind of latent the frozen
407            // decoder has ever answered for. The posterior is wide enough for
408            // that to matter — sigma about twice the magnitude of the mean —
409            // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410            // a draw, so a flow trained towards the mean would be aiming at the
411            // one latent the decoder is worst at.
412            let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413            let width = shape.latents * shape.latent_dim;
414            for (slot, value) in clean.data[index * width..(index + 1) * width]
415                .iter_mut()
416                .zip(&latent.data)
417            {
418                *slot = value * args.latent_scale;
419            }
420        }
421        if step == 0 {
422            println!("latent spread after scaling: {:.3}", spread(&clean));
423        }
424
425        running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426        step += 1;
427        model.step_clipped(1.0, 1.0)?;
428        model.zero_grad();
429
430        if step % args.report == 0 {
431            println!(
432                "step {step}: velocity mse {:.5}",
433                running / args.report as f32
434            );
435            running = 0.0;
436        }
437        if step % args.checkpoint_every == 0 || step == args.steps {
438            model.save(
439                &args.out,
440                &BTreeMap::from([
441                    step_at(step),
442                    ("latent_scale".to_string(), args.latent_scale.to_string()),
443                ]),
444            )?;
445            println!("wrote {} at step {step}", args.out.display());
446        }
447        if step >= args.steps {
448            break;
449        }
450    }
451    Ok(())
452}
Source

pub fn grad_norm(&mut self) -> Result<f32, NetworkError>

Source

pub fn params_mut(&mut self) -> Vec<&mut Param>

Source

pub fn num_parameters(&mut self) -> usize

Examples found in repository?
examples/train_shape.rs (line 394)
340fn train_flow(
341    args: &Args,
342    (cond_tokens, cond_dim): (usize, usize),
343    stream: BatchStream,
344    rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346    // Frozen: the flow transformer is trained against a latent space that has
347    // already stopped moving, so only `encode` is ever called here.
348    let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349    let shape = *vae.config();
350
351    let (mut model, mut step) = match &args.resume {
352        Some(path) => {
353            let (model, metadata) = FlowTransformer::load(path, rng)?;
354            (model, started_at(&metadata))
355        }
356        None => {
357            // The latent shape is the autoencoder's and the condition's shape
358            // is the corpus's, whatever a supplied configuration says.
359            let config = FlowConfig {
360                latents: shape.latents,
361                latent_dim: shape.latent_dim,
362                cond_tokens,
363                cond_dim,
364                ..read_config(args, FlowConfig::default())?
365            };
366            (FlowTransformer::new(config, rng)?, 0)
367        }
368    };
369    if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370        return Err(format!(
371            "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372            model.config().cond_tokens,
373            model.config().cond_dim
374        )
375        .into());
376    }
377    model.set_optimizer(adam(args.learning_rate));
378    if let Some(device) = args.cuda {
379        #[cfg(feature = "cuda")]
380        {
381            // 0 as the budget: the caller picked the card, and the blocks'
382            // weights are a small part of what a run of this size needs.
383            model.to_cuda(device, 0)?;
384            println!("flow transformer: training on CUDA device {device}");
385        }
386        #[cfg(not(feature = "cuda"))]
387        return Err(format!(
388            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389        )
390        .into());
391    }
392    println!(
393        "flow transformer: {} parameters, from step {step}",
394        model.num_parameters()
395    );
396
397    let mut running = 0.0;
398    for batch in stream {
399        let batch = batch?;
400        let count = batch.len();
401        let points = batch.surface.rows / count;
402
403        let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404        for index in 0..count {
405            // A draw from the posterior, not its mean. Every autoencoder step
406            // decoded a draw, so a draw is the only kind of latent the frozen
407            // decoder has ever answered for. The posterior is wide enough for
408            // that to matter — sigma about twice the magnitude of the mean —
409            // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410            // a draw, so a flow trained towards the mean would be aiming at the
411            // one latent the decoder is worst at.
412            let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413            let width = shape.latents * shape.latent_dim;
414            for (slot, value) in clean.data[index * width..(index + 1) * width]
415                .iter_mut()
416                .zip(&latent.data)
417            {
418                *slot = value * args.latent_scale;
419            }
420        }
421        if step == 0 {
422            println!("latent spread after scaling: {:.3}", spread(&clean));
423        }
424
425        running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426        step += 1;
427        model.step_clipped(1.0, 1.0)?;
428        model.zero_grad();
429
430        if step % args.report == 0 {
431            println!(
432                "step {step}: velocity mse {:.5}",
433                running / args.report as f32
434            );
435            running = 0.0;
436        }
437        if step % args.checkpoint_every == 0 || step == args.steps {
438            model.save(
439                &args.out,
440                &BTreeMap::from([
441                    step_at(step),
442                    ("latent_scale".to_string(), args.latent_scale.to_string()),
443                ]),
444            )?;
445            println!("wrote {} at step {step}", args.out.display());
446        }
447        if step >= args.steps {
448            break;
449        }
450    }
451    Ok(())
452}
Source

pub fn save<P: AsRef<Path>>( &mut self, path: P, metadata: &BTreeMap<String, String>, ) -> Result<(), NetworkError>

Writes the weights, the Adam moments and the configuration, as ShapeVae::save does.

The optimizer’s step counter goes in the metadata as well, so a resumed run restores Adam’s bias correction exactly instead of taking a few oversized steps with a counter that restarted at zero.

Examples found in repository?
examples/train_shape.rs (lines 438-444)
340fn train_flow(
341    args: &Args,
342    (cond_tokens, cond_dim): (usize, usize),
343    stream: BatchStream,
344    rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346    // Frozen: the flow transformer is trained against a latent space that has
347    // already stopped moving, so only `encode` is ever called here.
348    let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349    let shape = *vae.config();
350
351    let (mut model, mut step) = match &args.resume {
352        Some(path) => {
353            let (model, metadata) = FlowTransformer::load(path, rng)?;
354            (model, started_at(&metadata))
355        }
356        None => {
357            // The latent shape is the autoencoder's and the condition's shape
358            // is the corpus's, whatever a supplied configuration says.
359            let config = FlowConfig {
360                latents: shape.latents,
361                latent_dim: shape.latent_dim,
362                cond_tokens,
363                cond_dim,
364                ..read_config(args, FlowConfig::default())?
365            };
366            (FlowTransformer::new(config, rng)?, 0)
367        }
368    };
369    if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370        return Err(format!(
371            "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372            model.config().cond_tokens,
373            model.config().cond_dim
374        )
375        .into());
376    }
377    model.set_optimizer(adam(args.learning_rate));
378    if let Some(device) = args.cuda {
379        #[cfg(feature = "cuda")]
380        {
381            // 0 as the budget: the caller picked the card, and the blocks'
382            // weights are a small part of what a run of this size needs.
383            model.to_cuda(device, 0)?;
384            println!("flow transformer: training on CUDA device {device}");
385        }
386        #[cfg(not(feature = "cuda"))]
387        return Err(format!(
388            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389        )
390        .into());
391    }
392    println!(
393        "flow transformer: {} parameters, from step {step}",
394        model.num_parameters()
395    );
396
397    let mut running = 0.0;
398    for batch in stream {
399        let batch = batch?;
400        let count = batch.len();
401        let points = batch.surface.rows / count;
402
403        let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404        for index in 0..count {
405            // A draw from the posterior, not its mean. Every autoencoder step
406            // decoded a draw, so a draw is the only kind of latent the frozen
407            // decoder has ever answered for. The posterior is wide enough for
408            // that to matter — sigma about twice the magnitude of the mean —
409            // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410            // a draw, so a flow trained towards the mean would be aiming at the
411            // one latent the decoder is worst at.
412            let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413            let width = shape.latents * shape.latent_dim;
414            for (slot, value) in clean.data[index * width..(index + 1) * width]
415                .iter_mut()
416                .zip(&latent.data)
417            {
418                *slot = value * args.latent_scale;
419            }
420        }
421        if step == 0 {
422            println!("latent spread after scaling: {:.3}", spread(&clean));
423        }
424
425        running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426        step += 1;
427        model.step_clipped(1.0, 1.0)?;
428        model.zero_grad();
429
430        if step % args.report == 0 {
431            println!(
432                "step {step}: velocity mse {:.5}",
433                running / args.report as f32
434            );
435            running = 0.0;
436        }
437        if step % args.checkpoint_every == 0 || step == args.steps {
438            model.save(
439                &args.out,
440                &BTreeMap::from([
441                    step_at(step),
442                    ("latent_scale".to_string(), args.latent_scale.to_string()),
443                ]),
444            )?;
445            println!("wrote {} at step {step}", args.out.display());
446        }
447        if step >= args.steps {
448            break;
449        }
450    }
451    Ok(())
452}
Source

pub fn load<P: AsRef<Path>>( path: P, rng: &mut StdRng, ) -> Result<(Self, BTreeMap<String, String>), NetworkError>

Rebuilds a model from what save wrote, and returns the rest of the metadata.

Examples found in repository?
examples/train_shape.rs (line 353)
340fn train_flow(
341    args: &Args,
342    (cond_tokens, cond_dim): (usize, usize),
343    stream: BatchStream,
344    rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346    // Frozen: the flow transformer is trained against a latent space that has
347    // already stopped moving, so only `encode` is ever called here.
348    let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349    let shape = *vae.config();
350
351    let (mut model, mut step) = match &args.resume {
352        Some(path) => {
353            let (model, metadata) = FlowTransformer::load(path, rng)?;
354            (model, started_at(&metadata))
355        }
356        None => {
357            // The latent shape is the autoencoder's and the condition's shape
358            // is the corpus's, whatever a supplied configuration says.
359            let config = FlowConfig {
360                latents: shape.latents,
361                latent_dim: shape.latent_dim,
362                cond_tokens,
363                cond_dim,
364                ..read_config(args, FlowConfig::default())?
365            };
366            (FlowTransformer::new(config, rng)?, 0)
367        }
368    };
369    if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370        return Err(format!(
371            "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372            model.config().cond_tokens,
373            model.config().cond_dim
374        )
375        .into());
376    }
377    model.set_optimizer(adam(args.learning_rate));
378    if let Some(device) = args.cuda {
379        #[cfg(feature = "cuda")]
380        {
381            // 0 as the budget: the caller picked the card, and the blocks'
382            // weights are a small part of what a run of this size needs.
383            model.to_cuda(device, 0)?;
384            println!("flow transformer: training on CUDA device {device}");
385        }
386        #[cfg(not(feature = "cuda"))]
387        return Err(format!(
388            "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389        )
390        .into());
391    }
392    println!(
393        "flow transformer: {} parameters, from step {step}",
394        model.num_parameters()
395    );
396
397    let mut running = 0.0;
398    for batch in stream {
399        let batch = batch?;
400        let count = batch.len();
401        let points = batch.surface.rows / count;
402
403        let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404        for index in 0..count {
405            // A draw from the posterior, not its mean. Every autoencoder step
406            // decoded a draw, so a draw is the only kind of latent the frozen
407            // decoder has ever answered for. The posterior is wide enough for
408            // that to matter — sigma about twice the magnitude of the mean —
409            // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410            // a draw, so a flow trained towards the mean would be aiming at the
411            // one latent the decoder is worst at.
412            let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413            let width = shape.latents * shape.latent_dim;
414            for (slot, value) in clean.data[index * width..(index + 1) * width]
415                .iter_mut()
416                .zip(&latent.data)
417            {
418                *slot = value * args.latent_scale;
419            }
420        }
421        if step == 0 {
422            println!("latent spread after scaling: {:.3}", spread(&clean));
423        }
424
425        running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426        step += 1;
427        model.step_clipped(1.0, 1.0)?;
428        model.zero_grad();
429
430        if step % args.report == 0 {
431            println!(
432                "step {step}: velocity mse {:.5}",
433                running / args.report as f32
434            );
435            running = 0.0;
436        }
437        if step % args.checkpoint_every == 0 || step == args.steps {
438            model.save(
439                &args.out,
440                &BTreeMap::from([
441                    step_at(step),
442                    ("latent_scale".to_string(), args.latent_scale.to_string()),
443                ]),
444            )?;
445            println!("wrote {} at step {step}", args.out.display());
446        }
447        if step >= args.steps {
448            break;
449        }
450    }
451    Ok(())
452}
Source

pub fn on_device(&self) -> bool

Whether the blocks currently live on a device.

Trait Implementations§

Source§

impl Clone for FlowTransformer

Source§

fn clone(&self) -> Self

Returns a duplicate of the value. Read more
1.0.0 (const: unstable) · Source§

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source. Read more
Source§

impl Debug for FlowTransformer

Source§

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter. Read more
Source§

impl<'de> Deserialize<'de> for FlowTransformer

Source§

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>
where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer. Read more
Source§

impl PartialEq for FlowTransformer

Compares the model, not where it happens to be running: two models with the same weights are equal whether or not one of them holds a device context.

Source§

fn eq(&self, other: &Self) -> bool

Equality operator ==. Read more
1.0.0 (const: unstable) · Source§

fn ne(&self, other: &Rhs) -> bool

Inequality operator !=. Read more
Source§

impl Serialize for FlowTransformer

Source§

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>
where __S: Serializer,

Serialize this value into the given Serde serializer. Read more

Auto Trait Implementations§

Blanket Implementations§

Source§

impl<T> Any for T
where T: 'static + ?Sized,

Source§

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more
Source§

impl<T> Borrow<T> for T
where T: ?Sized,

Source§

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more
Source§

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source§

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more
Source§

impl<T> CloneToUninit for T
where T: Clone,

Source§

unsafe fn clone_to_uninit(&self, dest: *mut u8)

🔬This is a nightly-only experimental API. (clone_to_uninit)
Performs copy-assignment from self to dest. Read more
Source§

impl<T> DeserializeOwned for T
where T: for<'de> Deserialize<'de>,

Source§

impl<T> From<T> for T

Source§

fn from(t: T) -> T

Returns the argument unchanged.

Source§

impl<T, U> Into<U> for T
where U: From<T>,

Source§

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source§

impl<T> IntoEither for T

Source§

fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ

Converts self into a Left variant of Either<Self, Self> if into_left is true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more
Source§

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
where F: FnOnce(&Self) -> bool,

Converts self into a Left variant of Either<Self, Self> if into_left(&self) returns true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more
Source§

impl<T> Pointable for T

Source§

const ALIGN: usize

The alignment of pointer.
Source§

type Init = T

The type for initializers.
Source§

unsafe fn init(init: <T as Pointable>::Init) -> usize

Initializes a with the given initializer. Read more
Source§

unsafe fn deref<'a>(ptr: usize) -> &'a T

Dereferences the given pointer. Read more
Source§

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

Mutably dereferences the given pointer. Read more
Source§

unsafe fn drop(ptr: usize)

Drops the object pointed to by the given pointer. Read more
Source§

impl<T> ToOwned for T
where T: Clone,

Source§

type Owned = T

The resulting type after obtaining ownership.
Source§

fn to_owned(&self) -> T

Creates owned data from borrowed data, usually by cloning. Read more
Source§

fn clone_into(&self, target: &mut T)

Uses borrowed data to replace owned data, usually by cloning. Read more
Source§

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source§

type Error = !

The type returned in the event of a conversion error.
Source§

fn try_from(value: U) -> Result<T, !>

Performs the conversion.
Source§

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source§

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.
Source§

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.
Source§

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

Source§

fn vzip(self) -> V