pub struct FlowTransformer { /* private fields */ }Expand description
Noise and a picture in, a shape latent out.
Implementations§
Source§impl FlowTransformer
impl FlowTransformer
Sourcepub fn new(config: FlowConfig, rng: &mut StdRng) -> Result<Self, NetworkError>
pub fn new(config: FlowConfig, rng: &mut StdRng) -> Result<Self, NetworkError>
Examples found in repository?
340fn train_flow(
341 args: &Args,
342 (cond_tokens, cond_dim): (usize, usize),
343 stream: BatchStream,
344 rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346 // Frozen: the flow transformer is trained against a latent space that has
347 // already stopped moving, so only `encode` is ever called here.
348 let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349 let shape = *vae.config();
350
351 let (mut model, mut step) = match &args.resume {
352 Some(path) => {
353 let (model, metadata) = FlowTransformer::load(path, rng)?;
354 (model, started_at(&metadata))
355 }
356 None => {
357 // The latent shape is the autoencoder's and the condition's shape
358 // is the corpus's, whatever a supplied configuration says.
359 let config = FlowConfig {
360 latents: shape.latents,
361 latent_dim: shape.latent_dim,
362 cond_tokens,
363 cond_dim,
364 ..read_config(args, FlowConfig::default())?
365 };
366 (FlowTransformer::new(config, rng)?, 0)
367 }
368 };
369 if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370 return Err(format!(
371 "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372 model.config().cond_tokens,
373 model.config().cond_dim
374 )
375 .into());
376 }
377 model.set_optimizer(adam(args.learning_rate));
378 if let Some(device) = args.cuda {
379 #[cfg(feature = "cuda")]
380 {
381 // 0 as the budget: the caller picked the card, and the blocks'
382 // weights are a small part of what a run of this size needs.
383 model.to_cuda(device, 0)?;
384 println!("flow transformer: training on CUDA device {device}");
385 }
386 #[cfg(not(feature = "cuda"))]
387 return Err(format!(
388 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389 )
390 .into());
391 }
392 println!(
393 "flow transformer: {} parameters, from step {step}",
394 model.num_parameters()
395 );
396
397 let mut running = 0.0;
398 for batch in stream {
399 let batch = batch?;
400 let count = batch.len();
401 let points = batch.surface.rows / count;
402
403 let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404 for index in 0..count {
405 // A draw from the posterior, not its mean. Every autoencoder step
406 // decoded a draw, so a draw is the only kind of latent the frozen
407 // decoder has ever answered for. The posterior is wide enough for
408 // that to matter — sigma about twice the magnitude of the mean —
409 // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410 // a draw, so a flow trained towards the mean would be aiming at the
411 // one latent the decoder is worst at.
412 let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413 let width = shape.latents * shape.latent_dim;
414 for (slot, value) in clean.data[index * width..(index + 1) * width]
415 .iter_mut()
416 .zip(&latent.data)
417 {
418 *slot = value * args.latent_scale;
419 }
420 }
421 if step == 0 {
422 println!("latent spread after scaling: {:.3}", spread(&clean));
423 }
424
425 running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426 step += 1;
427 model.step_clipped(1.0, 1.0)?;
428 model.zero_grad();
429
430 if step % args.report == 0 {
431 println!(
432 "step {step}: velocity mse {:.5}",
433 running / args.report as f32
434 );
435 running = 0.0;
436 }
437 if step % args.checkpoint_every == 0 || step == args.steps {
438 model.save(
439 &args.out,
440 &BTreeMap::from([
441 step_at(step),
442 ("latent_scale".to_string(), args.latent_scale.to_string()),
443 ]),
444 )?;
445 println!("wrote {} at step {step}", args.out.display());
446 }
447 if step >= args.steps {
448 break;
449 }
450 }
451 Ok(())
452}Sourcepub fn config(&self) -> &FlowConfig
pub fn config(&self) -> &FlowConfig
Examples found in repository?
340fn train_flow(
341 args: &Args,
342 (cond_tokens, cond_dim): (usize, usize),
343 stream: BatchStream,
344 rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346 // Frozen: the flow transformer is trained against a latent space that has
347 // already stopped moving, so only `encode` is ever called here.
348 let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349 let shape = *vae.config();
350
351 let (mut model, mut step) = match &args.resume {
352 Some(path) => {
353 let (model, metadata) = FlowTransformer::load(path, rng)?;
354 (model, started_at(&metadata))
355 }
356 None => {
357 // The latent shape is the autoencoder's and the condition's shape
358 // is the corpus's, whatever a supplied configuration says.
359 let config = FlowConfig {
360 latents: shape.latents,
361 latent_dim: shape.latent_dim,
362 cond_tokens,
363 cond_dim,
364 ..read_config(args, FlowConfig::default())?
365 };
366 (FlowTransformer::new(config, rng)?, 0)
367 }
368 };
369 if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370 return Err(format!(
371 "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372 model.config().cond_tokens,
373 model.config().cond_dim
374 )
375 .into());
376 }
377 model.set_optimizer(adam(args.learning_rate));
378 if let Some(device) = args.cuda {
379 #[cfg(feature = "cuda")]
380 {
381 // 0 as the budget: the caller picked the card, and the blocks'
382 // weights are a small part of what a run of this size needs.
383 model.to_cuda(device, 0)?;
384 println!("flow transformer: training on CUDA device {device}");
385 }
386 #[cfg(not(feature = "cuda"))]
387 return Err(format!(
388 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389 )
390 .into());
391 }
392 println!(
393 "flow transformer: {} parameters, from step {step}",
394 model.num_parameters()
395 );
396
397 let mut running = 0.0;
398 for batch in stream {
399 let batch = batch?;
400 let count = batch.len();
401 let points = batch.surface.rows / count;
402
403 let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404 for index in 0..count {
405 // A draw from the posterior, not its mean. Every autoencoder step
406 // decoded a draw, so a draw is the only kind of latent the frozen
407 // decoder has ever answered for. The posterior is wide enough for
408 // that to matter — sigma about twice the magnitude of the mean —
409 // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410 // a draw, so a flow trained towards the mean would be aiming at the
411 // one latent the decoder is worst at.
412 let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413 let width = shape.latents * shape.latent_dim;
414 for (slot, value) in clean.data[index * width..(index + 1) * width]
415 .iter_mut()
416 .zip(&latent.data)
417 {
418 *slot = value * args.latent_scale;
419 }
420 }
421 if step == 0 {
422 println!("latent spread after scaling: {:.3}", spread(&clean));
423 }
424
425 running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426 step += 1;
427 model.step_clipped(1.0, 1.0)?;
428 model.zero_grad();
429
430 if step % args.report == 0 {
431 println!(
432 "step {step}: velocity mse {:.5}",
433 running / args.report as f32
434 );
435 running = 0.0;
436 }
437 if step % args.checkpoint_every == 0 || step == args.steps {
438 model.save(
439 &args.out,
440 &BTreeMap::from([
441 step_at(step),
442 ("latent_scale".to_string(), args.latent_scale.to_string()),
443 ]),
444 )?;
445 println!("wrote {} at step {step}", args.out.display());
446 }
447 if step >= args.steps {
448 break;
449 }
450 }
451 Ok(())
452}Sourcepub fn set_optimizer(&mut self, optimizer: Optimizer)
pub fn set_optimizer(&mut self, optimizer: Optimizer)
Examples found in repository?
340fn train_flow(
341 args: &Args,
342 (cond_tokens, cond_dim): (usize, usize),
343 stream: BatchStream,
344 rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346 // Frozen: the flow transformer is trained against a latent space that has
347 // already stopped moving, so only `encode` is ever called here.
348 let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349 let shape = *vae.config();
350
351 let (mut model, mut step) = match &args.resume {
352 Some(path) => {
353 let (model, metadata) = FlowTransformer::load(path, rng)?;
354 (model, started_at(&metadata))
355 }
356 None => {
357 // The latent shape is the autoencoder's and the condition's shape
358 // is the corpus's, whatever a supplied configuration says.
359 let config = FlowConfig {
360 latents: shape.latents,
361 latent_dim: shape.latent_dim,
362 cond_tokens,
363 cond_dim,
364 ..read_config(args, FlowConfig::default())?
365 };
366 (FlowTransformer::new(config, rng)?, 0)
367 }
368 };
369 if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370 return Err(format!(
371 "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372 model.config().cond_tokens,
373 model.config().cond_dim
374 )
375 .into());
376 }
377 model.set_optimizer(adam(args.learning_rate));
378 if let Some(device) = args.cuda {
379 #[cfg(feature = "cuda")]
380 {
381 // 0 as the budget: the caller picked the card, and the blocks'
382 // weights are a small part of what a run of this size needs.
383 model.to_cuda(device, 0)?;
384 println!("flow transformer: training on CUDA device {device}");
385 }
386 #[cfg(not(feature = "cuda"))]
387 return Err(format!(
388 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389 )
390 .into());
391 }
392 println!(
393 "flow transformer: {} parameters, from step {step}",
394 model.num_parameters()
395 );
396
397 let mut running = 0.0;
398 for batch in stream {
399 let batch = batch?;
400 let count = batch.len();
401 let points = batch.surface.rows / count;
402
403 let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404 for index in 0..count {
405 // A draw from the posterior, not its mean. Every autoencoder step
406 // decoded a draw, so a draw is the only kind of latent the frozen
407 // decoder has ever answered for. The posterior is wide enough for
408 // that to matter — sigma about twice the magnitude of the mean —
409 // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410 // a draw, so a flow trained towards the mean would be aiming at the
411 // one latent the decoder is worst at.
412 let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413 let width = shape.latents * shape.latent_dim;
414 for (slot, value) in clean.data[index * width..(index + 1) * width]
415 .iter_mut()
416 .zip(&latent.data)
417 {
418 *slot = value * args.latent_scale;
419 }
420 }
421 if step == 0 {
422 println!("latent spread after scaling: {:.3}", spread(&clean));
423 }
424
425 running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426 step += 1;
427 model.step_clipped(1.0, 1.0)?;
428 model.zero_grad();
429
430 if step % args.report == 0 {
431 println!(
432 "step {step}: velocity mse {:.5}",
433 running / args.report as f32
434 );
435 running = 0.0;
436 }
437 if step % args.checkpoint_every == 0 || step == args.steps {
438 model.save(
439 &args.out,
440 &BTreeMap::from([
441 step_at(step),
442 ("latent_scale".to_string(), args.latent_scale.to_string()),
443 ]),
444 )?;
445 println!("wrote {} at step {step}", args.out.display());
446 }
447 if step >= args.steps {
448 break;
449 }
450 }
451 Ok(())
452}Sourcepub fn null_tokens(&self) -> &Matrix
pub fn null_tokens(&self) -> &Matrix
The learned stand-in for a dropped condition, [cond_tokens, cond_dim].
Sourcepub fn forward(
&self,
latent: &Matrix,
tokens: &Matrix,
times: &[f32],
) -> Result<Matrix, NetworkError>
pub fn forward( &self, latent: &Matrix, tokens: &Matrix, times: &[f32], ) -> Result<Matrix, NetworkError>
The predicted velocity at time, [sequences * latents, latent_dim].
time runs from 1.0 at pure noise down to 0.0 at a clean latent, which
is the same quantity Scheduler::sigmas produces.
Sourcepub fn forward_train(
&self,
latent: &Matrix,
tokens: &Matrix,
times: &[f32],
) -> Result<(Matrix, FlowCache), NetworkError>
pub fn forward_train( &self, latent: &Matrix, tokens: &Matrix, times: &[f32], ) -> Result<(Matrix, FlowCache), NetworkError>
The same, keeping what the backward pass needs.
Sourcepub fn backward(
&mut self,
cache: &FlowCache,
grad_velocity: &Matrix,
) -> Result<Matrix, NetworkError>
pub fn backward( &mut self, cache: &FlowCache, grad_velocity: &Matrix, ) -> Result<Matrix, NetworkError>
Accumulates every weight gradient and returns dL/dtokens.
The image tower is frozen, so the caller normally drops what comes back; it is returned rather than swallowed because the gradient is what a later stage would need to fine-tune the tower.
Sourcepub fn train_step<R: Rng>(
&mut self,
clean: &Matrix,
tokens: &Matrix,
dropout: f32,
rng: &mut R,
) -> Result<f32, NetworkError>
pub fn train_step<R: Rng>( &mut self, clean: &Matrix, tokens: &Matrix, dropout: f32, rng: &mut R, ) -> Result<f32, NetworkError>
One flow-matching step: mix, predict, score, and accumulate gradients.
clean is the shape autoencoder’s latent for the target mesh and
tokens the frozen image tokens that describe it. dropout is the
probability that a sequence’s condition is replaced by
null_tokens, which is what makes
classifier-free guidance possible at sampling time; 0.1 is the usual
value and 0.0 turns it off.
Returns the mean squared error against the velocity that actually joins the two endpoints.
Examples found in repository?
340fn train_flow(
341 args: &Args,
342 (cond_tokens, cond_dim): (usize, usize),
343 stream: BatchStream,
344 rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346 // Frozen: the flow transformer is trained against a latent space that has
347 // already stopped moving, so only `encode` is ever called here.
348 let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349 let shape = *vae.config();
350
351 let (mut model, mut step) = match &args.resume {
352 Some(path) => {
353 let (model, metadata) = FlowTransformer::load(path, rng)?;
354 (model, started_at(&metadata))
355 }
356 None => {
357 // The latent shape is the autoencoder's and the condition's shape
358 // is the corpus's, whatever a supplied configuration says.
359 let config = FlowConfig {
360 latents: shape.latents,
361 latent_dim: shape.latent_dim,
362 cond_tokens,
363 cond_dim,
364 ..read_config(args, FlowConfig::default())?
365 };
366 (FlowTransformer::new(config, rng)?, 0)
367 }
368 };
369 if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370 return Err(format!(
371 "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372 model.config().cond_tokens,
373 model.config().cond_dim
374 )
375 .into());
376 }
377 model.set_optimizer(adam(args.learning_rate));
378 if let Some(device) = args.cuda {
379 #[cfg(feature = "cuda")]
380 {
381 // 0 as the budget: the caller picked the card, and the blocks'
382 // weights are a small part of what a run of this size needs.
383 model.to_cuda(device, 0)?;
384 println!("flow transformer: training on CUDA device {device}");
385 }
386 #[cfg(not(feature = "cuda"))]
387 return Err(format!(
388 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389 )
390 .into());
391 }
392 println!(
393 "flow transformer: {} parameters, from step {step}",
394 model.num_parameters()
395 );
396
397 let mut running = 0.0;
398 for batch in stream {
399 let batch = batch?;
400 let count = batch.len();
401 let points = batch.surface.rows / count;
402
403 let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404 for index in 0..count {
405 // A draw from the posterior, not its mean. Every autoencoder step
406 // decoded a draw, so a draw is the only kind of latent the frozen
407 // decoder has ever answered for. The posterior is wide enough for
408 // that to matter — sigma about twice the magnitude of the mean —
409 // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410 // a draw, so a flow trained towards the mean would be aiming at the
411 // one latent the decoder is worst at.
412 let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413 let width = shape.latents * shape.latent_dim;
414 for (slot, value) in clean.data[index * width..(index + 1) * width]
415 .iter_mut()
416 .zip(&latent.data)
417 {
418 *slot = value * args.latent_scale;
419 }
420 }
421 if step == 0 {
422 println!("latent spread after scaling: {:.3}", spread(&clean));
423 }
424
425 running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426 step += 1;
427 model.step_clipped(1.0, 1.0)?;
428 model.zero_grad();
429
430 if step % args.report == 0 {
431 println!(
432 "step {step}: velocity mse {:.5}",
433 running / args.report as f32
434 );
435 running = 0.0;
436 }
437 if step % args.checkpoint_every == 0 || step == args.steps {
438 model.save(
439 &args.out,
440 &BTreeMap::from([
441 step_at(step),
442 ("latent_scale".to_string(), args.latent_scale.to_string()),
443 ]),
444 )?;
445 println!("wrote {} at step {step}", args.out.display());
446 }
447 if step >= args.steps {
448 break;
449 }
450 }
451 Ok(())
452}Sourcepub fn drop_condition<R: Rng>(
&self,
tokens: &Matrix,
dropout: f32,
rng: &mut R,
) -> Result<(Matrix, Vec<bool>), NetworkError>
pub fn drop_condition<R: Rng>( &self, tokens: &Matrix, dropout: f32, rng: &mut R, ) -> Result<(Matrix, Vec<bool>), NetworkError>
Replaces each sequence’s conditioning tokens with the learned null
embedding, independently, with probability dropout.
The flags say which sequences were replaced, which is what lets the backward pass send those sequences’ token gradient to the null embedding instead of throwing it away.
Sourcepub fn step_clipped(
&mut self,
scale: f32,
max_norm: f32,
) -> Result<f32, NetworkError>
pub fn step_clipped( &mut self, scale: f32, max_norm: f32, ) -> Result<f32, NetworkError>
Applies the optimizer with the gradients clipped to a global norm, and returns the pre-clip norm.
Examples found in repository?
340fn train_flow(
341 args: &Args,
342 (cond_tokens, cond_dim): (usize, usize),
343 stream: BatchStream,
344 rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346 // Frozen: the flow transformer is trained against a latent space that has
347 // already stopped moving, so only `encode` is ever called here.
348 let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349 let shape = *vae.config();
350
351 let (mut model, mut step) = match &args.resume {
352 Some(path) => {
353 let (model, metadata) = FlowTransformer::load(path, rng)?;
354 (model, started_at(&metadata))
355 }
356 None => {
357 // The latent shape is the autoencoder's and the condition's shape
358 // is the corpus's, whatever a supplied configuration says.
359 let config = FlowConfig {
360 latents: shape.latents,
361 latent_dim: shape.latent_dim,
362 cond_tokens,
363 cond_dim,
364 ..read_config(args, FlowConfig::default())?
365 };
366 (FlowTransformer::new(config, rng)?, 0)
367 }
368 };
369 if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370 return Err(format!(
371 "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372 model.config().cond_tokens,
373 model.config().cond_dim
374 )
375 .into());
376 }
377 model.set_optimizer(adam(args.learning_rate));
378 if let Some(device) = args.cuda {
379 #[cfg(feature = "cuda")]
380 {
381 // 0 as the budget: the caller picked the card, and the blocks'
382 // weights are a small part of what a run of this size needs.
383 model.to_cuda(device, 0)?;
384 println!("flow transformer: training on CUDA device {device}");
385 }
386 #[cfg(not(feature = "cuda"))]
387 return Err(format!(
388 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389 )
390 .into());
391 }
392 println!(
393 "flow transformer: {} parameters, from step {step}",
394 model.num_parameters()
395 );
396
397 let mut running = 0.0;
398 for batch in stream {
399 let batch = batch?;
400 let count = batch.len();
401 let points = batch.surface.rows / count;
402
403 let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404 for index in 0..count {
405 // A draw from the posterior, not its mean. Every autoencoder step
406 // decoded a draw, so a draw is the only kind of latent the frozen
407 // decoder has ever answered for. The posterior is wide enough for
408 // that to matter — sigma about twice the magnitude of the mean —
409 // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410 // a draw, so a flow trained towards the mean would be aiming at the
411 // one latent the decoder is worst at.
412 let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413 let width = shape.latents * shape.latent_dim;
414 for (slot, value) in clean.data[index * width..(index + 1) * width]
415 .iter_mut()
416 .zip(&latent.data)
417 {
418 *slot = value * args.latent_scale;
419 }
420 }
421 if step == 0 {
422 println!("latent spread after scaling: {:.3}", spread(&clean));
423 }
424
425 running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426 step += 1;
427 model.step_clipped(1.0, 1.0)?;
428 model.zero_grad();
429
430 if step % args.report == 0 {
431 println!(
432 "step {step}: velocity mse {:.5}",
433 running / args.report as f32
434 );
435 running = 0.0;
436 }
437 if step % args.checkpoint_every == 0 || step == args.steps {
438 model.save(
439 &args.out,
440 &BTreeMap::from([
441 step_at(step),
442 ("latent_scale".to_string(), args.latent_scale.to_string()),
443 ]),
444 )?;
445 println!("wrote {} at step {step}", args.out.display());
446 }
447 if step >= args.steps {
448 break;
449 }
450 }
451 Ok(())
452}Sourcepub fn zero_grad(&mut self)
pub fn zero_grad(&mut self)
Examples found in repository?
340fn train_flow(
341 args: &Args,
342 (cond_tokens, cond_dim): (usize, usize),
343 stream: BatchStream,
344 rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346 // Frozen: the flow transformer is trained against a latent space that has
347 // already stopped moving, so only `encode` is ever called here.
348 let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349 let shape = *vae.config();
350
351 let (mut model, mut step) = match &args.resume {
352 Some(path) => {
353 let (model, metadata) = FlowTransformer::load(path, rng)?;
354 (model, started_at(&metadata))
355 }
356 None => {
357 // The latent shape is the autoencoder's and the condition's shape
358 // is the corpus's, whatever a supplied configuration says.
359 let config = FlowConfig {
360 latents: shape.latents,
361 latent_dim: shape.latent_dim,
362 cond_tokens,
363 cond_dim,
364 ..read_config(args, FlowConfig::default())?
365 };
366 (FlowTransformer::new(config, rng)?, 0)
367 }
368 };
369 if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370 return Err(format!(
371 "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372 model.config().cond_tokens,
373 model.config().cond_dim
374 )
375 .into());
376 }
377 model.set_optimizer(adam(args.learning_rate));
378 if let Some(device) = args.cuda {
379 #[cfg(feature = "cuda")]
380 {
381 // 0 as the budget: the caller picked the card, and the blocks'
382 // weights are a small part of what a run of this size needs.
383 model.to_cuda(device, 0)?;
384 println!("flow transformer: training on CUDA device {device}");
385 }
386 #[cfg(not(feature = "cuda"))]
387 return Err(format!(
388 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389 )
390 .into());
391 }
392 println!(
393 "flow transformer: {} parameters, from step {step}",
394 model.num_parameters()
395 );
396
397 let mut running = 0.0;
398 for batch in stream {
399 let batch = batch?;
400 let count = batch.len();
401 let points = batch.surface.rows / count;
402
403 let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404 for index in 0..count {
405 // A draw from the posterior, not its mean. Every autoencoder step
406 // decoded a draw, so a draw is the only kind of latent the frozen
407 // decoder has ever answered for. The posterior is wide enough for
408 // that to matter — sigma about twice the magnitude of the mean —
409 // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410 // a draw, so a flow trained towards the mean would be aiming at the
411 // one latent the decoder is worst at.
412 let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413 let width = shape.latents * shape.latent_dim;
414 for (slot, value) in clean.data[index * width..(index + 1) * width]
415 .iter_mut()
416 .zip(&latent.data)
417 {
418 *slot = value * args.latent_scale;
419 }
420 }
421 if step == 0 {
422 println!("latent spread after scaling: {:.3}", spread(&clean));
423 }
424
425 running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426 step += 1;
427 model.step_clipped(1.0, 1.0)?;
428 model.zero_grad();
429
430 if step % args.report == 0 {
431 println!(
432 "step {step}: velocity mse {:.5}",
433 running / args.report as f32
434 );
435 running = 0.0;
436 }
437 if step % args.checkpoint_every == 0 || step == args.steps {
438 model.save(
439 &args.out,
440 &BTreeMap::from([
441 step_at(step),
442 ("latent_scale".to_string(), args.latent_scale.to_string()),
443 ]),
444 )?;
445 println!("wrote {} at step {step}", args.out.display());
446 }
447 if step >= args.steps {
448 break;
449 }
450 }
451 Ok(())
452}pub fn grad_norm(&mut self) -> Result<f32, NetworkError>
pub fn params_mut(&mut self) -> Vec<&mut Param>
Sourcepub fn num_parameters(&mut self) -> usize
pub fn num_parameters(&mut self) -> usize
Examples found in repository?
340fn train_flow(
341 args: &Args,
342 (cond_tokens, cond_dim): (usize, usize),
343 stream: BatchStream,
344 rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346 // Frozen: the flow transformer is trained against a latent space that has
347 // already stopped moving, so only `encode` is ever called here.
348 let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349 let shape = *vae.config();
350
351 let (mut model, mut step) = match &args.resume {
352 Some(path) => {
353 let (model, metadata) = FlowTransformer::load(path, rng)?;
354 (model, started_at(&metadata))
355 }
356 None => {
357 // The latent shape is the autoencoder's and the condition's shape
358 // is the corpus's, whatever a supplied configuration says.
359 let config = FlowConfig {
360 latents: shape.latents,
361 latent_dim: shape.latent_dim,
362 cond_tokens,
363 cond_dim,
364 ..read_config(args, FlowConfig::default())?
365 };
366 (FlowTransformer::new(config, rng)?, 0)
367 }
368 };
369 if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370 return Err(format!(
371 "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372 model.config().cond_tokens,
373 model.config().cond_dim
374 )
375 .into());
376 }
377 model.set_optimizer(adam(args.learning_rate));
378 if let Some(device) = args.cuda {
379 #[cfg(feature = "cuda")]
380 {
381 // 0 as the budget: the caller picked the card, and the blocks'
382 // weights are a small part of what a run of this size needs.
383 model.to_cuda(device, 0)?;
384 println!("flow transformer: training on CUDA device {device}");
385 }
386 #[cfg(not(feature = "cuda"))]
387 return Err(format!(
388 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389 )
390 .into());
391 }
392 println!(
393 "flow transformer: {} parameters, from step {step}",
394 model.num_parameters()
395 );
396
397 let mut running = 0.0;
398 for batch in stream {
399 let batch = batch?;
400 let count = batch.len();
401 let points = batch.surface.rows / count;
402
403 let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404 for index in 0..count {
405 // A draw from the posterior, not its mean. Every autoencoder step
406 // decoded a draw, so a draw is the only kind of latent the frozen
407 // decoder has ever answered for. The posterior is wide enough for
408 // that to matter — sigma about twice the magnitude of the mean —
409 // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410 // a draw, so a flow trained towards the mean would be aiming at the
411 // one latent the decoder is worst at.
412 let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413 let width = shape.latents * shape.latent_dim;
414 for (slot, value) in clean.data[index * width..(index + 1) * width]
415 .iter_mut()
416 .zip(&latent.data)
417 {
418 *slot = value * args.latent_scale;
419 }
420 }
421 if step == 0 {
422 println!("latent spread after scaling: {:.3}", spread(&clean));
423 }
424
425 running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426 step += 1;
427 model.step_clipped(1.0, 1.0)?;
428 model.zero_grad();
429
430 if step % args.report == 0 {
431 println!(
432 "step {step}: velocity mse {:.5}",
433 running / args.report as f32
434 );
435 running = 0.0;
436 }
437 if step % args.checkpoint_every == 0 || step == args.steps {
438 model.save(
439 &args.out,
440 &BTreeMap::from([
441 step_at(step),
442 ("latent_scale".to_string(), args.latent_scale.to_string()),
443 ]),
444 )?;
445 println!("wrote {} at step {step}", args.out.display());
446 }
447 if step >= args.steps {
448 break;
449 }
450 }
451 Ok(())
452}Sourcepub fn save<P: AsRef<Path>>(
&mut self,
path: P,
metadata: &BTreeMap<String, String>,
) -> Result<(), NetworkError>
pub fn save<P: AsRef<Path>>( &mut self, path: P, metadata: &BTreeMap<String, String>, ) -> Result<(), NetworkError>
Writes the weights, the Adam moments and the configuration, as
ShapeVae::save does.
The optimizer’s step counter goes in the metadata as well, so a resumed run restores Adam’s bias correction exactly instead of taking a few oversized steps with a counter that restarted at zero.
Examples found in repository?
340fn train_flow(
341 args: &Args,
342 (cond_tokens, cond_dim): (usize, usize),
343 stream: BatchStream,
344 rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346 // Frozen: the flow transformer is trained against a latent space that has
347 // already stopped moving, so only `encode` is ever called here.
348 let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349 let shape = *vae.config();
350
351 let (mut model, mut step) = match &args.resume {
352 Some(path) => {
353 let (model, metadata) = FlowTransformer::load(path, rng)?;
354 (model, started_at(&metadata))
355 }
356 None => {
357 // The latent shape is the autoencoder's and the condition's shape
358 // is the corpus's, whatever a supplied configuration says.
359 let config = FlowConfig {
360 latents: shape.latents,
361 latent_dim: shape.latent_dim,
362 cond_tokens,
363 cond_dim,
364 ..read_config(args, FlowConfig::default())?
365 };
366 (FlowTransformer::new(config, rng)?, 0)
367 }
368 };
369 if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370 return Err(format!(
371 "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372 model.config().cond_tokens,
373 model.config().cond_dim
374 )
375 .into());
376 }
377 model.set_optimizer(adam(args.learning_rate));
378 if let Some(device) = args.cuda {
379 #[cfg(feature = "cuda")]
380 {
381 // 0 as the budget: the caller picked the card, and the blocks'
382 // weights are a small part of what a run of this size needs.
383 model.to_cuda(device, 0)?;
384 println!("flow transformer: training on CUDA device {device}");
385 }
386 #[cfg(not(feature = "cuda"))]
387 return Err(format!(
388 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389 )
390 .into());
391 }
392 println!(
393 "flow transformer: {} parameters, from step {step}",
394 model.num_parameters()
395 );
396
397 let mut running = 0.0;
398 for batch in stream {
399 let batch = batch?;
400 let count = batch.len();
401 let points = batch.surface.rows / count;
402
403 let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404 for index in 0..count {
405 // A draw from the posterior, not its mean. Every autoencoder step
406 // decoded a draw, so a draw is the only kind of latent the frozen
407 // decoder has ever answered for. The posterior is wide enough for
408 // that to matter — sigma about twice the magnitude of the mean —
409 // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410 // a draw, so a flow trained towards the mean would be aiming at the
411 // one latent the decoder is worst at.
412 let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413 let width = shape.latents * shape.latent_dim;
414 for (slot, value) in clean.data[index * width..(index + 1) * width]
415 .iter_mut()
416 .zip(&latent.data)
417 {
418 *slot = value * args.latent_scale;
419 }
420 }
421 if step == 0 {
422 println!("latent spread after scaling: {:.3}", spread(&clean));
423 }
424
425 running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426 step += 1;
427 model.step_clipped(1.0, 1.0)?;
428 model.zero_grad();
429
430 if step % args.report == 0 {
431 println!(
432 "step {step}: velocity mse {:.5}",
433 running / args.report as f32
434 );
435 running = 0.0;
436 }
437 if step % args.checkpoint_every == 0 || step == args.steps {
438 model.save(
439 &args.out,
440 &BTreeMap::from([
441 step_at(step),
442 ("latent_scale".to_string(), args.latent_scale.to_string()),
443 ]),
444 )?;
445 println!("wrote {} at step {step}", args.out.display());
446 }
447 if step >= args.steps {
448 break;
449 }
450 }
451 Ok(())
452}Sourcepub fn load<P: AsRef<Path>>(
path: P,
rng: &mut StdRng,
) -> Result<(Self, BTreeMap<String, String>), NetworkError>
pub fn load<P: AsRef<Path>>( path: P, rng: &mut StdRng, ) -> Result<(Self, BTreeMap<String, String>), NetworkError>
Rebuilds a model from what save wrote, and returns the
rest of the metadata.
Examples found in repository?
340fn train_flow(
341 args: &Args,
342 (cond_tokens, cond_dim): (usize, usize),
343 stream: BatchStream,
344 rng: &mut StdRng,
345) -> Result<(), Box<dyn std::error::Error>> {
346 // Frozen: the flow transformer is trained against a latent space that has
347 // already stopped moving, so only `encode` is ever called here.
348 let (vae, _) = ShapeVae::load(&args.vae, rng)?;
349 let shape = *vae.config();
350
351 let (mut model, mut step) = match &args.resume {
352 Some(path) => {
353 let (model, metadata) = FlowTransformer::load(path, rng)?;
354 (model, started_at(&metadata))
355 }
356 None => {
357 // The latent shape is the autoencoder's and the condition's shape
358 // is the corpus's, whatever a supplied configuration says.
359 let config = FlowConfig {
360 latents: shape.latents,
361 latent_dim: shape.latent_dim,
362 cond_tokens,
363 cond_dim,
364 ..read_config(args, FlowConfig::default())?
365 };
366 (FlowTransformer::new(config, rng)?, 0)
367 }
368 };
369 if model.config().cond_dim != cond_dim || model.config().cond_tokens != cond_tokens {
370 return Err(format!(
371 "the checkpoint is conditioned on [{}, {}] tokens and the corpus has [{cond_tokens}, {cond_dim}]",
372 model.config().cond_tokens,
373 model.config().cond_dim
374 )
375 .into());
376 }
377 model.set_optimizer(adam(args.learning_rate));
378 if let Some(device) = args.cuda {
379 #[cfg(feature = "cuda")]
380 {
381 // 0 as the budget: the caller picked the card, and the blocks'
382 // weights are a small part of what a run of this size needs.
383 model.to_cuda(device, 0)?;
384 println!("flow transformer: training on CUDA device {device}");
385 }
386 #[cfg(not(feature = "cuda"))]
387 return Err(format!(
388 "--cuda {device} needs the crate's `cuda` feature: rebuild with --features cuda"
389 )
390 .into());
391 }
392 println!(
393 "flow transformer: {} parameters, from step {step}",
394 model.num_parameters()
395 );
396
397 let mut running = 0.0;
398 for batch in stream {
399 let batch = batch?;
400 let count = batch.len();
401 let points = batch.surface.rows / count;
402
403 let mut clean = Matrix::new(count * shape.latents, shape.latent_dim);
404 for index in 0..count {
405 // A draw from the posterior, not its mean. Every autoencoder step
406 // decoded a draw, so a draw is the only kind of latent the frozen
407 // decoder has ever answered for. The posterior is wide enough for
408 // that to matter — sigma about twice the magnitude of the mean —
409 // and the mean reconstructs at clamped L1 0.029 against 0.0025 for
410 // a draw, so a flow trained towards the mean would be aiming at the
411 // one latent the decoder is worst at.
412 let latent = vae.encode_sample(&slice(&batch.surface, index, points), rng)?;
413 let width = shape.latents * shape.latent_dim;
414 for (slot, value) in clean.data[index * width..(index + 1) * width]
415 .iter_mut()
416 .zip(&latent.data)
417 {
418 *slot = value * args.latent_scale;
419 }
420 }
421 if step == 0 {
422 println!("latent spread after scaling: {:.3}", spread(&clean));
423 }
424
425 running += model.train_step(&clean, &batch.tokens, args.dropout, rng)?;
426 step += 1;
427 model.step_clipped(1.0, 1.0)?;
428 model.zero_grad();
429
430 if step % args.report == 0 {
431 println!(
432 "step {step}: velocity mse {:.5}",
433 running / args.report as f32
434 );
435 running = 0.0;
436 }
437 if step % args.checkpoint_every == 0 || step == args.steps {
438 model.save(
439 &args.out,
440 &BTreeMap::from([
441 step_at(step),
442 ("latent_scale".to_string(), args.latent_scale.to_string()),
443 ]),
444 )?;
445 println!("wrote {} at step {step}", args.out.display());
446 }
447 if step >= args.steps {
448 break;
449 }
450 }
451 Ok(())
452}Trait Implementations§
Source§impl Clone for FlowTransformer
impl Clone for FlowTransformer
Source§impl Debug for FlowTransformer
impl Debug for FlowTransformer
Source§impl<'de> Deserialize<'de> for FlowTransformer
impl<'de> Deserialize<'de> for FlowTransformer
Source§fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where
__D: Deserializer<'de>,
fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where
__D: Deserializer<'de>,
Source§impl PartialEq for FlowTransformer
Compares the model, not where it happens to be running: two models with the
same weights are equal whether or not one of them holds a device context.
impl PartialEq for FlowTransformer
Compares the model, not where it happens to be running: two models with the same weights are equal whether or not one of them holds a device context.
Auto Trait Implementations§
impl Forget for FlowTransformer
impl Freeze for FlowTransformer
impl Move for FlowTransformer
impl RefUnwindSafe for FlowTransformer
impl Send for FlowTransformer
impl Sync for FlowTransformer
impl Unpin for FlowTransformer
impl UnsafeUnpin for FlowTransformer
impl UnwindSafe for FlowTransformer
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Source§impl<T> CloneToUninit for Twhere
T: Clone,
impl<T> CloneToUninit for Twhere
T: Clone,
impl<T> DeserializeOwned for Twhere
T: for<'de> Deserialize<'de>,
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more