Struct TransformerTrainConfig 

pub struct TransformerTrainConfig {

    pub base: TrainConfig,
    pub model_config: TransformerConfig,
    pub checkpoint_config: CheckpointConfig,
    pub precision_config: MixedPrecisionConfig,
    pub max_seq_len: usize,
    pub accumulation_steps: usize,
    pub warmup_steps: usize,
    pub lr: f32,
    pub max_steps: Option<usize>,
    pub use_cuda: bool,
    pub beta1: f32,
    pub beta2: f32,
    pub weight_decay: f32,
    pub distributed: Option<DistributedTrainConfig>,
    pub deterministic: bool,
    pub seed: u64,
    pub profile_interval: usize,
    pub lora_rank: Option<usize>,
    pub lora_alpha: Option<f32>,
    pub lora_target_modules: Option<Vec<String>>,
    pub lora_plus_ratio: f32,
    pub double_quantize: bool,
    pub quantize_nf4: bool,
}

Configuration for transformer training

Fields

base: TrainConfig

Base training configuration

model_config: TransformerConfig

Transformer architecture configuration

checkpoint_config: CheckpointConfig

Checkpoint configuration for memory efficiency

precision_config: MixedPrecisionConfig

Mixed-precision configuration

max_seq_len: usize

Maximum sequence length

accumulation_steps: usize

Accumulation steps for gradient accumulation

warmup_steps: usize

Warmup steps for learning rate scheduler

lr: f32

Learning rate

max_steps: Option<usize>

Maximum training steps (stop after this many optimizer steps)

use_cuda: bool

Use CUDA GPU training when available (default: true = auto-detect)

beta1: f32

AdamW beta1 (default: 0.9)

beta2: f32

AdamW beta2 (default: 0.999)

weight_decay: f32

AdamW weight decay (default: 0.01)

distributed: Option<DistributedTrainConfig>

Distributed training configuration (None = single-GPU)

deterministic: bool

Enable bitwise deterministic training (CUBLAS_WORKSPACE_CONFIG, cuDNN deterministic) Contract: C-DETERM-001

seed: u64

Random seed for reproducibility

profile_interval: usize

KAIZEN-047: Step profiler report interval (0 = disabled, N = print every N steps)

lora_rank: Option<usize>

LoRA rank (None = full fine-tuning, Some(r) = LoRA with rank r)

lora_alpha: Option<f32>

LoRA alpha scaling factor (default: 2 * rank)

lora_target_modules: Option<Vec<String>>

LoRA target modules (e.g., q_proj, v_proj)

lora_plus_ratio: f32

LoRA+ ratio: LR multiplier for B matrices (ENT-LoRA-006) Default 1.0 = standard LoRA. 16.0 = LoRA+ (Hayou et al. ICML 2024)

double_quantize: bool

Double quantization for QLoRA (ENT-LoRA-008) Quantizes FP32 absmax constants to 8-bit, saving ~0.37 bits/param

quantize_nf4: bool

Quantize frozen base weights to NF4 (4-bit) for QLoRA pretraining (ENT-263)

When enabled with LoRA, uses CudaNf4TransformerBlock instead of CudaTransformerBlock, achieving ~8x VRAM compression on frozen weights. Only LoRA adapters and norm weights remain trainable in fp32.

Implementations

impl TransformerTrainConfig

pub fn new(model_config: TransformerConfig) -> Self

Create new config with defaults

pub fn with_checkpointing(self, num_segments: usize) -> Self

Enable gradient checkpointing

pub fn with_bf16(self) -> Self

Enable bf16 mixed precision

pub fn with_fp16(self) -> Self

Enable fp16 mixed precision with dynamic loss scaling

pub fn with_max_seq_len(self, len: usize) -> Self

Set maximum sequence length

pub fn with_accumulation_steps(self, steps: usize) -> Self

Set gradient accumulation steps

pub fn with_warmup_steps(self, steps: usize) -> Self

Set warmup steps

pub fn with_lr(self, lr: f32) -> Self

Set learning rate

pub fn with_grad_clip(self, clip: f32) -> Self

Set gradient clipping

pub fn with_max_steps(self, steps: usize) -> Self

Set maximum training steps

pub fn with_use_cuda(self, use_cuda: bool) -> Self

Enable or disable CUDA GPU training (default: true = auto-detect)

pub fn with_beta2(self, beta2: f32) -> Self

Set AdamW beta2 (default: 0.999)

pub fn with_weight_decay(self, wd: f32) -> Self

Set AdamW weight decay (default: 0.01)

pub fn with_deterministic(self, deterministic: bool) -> Self

Enable bitwise deterministic training (C-DETERM-001)

Sets CUBLAS_WORKSPACE_CONFIG, cuDNN deterministic mode, and disables cuDNN benchmark. May reduce throughput but guarantees reproducibility.

pub fn with_seed(self, seed: u64) -> Self

Set random seed for reproducibility

pub fn apply_deterministic_settings(&self)

Apply deterministic settings to the CUDA environment.

Must be called before any cuBLAS/cuDNN operations. Uses ReproducibilityConfig from finetune infrastructure.

Contract (C-DETERM-001)

After calling this, CUBLAS_WORKSPACE_CONFIG=:4096:8 and CUDNN_DETERMINISTIC=1 are guaranteed set in the process environment.

pub fn with_profile_interval(self, interval: usize) -> Self

Set step profiler report interval (0 = disabled, N = print every N steps)

pub fn with_lora( self, rank: usize, alpha: f32, target_modules: Vec<String>, ) -> Self

Enable LoRA fine-tuning with rank, alpha, and target modules

When LoRA is enabled, only LoRA adapter weights (A, B matrices) and layer norms are trainable. Base model weights are frozen.

Contract (ENT-LoRA-001)

• Base weights frozen (requires_grad=false)
• Only LoRA A/B + norms are optimizer targets
• scale = alpha / rank

pub fn with_lora_plus_ratio(self, ratio: f32) -> Self

Set LoRA+ ratio (ENT-LoRA-006)

LR multiplier for B matrices. Default 1.0 = standard LoRA. 16.0 = LoRA+ (Hayou et al. ICML 2024) — B learns 16x faster than A.

pub fn with_double_quantize(self, enabled: bool) -> Self

Enable double quantization for QLoRA (ENT-LoRA-008)

pub fn with_quantize_nf4(self, enabled: bool) -> Self

Enable NF4 quantization for QLoRA pretraining (ENT-263)

When enabled with LoRA, frozen base weights are quantized to 4-bit NF4, achieving ~8x VRAM compression. Only LoRA adapters and norm weights are trainable. Requires lora_rank to be set.

pub fn is_nf4(&self) -> bool

Check if NF4 quantization is enabled for QLoRA

pub fn is_lora(&self) -> bool

Check if LoRA fine-tuning is enabled

pub fn with_distributed(self, config: DistributedTrainConfig) -> Self

Enable distributed training with the given configuration

pub fn is_distributed(&self) -> bool

Check if distributed training is enabled

pub fn world_size(&self) -> usize

Get world size (1 for single-GPU)

pub fn rank(&self) -> usize

Get this worker’s rank (0 for single-GPU)

Trait Implementations

impl Clone for TransformerTrainConfig

fn clone(&self) -> TransformerTrainConfig

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TransformerTrainConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.