entrenar 0.7.11

Training & Optimization library with autograd, LoRA, quantization, and model merging
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

//! Memory estimation and requirements for fine-tuning
//!
//! Provides utilities for estimating memory usage during training.

/// Mixed precision training options
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MixedPrecision {
    /// FP16 mixed precision
    Fp16,
    /// BF16 mixed precision (better for training)
    Bf16,
}

/// Memory requirements breakdown
#[derive(Debug, Clone, Copy)]
pub struct MemoryRequirement {
    /// Model weights
    pub model: u64,
    /// Optimizer states
    pub optimizer: u64,
    /// Gradients
    pub gradients: u64,
    /// Activations
    pub activations: u64,
}

impl MemoryRequirement {
    /// Total memory required
    #[must_use]
    pub fn total(&self) -> u64 {
        self.model + self.optimizer + self.gradients + self.activations
    }

    /// Check if fits in available memory
    #[must_use]
    pub fn fits_in(&self, available: u64) -> bool {
        self.total() <= available
    }

    /// Memory savings compared to full fine-tuning
    #[must_use]
    pub fn savings_vs_full(&self, full_params: u64) -> f32 {
        let full_memory = full_params * 4 + full_params * 4 * 2 + full_params * 4;
        1.0 - (self.total() as f32 / full_memory as f32)
    }

    /// Format as human-readable string
    #[must_use]
    pub fn format_human(&self) -> String {
        format!(
            "Model: {:.1}GB, Optimizer: {:.1}GB, Gradients: {:.1}GB, Activations: {:.1}GB, Total: {:.1}GB",
            self.model as f64 / 1e9,
            self.optimizer as f64 / 1e9,
            self.gradients as f64 / 1e9,
            self.activations as f64 / 1e9,
            self.total() as f64 / 1e9
        )
    }
}