tensorlogic-train 0.1.0

Training loops, loss composition, and optimization schedules for TensorLogic
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

//! LoRA configuration.

/// Configuration for a LoRA adapter layer.
///
/// LoRA (Low-Rank Adaptation, Hu et al., 2021) decomposes weight updates as
/// `dW = B @ A` where `B in R^{d x r}`, `A in R^{r x k}`, and `r << min(d, k)`.
/// The effective scaling factor applied is `alpha / rank`.
#[derive(Debug, Clone)]
pub struct LoraConfig {
    /// Low-rank dimension r.
    pub rank: usize,
    /// Scaling factor: effective scaling = alpha / rank.
    pub alpha: f64,
    /// Dropout probability for training regularisation (0.0 = no dropout).
    pub dropout: f64,
    /// Which weight matrices to adapt (matched by name).
    pub target_modules: Vec<String>,
    /// RNG seed for reproducible A-matrix initialisation.
    pub seed: u64,
}

impl Default for LoraConfig {
    fn default() -> Self {
        Self {
            rank: 8,
            alpha: 8.0,
            dropout: 0.0,
            target_modules: Vec::new(),
            seed: 42,
        }
    }
}