entrenar 0.7.10

Training & Optimization library with autograd, LoRA, quantization, and model merging
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130

//! Model Configuration
//!
//! Contains model-related configuration types for training manifests.

use serde::{Deserialize, Serialize};
use std::collections::HashMap;

use super::shorthand::deserialize_human_usize_opt;

/// Model configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelConfig {
    /// Model source URI (pacha://, hf://, or local path)
    pub source: String,

    /// Model format (safetensors, gguf, apr, pt)
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub format: Option<String>,

    /// Architecture override
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub architecture: Option<ArchitectureConfig>,

    /// Layers to freeze
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub freeze: Option<Vec<String>>,

    /// Device placement (auto, cpu, cuda, cuda:0, mps)
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub device: Option<String>,

    /// Data type (float32, float16, bfloat16)
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub dtype: Option<String>,
}

/// Model architecture configuration
///
/// Supports both preset names and custom architecture parameters.
/// Custom params override values from config.json or preset defaults.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ArchitectureConfig {
    /// Architecture type (transformer, sequential)
    #[serde(rename = "type")]
    pub arch_type: String,

    /// Hidden size (embedding dimension). Accepts shorthand: `"1K"` = 1024.
    #[serde(
        default,
        skip_serializing_if = "Option::is_none",
        deserialize_with = "deserialize_human_usize_opt"
    )]
    pub hidden_size: Option<usize>,

    /// Number of transformer layers
    #[serde(
        default,
        skip_serializing_if = "Option::is_none",
        alias = "num_hidden_layers",
        deserialize_with = "deserialize_human_usize_opt"
    )]
    pub num_layers: Option<usize>,

    /// Number of attention heads
    #[serde(
        default,
        skip_serializing_if = "Option::is_none",
        alias = "num_attention_heads",
        deserialize_with = "deserialize_human_usize_opt"
    )]
    pub num_heads: Option<usize>,

    /// Number of key-value heads (for grouped-query attention)
    #[serde(
        default,
        skip_serializing_if = "Option::is_none",
        alias = "num_key_value_heads",
        deserialize_with = "deserialize_human_usize_opt"
    )]
    pub num_kv_heads: Option<usize>,

    /// FFN intermediate dimension. Accepts shorthand: `"4K"` = 4096.
    #[serde(
        default,
        skip_serializing_if = "Option::is_none",
        deserialize_with = "deserialize_human_usize_opt"
    )]
    pub intermediate_size: Option<usize>,

    /// Vocabulary size. Accepts shorthand: `"32K"` = 32768.
    #[serde(
        default,
        skip_serializing_if = "Option::is_none",
        deserialize_with = "deserialize_human_usize_opt"
    )]
    pub vocab_size: Option<usize>,

    /// Maximum sequence/position length. Accepts shorthand: `"2K"` = 2048, `"128K"` = 131072.
    #[serde(
        default,
        skip_serializing_if = "Option::is_none",
        alias = "max_position_embeddings",
        deserialize_with = "deserialize_human_usize_opt"
    )]
    pub max_seq_length: Option<usize>,

    /// RMS normalization epsilon
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub rms_norm_eps: Option<f32>,

    /// RoPE theta (rotary positional encoding base)
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub rope_theta: Option<f32>,

    /// Whether to use bias in linear layers
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub use_bias: Option<bool>,

    /// Per-head dimension override (for models where head_dim != hidden_size / num_heads)
    #[serde(
        default,
        skip_serializing_if = "Option::is_none",
        deserialize_with = "deserialize_human_usize_opt"
    )]
    pub head_dim: Option<usize>,

    /// Sequential layers (for sequential architecture type)
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub layers: Option<Vec<HashMap<String, serde_json::Value>>>,
}