Skip to main content

trustformers_models/recursive/
config.rs

1use serde::{Deserialize, Serialize};
2use trustformers_core::traits::Config;
3
4/// Recursive Transformer configuration
5/// Reference: "Recursive Transformers for Long Sequences" (Universal Transformers, PaLM)
6#[derive(Debug, Clone, Serialize, Deserialize)]
7pub struct RecursiveConfig {
8    // Base transformer configuration
9    pub vocab_size: usize,
10    pub hidden_size: usize,
11    pub intermediate_size: usize,
12    pub num_attention_heads: usize,
13    pub num_key_value_heads: Option<usize>,
14    pub hidden_act: String,
15    pub max_position_embeddings: usize,
16    pub initializer_range: f32,
17    pub layer_norm_eps: f32,
18    pub use_cache: bool,
19    pub pad_token_id: Option<u32>,
20    pub bos_token_id: u32,
21    pub eos_token_id: u32,
22
23    // Recursive configuration
24    pub num_recursive_layers: usize,
25    pub recursion_depth: usize,
26    pub chunk_size: usize,
27    pub overlap_size: usize,
28    pub use_adaptive_depth: bool,
29    pub min_depth: usize,
30    pub max_depth: usize,
31    pub depth_threshold: f32,
32
33    // Memory configuration
34    pub memory_size: usize,
35    pub use_memory_compression: bool,
36    pub compression_ratio: f32,
37    pub memory_update_strategy: String,
38
39    // Hierarchical processing
40    pub use_hierarchical_attention: bool,
41    pub hierarchy_levels: usize,
42    pub level_compression_ratios: Vec<f32>,
43    pub cross_level_attention: bool,
44
45    // Universal Transformer features
46    pub use_universal_transformer: bool,
47    pub max_steps: usize,
48    pub adaptive_computation_time: bool,
49    pub act_threshold: f32,
50    pub act_loss_weight: f32,
51
52    // Optimization
53    pub use_gradient_checkpointing: bool,
54    pub use_flash_attention: bool,
55    pub dropout: f32,
56    pub attention_dropout: f32,
57
58    // Model type
59    pub model_type: String,
60}
61
62impl Default for RecursiveConfig {
63    fn default() -> Self {
64        Self {
65            // Base transformer defaults
66            vocab_size: 32000,
67            hidden_size: 768,
68            intermediate_size: 3072,
69            num_attention_heads: 12,
70            num_key_value_heads: None,
71            hidden_act: "gelu".to_string(),
72            max_position_embeddings: 16384, // Long sequences
73            initializer_range: 0.02,
74            layer_norm_eps: 1e-12,
75            use_cache: true,
76            pad_token_id: Some(0),
77            bos_token_id: 1,
78            eos_token_id: 2,
79
80            // Recursive defaults
81            num_recursive_layers: 6,
82            recursion_depth: 3,
83            chunk_size: 512,
84            overlap_size: 64,
85            use_adaptive_depth: true,
86            min_depth: 1,
87            max_depth: 5,
88            depth_threshold: 0.5,
89
90            // Memory defaults
91            memory_size: 1024,
92            use_memory_compression: true,
93            compression_ratio: 0.5,
94            memory_update_strategy: "gated".to_string(),
95
96            // Hierarchical defaults
97            use_hierarchical_attention: true,
98            hierarchy_levels: 3,
99            level_compression_ratios: vec![1.0, 0.5, 0.25],
100            cross_level_attention: true,
101
102            // Universal Transformer defaults
103            use_universal_transformer: false,
104            max_steps: 10,
105            adaptive_computation_time: false,
106            act_threshold: 0.01,
107            act_loss_weight: 0.01,
108
109            // Optimization defaults
110            use_gradient_checkpointing: true,
111            use_flash_attention: true,
112            dropout: 0.1,
113            attention_dropout: 0.1,
114
115            model_type: "recursive".to_string(),
116        }
117    }
118}
119
120impl Config for RecursiveConfig {
121    fn validate(&self) -> trustformers_core::errors::Result<()> {
122        // Validate base transformer configuration
123        if !self.hidden_size.is_multiple_of(self.num_attention_heads) {
124            return Err(
125                trustformers_core::errors::TrustformersError::invalid_config(
126                    "hidden_size must be divisible by num_attention_heads".to_string(),
127                ),
128            );
129        }
130
131        if let Some(num_kv_heads) = self.num_key_value_heads {
132            if !self.num_attention_heads.is_multiple_of(num_kv_heads) {
133                return Err(
134                    trustformers_core::errors::TrustformersError::invalid_config(
135                        "num_attention_heads must be divisible by num_key_value_heads".to_string(),
136                    ),
137                );
138            }
139        }
140
141        // Validate recursive configuration
142        if self.recursion_depth == 0 {
143            return Err(
144                trustformers_core::errors::TrustformersError::invalid_config(
145                    "recursion_depth must be greater than 0".to_string(),
146                ),
147            );
148        }
149
150        if self.chunk_size == 0 {
151            return Err(
152                trustformers_core::errors::TrustformersError::invalid_config(
153                    "chunk_size must be greater than 0".to_string(),
154                ),
155            );
156        }
157
158        if self.overlap_size >= self.chunk_size {
159            return Err(
160                trustformers_core::errors::TrustformersError::invalid_config(
161                    "overlap_size must be less than chunk_size".to_string(),
162                ),
163            );
164        }
165
166        if self.use_adaptive_depth && self.min_depth >= self.max_depth {
167            return Err(
168                trustformers_core::errors::TrustformersError::invalid_config(
169                    "min_depth must be less than max_depth for adaptive depth".to_string(),
170                ),
171            );
172        }
173
174        // Validate hierarchical configuration
175        if self.use_hierarchical_attention {
176            if self.hierarchy_levels == 0 {
177                return Err(
178                    trustformers_core::errors::TrustformersError::invalid_config(
179                        "hierarchy_levels must be greater than 0".to_string(),
180                    ),
181                );
182            }
183
184            if self.level_compression_ratios.len() != self.hierarchy_levels {
185                return Err(
186                    trustformers_core::errors::TrustformersError::invalid_config(
187                        "level_compression_ratios length must equal hierarchy_levels".to_string(),
188                    ),
189                );
190            }
191
192            for ratio in &self.level_compression_ratios {
193                if *ratio <= 0.0 || *ratio > 1.0 {
194                    return Err(
195                        trustformers_core::errors::TrustformersError::invalid_config(
196                            "compression ratios must be between 0.0 and 1.0".to_string(),
197                        ),
198                    );
199                }
200            }
201        }
202
203        // Validate Universal Transformer configuration
204        if self.use_universal_transformer && self.max_steps == 0 {
205            return Err(
206                trustformers_core::errors::TrustformersError::invalid_config(
207                    "max_steps must be greater than 0 for Universal Transformer".to_string(),
208                ),
209            );
210        }
211
212        Ok(())
213    }
214
215    fn architecture(&self) -> &'static str {
216        "RecursiveTransformer"
217    }
218}
219
220impl RecursiveConfig {
221    /// Create a recursive transformer for long documents
222    pub fn long_document() -> Self {
223        Self {
224            hidden_size: 1024,
225            intermediate_size: 4096,
226            num_attention_heads: 16,
227            max_position_embeddings: 32768,
228            recursion_depth: 4,
229            chunk_size: 1024,
230            overlap_size: 128,
231            memory_size: 2048,
232            hierarchy_levels: 4,
233            level_compression_ratios: vec![1.0, 0.75, 0.5, 0.25],
234            model_type: "recursive-long-document".to_string(),
235            ..Self::default()
236        }
237    }
238
239    /// Create a recursive transformer with Universal Transformer features
240    pub fn universal() -> Self {
241        Self {
242            hidden_size: 768,
243            intermediate_size: 3072,
244            num_attention_heads: 12,
245            num_recursive_layers: 1, // Single layer, repeated
246            use_universal_transformer: true,
247            max_steps: 8,
248            adaptive_computation_time: true,
249            act_threshold: 0.01,
250            recursion_depth: 1, // Depth through time, not hierarchy
251            model_type: "recursive-universal".to_string(),
252            ..Self::default()
253        }
254    }
255
256    /// Create a memory-efficient recursive transformer
257    pub fn memory_efficient() -> Self {
258        Self {
259            hidden_size: 512,
260            intermediate_size: 2048,
261            num_attention_heads: 8,
262            use_gradient_checkpointing: true,
263            use_memory_compression: true,
264            compression_ratio: 0.25,
265            chunk_size: 256,
266            overlap_size: 32,
267            memory_size: 512,
268            model_type: "recursive-memory-efficient".to_string(),
269            ..Self::default()
270        }
271    }
272
273    /// Create a hierarchical recursive transformer
274    pub fn hierarchical() -> Self {
275        Self {
276            hidden_size: 768,
277            intermediate_size: 3072,
278            num_attention_heads: 12,
279            use_hierarchical_attention: true,
280            hierarchy_levels: 3,
281            level_compression_ratios: vec![1.0, 0.5, 0.25],
282            cross_level_attention: true,
283            recursion_depth: 3,
284            chunk_size: 512,
285            model_type: "recursive-hierarchical".to_string(),
286            ..Self::default()
287        }
288    }
289
290    /// Create configuration for code understanding
291    pub fn code_understanding() -> Self {
292        Self {
293            vocab_size: 50000, // Larger vocabulary for code
294            hidden_size: 1024,
295            intermediate_size: 4096,
296            num_attention_heads: 16,
297            max_position_embeddings: 8192,
298            recursion_depth: 3,
299            chunk_size: 512,
300            overlap_size: 64,
301            use_hierarchical_attention: true,
302            hierarchy_levels: 3, // File -> Function -> Statement level
303            memory_size: 1024,
304            model_type: "recursive-code".to_string(),
305            ..Self::default()
306        }
307    }
308
309    /// Get the head dimension
310    pub fn head_dim(&self) -> usize {
311        self.hidden_size / self.num_attention_heads
312    }
313
314    /// Get the number of key-value heads
315    pub fn num_kv_heads(&self) -> usize {
316        self.num_key_value_heads.unwrap_or(self.num_attention_heads)
317    }
318
319    /// Get effective sequence length per chunk
320    pub fn effective_chunk_size(&self) -> usize {
321        self.chunk_size - self.overlap_size
322    }
323
324    /// Get total memory capacity across all levels
325    pub fn total_memory_capacity(&self) -> usize {
326        if self.use_hierarchical_attention {
327            self.level_compression_ratios
328                .iter()
329                .map(|ratio| (self.memory_size as f32 * ratio) as usize)
330                .sum()
331        } else {
332            self.memory_size
333        }
334    }
335
336    /// Create configuration from pretrained model name
337    pub fn from_pretrained_name(name: &str) -> Option<Self> {
338        match name {
339            "recursive-long-document" => Some(Self::long_document()),
340            "recursive-universal" => Some(Self::universal()),
341            "recursive-memory-efficient" => Some(Self::memory_efficient()),
342            "recursive-hierarchical" => Some(Self::hierarchical()),
343            "recursive-code" => Some(Self::code_understanding()),
344            _ => None,
345        }
346    }
347
348    /// Configure memory settings
349    pub fn with_memory(&mut self, size: usize, compression: bool, ratio: f32) -> &mut Self {
350        self.memory_size = size;
351        self.use_memory_compression = compression;
352        self.compression_ratio = ratio;
353        self
354    }
355
356    /// Configure chunking settings
357    pub fn with_chunks(&mut self, chunk_size: usize, overlap_size: usize) -> &mut Self {
358        self.chunk_size = chunk_size;
359        self.overlap_size = overlap_size;
360        self
361    }
362
363    /// Configure recursion depth
364    pub fn with_depth(&mut self, depth: usize, adaptive: bool) -> &mut Self {
365        self.recursion_depth = depth;
366        self.use_adaptive_depth = adaptive;
367        if adaptive {
368            self.max_depth = depth * 2;
369        }
370        self
371    }
372
373    /// Configure hierarchical attention
374    pub fn with_hierarchy(&mut self, levels: usize, ratios: Vec<f32>) -> &mut Self {
375        self.use_hierarchical_attention = true;
376        self.hierarchy_levels = levels;
377        self.level_compression_ratios = ratios;
378        self
379    }
380
381    /// Configure Universal Transformer settings
382    pub fn with_universal(&mut self, max_steps: usize, act: bool) -> &mut Self {
383        self.use_universal_transformer = true;
384        self.max_steps = max_steps;
385        self.adaptive_computation_time = act;
386        self
387    }
388}
389
390/// Recursive processing strategy
391#[derive(Debug, Clone, Serialize, Deserialize, Default)]
392pub enum RecursionStrategy {
393    /// Divide and conquer - split input into chunks
394    #[default]
395    DivideAndConquer,
396    /// Hierarchical - process at multiple resolutions
397    Hierarchical,
398    /// Universal - repeat same layer multiple times
399    Universal,
400    /// Memory-augmented - use external memory
401    MemoryAugmented,
402    /// Adaptive - dynamically determine recursion depth
403    Adaptive,
404}
405
406/// Memory update strategy for recursive processing
407#[derive(Debug, Clone, Serialize, Deserialize, Default)]
408pub enum MemoryUpdateStrategy {
409    /// Simple concatenation
410    Concatenate,
411    /// Gated update mechanism
412    #[default]
413    Gated,
414    /// Attention-based update
415    Attention,
416    /// LSTM-style update
417    LSTM,
418    /// Differentiable Neural Computer style
419    DNC,
420}