1use serde::{Deserialize, Serialize};
2use trustformers_core::traits::Config;
3
4#[derive(Debug, Clone, Serialize, Deserialize)]
7pub struct RecursiveConfig {
8 pub vocab_size: usize,
10 pub hidden_size: usize,
11 pub intermediate_size: usize,
12 pub num_attention_heads: usize,
13 pub num_key_value_heads: Option<usize>,
14 pub hidden_act: String,
15 pub max_position_embeddings: usize,
16 pub initializer_range: f32,
17 pub layer_norm_eps: f32,
18 pub use_cache: bool,
19 pub pad_token_id: Option<u32>,
20 pub bos_token_id: u32,
21 pub eos_token_id: u32,
22
23 pub num_recursive_layers: usize,
25 pub recursion_depth: usize,
26 pub chunk_size: usize,
27 pub overlap_size: usize,
28 pub use_adaptive_depth: bool,
29 pub min_depth: usize,
30 pub max_depth: usize,
31 pub depth_threshold: f32,
32
33 pub memory_size: usize,
35 pub use_memory_compression: bool,
36 pub compression_ratio: f32,
37 pub memory_update_strategy: String,
38
39 pub use_hierarchical_attention: bool,
41 pub hierarchy_levels: usize,
42 pub level_compression_ratios: Vec<f32>,
43 pub cross_level_attention: bool,
44
45 pub use_universal_transformer: bool,
47 pub max_steps: usize,
48 pub adaptive_computation_time: bool,
49 pub act_threshold: f32,
50 pub act_loss_weight: f32,
51
52 pub use_gradient_checkpointing: bool,
54 pub use_flash_attention: bool,
55 pub dropout: f32,
56 pub attention_dropout: f32,
57
58 pub model_type: String,
60}
61
62impl Default for RecursiveConfig {
63 fn default() -> Self {
64 Self {
65 vocab_size: 32000,
67 hidden_size: 768,
68 intermediate_size: 3072,
69 num_attention_heads: 12,
70 num_key_value_heads: None,
71 hidden_act: "gelu".to_string(),
72 max_position_embeddings: 16384, initializer_range: 0.02,
74 layer_norm_eps: 1e-12,
75 use_cache: true,
76 pad_token_id: Some(0),
77 bos_token_id: 1,
78 eos_token_id: 2,
79
80 num_recursive_layers: 6,
82 recursion_depth: 3,
83 chunk_size: 512,
84 overlap_size: 64,
85 use_adaptive_depth: true,
86 min_depth: 1,
87 max_depth: 5,
88 depth_threshold: 0.5,
89
90 memory_size: 1024,
92 use_memory_compression: true,
93 compression_ratio: 0.5,
94 memory_update_strategy: "gated".to_string(),
95
96 use_hierarchical_attention: true,
98 hierarchy_levels: 3,
99 level_compression_ratios: vec![1.0, 0.5, 0.25],
100 cross_level_attention: true,
101
102 use_universal_transformer: false,
104 max_steps: 10,
105 adaptive_computation_time: false,
106 act_threshold: 0.01,
107 act_loss_weight: 0.01,
108
109 use_gradient_checkpointing: true,
111 use_flash_attention: true,
112 dropout: 0.1,
113 attention_dropout: 0.1,
114
115 model_type: "recursive".to_string(),
116 }
117 }
118}
119
120impl Config for RecursiveConfig {
121 fn validate(&self) -> trustformers_core::errors::Result<()> {
122 if !self.hidden_size.is_multiple_of(self.num_attention_heads) {
124 return Err(
125 trustformers_core::errors::TrustformersError::invalid_config(
126 "hidden_size must be divisible by num_attention_heads".to_string(),
127 ),
128 );
129 }
130
131 if let Some(num_kv_heads) = self.num_key_value_heads {
132 if !self.num_attention_heads.is_multiple_of(num_kv_heads) {
133 return Err(
134 trustformers_core::errors::TrustformersError::invalid_config(
135 "num_attention_heads must be divisible by num_key_value_heads".to_string(),
136 ),
137 );
138 }
139 }
140
141 if self.recursion_depth == 0 {
143 return Err(
144 trustformers_core::errors::TrustformersError::invalid_config(
145 "recursion_depth must be greater than 0".to_string(),
146 ),
147 );
148 }
149
150 if self.chunk_size == 0 {
151 return Err(
152 trustformers_core::errors::TrustformersError::invalid_config(
153 "chunk_size must be greater than 0".to_string(),
154 ),
155 );
156 }
157
158 if self.overlap_size >= self.chunk_size {
159 return Err(
160 trustformers_core::errors::TrustformersError::invalid_config(
161 "overlap_size must be less than chunk_size".to_string(),
162 ),
163 );
164 }
165
166 if self.use_adaptive_depth && self.min_depth >= self.max_depth {
167 return Err(
168 trustformers_core::errors::TrustformersError::invalid_config(
169 "min_depth must be less than max_depth for adaptive depth".to_string(),
170 ),
171 );
172 }
173
174 if self.use_hierarchical_attention {
176 if self.hierarchy_levels == 0 {
177 return Err(
178 trustformers_core::errors::TrustformersError::invalid_config(
179 "hierarchy_levels must be greater than 0".to_string(),
180 ),
181 );
182 }
183
184 if self.level_compression_ratios.len() != self.hierarchy_levels {
185 return Err(
186 trustformers_core::errors::TrustformersError::invalid_config(
187 "level_compression_ratios length must equal hierarchy_levels".to_string(),
188 ),
189 );
190 }
191
192 for ratio in &self.level_compression_ratios {
193 if *ratio <= 0.0 || *ratio > 1.0 {
194 return Err(
195 trustformers_core::errors::TrustformersError::invalid_config(
196 "compression ratios must be between 0.0 and 1.0".to_string(),
197 ),
198 );
199 }
200 }
201 }
202
203 if self.use_universal_transformer && self.max_steps == 0 {
205 return Err(
206 trustformers_core::errors::TrustformersError::invalid_config(
207 "max_steps must be greater than 0 for Universal Transformer".to_string(),
208 ),
209 );
210 }
211
212 Ok(())
213 }
214
215 fn architecture(&self) -> &'static str {
216 "RecursiveTransformer"
217 }
218}
219
220impl RecursiveConfig {
221 pub fn long_document() -> Self {
223 Self {
224 hidden_size: 1024,
225 intermediate_size: 4096,
226 num_attention_heads: 16,
227 max_position_embeddings: 32768,
228 recursion_depth: 4,
229 chunk_size: 1024,
230 overlap_size: 128,
231 memory_size: 2048,
232 hierarchy_levels: 4,
233 level_compression_ratios: vec![1.0, 0.75, 0.5, 0.25],
234 model_type: "recursive-long-document".to_string(),
235 ..Self::default()
236 }
237 }
238
239 pub fn universal() -> Self {
241 Self {
242 hidden_size: 768,
243 intermediate_size: 3072,
244 num_attention_heads: 12,
245 num_recursive_layers: 1, use_universal_transformer: true,
247 max_steps: 8,
248 adaptive_computation_time: true,
249 act_threshold: 0.01,
250 recursion_depth: 1, model_type: "recursive-universal".to_string(),
252 ..Self::default()
253 }
254 }
255
256 pub fn memory_efficient() -> Self {
258 Self {
259 hidden_size: 512,
260 intermediate_size: 2048,
261 num_attention_heads: 8,
262 use_gradient_checkpointing: true,
263 use_memory_compression: true,
264 compression_ratio: 0.25,
265 chunk_size: 256,
266 overlap_size: 32,
267 memory_size: 512,
268 model_type: "recursive-memory-efficient".to_string(),
269 ..Self::default()
270 }
271 }
272
273 pub fn hierarchical() -> Self {
275 Self {
276 hidden_size: 768,
277 intermediate_size: 3072,
278 num_attention_heads: 12,
279 use_hierarchical_attention: true,
280 hierarchy_levels: 3,
281 level_compression_ratios: vec![1.0, 0.5, 0.25],
282 cross_level_attention: true,
283 recursion_depth: 3,
284 chunk_size: 512,
285 model_type: "recursive-hierarchical".to_string(),
286 ..Self::default()
287 }
288 }
289
290 pub fn code_understanding() -> Self {
292 Self {
293 vocab_size: 50000, hidden_size: 1024,
295 intermediate_size: 4096,
296 num_attention_heads: 16,
297 max_position_embeddings: 8192,
298 recursion_depth: 3,
299 chunk_size: 512,
300 overlap_size: 64,
301 use_hierarchical_attention: true,
302 hierarchy_levels: 3, memory_size: 1024,
304 model_type: "recursive-code".to_string(),
305 ..Self::default()
306 }
307 }
308
309 pub fn head_dim(&self) -> usize {
311 self.hidden_size / self.num_attention_heads
312 }
313
314 pub fn num_kv_heads(&self) -> usize {
316 self.num_key_value_heads.unwrap_or(self.num_attention_heads)
317 }
318
319 pub fn effective_chunk_size(&self) -> usize {
321 self.chunk_size - self.overlap_size
322 }
323
324 pub fn total_memory_capacity(&self) -> usize {
326 if self.use_hierarchical_attention {
327 self.level_compression_ratios
328 .iter()
329 .map(|ratio| (self.memory_size as f32 * ratio) as usize)
330 .sum()
331 } else {
332 self.memory_size
333 }
334 }
335
336 pub fn from_pretrained_name(name: &str) -> Option<Self> {
338 match name {
339 "recursive-long-document" => Some(Self::long_document()),
340 "recursive-universal" => Some(Self::universal()),
341 "recursive-memory-efficient" => Some(Self::memory_efficient()),
342 "recursive-hierarchical" => Some(Self::hierarchical()),
343 "recursive-code" => Some(Self::code_understanding()),
344 _ => None,
345 }
346 }
347
348 pub fn with_memory(&mut self, size: usize, compression: bool, ratio: f32) -> &mut Self {
350 self.memory_size = size;
351 self.use_memory_compression = compression;
352 self.compression_ratio = ratio;
353 self
354 }
355
356 pub fn with_chunks(&mut self, chunk_size: usize, overlap_size: usize) -> &mut Self {
358 self.chunk_size = chunk_size;
359 self.overlap_size = overlap_size;
360 self
361 }
362
363 pub fn with_depth(&mut self, depth: usize, adaptive: bool) -> &mut Self {
365 self.recursion_depth = depth;
366 self.use_adaptive_depth = adaptive;
367 if adaptive {
368 self.max_depth = depth * 2;
369 }
370 self
371 }
372
373 pub fn with_hierarchy(&mut self, levels: usize, ratios: Vec<f32>) -> &mut Self {
375 self.use_hierarchical_attention = true;
376 self.hierarchy_levels = levels;
377 self.level_compression_ratios = ratios;
378 self
379 }
380
381 pub fn with_universal(&mut self, max_steps: usize, act: bool) -> &mut Self {
383 self.use_universal_transformer = true;
384 self.max_steps = max_steps;
385 self.adaptive_computation_time = act;
386 self
387 }
388}
389
390#[derive(Debug, Clone, Serialize, Deserialize, Default)]
392pub enum RecursionStrategy {
393 #[default]
395 DivideAndConquer,
396 Hierarchical,
398 Universal,
400 MemoryAugmented,
402 Adaptive,
404}
405
406#[derive(Debug, Clone, Serialize, Deserialize, Default)]
408pub enum MemoryUpdateStrategy {
409 Concatenate,
411 #[default]
413 Gated,
414 Attention,
416 LSTM,
418 DNC,
420}