Skip to main content

trustformers_models/recursive/
model.rs

1use crate::recursive::config::RecursiveConfig;
2use trustformers_core::{
3    errors::{Result, TrustformersError},
4    layers::{Embedding, FeedForward, LayerNorm, Linear, MultiHeadAttention},
5    tensor::Tensor,
6    traits::{Layer, Model},
7};
8
9/// Main Recursive Transformer model
10pub struct RecursiveTransformer {
11    config: RecursiveConfig,
12    embeddings: Embedding,
13    position_embeddings: Embedding,
14    recursive_layers: Vec<RecursiveLayer>,
15    memory_manager: MemoryManager,
16    depth_predictor: Option<DepthPredictor>,
17    hierarchy_manager: Option<HierarchyManager>,
18    universal_controller: Option<UniversalController>,
19    norm: LayerNorm,
20    lm_head: Linear,
21}
22
23impl RecursiveTransformer {
24    pub fn new(config: RecursiveConfig) -> Result<Self> {
25        let embeddings = Embedding::new(config.vocab_size, config.hidden_size, None)?;
26        let position_embeddings =
27            Embedding::new(config.max_position_embeddings, config.hidden_size, None)?;
28
29        let mut recursive_layers = Vec::new();
30        for _ in 0..config.num_recursive_layers {
31            recursive_layers.push(RecursiveLayer::new(config.clone())?);
32        }
33
34        let memory_manager = MemoryManager::new(config.clone())?;
35
36        let depth_predictor = if config.use_adaptive_depth {
37            Some(DepthPredictor::new(config.clone())?)
38        } else {
39            None
40        };
41
42        let hierarchy_manager = if config.use_hierarchical_attention {
43            Some(HierarchyManager::new(config.clone())?)
44        } else {
45            None
46        };
47
48        let universal_controller = if config.use_universal_transformer {
49            Some(UniversalController::new(config.clone())?)
50        } else {
51            None
52        };
53
54        let norm = LayerNorm::new(vec![config.hidden_size], config.layer_norm_eps)?;
55        let lm_head = Linear::new(config.hidden_size, config.vocab_size, false);
56
57        Ok(Self {
58            config,
59            embeddings,
60            position_embeddings,
61            recursive_layers,
62            memory_manager,
63            depth_predictor,
64            hierarchy_manager,
65            universal_controller,
66            norm,
67            lm_head,
68        })
69    }
70
71    fn chunk_input(&self, input_ids: &Tensor) -> Result<Vec<Tensor>> {
72        let seq_len = input_ids.shape()[1];
73        let chunk_size = self.config.chunk_size;
74        let overlap_size = self.config.overlap_size;
75        let effective_chunk_size = chunk_size - overlap_size;
76
77        let mut chunks = Vec::new();
78        let mut start = 0;
79
80        while start < seq_len {
81            let end = std::cmp::min(start + chunk_size, seq_len);
82            let chunk = input_ids.slice(1, start, end)?;
83            chunks.push(chunk);
84
85            if end == seq_len {
86                break;
87            }
88
89            start += effective_chunk_size;
90        }
91
92        Ok(chunks)
93    }
94
95    fn recursive_process(
96        &self,
97        chunks: Vec<Tensor>,
98        depth: usize,
99        memory: &mut MemoryState,
100    ) -> Result<Tensor> {
101        if chunks.len() == 1 {
102            // Base case: process single chunk
103            return self.process_single_chunk(&chunks[0], memory);
104        }
105
106        if depth == 0 {
107            // Depth exhausted but multiple chunks remain - process and combine all
108            let processed: Result<Vec<Tensor>> =
109                chunks.iter().map(|chunk| self.process_single_chunk(chunk, memory)).collect();
110            return self.combine_chunks(processed?, memory);
111        }
112
113        // Divide chunks into groups for recursive processing
114        let group_size = std::cmp::max(2, chunks.len() / 2);
115        let mut processed_chunks = Vec::new();
116
117        for chunk_group in chunks.chunks(group_size) {
118            if chunk_group.len() == 1 {
119                let processed = self.process_single_chunk(&chunk_group[0], memory)?;
120                processed_chunks.push(processed);
121            } else {
122                // Recursively process the group
123                let sub_result = self.recursive_process(chunk_group.to_vec(), depth - 1, memory)?;
124                processed_chunks.push(sub_result);
125            }
126        }
127
128        // Combine processed chunks
129        if processed_chunks.len() == 1 {
130            processed_chunks.into_iter().next().ok_or_else(|| {
131                TrustformersError::tensor_op_error(
132                    "forward_with_memory",
133                    "Unexpected empty processed_chunks",
134                )
135            })
136        } else {
137            self.combine_chunks(processed_chunks, memory)
138        }
139    }
140
141    fn process_single_chunk(&self, chunk: &Tensor, memory: &mut MemoryState) -> Result<Tensor> {
142        // Embed the chunk - handle both 1D and 2D chunks
143        let (batch_size, seq_len, chunk_vec) = match chunk {
144            Tensor::F32(array) => {
145                if array.ndim() == 1 {
146                    // 1D chunk [seq_len]
147                    let seq_len = array.len();
148                    let chunk_vec: Vec<u32> = array.iter().map(|&x| x as u32).collect();
149                    (1, seq_len, chunk_vec)
150                } else if array.ndim() == 2 {
151                    // 2D chunk [batch_size, seq_len]
152                    let batch_size = array.shape()[0];
153                    let seq_len = array.shape()[1];
154                    // Flatten and process first batch for now
155                    // Full batch processing would need batched embedding support
156                    let chunk_vec: Vec<u32> = array.iter().map(|&x| x as u32).collect();
157                    (batch_size, seq_len, chunk_vec)
158                } else {
159                    return Err(TrustformersError::tensor_op_error(
160                        "Chunk must be 1D or 2D",
161                        "recursive_forward",
162                    ));
163                }
164            },
165            _ => {
166                return Err(TrustformersError::tensor_op_error(
167                    "Unsupported tensor type for chunk",
168                    "recursive_forward",
169                ))
170            },
171        };
172
173        // Get embeddings - returns [total_tokens, hidden_size]
174        let embedded = self.embeddings.forward(chunk_vec)?;
175
176        // Reshape to 3D [batch_size, seq_len, hidden_size] if needed
177        let embedded = if embedded.shape().len() == 2 {
178            let total_tokens = embedded.shape()[0];
179            let hidden_size = embedded.shape()[1];
180            if total_tokens == batch_size * seq_len {
181                embedded.reshape(&[batch_size, seq_len, hidden_size])?
182            } else {
183                // Single sequence case - add batch dimension
184                embedded.reshape(&[1, total_tokens, hidden_size])?
185            }
186        } else {
187            embedded
188        };
189
190        // Add positional embeddings
191        let actual_seq_len = embedded.shape()[1];
192        let position_ids: Vec<u32> = (0..actual_seq_len).map(|i| i as u32).collect();
193        let pos_embedded = self.position_embeddings.forward(position_ids)?;
194
195        // Reshape position embeddings to match [batch, seq, hidden]
196        let pos_embedded = if pos_embedded.shape().len() == 2 {
197            let hidden_size = pos_embedded.shape()[1];
198            pos_embedded.reshape(&[1, actual_seq_len, hidden_size])?
199        } else {
200            pos_embedded
201        };
202
203        // Broadcast position embeddings to batch size
204        let pos_embedded = if pos_embedded.shape()[0] == 1 && embedded.shape()[0] > 1 {
205            pos_embedded.broadcast_to(&[
206                embedded.shape()[0],
207                actual_seq_len,
208                embedded.shape()[2],
209            ])?
210        } else {
211            pos_embedded
212        };
213
214        let mut hidden_states = embedded.add(&pos_embedded)?;
215
216        // Process through recursive layers
217        if let Some(ref universal) = self.universal_controller {
218            // Universal Transformer processing
219            hidden_states = universal.process(hidden_states, &self.recursive_layers[0], memory)?;
220        } else {
221            // Standard recursive processing
222            for layer in &self.recursive_layers {
223                hidden_states = layer.forward(RecursiveLayerInput {
224                    hidden_states,
225                    memory_state: memory.clone(),
226                    depth: 0,
227                })?;
228            }
229        }
230
231        // Update memory with chunk representation
232        let chunk_summary = self.summarize_chunk(&hidden_states)?;
233        memory.update(chunk_summary)?;
234
235        Ok(hidden_states)
236    }
237
238    fn combine_chunks(&self, chunks: Vec<Tensor>, memory: &mut MemoryState) -> Result<Tensor> {
239        // Concatenate chunks along sequence dimension
240        let combined = Tensor::concat(&chunks, 1)?;
241
242        // Apply cross-chunk attention if hierarchical
243        if let Some(ref hierarchy) = self.hierarchy_manager {
244            hierarchy.cross_chunk_attention(combined, memory)
245        } else {
246            Ok(combined)
247        }
248    }
249
250    fn summarize_chunk(&self, hidden_states: &Tensor) -> Result<Tensor> {
251        // Create a summary representation of the chunk
252        // Take the last position from each sequence as the summary
253        let shape = hidden_states.shape();
254        if shape.len() == 3 {
255            // Input is [batch, seq, hidden] -> output [batch, 1, hidden]
256            let seq_len = shape[1];
257            // Get last position
258            let last_pos = hidden_states.slice(1, seq_len - 1, seq_len)?;
259            Ok(last_pos)
260        } else if shape.len() == 2 {
261            // Input is [seq, hidden] -> output [1, 1, hidden]
262            let seq_len = shape[0];
263            let hidden_size = shape[1];
264            // Get last position and reshape
265            let last_pos = hidden_states.slice(0, seq_len - 1, seq_len)?;
266            last_pos.reshape(&[1, 1, hidden_size])
267        } else {
268            // Return a minimal valid tensor
269            Ok(hidden_states.clone())
270        }
271    }
272}
273
274impl Model for RecursiveTransformer {
275    type Config = RecursiveConfig;
276    type Input = RecursiveInput;
277    type Output = RecursiveOutput;
278
279    fn forward(&self, input: Self::Input) -> Result<Self::Output> {
280        let RecursiveInput {
281            input_ids,
282            attention_mask: _,
283            position_ids: _,
284            memory_state: initial_memory,
285        } = input;
286
287        let batch_size = input_ids.shape()[0];
288        let seq_len = input_ids.shape()[1];
289
290        // Initialize memory state
291        let mut memory = match initial_memory {
292            Some(m) => m,
293            None => MemoryState::new(batch_size, self.config.memory_size, self.config.hidden_size)?,
294        };
295
296        // Determine recursion depth
297        let depth = if let Some(ref predictor) = self.depth_predictor {
298            predictor.predict_depth(&input_ids, &memory)?
299        } else {
300            self.config.recursion_depth
301        };
302
303        // Chunk the input if necessary
304        let chunks = if seq_len > self.config.chunk_size {
305            self.chunk_input(&input_ids)?
306        } else {
307            vec![input_ids.clone()]
308        };
309
310        // Process recursively
311        let hidden_states = self.recursive_process(chunks, depth, &mut memory)?;
312
313        // Final normalization and prediction
314        let normalized = self.norm.forward(hidden_states)?;
315        let logits = self.lm_head.forward(normalized.clone())?;
316
317        Ok(RecursiveOutput {
318            last_hidden_state: normalized,
319            logits,
320            memory_state: memory,
321            recursion_depth: depth,
322            computation_steps: 0, // Would be computed in real implementation
323        })
324    }
325
326    fn load_pretrained(&mut self, _reader: &mut dyn std::io::Read) -> Result<()> {
327        Err(TrustformersError::not_implemented(
328            "Use load_from_path or load_from_huggingface for enhanced weight loading".to_string(),
329        ))
330    }
331
332    fn get_config(&self) -> &Self::Config {
333        &self.config
334    }
335
336    fn num_parameters(&self) -> usize {
337        let mut total = 0;
338
339        // Core components
340        total += self.embeddings.parameter_count();
341        total += self.position_embeddings.parameter_count();
342        total += self.recursive_layers.iter().map(|layer| layer.parameter_count()).sum::<usize>();
343        total += self.memory_manager.parameter_count();
344        total += self.norm.parameter_count();
345        total += self.lm_head.parameter_count();
346
347        // Optional components
348        if let Some(ref predictor) = self.depth_predictor {
349            total += predictor.parameter_count();
350        }
351        if let Some(ref manager) = self.hierarchy_manager {
352            total += manager.parameter_count();
353        }
354        if let Some(ref controller) = self.universal_controller {
355            total += controller.parameter_count();
356        }
357
358        total
359    }
360}
361
362impl RecursiveTransformer {
363    /// Enhanced weight loading from local path
364    pub fn load_from_path(&mut self, model_path: impl AsRef<std::path::Path>) -> Result<()> {
365        use crate::weight_loading::{auto_create_loader, WeightLoadingConfig};
366
367        let config = WeightLoadingConfig {
368            lazy_loading: true,
369            memory_mapped: false,
370            ..Default::default()
371        };
372
373        let mut loader = auto_create_loader(model_path, Some(config))?;
374
375        // Load embeddings
376        if let Ok(embed_weights) = loader.load_tensor("embeddings.weight") {
377            self.embeddings.set_weight(embed_weights)?;
378        }
379
380        // Load position embeddings
381        if let Ok(pos_embed_weights) = loader.load_tensor("position_embeddings.weight") {
382            self.position_embeddings.set_weight(pos_embed_weights)?;
383        }
384
385        // Load recursive layers
386        for (i, layer) in self.recursive_layers.iter_mut().enumerate() {
387            let layer_prefix = format!("recursive_layers.{}", i);
388
389            // Note: MultiHeadAttention weight loading is not accessible due to private fields
390            // This would need to be implemented at the core level or through a different interface
391
392            // Note: FeedForward weight loading is not accessible due to private fields
393            // This would need to be implemented at the core level or through accessor methods
394
395            // Load layer norm weights
396            if let Ok(ln1_weight) =
397                loader.load_tensor(&format!("{}.layer_norm1.weight", layer_prefix))
398            {
399                layer.layer_norm1.set_weight(ln1_weight)?;
400            }
401            if let Ok(ln2_weight) =
402                loader.load_tensor(&format!("{}.layer_norm2.weight", layer_prefix))
403            {
404                layer.layer_norm2.set_weight(ln2_weight)?;
405            }
406
407            // Load layer norm3 if present
408            if let Some(ref mut ln3) = layer.layer_norm3 {
409                if let Ok(ln3_weight) =
410                    loader.load_tensor(&format!("{}.layer_norm3.weight", layer_prefix))
411                {
412                    ln3.set_weight(ln3_weight)?;
413                }
414            }
415
416            // Note: Memory gate weight loading would need to be implemented
417            // based on the MemoryGate structure
418        }
419
420        // Load final normalization
421        if let Ok(norm_weight) = loader.load_tensor("norm.weight") {
422            self.norm.set_weight(norm_weight)?;
423        }
424
425        // Load LM head
426        if let Ok(lm_head_weight) = loader.load_tensor("lm_head.weight") {
427            self.lm_head.set_weight(lm_head_weight)?;
428        }
429
430        // Load optional components
431        // Note: DepthPredictor weight loading would need to be implemented
432        // based on the DepthPredictor structure and available methods
433
434        loader.close()?;
435        Ok(())
436    }
437
438    /// Enhanced weight loading from HuggingFace Hub
439    pub fn load_from_huggingface(&mut self, model_name: &str) -> Result<()> {
440        // Check if model is cached locally
441        let cache_dir = std::env::var("HF_HOME")
442            .or_else(|_| std::env::var("HUGGINGFACE_HUB_CACHE"))
443            .unwrap_or_else(|_| {
444                std::env::var("HOME").unwrap_or_else(|_| ".".to_string())
445                    + "/.cache/huggingface/hub"
446            });
447
448        let model_path = std::path::Path::new(&cache_dir)
449            .join(format!("models--{}", model_name.replace("/", "--")));
450
451        if model_path.exists() {
452            self.load_from_path(&model_path)
453        } else {
454            // Attempt to download the model from HuggingFace Hub
455            self.download_from_huggingface_hub(model_name, &model_path)?;
456            self.load_from_path(&model_path)
457        }
458    }
459
460    /// Download model from HuggingFace Hub
461    fn download_from_huggingface_hub(
462        &self,
463        model_name: &str,
464        model_path: &std::path::Path,
465    ) -> Result<()> {
466        use std::process::Command;
467
468        tracing::info!(
469            "Downloading model {} from HuggingFace Hub to {:?}",
470            model_name,
471            model_path
472        );
473
474        // Create the model directory
475        std::fs::create_dir_all(model_path).map_err(|e| {
476            TrustformersError::io_error(format!("Failed to create model directory: {}", e))
477        })?;
478
479        // List of essential files for Recursive models
480        let essential_files = vec![
481            "config.json",
482            "tokenizer.json",
483            "tokenizer_config.json",
484            "pytorch_model.bin", // Try .bin first
485            "model.safetensors", // Fall back to safetensors
486        ];
487
488        let base_url = format!("https://huggingface.co/{}/resolve/main", model_name);
489
490        // Try to download each essential file
491        for file_name in &essential_files {
492            let file_url = format!("{}/{}", base_url, file_name);
493            let file_path = model_path.join(file_name);
494
495            tracing::info!("Attempting to download {}", file_url);
496
497            // Try using curl first
498            let file_path_str = file_path.to_str().ok_or_else(|| {
499                TrustformersError::invalid_config(format!("Invalid UTF-8 in path: {:?}", file_path))
500            })?;
501            let curl_result = Command::new("curl")
502                .args([
503                    "-L", // Follow redirects
504                    "-f", // Fail on HTTP errors
505                    "-o",
506                    file_path_str,
507                    &file_url,
508                ])
509                .output();
510
511            match curl_result {
512                Ok(output) if output.status.success() => {
513                    tracing::info!("Successfully downloaded {}", file_name);
514                    continue;
515                },
516                Ok(output) => {
517                    tracing::warn!(
518                        "Failed to download {} with curl: {}",
519                        file_name,
520                        String::from_utf8_lossy(&output.stderr)
521                    );
522                },
523                Err(e) => {
524                    tracing::info!("curl not available: {}", e);
525                },
526            }
527
528            // Try using wget as fallback
529            let wget_result = Command::new("wget").args(["-O", file_path_str, &file_url]).output();
530
531            match wget_result {
532                Ok(output) if output.status.success() => {
533                    tracing::info!("Successfully downloaded {} with wget", file_name);
534                    continue;
535                },
536                Ok(output) => {
537                    tracing::warn!(
538                        "Failed to download {} with wget: {}",
539                        file_name,
540                        String::from_utf8_lossy(&output.stderr)
541                    );
542                },
543                Err(e) => {
544                    tracing::info!("wget not available: {}", e);
545                },
546            }
547
548            // If essential files like config.json or pytorch_model.bin fail, return error
549            if matches!(file_name, &"config.json" | &"pytorch_model.bin") {
550                return Err(TrustformersError::io_error(format!(
551                    "Failed to download essential file {} for model {}. Please ensure curl or wget is installed and you have internet access.",
552                    file_name, model_name
553                )));
554            }
555        }
556
557        tracing::info!(
558            "Successfully downloaded model {} from HuggingFace Hub",
559            model_name
560        );
561        Ok(())
562    }
563
564    /// Load weights with lazy loading for large models
565    pub fn load_with_lazy_loading(
566        &mut self,
567        model_path: impl AsRef<std::path::Path>,
568    ) -> Result<()> {
569        // For now, delegate to regular loading
570        self.load_from_path(model_path)
571    }
572}
573
574/// Single recursive layer
575pub struct RecursiveLayer {
576    #[allow(dead_code)]
577    config: RecursiveConfig,
578    self_attention: MultiHeadAttention,
579    cross_attention: Option<MultiHeadAttention>,
580    feed_forward: FeedForward,
581    layer_norm1: LayerNorm,
582    layer_norm2: LayerNorm,
583    layer_norm3: Option<LayerNorm>,
584    memory_gate: MemoryGate,
585}
586
587impl RecursiveLayer {
588    pub fn new(config: RecursiveConfig) -> Result<Self> {
589        let self_attention = MultiHeadAttention::new(
590            config.hidden_size,
591            config.num_attention_heads,
592            config.attention_dropout,
593            true,
594        )?;
595
596        let cross_attention = if config.use_hierarchical_attention {
597            Some(MultiHeadAttention::new(
598                config.hidden_size,
599                config.num_attention_heads,
600                config.attention_dropout,
601                false,
602            )?)
603        } else {
604            None
605        };
606
607        let feed_forward =
608            FeedForward::new(config.hidden_size, config.intermediate_size, config.dropout)?;
609
610        let layer_norm1 = LayerNorm::new(vec![config.hidden_size], config.layer_norm_eps)?;
611        let layer_norm2 = LayerNorm::new(vec![config.hidden_size], config.layer_norm_eps)?;
612        let layer_norm3 = if cross_attention.is_some() {
613            Some(LayerNorm::new(
614                vec![config.hidden_size],
615                config.layer_norm_eps,
616            )?)
617        } else {
618            None
619        };
620
621        let memory_gate = MemoryGate::new(config.clone())?;
622
623        Ok(Self {
624            config,
625            self_attention,
626            cross_attention,
627            feed_forward,
628            layer_norm1,
629            layer_norm2,
630            layer_norm3,
631            memory_gate,
632        })
633    }
634
635    pub fn parameter_count(&self) -> usize {
636        let mut total = self.self_attention.parameter_count()
637            + self.feed_forward.parameter_count()
638            + self.layer_norm1.parameter_count()
639            + self.layer_norm2.parameter_count()
640            + self.memory_gate.parameter_count();
641
642        if let Some(ref cross_attn) = self.cross_attention {
643            total += cross_attn.parameter_count();
644        }
645        if let Some(ref norm3) = self.layer_norm3 {
646            total += norm3.parameter_count();
647        }
648
649        total
650    }
651}
652
653impl Layer for RecursiveLayer {
654    type Input = RecursiveLayerInput;
655    type Output = Tensor;
656
657    fn forward(&self, input: Self::Input) -> Result<Self::Output> {
658        let RecursiveLayerInput {
659            hidden_states,
660            memory_state,
661            depth: _,
662        } = input;
663
664        let mut hidden_states = hidden_states;
665
666        // Self-attention
667        let normed = self.layer_norm1.forward(hidden_states.clone())?;
668        let attn_output = self.self_attention.forward(normed)?;
669        hidden_states = hidden_states.add(&attn_output)?;
670
671        // Cross-attention with memory (if enabled)
672        if let (Some(cross_attn), Some(layer_norm)) = (&self.cross_attention, &self.layer_norm3) {
673            let normed = layer_norm.forward(hidden_states.clone())?;
674            let _memory_content = memory_state.get_content()?;
675            let cross_output = cross_attn.forward(normed)?;
676            hidden_states = hidden_states.add(&cross_output)?;
677        }
678
679        // Feed-forward
680        let normed = self.layer_norm2.forward(hidden_states.clone())?;
681        let ff_output = self.feed_forward.forward(normed)?;
682        hidden_states = hidden_states.add(&ff_output)?;
683
684        // Memory gating
685        hidden_states = self.memory_gate.forward((hidden_states, memory_state))?;
686
687        Ok(hidden_states)
688    }
689}
690
691/// Memory management for recursive processing
692pub struct MemoryManager {
693    #[allow(dead_code)]
694    config: RecursiveConfig,
695    memory_projection: Linear,
696    compression_layer: Option<Linear>,
697}
698
699impl MemoryManager {
700    pub fn new(config: RecursiveConfig) -> Result<Self> {
701        let memory_projection = Linear::new(config.hidden_size, config.memory_size, true);
702
703        let compression_layer = if config.use_memory_compression {
704            Some(Linear::new(
705                config.memory_size,
706                (config.memory_size as f32 * config.compression_ratio) as usize,
707                true,
708            ))
709        } else {
710            None
711        };
712
713        Ok(Self {
714            config,
715            memory_projection,
716            compression_layer,
717        })
718    }
719
720    pub fn parameter_count(&self) -> usize {
721        let mut total = self.memory_projection.parameter_count();
722        if let Some(ref compression) = self.compression_layer {
723            total += compression.parameter_count();
724        }
725        total
726    }
727}
728
729/// Memory state for tracking information across recursive calls
730#[derive(Debug, Clone)]
731pub struct MemoryState {
732    content: Tensor,
733    write_head: usize,
734    read_head: usize,
735    capacity: usize,
736}
737
738impl MemoryState {
739    pub fn new(batch_size: usize, memory_size: usize, hidden_size: usize) -> Result<Self> {
740        let content = Tensor::zeros(&[batch_size, memory_size, hidden_size])?;
741        Ok(Self {
742            content,
743            write_head: 0,
744            read_head: 0,
745            capacity: memory_size,
746        })
747    }
748
749    pub fn update(&mut self, new_content: Tensor) -> Result<()> {
750        let content_size = new_content.shape()[1];
751
752        // If new content is larger than or equal to capacity, just use the last `capacity` elements
753        if content_size >= self.capacity {
754            self.content = new_content.slice(1, content_size - self.capacity, content_size)?;
755            self.write_head = 0;
756            return Ok(());
757        }
758
759        // Simple append if there's room
760        let end_pos = self.write_head + content_size;
761
762        if end_pos <= self.capacity {
763            // Content fits without wrapping
764            let before = if self.write_head > 0 {
765                Some(self.content.slice(1, 0, self.write_head)?)
766            } else {
767                None
768            };
769
770            let after = if end_pos < self.capacity {
771                Some(self.content.slice(1, end_pos, self.capacity)?)
772            } else {
773                None
774            };
775
776            // Reconstruct memory with new content (concatenate on dimension 1)
777            match (before, after) {
778                (Some(b), Some(a)) => {
779                    self.content = Tensor::concat(&[b, new_content, a], 1)?;
780                },
781                (Some(b), None) => {
782                    self.content = Tensor::concat(&[b, new_content], 1)?;
783                },
784                (None, Some(a)) => {
785                    self.content = Tensor::concat(&[new_content, a], 1)?;
786                },
787                (None, None) => {
788                    self.content = new_content;
789                },
790            }
791            self.write_head = end_pos;
792        } else {
793            // Content wraps around
794            let available = self.capacity - self.write_head;
795            let first_part = new_content.slice(1, 0, available)?;
796            let second_part = new_content.slice(1, available, content_size)?;
797            let remaining = content_size - available;
798
799            // Get any old content between where second_part ends and write_head
800            let middle = if remaining < self.write_head {
801                Some(self.content.slice(1, remaining, self.write_head)?)
802            } else {
803                None
804            };
805
806            // Reconstruct: [second_part, middle (if any), first_part]
807            match middle {
808                Some(m) => {
809                    self.content = Tensor::concat(&[second_part, m, first_part], 1)?;
810                },
811                None => {
812                    self.content = Tensor::concat(&[second_part, first_part], 1)?;
813                },
814            }
815            self.write_head = remaining;
816        }
817
818        Ok(())
819    }
820
821    pub fn get_content(&self) -> Result<Tensor> {
822        Ok(self.content.clone())
823    }
824
825    pub fn read(&mut self, size: usize) -> Result<Tensor> {
826        let end_pos = std::cmp::min(self.read_head + size, self.capacity);
827        let content = self.content.slice(1, self.read_head, end_pos)?;
828        self.read_head = (self.read_head + size) % self.capacity;
829        Ok(content)
830    }
831}
832
833/// Memory gate for controlling information flow
834pub struct MemoryGate {
835    gate_projection: Linear,
836    memory_projection: Linear,
837}
838
839impl MemoryGate {
840    pub fn new(config: RecursiveConfig) -> Result<Self> {
841        let gate_projection = Linear::new(config.hidden_size * 2, config.hidden_size, true);
842        let memory_projection = Linear::new(config.memory_size, config.hidden_size, false);
843
844        Ok(Self {
845            gate_projection,
846            memory_projection,
847        })
848    }
849
850    pub fn parameter_count(&self) -> usize {
851        self.gate_projection.parameter_count() + self.memory_projection.parameter_count()
852    }
853}
854
855impl Layer for MemoryGate {
856    type Input = (Tensor, MemoryState);
857    type Output = Tensor;
858
859    fn forward(&self, input: Self::Input) -> Result<Self::Output> {
860        let (hidden_states, _memory_state) = input;
861
862        // Simplified memory gating - just pass through hidden states
863        // Full memory integration requires:
864        // 1. Proper memory content projection
865        // 2. Gate computation with concatenated features
866        // 3. Gated combination of hidden states and memory features
867        //
868        // For now, apply a simple learned residual connection
869        // This preserves gradient flow while avoiding complex shape manipulations
870
871        Ok(hidden_states)
872    }
873}
874
875/// Depth predictor for adaptive recursion
876pub struct DepthPredictor {
877    predictor: Linear,
878    threshold: f32,
879}
880
881impl DepthPredictor {
882    pub fn new(config: RecursiveConfig) -> Result<Self> {
883        let predictor = Linear::new(config.hidden_size, 1, true);
884        Ok(Self {
885            predictor,
886            threshold: config.depth_threshold,
887        })
888    }
889
890    pub fn predict_depth(&self, input_ids: &Tensor, memory: &MemoryState) -> Result<usize> {
891        // Simple heuristic based on sequence length and memory state
892        let seq_len = input_ids.shape()[1] as f32;
893        let memory_usage = memory.write_head as f32 / memory.capacity as f32;
894
895        let complexity_score = (seq_len / 1000.0) + memory_usage;
896
897        if complexity_score > self.threshold {
898            Ok(5) // High complexity, use more depth
899        } else if complexity_score > self.threshold / 2.0 {
900            Ok(3) // Medium complexity
901        } else {
902            Ok(1) // Low complexity
903        }
904    }
905
906    pub fn parameter_count(&self) -> usize {
907        self.predictor.parameter_count()
908    }
909}
910
911/// Hierarchy manager for multi-level processing
912pub struct HierarchyManager {
913    #[allow(dead_code)]
914    config: RecursiveConfig,
915    level_projections: Vec<Linear>,
916    cross_level_attention: MultiHeadAttention,
917}
918
919impl HierarchyManager {
920    pub fn new(config: RecursiveConfig) -> Result<Self> {
921        let mut level_projections = Vec::new();
922
923        for &ratio in &config.level_compression_ratios {
924            let compressed_size = (config.hidden_size as f32 * ratio) as usize;
925            level_projections.push(Linear::new(config.hidden_size, compressed_size, false));
926        }
927
928        let cross_level_attention = MultiHeadAttention::new(
929            config.hidden_size,
930            config.num_attention_heads,
931            config.attention_dropout,
932            false,
933        )?;
934
935        Ok(Self {
936            config,
937            level_projections,
938            cross_level_attention,
939        })
940    }
941
942    pub fn cross_chunk_attention(&self, combined: Tensor, memory: &MemoryState) -> Result<Tensor> {
943        let _memory_content = memory.get_content()?;
944        let attended = self.cross_level_attention.forward(combined.clone())?;
945        combined.add(&attended)
946    }
947
948    pub fn parameter_count(&self) -> usize {
949        let projections_count: usize =
950            self.level_projections.iter().map(|proj| proj.parameter_count()).sum();
951        projections_count + self.cross_level_attention.parameter_count()
952    }
953}
954
955/// Universal Transformer controller
956pub struct UniversalController {
957    config: RecursiveConfig,
958    step_embedding: Embedding,
959    halting_predictor: Linear,
960}
961
962impl UniversalController {
963    pub fn new(config: RecursiveConfig) -> Result<Self> {
964        let step_embedding = Embedding::new(config.max_steps, config.hidden_size, None)?;
965        let halting_predictor = Linear::new(config.hidden_size, 1, true);
966
967        Ok(Self {
968            config,
969            step_embedding,
970            halting_predictor,
971        })
972    }
973
974    pub fn process(
975        &self,
976        mut hidden_states: Tensor,
977        layer: &RecursiveLayer,
978        memory: &mut MemoryState,
979    ) -> Result<Tensor> {
980        let mut total_halting_prob =
981            Tensor::zeros(&[hidden_states.shape()[0], hidden_states.shape()[1]])?;
982        let mut step = 0;
983
984        while step < self.config.max_steps {
985            // Add step embedding
986            let step_vec = vec![step as u32];
987            let step_emb = self.step_embedding.forward(step_vec)?;
988            hidden_states = hidden_states.add(&step_emb.unsqueeze(0)?.unsqueeze(0)?)?;
989
990            // Process through layer
991            hidden_states = layer.forward(RecursiveLayerInput {
992                hidden_states,
993                memory_state: memory.clone(),
994                depth: step,
995            })?;
996
997            // Compute halting probability
998            let halting_logits = self.halting_predictor.forward(hidden_states.clone())?;
999            let halting_prob = halting_logits.sigmoid()?;
1000
1001            total_halting_prob = total_halting_prob.add(&halting_prob.squeeze(1)?)?;
1002
1003            // Check if we should halt using Adaptive Computation Time (ACT)
1004            if self.config.adaptive_computation_time {
1005                // Check individual tokens - if any token has high enough probability, consider halting
1006                let batch_size = total_halting_prob.shape()[0];
1007                let seq_len = total_halting_prob.shape()[1];
1008                let mut should_halt = false;
1009
1010                for batch_idx in 0..batch_size {
1011                    for seq_idx in 0..seq_len {
1012                        if let Ok(prob) = total_halting_prob.get_scalar(&[batch_idx, seq_idx]) {
1013                            if prob >= self.config.act_threshold {
1014                                should_halt = true;
1015                                break;
1016                            }
1017                        }
1018                    }
1019                    if should_halt {
1020                        break;
1021                    }
1022                }
1023
1024                if should_halt {
1025                    break;
1026                }
1027            } else {
1028                // Fallback to simple step threshold when ACT is disabled
1029                if step >= 5 {
1030                    break;
1031                }
1032            }
1033
1034            step += 1;
1035        }
1036
1037        Ok(hidden_states)
1038    }
1039
1040    pub fn parameter_count(&self) -> usize {
1041        self.step_embedding.parameter_count() + self.halting_predictor.parameter_count()
1042    }
1043}
1044
1045// Input/Output structures
1046#[derive(Debug)]
1047pub struct RecursiveInput {
1048    pub input_ids: Tensor,
1049    pub attention_mask: Option<Tensor>,
1050    pub position_ids: Option<Tensor>,
1051    pub memory_state: Option<MemoryState>,
1052}
1053
1054#[derive(Debug)]
1055pub struct RecursiveOutput {
1056    pub last_hidden_state: Tensor,
1057    pub logits: Tensor,
1058    pub memory_state: MemoryState,
1059    pub recursion_depth: usize,
1060    pub computation_steps: usize,
1061}
1062
1063#[derive(Debug)]
1064pub struct RecursiveLayerInput {
1065    pub hidden_states: Tensor,
1066    pub memory_state: MemoryState,
1067    pub depth: usize,
1068}
1069
1070/// For language modeling tasks
1071pub struct RecursiveForCausalLM {
1072    base_model: RecursiveTransformer,
1073}
1074
1075impl RecursiveForCausalLM {
1076    pub fn new(config: RecursiveConfig) -> Result<Self> {
1077        let base_model = RecursiveTransformer::new(config)?;
1078        Ok(Self { base_model })
1079    }
1080}
1081
1082impl Model for RecursiveForCausalLM {
1083    type Config = RecursiveConfig;
1084    type Input = RecursiveInput;
1085    type Output = RecursiveOutput;
1086
1087    fn forward(&self, input: Self::Input) -> Result<Self::Output> {
1088        self.base_model.forward(input)
1089    }
1090
1091    fn load_pretrained(&mut self, _reader: &mut dyn std::io::Read) -> Result<()> {
1092        Err(TrustformersError::not_implemented(
1093            "Use load_from_path or load_from_huggingface for enhanced weight loading".to_string(),
1094        ))
1095    }
1096
1097    fn get_config(&self) -> &Self::Config {
1098        &self.base_model.config
1099    }
1100
1101    fn num_parameters(&self) -> usize {
1102        self.base_model.num_parameters()
1103    }
1104}
1105
1106impl RecursiveForCausalLM {
1107    /// Enhanced weight loading from local path
1108    pub fn load_from_path(&mut self, model_path: impl AsRef<std::path::Path>) -> Result<()> {
1109        // Delegate to base model weight loading
1110        self.base_model.load_from_path(model_path)
1111    }
1112
1113    /// Enhanced weight loading from HuggingFace Hub
1114    pub fn load_from_huggingface(&mut self, model_name: &str) -> Result<()> {
1115        // Delegate to base model weight loading
1116        self.base_model.load_from_huggingface(model_name)
1117    }
1118
1119    /// Load weights with lazy loading for large models
1120    pub fn load_with_lazy_loading(
1121        &mut self,
1122        model_path: impl AsRef<std::path::Path>,
1123    ) -> Result<()> {
1124        // Delegate to base model weight loading
1125        self.base_model.load_with_lazy_loading(model_path)
1126    }
1127}
1128
1129/// For sequence classification tasks
1130pub struct RecursiveForSequenceClassification {
1131    base_model: RecursiveTransformer,
1132    classifier: Linear,
1133    #[allow(dead_code)]
1134    num_labels: usize,
1135}
1136
1137impl RecursiveForSequenceClassification {
1138    pub fn new(config: RecursiveConfig, num_labels: usize) -> Result<Self> {
1139        let base_config = config;
1140        // Don't need LM head for classification
1141        let base_model = RecursiveTransformer::new(base_config.clone())?;
1142        let classifier = Linear::new(base_config.hidden_size, num_labels, false);
1143
1144        Ok(Self {
1145            base_model,
1146            classifier,
1147            num_labels,
1148        })
1149    }
1150}
1151
1152impl Model for RecursiveForSequenceClassification {
1153    type Config = RecursiveConfig;
1154    type Input = RecursiveInput;
1155    type Output = RecursiveClassificationOutput;
1156
1157    fn forward(&self, input: Self::Input) -> Result<Self::Output> {
1158        let output = self.base_model.forward(input)?;
1159
1160        // Use final hidden state for classification
1161        // Take the last position instead of mean to avoid 0D tensor issues
1162        let hidden_shape = output.last_hidden_state.shape();
1163        let pooled = if hidden_shape.len() == 3 {
1164            // [batch, seq, hidden] -> take last position -> [batch, hidden]
1165            let batch_size = hidden_shape[0];
1166            let seq_len = hidden_shape[1];
1167            let hidden_size = hidden_shape[2];
1168            let last_pos = output.last_hidden_state.slice(1, seq_len - 1, seq_len)?;
1169            last_pos.reshape(&[batch_size, hidden_size])?
1170        } else if hidden_shape.len() == 2 {
1171            // [seq, hidden] -> take last position -> [1, hidden]
1172            let seq_len = hidden_shape[0];
1173            let hidden_size = hidden_shape[1];
1174            let last_pos = output.last_hidden_state.slice(0, seq_len - 1, seq_len)?;
1175            last_pos.reshape(&[1, hidden_size])?
1176        } else {
1177            output.last_hidden_state.clone()
1178        };
1179
1180        let logits = self.classifier.forward(pooled)?;
1181
1182        Ok(RecursiveClassificationOutput {
1183            logits,
1184            hidden_states: output.last_hidden_state,
1185            memory_state: output.memory_state,
1186            recursion_depth: output.recursion_depth,
1187        })
1188    }
1189
1190    fn load_pretrained(&mut self, _reader: &mut dyn std::io::Read) -> Result<()> {
1191        Err(TrustformersError::not_implemented(
1192            "Use load_from_path or load_from_huggingface for enhanced weight loading".to_string(),
1193        ))
1194    }
1195
1196    fn get_config(&self) -> &Self::Config {
1197        &self.base_model.config
1198    }
1199
1200    fn num_parameters(&self) -> usize {
1201        self.base_model.num_parameters() + self.classifier.parameter_count()
1202    }
1203}
1204
1205impl RecursiveForSequenceClassification {
1206    /// Enhanced weight loading from local path
1207    pub fn load_from_path(&mut self, model_path: impl AsRef<std::path::Path>) -> Result<()> {
1208        tracing::info!(
1209            "Loading sequence classification weights from: {:?}",
1210            model_path.as_ref()
1211        );
1212
1213        // Load base model weights first
1214        self.base_model.load_from_path(model_path)?;
1215
1216        // In a complete implementation, this would also load classifier weights
1217
1218        Ok(())
1219    }
1220
1221    /// Enhanced weight loading from HuggingFace Hub
1222    pub fn load_from_huggingface(&mut self, model_name: &str) -> Result<()> {
1223        // Check if model is cached locally
1224        let cache_dir = std::env::var("HF_HOME")
1225            .or_else(|_| std::env::var("HUGGINGFACE_HUB_CACHE"))
1226            .unwrap_or_else(|_| {
1227                std::env::var("HOME").unwrap_or_else(|_| ".".to_string())
1228                    + "/.cache/huggingface/hub"
1229            });
1230
1231        let model_path = std::path::Path::new(&cache_dir)
1232            .join(format!("models--{}", model_name.replace("/", "--")));
1233
1234        if model_path.exists() {
1235            self.load_from_path(&model_path)
1236        } else {
1237            // Attempt to download the model from HuggingFace Hub
1238            self.download_from_huggingface_hub(model_name, &model_path)?;
1239            self.load_from_path(&model_path)
1240        }
1241    }
1242
1243    /// Download model from HuggingFace Hub
1244    fn download_from_huggingface_hub(
1245        &self,
1246        model_name: &str,
1247        model_path: &std::path::Path,
1248    ) -> Result<()> {
1249        use std::process::Command;
1250
1251        tracing::info!(
1252            "Downloading model {} from HuggingFace Hub to {:?}",
1253            model_name,
1254            model_path
1255        );
1256
1257        // Create the model directory
1258        std::fs::create_dir_all(model_path).map_err(|e| {
1259            TrustformersError::io_error(format!("Failed to create model directory: {}", e))
1260        })?;
1261
1262        // List of essential files for Recursive models
1263        let essential_files = vec![
1264            "config.json",
1265            "tokenizer.json",
1266            "tokenizer_config.json",
1267            "pytorch_model.bin", // Try .bin first
1268            "model.safetensors", // Fall back to safetensors
1269        ];
1270
1271        let base_url = format!("https://huggingface.co/{}/resolve/main", model_name);
1272
1273        // Try to download each essential file
1274        for file_name in &essential_files {
1275            let file_url = format!("{}/{}", base_url, file_name);
1276            let file_path = model_path.join(file_name);
1277
1278            tracing::info!("Attempting to download {}", file_url);
1279
1280            // Try using curl first
1281            let file_path_str = file_path.to_str().ok_or_else(|| {
1282                TrustformersError::invalid_config(format!("Invalid UTF-8 in path: {:?}", file_path))
1283            })?;
1284            let curl_result = Command::new("curl")
1285                .args([
1286                    "-L", // Follow redirects
1287                    "-f", // Fail on HTTP errors
1288                    "-o",
1289                    file_path_str,
1290                    &file_url,
1291                ])
1292                .output();
1293
1294            match curl_result {
1295                Ok(output) if output.status.success() => {
1296                    tracing::info!("Successfully downloaded {}", file_name);
1297                    continue;
1298                },
1299                Ok(output) => {
1300                    tracing::warn!(
1301                        "Failed to download {} with curl: {}",
1302                        file_name,
1303                        String::from_utf8_lossy(&output.stderr)
1304                    );
1305                },
1306                Err(e) => {
1307                    tracing::info!("curl not available: {}", e);
1308                },
1309            }
1310
1311            // Try using wget as fallback
1312            let wget_result = Command::new("wget").args(["-O", file_path_str, &file_url]).output();
1313
1314            match wget_result {
1315                Ok(output) if output.status.success() => {
1316                    tracing::info!("Successfully downloaded {} with wget", file_name);
1317                    continue;
1318                },
1319                Ok(output) => {
1320                    tracing::warn!(
1321                        "Failed to download {} with wget: {}",
1322                        file_name,
1323                        String::from_utf8_lossy(&output.stderr)
1324                    );
1325                },
1326                Err(e) => {
1327                    tracing::info!("wget not available: {}", e);
1328                },
1329            }
1330
1331            // If essential files like config.json or pytorch_model.bin fail, return error
1332            if matches!(file_name, &"config.json" | &"pytorch_model.bin") {
1333                return Err(TrustformersError::io_error(format!(
1334                    "Failed to download essential file {} for model {}. Please ensure curl or wget is installed and you have internet access.",
1335                    file_name, model_name
1336                )));
1337            }
1338        }
1339
1340        tracing::info!(
1341            "Successfully downloaded model {} from HuggingFace Hub",
1342            model_name
1343        );
1344        Ok(())
1345    }
1346
1347    /// Load weights with lazy loading for large models
1348    pub fn load_with_lazy_loading(
1349        &mut self,
1350        model_path: impl AsRef<std::path::Path>,
1351    ) -> Result<()> {
1352        // For now, delegate to regular loading
1353        self.load_from_path(model_path)
1354    }
1355}
1356
1357#[derive(Debug)]
1358pub struct RecursiveClassificationOutput {
1359    pub logits: Tensor,
1360    pub hidden_states: Tensor,
1361    pub memory_state: MemoryState,
1362    pub recursion_depth: usize,
1363}