1use crate::recursive::config::RecursiveConfig;
2use trustformers_core::{
3 errors::{Result, TrustformersError},
4 layers::{Embedding, FeedForward, LayerNorm, Linear, MultiHeadAttention},
5 tensor::Tensor,
6 traits::{Layer, Model},
7};
8
9pub struct RecursiveTransformer {
11 config: RecursiveConfig,
12 embeddings: Embedding,
13 position_embeddings: Embedding,
14 recursive_layers: Vec<RecursiveLayer>,
15 memory_manager: MemoryManager,
16 depth_predictor: Option<DepthPredictor>,
17 hierarchy_manager: Option<HierarchyManager>,
18 universal_controller: Option<UniversalController>,
19 norm: LayerNorm,
20 lm_head: Linear,
21}
22
23impl RecursiveTransformer {
24 pub fn new(config: RecursiveConfig) -> Result<Self> {
25 let embeddings = Embedding::new(config.vocab_size, config.hidden_size, None)?;
26 let position_embeddings =
27 Embedding::new(config.max_position_embeddings, config.hidden_size, None)?;
28
29 let mut recursive_layers = Vec::new();
30 for _ in 0..config.num_recursive_layers {
31 recursive_layers.push(RecursiveLayer::new(config.clone())?);
32 }
33
34 let memory_manager = MemoryManager::new(config.clone())?;
35
36 let depth_predictor = if config.use_adaptive_depth {
37 Some(DepthPredictor::new(config.clone())?)
38 } else {
39 None
40 };
41
42 let hierarchy_manager = if config.use_hierarchical_attention {
43 Some(HierarchyManager::new(config.clone())?)
44 } else {
45 None
46 };
47
48 let universal_controller = if config.use_universal_transformer {
49 Some(UniversalController::new(config.clone())?)
50 } else {
51 None
52 };
53
54 let norm = LayerNorm::new(vec![config.hidden_size], config.layer_norm_eps)?;
55 let lm_head = Linear::new(config.hidden_size, config.vocab_size, false);
56
57 Ok(Self {
58 config,
59 embeddings,
60 position_embeddings,
61 recursive_layers,
62 memory_manager,
63 depth_predictor,
64 hierarchy_manager,
65 universal_controller,
66 norm,
67 lm_head,
68 })
69 }
70
71 fn chunk_input(&self, input_ids: &Tensor) -> Result<Vec<Tensor>> {
72 let seq_len = input_ids.shape()[1];
73 let chunk_size = self.config.chunk_size;
74 let overlap_size = self.config.overlap_size;
75 let effective_chunk_size = chunk_size - overlap_size;
76
77 let mut chunks = Vec::new();
78 let mut start = 0;
79
80 while start < seq_len {
81 let end = std::cmp::min(start + chunk_size, seq_len);
82 let chunk = input_ids.slice(1, start, end)?;
83 chunks.push(chunk);
84
85 if end == seq_len {
86 break;
87 }
88
89 start += effective_chunk_size;
90 }
91
92 Ok(chunks)
93 }
94
95 fn recursive_process(
96 &self,
97 chunks: Vec<Tensor>,
98 depth: usize,
99 memory: &mut MemoryState,
100 ) -> Result<Tensor> {
101 if chunks.len() == 1 {
102 return self.process_single_chunk(&chunks[0], memory);
104 }
105
106 if depth == 0 {
107 let processed: Result<Vec<Tensor>> =
109 chunks.iter().map(|chunk| self.process_single_chunk(chunk, memory)).collect();
110 return self.combine_chunks(processed?, memory);
111 }
112
113 let group_size = std::cmp::max(2, chunks.len() / 2);
115 let mut processed_chunks = Vec::new();
116
117 for chunk_group in chunks.chunks(group_size) {
118 if chunk_group.len() == 1 {
119 let processed = self.process_single_chunk(&chunk_group[0], memory)?;
120 processed_chunks.push(processed);
121 } else {
122 let sub_result = self.recursive_process(chunk_group.to_vec(), depth - 1, memory)?;
124 processed_chunks.push(sub_result);
125 }
126 }
127
128 if processed_chunks.len() == 1 {
130 processed_chunks.into_iter().next().ok_or_else(|| {
131 TrustformersError::tensor_op_error(
132 "forward_with_memory",
133 "Unexpected empty processed_chunks",
134 )
135 })
136 } else {
137 self.combine_chunks(processed_chunks, memory)
138 }
139 }
140
141 fn process_single_chunk(&self, chunk: &Tensor, memory: &mut MemoryState) -> Result<Tensor> {
142 let (batch_size, seq_len, chunk_vec) = match chunk {
144 Tensor::F32(array) => {
145 if array.ndim() == 1 {
146 let seq_len = array.len();
148 let chunk_vec: Vec<u32> = array.iter().map(|&x| x as u32).collect();
149 (1, seq_len, chunk_vec)
150 } else if array.ndim() == 2 {
151 let batch_size = array.shape()[0];
153 let seq_len = array.shape()[1];
154 let chunk_vec: Vec<u32> = array.iter().map(|&x| x as u32).collect();
157 (batch_size, seq_len, chunk_vec)
158 } else {
159 return Err(TrustformersError::tensor_op_error(
160 "Chunk must be 1D or 2D",
161 "recursive_forward",
162 ));
163 }
164 },
165 _ => {
166 return Err(TrustformersError::tensor_op_error(
167 "Unsupported tensor type for chunk",
168 "recursive_forward",
169 ))
170 },
171 };
172
173 let embedded = self.embeddings.forward(chunk_vec)?;
175
176 let embedded = if embedded.shape().len() == 2 {
178 let total_tokens = embedded.shape()[0];
179 let hidden_size = embedded.shape()[1];
180 if total_tokens == batch_size * seq_len {
181 embedded.reshape(&[batch_size, seq_len, hidden_size])?
182 } else {
183 embedded.reshape(&[1, total_tokens, hidden_size])?
185 }
186 } else {
187 embedded
188 };
189
190 let actual_seq_len = embedded.shape()[1];
192 let position_ids: Vec<u32> = (0..actual_seq_len).map(|i| i as u32).collect();
193 let pos_embedded = self.position_embeddings.forward(position_ids)?;
194
195 let pos_embedded = if pos_embedded.shape().len() == 2 {
197 let hidden_size = pos_embedded.shape()[1];
198 pos_embedded.reshape(&[1, actual_seq_len, hidden_size])?
199 } else {
200 pos_embedded
201 };
202
203 let pos_embedded = if pos_embedded.shape()[0] == 1 && embedded.shape()[0] > 1 {
205 pos_embedded.broadcast_to(&[
206 embedded.shape()[0],
207 actual_seq_len,
208 embedded.shape()[2],
209 ])?
210 } else {
211 pos_embedded
212 };
213
214 let mut hidden_states = embedded.add(&pos_embedded)?;
215
216 if let Some(ref universal) = self.universal_controller {
218 hidden_states = universal.process(hidden_states, &self.recursive_layers[0], memory)?;
220 } else {
221 for layer in &self.recursive_layers {
223 hidden_states = layer.forward(RecursiveLayerInput {
224 hidden_states,
225 memory_state: memory.clone(),
226 depth: 0,
227 })?;
228 }
229 }
230
231 let chunk_summary = self.summarize_chunk(&hidden_states)?;
233 memory.update(chunk_summary)?;
234
235 Ok(hidden_states)
236 }
237
238 fn combine_chunks(&self, chunks: Vec<Tensor>, memory: &mut MemoryState) -> Result<Tensor> {
239 let combined = Tensor::concat(&chunks, 1)?;
241
242 if let Some(ref hierarchy) = self.hierarchy_manager {
244 hierarchy.cross_chunk_attention(combined, memory)
245 } else {
246 Ok(combined)
247 }
248 }
249
250 fn summarize_chunk(&self, hidden_states: &Tensor) -> Result<Tensor> {
251 let shape = hidden_states.shape();
254 if shape.len() == 3 {
255 let seq_len = shape[1];
257 let last_pos = hidden_states.slice(1, seq_len - 1, seq_len)?;
259 Ok(last_pos)
260 } else if shape.len() == 2 {
261 let seq_len = shape[0];
263 let hidden_size = shape[1];
264 let last_pos = hidden_states.slice(0, seq_len - 1, seq_len)?;
266 last_pos.reshape(&[1, 1, hidden_size])
267 } else {
268 Ok(hidden_states.clone())
270 }
271 }
272}
273
274impl Model for RecursiveTransformer {
275 type Config = RecursiveConfig;
276 type Input = RecursiveInput;
277 type Output = RecursiveOutput;
278
279 fn forward(&self, input: Self::Input) -> Result<Self::Output> {
280 let RecursiveInput {
281 input_ids,
282 attention_mask: _,
283 position_ids: _,
284 memory_state: initial_memory,
285 } = input;
286
287 let batch_size = input_ids.shape()[0];
288 let seq_len = input_ids.shape()[1];
289
290 let mut memory = match initial_memory {
292 Some(m) => m,
293 None => MemoryState::new(batch_size, self.config.memory_size, self.config.hidden_size)?,
294 };
295
296 let depth = if let Some(ref predictor) = self.depth_predictor {
298 predictor.predict_depth(&input_ids, &memory)?
299 } else {
300 self.config.recursion_depth
301 };
302
303 let chunks = if seq_len > self.config.chunk_size {
305 self.chunk_input(&input_ids)?
306 } else {
307 vec![input_ids.clone()]
308 };
309
310 let hidden_states = self.recursive_process(chunks, depth, &mut memory)?;
312
313 let normalized = self.norm.forward(hidden_states)?;
315 let logits = self.lm_head.forward(normalized.clone())?;
316
317 Ok(RecursiveOutput {
318 last_hidden_state: normalized,
319 logits,
320 memory_state: memory,
321 recursion_depth: depth,
322 computation_steps: 0, })
324 }
325
326 fn load_pretrained(&mut self, _reader: &mut dyn std::io::Read) -> Result<()> {
327 Err(TrustformersError::not_implemented(
328 "Use load_from_path or load_from_huggingface for enhanced weight loading".to_string(),
329 ))
330 }
331
332 fn get_config(&self) -> &Self::Config {
333 &self.config
334 }
335
336 fn num_parameters(&self) -> usize {
337 let mut total = 0;
338
339 total += self.embeddings.parameter_count();
341 total += self.position_embeddings.parameter_count();
342 total += self.recursive_layers.iter().map(|layer| layer.parameter_count()).sum::<usize>();
343 total += self.memory_manager.parameter_count();
344 total += self.norm.parameter_count();
345 total += self.lm_head.parameter_count();
346
347 if let Some(ref predictor) = self.depth_predictor {
349 total += predictor.parameter_count();
350 }
351 if let Some(ref manager) = self.hierarchy_manager {
352 total += manager.parameter_count();
353 }
354 if let Some(ref controller) = self.universal_controller {
355 total += controller.parameter_count();
356 }
357
358 total
359 }
360}
361
362impl RecursiveTransformer {
363 pub fn load_from_path(&mut self, model_path: impl AsRef<std::path::Path>) -> Result<()> {
365 use crate::weight_loading::{auto_create_loader, WeightLoadingConfig};
366
367 let config = WeightLoadingConfig {
368 lazy_loading: true,
369 memory_mapped: false,
370 ..Default::default()
371 };
372
373 let mut loader = auto_create_loader(model_path, Some(config))?;
374
375 if let Ok(embed_weights) = loader.load_tensor("embeddings.weight") {
377 self.embeddings.set_weight(embed_weights)?;
378 }
379
380 if let Ok(pos_embed_weights) = loader.load_tensor("position_embeddings.weight") {
382 self.position_embeddings.set_weight(pos_embed_weights)?;
383 }
384
385 for (i, layer) in self.recursive_layers.iter_mut().enumerate() {
387 let layer_prefix = format!("recursive_layers.{}", i);
388
389 if let Ok(ln1_weight) =
397 loader.load_tensor(&format!("{}.layer_norm1.weight", layer_prefix))
398 {
399 layer.layer_norm1.set_weight(ln1_weight)?;
400 }
401 if let Ok(ln2_weight) =
402 loader.load_tensor(&format!("{}.layer_norm2.weight", layer_prefix))
403 {
404 layer.layer_norm2.set_weight(ln2_weight)?;
405 }
406
407 if let Some(ref mut ln3) = layer.layer_norm3 {
409 if let Ok(ln3_weight) =
410 loader.load_tensor(&format!("{}.layer_norm3.weight", layer_prefix))
411 {
412 ln3.set_weight(ln3_weight)?;
413 }
414 }
415
416 }
419
420 if let Ok(norm_weight) = loader.load_tensor("norm.weight") {
422 self.norm.set_weight(norm_weight)?;
423 }
424
425 if let Ok(lm_head_weight) = loader.load_tensor("lm_head.weight") {
427 self.lm_head.set_weight(lm_head_weight)?;
428 }
429
430 loader.close()?;
435 Ok(())
436 }
437
438 pub fn load_from_huggingface(&mut self, model_name: &str) -> Result<()> {
440 let cache_dir = std::env::var("HF_HOME")
442 .or_else(|_| std::env::var("HUGGINGFACE_HUB_CACHE"))
443 .unwrap_or_else(|_| {
444 std::env::var("HOME").unwrap_or_else(|_| ".".to_string())
445 + "/.cache/huggingface/hub"
446 });
447
448 let model_path = std::path::Path::new(&cache_dir)
449 .join(format!("models--{}", model_name.replace("/", "--")));
450
451 if model_path.exists() {
452 self.load_from_path(&model_path)
453 } else {
454 self.download_from_huggingface_hub(model_name, &model_path)?;
456 self.load_from_path(&model_path)
457 }
458 }
459
460 fn download_from_huggingface_hub(
462 &self,
463 model_name: &str,
464 model_path: &std::path::Path,
465 ) -> Result<()> {
466 use std::process::Command;
467
468 tracing::info!(
469 "Downloading model {} from HuggingFace Hub to {:?}",
470 model_name,
471 model_path
472 );
473
474 std::fs::create_dir_all(model_path).map_err(|e| {
476 TrustformersError::io_error(format!("Failed to create model directory: {}", e))
477 })?;
478
479 let essential_files = vec![
481 "config.json",
482 "tokenizer.json",
483 "tokenizer_config.json",
484 "pytorch_model.bin", "model.safetensors", ];
487
488 let base_url = format!("https://huggingface.co/{}/resolve/main", model_name);
489
490 for file_name in &essential_files {
492 let file_url = format!("{}/{}", base_url, file_name);
493 let file_path = model_path.join(file_name);
494
495 tracing::info!("Attempting to download {}", file_url);
496
497 let file_path_str = file_path.to_str().ok_or_else(|| {
499 TrustformersError::invalid_config(format!("Invalid UTF-8 in path: {:?}", file_path))
500 })?;
501 let curl_result = Command::new("curl")
502 .args([
503 "-L", "-f", "-o",
506 file_path_str,
507 &file_url,
508 ])
509 .output();
510
511 match curl_result {
512 Ok(output) if output.status.success() => {
513 tracing::info!("Successfully downloaded {}", file_name);
514 continue;
515 },
516 Ok(output) => {
517 tracing::warn!(
518 "Failed to download {} with curl: {}",
519 file_name,
520 String::from_utf8_lossy(&output.stderr)
521 );
522 },
523 Err(e) => {
524 tracing::info!("curl not available: {}", e);
525 },
526 }
527
528 let wget_result = Command::new("wget").args(["-O", file_path_str, &file_url]).output();
530
531 match wget_result {
532 Ok(output) if output.status.success() => {
533 tracing::info!("Successfully downloaded {} with wget", file_name);
534 continue;
535 },
536 Ok(output) => {
537 tracing::warn!(
538 "Failed to download {} with wget: {}",
539 file_name,
540 String::from_utf8_lossy(&output.stderr)
541 );
542 },
543 Err(e) => {
544 tracing::info!("wget not available: {}", e);
545 },
546 }
547
548 if matches!(file_name, &"config.json" | &"pytorch_model.bin") {
550 return Err(TrustformersError::io_error(format!(
551 "Failed to download essential file {} for model {}. Please ensure curl or wget is installed and you have internet access.",
552 file_name, model_name
553 )));
554 }
555 }
556
557 tracing::info!(
558 "Successfully downloaded model {} from HuggingFace Hub",
559 model_name
560 );
561 Ok(())
562 }
563
564 pub fn load_with_lazy_loading(
566 &mut self,
567 model_path: impl AsRef<std::path::Path>,
568 ) -> Result<()> {
569 self.load_from_path(model_path)
571 }
572}
573
574pub struct RecursiveLayer {
576 #[allow(dead_code)]
577 config: RecursiveConfig,
578 self_attention: MultiHeadAttention,
579 cross_attention: Option<MultiHeadAttention>,
580 feed_forward: FeedForward,
581 layer_norm1: LayerNorm,
582 layer_norm2: LayerNorm,
583 layer_norm3: Option<LayerNorm>,
584 memory_gate: MemoryGate,
585}
586
587impl RecursiveLayer {
588 pub fn new(config: RecursiveConfig) -> Result<Self> {
589 let self_attention = MultiHeadAttention::new(
590 config.hidden_size,
591 config.num_attention_heads,
592 config.attention_dropout,
593 true,
594 )?;
595
596 let cross_attention = if config.use_hierarchical_attention {
597 Some(MultiHeadAttention::new(
598 config.hidden_size,
599 config.num_attention_heads,
600 config.attention_dropout,
601 false,
602 )?)
603 } else {
604 None
605 };
606
607 let feed_forward =
608 FeedForward::new(config.hidden_size, config.intermediate_size, config.dropout)?;
609
610 let layer_norm1 = LayerNorm::new(vec![config.hidden_size], config.layer_norm_eps)?;
611 let layer_norm2 = LayerNorm::new(vec![config.hidden_size], config.layer_norm_eps)?;
612 let layer_norm3 = if cross_attention.is_some() {
613 Some(LayerNorm::new(
614 vec![config.hidden_size],
615 config.layer_norm_eps,
616 )?)
617 } else {
618 None
619 };
620
621 let memory_gate = MemoryGate::new(config.clone())?;
622
623 Ok(Self {
624 config,
625 self_attention,
626 cross_attention,
627 feed_forward,
628 layer_norm1,
629 layer_norm2,
630 layer_norm3,
631 memory_gate,
632 })
633 }
634
635 pub fn parameter_count(&self) -> usize {
636 let mut total = self.self_attention.parameter_count()
637 + self.feed_forward.parameter_count()
638 + self.layer_norm1.parameter_count()
639 + self.layer_norm2.parameter_count()
640 + self.memory_gate.parameter_count();
641
642 if let Some(ref cross_attn) = self.cross_attention {
643 total += cross_attn.parameter_count();
644 }
645 if let Some(ref norm3) = self.layer_norm3 {
646 total += norm3.parameter_count();
647 }
648
649 total
650 }
651}
652
653impl Layer for RecursiveLayer {
654 type Input = RecursiveLayerInput;
655 type Output = Tensor;
656
657 fn forward(&self, input: Self::Input) -> Result<Self::Output> {
658 let RecursiveLayerInput {
659 hidden_states,
660 memory_state,
661 depth: _,
662 } = input;
663
664 let mut hidden_states = hidden_states;
665
666 let normed = self.layer_norm1.forward(hidden_states.clone())?;
668 let attn_output = self.self_attention.forward(normed)?;
669 hidden_states = hidden_states.add(&attn_output)?;
670
671 if let (Some(cross_attn), Some(layer_norm)) = (&self.cross_attention, &self.layer_norm3) {
673 let normed = layer_norm.forward(hidden_states.clone())?;
674 let _memory_content = memory_state.get_content()?;
675 let cross_output = cross_attn.forward(normed)?;
676 hidden_states = hidden_states.add(&cross_output)?;
677 }
678
679 let normed = self.layer_norm2.forward(hidden_states.clone())?;
681 let ff_output = self.feed_forward.forward(normed)?;
682 hidden_states = hidden_states.add(&ff_output)?;
683
684 hidden_states = self.memory_gate.forward((hidden_states, memory_state))?;
686
687 Ok(hidden_states)
688 }
689}
690
691pub struct MemoryManager {
693 #[allow(dead_code)]
694 config: RecursiveConfig,
695 memory_projection: Linear,
696 compression_layer: Option<Linear>,
697}
698
699impl MemoryManager {
700 pub fn new(config: RecursiveConfig) -> Result<Self> {
701 let memory_projection = Linear::new(config.hidden_size, config.memory_size, true);
702
703 let compression_layer = if config.use_memory_compression {
704 Some(Linear::new(
705 config.memory_size,
706 (config.memory_size as f32 * config.compression_ratio) as usize,
707 true,
708 ))
709 } else {
710 None
711 };
712
713 Ok(Self {
714 config,
715 memory_projection,
716 compression_layer,
717 })
718 }
719
720 pub fn parameter_count(&self) -> usize {
721 let mut total = self.memory_projection.parameter_count();
722 if let Some(ref compression) = self.compression_layer {
723 total += compression.parameter_count();
724 }
725 total
726 }
727}
728
729#[derive(Debug, Clone)]
731pub struct MemoryState {
732 content: Tensor,
733 write_head: usize,
734 read_head: usize,
735 capacity: usize,
736}
737
738impl MemoryState {
739 pub fn new(batch_size: usize, memory_size: usize, hidden_size: usize) -> Result<Self> {
740 let content = Tensor::zeros(&[batch_size, memory_size, hidden_size])?;
741 Ok(Self {
742 content,
743 write_head: 0,
744 read_head: 0,
745 capacity: memory_size,
746 })
747 }
748
749 pub fn update(&mut self, new_content: Tensor) -> Result<()> {
750 let content_size = new_content.shape()[1];
751
752 if content_size >= self.capacity {
754 self.content = new_content.slice(1, content_size - self.capacity, content_size)?;
755 self.write_head = 0;
756 return Ok(());
757 }
758
759 let end_pos = self.write_head + content_size;
761
762 if end_pos <= self.capacity {
763 let before = if self.write_head > 0 {
765 Some(self.content.slice(1, 0, self.write_head)?)
766 } else {
767 None
768 };
769
770 let after = if end_pos < self.capacity {
771 Some(self.content.slice(1, end_pos, self.capacity)?)
772 } else {
773 None
774 };
775
776 match (before, after) {
778 (Some(b), Some(a)) => {
779 self.content = Tensor::concat(&[b, new_content, a], 1)?;
780 },
781 (Some(b), None) => {
782 self.content = Tensor::concat(&[b, new_content], 1)?;
783 },
784 (None, Some(a)) => {
785 self.content = Tensor::concat(&[new_content, a], 1)?;
786 },
787 (None, None) => {
788 self.content = new_content;
789 },
790 }
791 self.write_head = end_pos;
792 } else {
793 let available = self.capacity - self.write_head;
795 let first_part = new_content.slice(1, 0, available)?;
796 let second_part = new_content.slice(1, available, content_size)?;
797 let remaining = content_size - available;
798
799 let middle = if remaining < self.write_head {
801 Some(self.content.slice(1, remaining, self.write_head)?)
802 } else {
803 None
804 };
805
806 match middle {
808 Some(m) => {
809 self.content = Tensor::concat(&[second_part, m, first_part], 1)?;
810 },
811 None => {
812 self.content = Tensor::concat(&[second_part, first_part], 1)?;
813 },
814 }
815 self.write_head = remaining;
816 }
817
818 Ok(())
819 }
820
821 pub fn get_content(&self) -> Result<Tensor> {
822 Ok(self.content.clone())
823 }
824
825 pub fn read(&mut self, size: usize) -> Result<Tensor> {
826 let end_pos = std::cmp::min(self.read_head + size, self.capacity);
827 let content = self.content.slice(1, self.read_head, end_pos)?;
828 self.read_head = (self.read_head + size) % self.capacity;
829 Ok(content)
830 }
831}
832
833pub struct MemoryGate {
835 gate_projection: Linear,
836 memory_projection: Linear,
837}
838
839impl MemoryGate {
840 pub fn new(config: RecursiveConfig) -> Result<Self> {
841 let gate_projection = Linear::new(config.hidden_size * 2, config.hidden_size, true);
842 let memory_projection = Linear::new(config.memory_size, config.hidden_size, false);
843
844 Ok(Self {
845 gate_projection,
846 memory_projection,
847 })
848 }
849
850 pub fn parameter_count(&self) -> usize {
851 self.gate_projection.parameter_count() + self.memory_projection.parameter_count()
852 }
853}
854
855impl Layer for MemoryGate {
856 type Input = (Tensor, MemoryState);
857 type Output = Tensor;
858
859 fn forward(&self, input: Self::Input) -> Result<Self::Output> {
860 let (hidden_states, _memory_state) = input;
861
862 Ok(hidden_states)
872 }
873}
874
875pub struct DepthPredictor {
877 predictor: Linear,
878 threshold: f32,
879}
880
881impl DepthPredictor {
882 pub fn new(config: RecursiveConfig) -> Result<Self> {
883 let predictor = Linear::new(config.hidden_size, 1, true);
884 Ok(Self {
885 predictor,
886 threshold: config.depth_threshold,
887 })
888 }
889
890 pub fn predict_depth(&self, input_ids: &Tensor, memory: &MemoryState) -> Result<usize> {
891 let seq_len = input_ids.shape()[1] as f32;
893 let memory_usage = memory.write_head as f32 / memory.capacity as f32;
894
895 let complexity_score = (seq_len / 1000.0) + memory_usage;
896
897 if complexity_score > self.threshold {
898 Ok(5) } else if complexity_score > self.threshold / 2.0 {
900 Ok(3) } else {
902 Ok(1) }
904 }
905
906 pub fn parameter_count(&self) -> usize {
907 self.predictor.parameter_count()
908 }
909}
910
911pub struct HierarchyManager {
913 #[allow(dead_code)]
914 config: RecursiveConfig,
915 level_projections: Vec<Linear>,
916 cross_level_attention: MultiHeadAttention,
917}
918
919impl HierarchyManager {
920 pub fn new(config: RecursiveConfig) -> Result<Self> {
921 let mut level_projections = Vec::new();
922
923 for &ratio in &config.level_compression_ratios {
924 let compressed_size = (config.hidden_size as f32 * ratio) as usize;
925 level_projections.push(Linear::new(config.hidden_size, compressed_size, false));
926 }
927
928 let cross_level_attention = MultiHeadAttention::new(
929 config.hidden_size,
930 config.num_attention_heads,
931 config.attention_dropout,
932 false,
933 )?;
934
935 Ok(Self {
936 config,
937 level_projections,
938 cross_level_attention,
939 })
940 }
941
942 pub fn cross_chunk_attention(&self, combined: Tensor, memory: &MemoryState) -> Result<Tensor> {
943 let _memory_content = memory.get_content()?;
944 let attended = self.cross_level_attention.forward(combined.clone())?;
945 combined.add(&attended)
946 }
947
948 pub fn parameter_count(&self) -> usize {
949 let projections_count: usize =
950 self.level_projections.iter().map(|proj| proj.parameter_count()).sum();
951 projections_count + self.cross_level_attention.parameter_count()
952 }
953}
954
955pub struct UniversalController {
957 config: RecursiveConfig,
958 step_embedding: Embedding,
959 halting_predictor: Linear,
960}
961
962impl UniversalController {
963 pub fn new(config: RecursiveConfig) -> Result<Self> {
964 let step_embedding = Embedding::new(config.max_steps, config.hidden_size, None)?;
965 let halting_predictor = Linear::new(config.hidden_size, 1, true);
966
967 Ok(Self {
968 config,
969 step_embedding,
970 halting_predictor,
971 })
972 }
973
974 pub fn process(
975 &self,
976 mut hidden_states: Tensor,
977 layer: &RecursiveLayer,
978 memory: &mut MemoryState,
979 ) -> Result<Tensor> {
980 let mut total_halting_prob =
981 Tensor::zeros(&[hidden_states.shape()[0], hidden_states.shape()[1]])?;
982 let mut step = 0;
983
984 while step < self.config.max_steps {
985 let step_vec = vec![step as u32];
987 let step_emb = self.step_embedding.forward(step_vec)?;
988 hidden_states = hidden_states.add(&step_emb.unsqueeze(0)?.unsqueeze(0)?)?;
989
990 hidden_states = layer.forward(RecursiveLayerInput {
992 hidden_states,
993 memory_state: memory.clone(),
994 depth: step,
995 })?;
996
997 let halting_logits = self.halting_predictor.forward(hidden_states.clone())?;
999 let halting_prob = halting_logits.sigmoid()?;
1000
1001 total_halting_prob = total_halting_prob.add(&halting_prob.squeeze(1)?)?;
1002
1003 if self.config.adaptive_computation_time {
1005 let batch_size = total_halting_prob.shape()[0];
1007 let seq_len = total_halting_prob.shape()[1];
1008 let mut should_halt = false;
1009
1010 for batch_idx in 0..batch_size {
1011 for seq_idx in 0..seq_len {
1012 if let Ok(prob) = total_halting_prob.get_scalar(&[batch_idx, seq_idx]) {
1013 if prob >= self.config.act_threshold {
1014 should_halt = true;
1015 break;
1016 }
1017 }
1018 }
1019 if should_halt {
1020 break;
1021 }
1022 }
1023
1024 if should_halt {
1025 break;
1026 }
1027 } else {
1028 if step >= 5 {
1030 break;
1031 }
1032 }
1033
1034 step += 1;
1035 }
1036
1037 Ok(hidden_states)
1038 }
1039
1040 pub fn parameter_count(&self) -> usize {
1041 self.step_embedding.parameter_count() + self.halting_predictor.parameter_count()
1042 }
1043}
1044
1045#[derive(Debug)]
1047pub struct RecursiveInput {
1048 pub input_ids: Tensor,
1049 pub attention_mask: Option<Tensor>,
1050 pub position_ids: Option<Tensor>,
1051 pub memory_state: Option<MemoryState>,
1052}
1053
1054#[derive(Debug)]
1055pub struct RecursiveOutput {
1056 pub last_hidden_state: Tensor,
1057 pub logits: Tensor,
1058 pub memory_state: MemoryState,
1059 pub recursion_depth: usize,
1060 pub computation_steps: usize,
1061}
1062
1063#[derive(Debug)]
1064pub struct RecursiveLayerInput {
1065 pub hidden_states: Tensor,
1066 pub memory_state: MemoryState,
1067 pub depth: usize,
1068}
1069
1070pub struct RecursiveForCausalLM {
1072 base_model: RecursiveTransformer,
1073}
1074
1075impl RecursiveForCausalLM {
1076 pub fn new(config: RecursiveConfig) -> Result<Self> {
1077 let base_model = RecursiveTransformer::new(config)?;
1078 Ok(Self { base_model })
1079 }
1080}
1081
1082impl Model for RecursiveForCausalLM {
1083 type Config = RecursiveConfig;
1084 type Input = RecursiveInput;
1085 type Output = RecursiveOutput;
1086
1087 fn forward(&self, input: Self::Input) -> Result<Self::Output> {
1088 self.base_model.forward(input)
1089 }
1090
1091 fn load_pretrained(&mut self, _reader: &mut dyn std::io::Read) -> Result<()> {
1092 Err(TrustformersError::not_implemented(
1093 "Use load_from_path or load_from_huggingface for enhanced weight loading".to_string(),
1094 ))
1095 }
1096
1097 fn get_config(&self) -> &Self::Config {
1098 &self.base_model.config
1099 }
1100
1101 fn num_parameters(&self) -> usize {
1102 self.base_model.num_parameters()
1103 }
1104}
1105
1106impl RecursiveForCausalLM {
1107 pub fn load_from_path(&mut self, model_path: impl AsRef<std::path::Path>) -> Result<()> {
1109 self.base_model.load_from_path(model_path)
1111 }
1112
1113 pub fn load_from_huggingface(&mut self, model_name: &str) -> Result<()> {
1115 self.base_model.load_from_huggingface(model_name)
1117 }
1118
1119 pub fn load_with_lazy_loading(
1121 &mut self,
1122 model_path: impl AsRef<std::path::Path>,
1123 ) -> Result<()> {
1124 self.base_model.load_with_lazy_loading(model_path)
1126 }
1127}
1128
1129pub struct RecursiveForSequenceClassification {
1131 base_model: RecursiveTransformer,
1132 classifier: Linear,
1133 #[allow(dead_code)]
1134 num_labels: usize,
1135}
1136
1137impl RecursiveForSequenceClassification {
1138 pub fn new(config: RecursiveConfig, num_labels: usize) -> Result<Self> {
1139 let base_config = config;
1140 let base_model = RecursiveTransformer::new(base_config.clone())?;
1142 let classifier = Linear::new(base_config.hidden_size, num_labels, false);
1143
1144 Ok(Self {
1145 base_model,
1146 classifier,
1147 num_labels,
1148 })
1149 }
1150}
1151
1152impl Model for RecursiveForSequenceClassification {
1153 type Config = RecursiveConfig;
1154 type Input = RecursiveInput;
1155 type Output = RecursiveClassificationOutput;
1156
1157 fn forward(&self, input: Self::Input) -> Result<Self::Output> {
1158 let output = self.base_model.forward(input)?;
1159
1160 let hidden_shape = output.last_hidden_state.shape();
1163 let pooled = if hidden_shape.len() == 3 {
1164 let batch_size = hidden_shape[0];
1166 let seq_len = hidden_shape[1];
1167 let hidden_size = hidden_shape[2];
1168 let last_pos = output.last_hidden_state.slice(1, seq_len - 1, seq_len)?;
1169 last_pos.reshape(&[batch_size, hidden_size])?
1170 } else if hidden_shape.len() == 2 {
1171 let seq_len = hidden_shape[0];
1173 let hidden_size = hidden_shape[1];
1174 let last_pos = output.last_hidden_state.slice(0, seq_len - 1, seq_len)?;
1175 last_pos.reshape(&[1, hidden_size])?
1176 } else {
1177 output.last_hidden_state.clone()
1178 };
1179
1180 let logits = self.classifier.forward(pooled)?;
1181
1182 Ok(RecursiveClassificationOutput {
1183 logits,
1184 hidden_states: output.last_hidden_state,
1185 memory_state: output.memory_state,
1186 recursion_depth: output.recursion_depth,
1187 })
1188 }
1189
1190 fn load_pretrained(&mut self, _reader: &mut dyn std::io::Read) -> Result<()> {
1191 Err(TrustformersError::not_implemented(
1192 "Use load_from_path or load_from_huggingface for enhanced weight loading".to_string(),
1193 ))
1194 }
1195
1196 fn get_config(&self) -> &Self::Config {
1197 &self.base_model.config
1198 }
1199
1200 fn num_parameters(&self) -> usize {
1201 self.base_model.num_parameters() + self.classifier.parameter_count()
1202 }
1203}
1204
1205impl RecursiveForSequenceClassification {
1206 pub fn load_from_path(&mut self, model_path: impl AsRef<std::path::Path>) -> Result<()> {
1208 tracing::info!(
1209 "Loading sequence classification weights from: {:?}",
1210 model_path.as_ref()
1211 );
1212
1213 self.base_model.load_from_path(model_path)?;
1215
1216 Ok(())
1219 }
1220
1221 pub fn load_from_huggingface(&mut self, model_name: &str) -> Result<()> {
1223 let cache_dir = std::env::var("HF_HOME")
1225 .or_else(|_| std::env::var("HUGGINGFACE_HUB_CACHE"))
1226 .unwrap_or_else(|_| {
1227 std::env::var("HOME").unwrap_or_else(|_| ".".to_string())
1228 + "/.cache/huggingface/hub"
1229 });
1230
1231 let model_path = std::path::Path::new(&cache_dir)
1232 .join(format!("models--{}", model_name.replace("/", "--")));
1233
1234 if model_path.exists() {
1235 self.load_from_path(&model_path)
1236 } else {
1237 self.download_from_huggingface_hub(model_name, &model_path)?;
1239 self.load_from_path(&model_path)
1240 }
1241 }
1242
1243 fn download_from_huggingface_hub(
1245 &self,
1246 model_name: &str,
1247 model_path: &std::path::Path,
1248 ) -> Result<()> {
1249 use std::process::Command;
1250
1251 tracing::info!(
1252 "Downloading model {} from HuggingFace Hub to {:?}",
1253 model_name,
1254 model_path
1255 );
1256
1257 std::fs::create_dir_all(model_path).map_err(|e| {
1259 TrustformersError::io_error(format!("Failed to create model directory: {}", e))
1260 })?;
1261
1262 let essential_files = vec![
1264 "config.json",
1265 "tokenizer.json",
1266 "tokenizer_config.json",
1267 "pytorch_model.bin", "model.safetensors", ];
1270
1271 let base_url = format!("https://huggingface.co/{}/resolve/main", model_name);
1272
1273 for file_name in &essential_files {
1275 let file_url = format!("{}/{}", base_url, file_name);
1276 let file_path = model_path.join(file_name);
1277
1278 tracing::info!("Attempting to download {}", file_url);
1279
1280 let file_path_str = file_path.to_str().ok_or_else(|| {
1282 TrustformersError::invalid_config(format!("Invalid UTF-8 in path: {:?}", file_path))
1283 })?;
1284 let curl_result = Command::new("curl")
1285 .args([
1286 "-L", "-f", "-o",
1289 file_path_str,
1290 &file_url,
1291 ])
1292 .output();
1293
1294 match curl_result {
1295 Ok(output) if output.status.success() => {
1296 tracing::info!("Successfully downloaded {}", file_name);
1297 continue;
1298 },
1299 Ok(output) => {
1300 tracing::warn!(
1301 "Failed to download {} with curl: {}",
1302 file_name,
1303 String::from_utf8_lossy(&output.stderr)
1304 );
1305 },
1306 Err(e) => {
1307 tracing::info!("curl not available: {}", e);
1308 },
1309 }
1310
1311 let wget_result = Command::new("wget").args(["-O", file_path_str, &file_url]).output();
1313
1314 match wget_result {
1315 Ok(output) if output.status.success() => {
1316 tracing::info!("Successfully downloaded {} with wget", file_name);
1317 continue;
1318 },
1319 Ok(output) => {
1320 tracing::warn!(
1321 "Failed to download {} with wget: {}",
1322 file_name,
1323 String::from_utf8_lossy(&output.stderr)
1324 );
1325 },
1326 Err(e) => {
1327 tracing::info!("wget not available: {}", e);
1328 },
1329 }
1330
1331 if matches!(file_name, &"config.json" | &"pytorch_model.bin") {
1333 return Err(TrustformersError::io_error(format!(
1334 "Failed to download essential file {} for model {}. Please ensure curl or wget is installed and you have internet access.",
1335 file_name, model_name
1336 )));
1337 }
1338 }
1339
1340 tracing::info!(
1341 "Successfully downloaded model {} from HuggingFace Hub",
1342 model_name
1343 );
1344 Ok(())
1345 }
1346
1347 pub fn load_with_lazy_loading(
1349 &mut self,
1350 model_path: impl AsRef<std::path::Path>,
1351 ) -> Result<()> {
1352 self.load_from_path(model_path)
1354 }
1355}
1356
1357#[derive(Debug)]
1358pub struct RecursiveClassificationOutput {
1359 pub logits: Tensor,
1360 pub hidden_states: Tensor,
1361 pub memory_state: MemoryState,
1362 pub recursion_depth: usize,
1363}