trustformers_debug/core/
session.rs

1//! Core debugging session and configuration management
2//!
3//! This module contains the fundamental components for TrustformeRS debugging including
4//! the main DebugSession coordinator, configuration structures, and session lifecycle management.
5// reason: debug/profiling scaffolding — structs are constructed and their fields/methods
6// are retained for the data model, serialization completeness, and future consumers that
7// do not yet read every member. Consolidated from many item-level #[allow(dead_code)].
8#![allow(dead_code)]
9
10use crate::*;
11use anyhow::Result;
12use serde::{Deserialize, Serialize};
13use std::fmt;
14use uuid::Uuid;
15
16/// Configuration for debugging session
17#[derive(Debug, Clone, Serialize, Deserialize)]
18pub struct DebugConfig {
19    /// Enable tensor inspection
20    pub enable_tensor_inspection: bool,
21    /// Enable gradient debugging
22    pub enable_gradient_debugging: bool,
23    /// Enable model diagnostics
24    pub enable_model_diagnostics: bool,
25    /// Enable visual debugging (requires display)
26    pub enable_visualization: bool,
27    /// Enable memory profiling
28    pub enable_memory_profiling: bool,
29    /// Enable computation graph analysis
30    pub enable_computation_graph_analysis: bool,
31    /// Maximum number of tensors to track
32    pub max_tracked_tensors: usize,
33    /// Maximum history length for gradients
34    pub max_gradient_history: usize,
35    /// Output directory for debug artifacts
36    pub output_dir: Option<String>,
37    /// Sampling rate for expensive operations (0.0 to 1.0)
38    pub sampling_rate: f32,
39    /// Memory profiling configuration
40    pub memory_profiling_config: MemoryProfilingConfig,
41    /// Computation graph analysis configuration
42    pub graph_analysis_config: GraphAnalysisConfig,
43    /// Architecture analysis configuration
44    pub architecture_analysis_config: architecture_analysis::ArchitectureAnalysisConfig,
45    /// Behavior analysis configuration
46    pub behavior_analysis_config: BehaviorAnalysisConfig,
47    /// Training dynamics analysis configuration
48    pub training_dynamics_config: TrainingDynamicsConfig,
49    /// Differential debugging configuration
50    pub differential_debugging_config: DifferentialDebuggingConfig,
51    /// Interpretability tools configuration
52    pub interpretability_config: InterpretabilityConfig,
53    /// Neural network debugging configuration
54    pub neural_network_debugging_config: Option<neural_network_debugging::TransformerDebugConfig>,
55    /// Advanced ML debugging configuration
56    pub advanced_ml_debugging_config: AdvancedMLDebuggingConfig,
57    /// Advanced GPU profiling configuration
58    pub advanced_gpu_profiling_config: AdvancedGpuProfilingConfig,
59    /// Kernel optimization configuration
60    pub kernel_optimization_config: KernelOptimizationConfig,
61    /// AI code analysis configuration
62    pub ai_code_analysis_config: AIAnalysisConfig,
63    /// Distributed debugging configuration
64    pub distributed_debugging_config: Option<DistributedDebugConfig>,
65    /// Environmental monitoring configuration
66    pub environmental_monitoring_config: EnvironmentalConfig,
67}
68
69impl Default for DebugConfig {
70    fn default() -> Self {
71        Self {
72            enable_tensor_inspection: true,
73            enable_gradient_debugging: true,
74            enable_model_diagnostics: true,
75            enable_visualization: false,
76            enable_memory_profiling: true,
77            enable_computation_graph_analysis: true,
78            max_tracked_tensors: 1000,
79            max_gradient_history: 100,
80            output_dir: None,
81            sampling_rate: 1.0,
82            memory_profiling_config: MemoryProfilingConfig::default(),
83            graph_analysis_config: GraphAnalysisConfig::default(),
84            architecture_analysis_config:
85                architecture_analysis::ArchitectureAnalysisConfig::default(),
86            behavior_analysis_config: BehaviorAnalysisConfig::default(),
87            training_dynamics_config: TrainingDynamicsConfig::default(),
88            differential_debugging_config: DifferentialDebuggingConfig::default(),
89            interpretability_config: InterpretabilityConfig::default(),
90            neural_network_debugging_config: None,
91            advanced_ml_debugging_config: AdvancedMLDebuggingConfig::default(),
92            advanced_gpu_profiling_config: AdvancedGpuProfilingConfig::default(),
93            kernel_optimization_config: KernelOptimizationConfig::default(),
94            ai_code_analysis_config: AIAnalysisConfig::default(),
95            distributed_debugging_config: None,
96            environmental_monitoring_config: EnvironmentalConfig::default(),
97        }
98    }
99}
100
101/// Main debugging session that coordinates all debugging tools
102#[derive(Debug)]
103pub struct DebugSession {
104    id: Uuid,
105    config: DebugConfig,
106    tensor_inspector: TensorInspector,
107    gradient_debugger: GradientDebugger,
108    model_diagnostics: ModelDiagnostics,
109    hooks: HookManager,
110    profiler: Profiler,
111    memory_profiler: Option<MemoryProfiler>,
112    interactive_debugger: InteractiveDebugger,
113    anomaly_detector: AnomalyDetector,
114    computation_graph_analyzer: ComputationGraphAnalyzer,
115    architecture_analyzer: architecture_analysis::ArchitectureAnalyzer,
116    behavior_analyzer: BehaviorAnalyzer,
117    training_dynamics_analyzer: TrainingDynamicsAnalyzer,
118    differential_debugger: DifferentialDebugger,
119    interpretability_analyzer: InterpretabilityAnalyzer,
120    health_checker: crate::health_checker::HealthChecker,
121    transformer_debugger: Option<neural_network_debugging::TransformerDebugger>,
122    advanced_ml_debugger: AdvancedMLDebugger,
123    advanced_gpu_profiler: Option<AdvancedGpuMemoryProfiler>,
124    kernel_optimizer: KernelOptimizationAnalyzer,
125    ai_code_analyzer: Option<AICodeAnalyzer>,
126    distributed_debugger: Option<DistributedDebugger>,
127    environmental_monitor: Option<EnvironmentalMonitor>,
128}
129
130impl DebugSession {
131    /// Create a new debugging session
132    pub fn new(config: DebugConfig) -> Self {
133        let id = Uuid::new_v4();
134
135        let memory_profiler = if config.enable_memory_profiling {
136            Some(MemoryProfiler::new(config.memory_profiling_config.clone()))
137        } else {
138            None
139        };
140
141        let transformer_debugger =
142            config.neural_network_debugging_config.as_ref().map(|neural_config| {
143                neural_network_debugging::TransformerDebugger::new(neural_config.clone())
144            });
145
146        let advanced_gpu_profiler = if config.advanced_gpu_profiling_config.enable_gpu_profiling {
147            AdvancedGpuMemoryProfiler::new(config.advanced_gpu_profiling_config.device_count).ok()
148        } else {
149            None
150        };
151
152        let ai_code_analyzer = if config.ai_code_analysis_config.enable_deep_analysis {
153            Some(AICodeAnalyzer::new(config.ai_code_analysis_config.clone()))
154        } else {
155            None
156        };
157
158        let distributed_debugger =
159            if let Some(ref dist_config) = config.distributed_debugging_config {
160                let node_id = NodeId::new(0, "debug-node".to_string());
161                Some(DistributedDebugger::new(dist_config.clone(), node_id))
162            } else {
163                None
164            };
165
166        let environmental_monitor = if config.environmental_monitoring_config.enable_carbon_tracking
167        {
168            Some(EnvironmentalMonitor::new(
169                config.environmental_monitoring_config.clone(),
170            ))
171        } else {
172            None
173        };
174
175        Self {
176            id,
177            tensor_inspector: TensorInspector::new(&config),
178            gradient_debugger: GradientDebugger::new(config.clone()),
179            model_diagnostics: ModelDiagnostics::new(&config),
180            hooks: HookManager::new(),
181            profiler: Profiler::new(&config),
182            memory_profiler,
183            interactive_debugger: InteractiveDebugger::new(&config),
184            anomaly_detector: AnomalyDetector::new(&config),
185            computation_graph_analyzer: ComputationGraphAnalyzer::new(
186                config.graph_analysis_config.clone(),
187            ),
188            architecture_analyzer: architecture_analysis::ArchitectureAnalyzer::new(
189                config.architecture_analysis_config.clone(),
190            ),
191            behavior_analyzer: BehaviorAnalyzer::new(config.behavior_analysis_config.clone()),
192            training_dynamics_analyzer: TrainingDynamicsAnalyzer::new(),
193            differential_debugger: DifferentialDebugger::new(
194                config.differential_debugging_config.clone(),
195            ),
196            interpretability_analyzer: InterpretabilityAnalyzer::new(
197                config.interpretability_config.clone(),
198            ),
199            health_checker: crate::health_checker::HealthChecker::new(&config),
200            transformer_debugger,
201            advanced_ml_debugger: AdvancedMLDebugger::new(
202                config.advanced_ml_debugging_config.clone(),
203            ),
204            advanced_gpu_profiler,
205            kernel_optimizer: match KernelOptimizationAnalyzer::new() {
206                Ok(analyzer) => analyzer,
207                Err(e) => {
208                    tracing::warn!(
209                        "Failed to initialize kernel optimizer sub-analyzers: {}; \
210                         continuing with an empty analyzer (fully functional, no prior history)",
211                        e
212                    );
213                    KernelOptimizationAnalyzer::new_empty()
214                },
215            },
216            ai_code_analyzer,
217            distributed_debugger,
218            environmental_monitor,
219            config,
220        }
221    }
222
223    /// Get session ID
224    pub fn id(&self) -> Uuid {
225        self.id
226    }
227
228    /// Get debug configuration
229    pub fn config(&self) -> &DebugConfig {
230        &self.config
231    }
232
233    /// Get tensor inspector
234    pub fn tensor_inspector(&self) -> &TensorInspector {
235        &self.tensor_inspector
236    }
237
238    /// Get mutable tensor inspector
239    pub fn tensor_inspector_mut(&mut self) -> &mut TensorInspector {
240        &mut self.tensor_inspector
241    }
242
243    /// Get gradient debugger
244    pub fn gradient_debugger(&self) -> &GradientDebugger {
245        &self.gradient_debugger
246    }
247
248    /// Get mutable gradient debugger
249    pub fn gradient_debugger_mut(&mut self) -> &mut GradientDebugger {
250        &mut self.gradient_debugger
251    }
252
253    /// Get model diagnostics
254    pub fn model_diagnostics(&self) -> &ModelDiagnostics {
255        &self.model_diagnostics
256    }
257
258    /// Get mutable model diagnostics
259    pub fn model_diagnostics_mut(&mut self) -> &mut ModelDiagnostics {
260        &mut self.model_diagnostics
261    }
262
263    /// Get hook manager
264    pub fn hooks(&self) -> &HookManager {
265        &self.hooks
266    }
267
268    /// Get mutable hook manager
269    pub fn hooks_mut(&mut self) -> &mut HookManager {
270        &mut self.hooks
271    }
272
273    /// Get profiler
274    pub fn profiler(&self) -> &Profiler {
275        &self.profiler
276    }
277
278    /// Get mutable profiler
279    pub fn profiler_mut(&mut self) -> &mut Profiler {
280        &mut self.profiler
281    }
282
283    /// Get memory profiler
284    pub fn memory_profiler(&self) -> Option<&MemoryProfiler> {
285        self.memory_profiler.as_ref()
286    }
287
288    /// Get mutable memory profiler
289    pub fn memory_profiler_mut(&mut self) -> Option<&mut MemoryProfiler> {
290        self.memory_profiler.as_mut()
291    }
292
293    /// Get interactive debugger
294    pub fn interactive_debugger(&self) -> &InteractiveDebugger {
295        &self.interactive_debugger
296    }
297
298    /// Get mutable interactive debugger
299    pub fn interactive_debugger_mut(&mut self) -> &mut InteractiveDebugger {
300        &mut self.interactive_debugger
301    }
302
303    /// Get anomaly detector
304    pub fn anomaly_detector(&self) -> &AnomalyDetector {
305        &self.anomaly_detector
306    }
307
308    /// Get mutable anomaly detector
309    pub fn anomaly_detector_mut(&mut self) -> &mut AnomalyDetector {
310        &mut self.anomaly_detector
311    }
312
313    /// Get computation graph analyzer
314    pub fn computation_graph_analyzer(&self) -> &ComputationGraphAnalyzer {
315        &self.computation_graph_analyzer
316    }
317
318    /// Get mutable computation graph analyzer
319    pub fn computation_graph_analyzer_mut(&mut self) -> &mut ComputationGraphAnalyzer {
320        &mut self.computation_graph_analyzer
321    }
322
323    /// Get architecture analyzer
324    pub fn architecture_analyzer(&self) -> &architecture_analysis::ArchitectureAnalyzer {
325        &self.architecture_analyzer
326    }
327
328    /// Get mutable architecture analyzer
329    pub fn architecture_analyzer_mut(
330        &mut self,
331    ) -> &mut architecture_analysis::ArchitectureAnalyzer {
332        &mut self.architecture_analyzer
333    }
334
335    /// Get behavior analyzer
336    pub fn behavior_analyzer(&self) -> &BehaviorAnalyzer {
337        &self.behavior_analyzer
338    }
339
340    /// Get mutable behavior analyzer
341    pub fn behavior_analyzer_mut(&mut self) -> &mut BehaviorAnalyzer {
342        &mut self.behavior_analyzer
343    }
344
345    /// Get training dynamics analyzer
346    pub fn training_dynamics_analyzer(&self) -> &TrainingDynamicsAnalyzer {
347        &self.training_dynamics_analyzer
348    }
349
350    /// Get mutable training dynamics analyzer
351    pub fn training_dynamics_analyzer_mut(&mut self) -> &mut TrainingDynamicsAnalyzer {
352        &mut self.training_dynamics_analyzer
353    }
354
355    /// Get differential debugger
356    pub fn differential_debugger(&self) -> &DifferentialDebugger {
357        &self.differential_debugger
358    }
359
360    /// Get mutable differential debugger
361    pub fn differential_debugger_mut(&mut self) -> &mut DifferentialDebugger {
362        &mut self.differential_debugger
363    }
364
365    /// Get interpretability analyzer
366    pub fn interpretability_analyzer(&self) -> &InterpretabilityAnalyzer {
367        &self.interpretability_analyzer
368    }
369
370    /// Get mutable interpretability analyzer
371    pub fn interpretability_analyzer_mut(&mut self) -> &mut InterpretabilityAnalyzer {
372        &mut self.interpretability_analyzer
373    }
374
375    /// Get health checker
376    pub fn health_checker(&self) -> &crate::health_checker::HealthChecker {
377        &self.health_checker
378    }
379
380    /// Get mutable health checker
381    pub fn health_checker_mut(&mut self) -> &mut crate::health_checker::HealthChecker {
382        &mut self.health_checker
383    }
384
385    /// Get transformer debugger
386    pub fn transformer_debugger(&self) -> Option<&neural_network_debugging::TransformerDebugger> {
387        self.transformer_debugger.as_ref()
388    }
389
390    /// Get mutable transformer debugger
391    pub fn transformer_debugger_mut(
392        &mut self,
393    ) -> Option<&mut neural_network_debugging::TransformerDebugger> {
394        self.transformer_debugger.as_mut()
395    }
396
397    /// Get advanced ML debugger
398    pub fn advanced_ml_debugger(&self) -> &AdvancedMLDebugger {
399        &self.advanced_ml_debugger
400    }
401
402    /// Get mutable advanced ML debugger
403    pub fn advanced_ml_debugger_mut(&mut self) -> &mut AdvancedMLDebugger {
404        &mut self.advanced_ml_debugger
405    }
406
407    /// Get AI code analyzer
408    pub fn ai_code_analyzer(&self) -> Option<&AICodeAnalyzer> {
409        self.ai_code_analyzer.as_ref()
410    }
411
412    /// Get mutable AI code analyzer
413    pub fn ai_code_analyzer_mut(&mut self) -> Option<&mut AICodeAnalyzer> {
414        self.ai_code_analyzer.as_mut()
415    }
416
417    /// Get distributed debugger
418    pub fn distributed_debugger(&self) -> Option<&DistributedDebugger> {
419        self.distributed_debugger.as_ref()
420    }
421
422    /// Get mutable distributed debugger
423    pub fn distributed_debugger_mut(&mut self) -> Option<&mut DistributedDebugger> {
424        self.distributed_debugger.as_mut()
425    }
426
427    /// Get environmental monitor
428    pub fn environmental_monitor(&self) -> Option<&EnvironmentalMonitor> {
429        self.environmental_monitor.as_ref()
430    }
431
432    /// Get mutable environmental monitor
433    pub fn environmental_monitor_mut(&mut self) -> Option<&mut EnvironmentalMonitor> {
434        self.environmental_monitor.as_mut()
435    }
436
437    /// Start debugging session
438    pub async fn start(&mut self) -> Result<()> {
439        tracing::info!("Starting debug session {}", self.id);
440
441        if self.config.enable_tensor_inspection {
442            self.tensor_inspector.start().await?;
443        }
444
445        if self.config.enable_gradient_debugging {
446            self.gradient_debugger.start().await?;
447        }
448
449        if self.config.enable_model_diagnostics {
450            self.model_diagnostics.start().await?;
451        }
452
453        self.profiler.start().await?;
454
455        if let Some(ref mut memory_profiler) = self.memory_profiler {
456            memory_profiler.start().await?;
457        }
458
459        self.interactive_debugger.start().await?;
460        self.anomaly_detector.start().await?;
461
462        Ok(())
463    }
464
465    /// Stop debugging session and generate report
466    pub async fn stop(&mut self) -> Result<DebugReport> {
467        tracing::info!("Stopping debug session {}", self.id);
468
469        let tensor_report = if self.config.enable_tensor_inspection {
470            Some(self.tensor_inspector.generate_report().await?)
471        } else {
472            None
473        };
474
475        let gradient_report = if self.config.enable_gradient_debugging {
476            Some(self.gradient_debugger.generate_report().await?)
477        } else {
478            None
479        };
480
481        let diagnostics_report = if self.config.enable_model_diagnostics {
482            Some(self.model_diagnostics.generate_report().await?)
483        } else {
484            None
485        };
486
487        let profiler_report = self.profiler.generate_report().await?;
488
489        let memory_profiler_report = if let Some(ref mut memory_profiler) = self.memory_profiler {
490            Some(memory_profiler.stop().await?)
491        } else {
492            None
493        };
494
495        let interactive_debugger_report = self.interactive_debugger.generate_report().await?;
496        let anomaly_report = self.anomaly_detector.generate_report().await?;
497
498        // `DebugSession` never drives `ComputationGraphAnalyzer` itself: graph
499        // analysis is an explicit, caller-initiated step
500        // (`crate::computation_graph`), so a session report has no graph to
501        // summarise. Honestly absent rather than an empty-looking report.
502        let computation_graph_report = None;
503
504        // Get new analyzer reports
505        let architecture_analysis_report =
506            Some(self.architecture_analyzer.generate_report().await?);
507        let behavior_analysis_report = Some(self.behavior_analyzer.generate_report().await?);
508        let training_dynamics_report =
509            Some(self.training_dynamics_analyzer.generate_report().await?);
510        let differential_debugging_report =
511            Some(self.differential_debugger.generate_report().await?);
512        let interpretability_report = Some(self.interpretability_analyzer.generate_report().await?);
513        let advanced_ml_debugging_report = Some(self.advanced_ml_debugger.generate_report().await?);
514
515        // Generate GPU profiling reports
516        let advanced_gpu_profiling_report = self
517            .advanced_gpu_profiler
518            .as_ref()
519            .map(|profiler| profiler.get_memory_analysis_report());
520
521        let kernel_optimization_report =
522            Some(self.generate_kernel_optimization_summary_report().await?);
523
524        Ok(DebugReport {
525            session_id: self.id,
526            tensor_report,
527            gradient_report,
528            diagnostics_report,
529            profiler_report,
530            memory_profiler_report,
531            interactive_debugger_report,
532            anomaly_report,
533            computation_graph_report,
534            architecture_analysis_report,
535            behavior_analysis_report,
536            training_dynamics_report,
537            differential_debugging_report,
538            interpretability_report,
539            advanced_ml_debugging_report,
540            advanced_gpu_profiling_report,
541            kernel_optimization_report,
542            config: self.config.clone(),
543        })
544    }
545
546    /// Export debug session to file
547    pub async fn export(&self, path: &str) -> Result<()> {
548        let report = self.generate_snapshot().await?;
549        let json = serde_json::to_string_pretty(&report)?;
550        tokio::fs::write(path, json).await?;
551        Ok(())
552    }
553
554    /// Generate a snapshot of current state
555    pub async fn generate_snapshot(&self) -> Result<DebugReport> {
556        let tensor_report = if self.config.enable_tensor_inspection {
557            Some(self.tensor_inspector.generate_report().await?)
558        } else {
559            None
560        };
561
562        let gradient_report = if self.config.enable_gradient_debugging {
563            Some(self.gradient_debugger.generate_report().await?)
564        } else {
565            None
566        };
567
568        let diagnostics_report = if self.config.enable_model_diagnostics {
569            Some(self.model_diagnostics.generate_report().await?)
570        } else {
571            None
572        };
573
574        let profiler_report = self.profiler.generate_report().await?;
575
576        // A snapshot must not stop the profiler, and `MemoryProfiler` only
577        // produces a report at `stop()` -- there is no borrow-a-report-mid-run
578        // API -- so a snapshot carries no memory report. `generate_report`
579        // (which does stop the profiler) is the call that returns one.
580        let memory_profiler_report = None;
581
582        let interactive_debugger_report = self.interactive_debugger.generate_report().await?;
583        let anomaly_report = self.anomaly_detector.generate_report().await?;
584
585        // `DebugSession` never drives `ComputationGraphAnalyzer` itself: graph
586        // analysis is an explicit, caller-initiated step
587        // (`crate::computation_graph`), so a session report has no graph to
588        // summarise. Honestly absent rather than an empty-looking report.
589        let computation_graph_report = None;
590
591        // Get new analyzer reports
592        let architecture_analysis_report =
593            Some(self.architecture_analyzer.generate_report().await?);
594        let behavior_analysis_report = Some(self.behavior_analyzer.generate_report().await?);
595        let training_dynamics_report =
596            Some(self.training_dynamics_analyzer.generate_report().await?);
597        let differential_debugging_report =
598            Some(self.differential_debugger.generate_report().await?);
599        let interpretability_report = Some(self.interpretability_analyzer.generate_report().await?);
600        let advanced_ml_debugging_report = Some(self.advanced_ml_debugger.generate_report().await?);
601
602        // Generate GPU profiling reports for snapshot
603        let advanced_gpu_profiling_report = self
604            .advanced_gpu_profiler
605            .as_ref()
606            .map(|profiler| profiler.get_memory_analysis_report());
607
608        let kernel_optimization_report =
609            Some(self.generate_kernel_optimization_summary_report().await?);
610
611        Ok(DebugReport {
612            session_id: self.id,
613            tensor_report,
614            gradient_report,
615            diagnostics_report,
616            profiler_report,
617            memory_profiler_report,
618            interactive_debugger_report,
619            anomaly_report,
620            computation_graph_report,
621            architecture_analysis_report,
622            behavior_analysis_report,
623            training_dynamics_report,
624            differential_debugging_report,
625            interpretability_report,
626            advanced_ml_debugging_report,
627            advanced_gpu_profiling_report,
628            kernel_optimization_report,
629            config: self.config.clone(),
630        })
631    }
632
633    /// Convenience method for debugging tensors (used by debug_tensor! macro)
634    pub fn debug_tensor<T>(&mut self, tensor: &ArrayD<T>, name: &str) -> Result<Uuid>
635    where
636        T: Clone + Into<f64> + fmt::Debug + 'static,
637    {
638        self.tensor_inspector.inspect_tensor(tensor, name, None, None)
639    }
640
641    /// Summarise every kernel this session's optimizer has really analysed.
642    ///
643    /// All counts come from the analyzer's own recorded profiles and
644    /// suggestions; `overall_optimization_score` is `None` when no kernel has
645    /// been analysed yet. The previous version returned
646    /// `total_kernels_analyzed: 0` alongside `overall_optimization_score: 85.0`
647    /// -- an 85-out-of-100 verdict on zero kernels.
648    async fn generate_kernel_optimization_summary_report(
649        &self,
650    ) -> Result<KernelOptimizationSummaryReport> {
651        /// A suggestion promising at least this much performance gain counts
652        /// as high impact.
653        const HIGH_IMPACT_GAIN_PERCENT: f64 = 10.0;
654
655        let kernel_names: Vec<String> = self
656            .kernel_optimizer
657            .analyzed_kernel_names()
658            .into_iter()
659            .map(str::to_string)
660            .collect();
661        let total_kernels_analyzed = kernel_names.len();
662        let suggestions = self.kernel_optimizer.optimization_suggestions();
663
664        let optimization_opportunities_found = suggestions.values().map(Vec::len).sum::<usize>();
665
666        let mut high_impact_optimizations: Vec<HighImpactOptimization> = suggestions
667            .iter()
668            .flat_map(|(kernel_name, opts)| {
669                opts.iter().filter_map(move |opt| {
670                    let gain = opt.expected_improvement.performance_gain_percentage;
671                    (gain >= HIGH_IMPACT_GAIN_PERCENT).then(|| HighImpactOptimization {
672                        kernel_name: kernel_name.clone(),
673                        optimization_type: format!("{:?}", opt.optimization_type),
674                        // Gain percentage expressed as a speedup factor.
675                        expected_speedup: 1.0 + gain / 100.0,
676                        implementation_difficulty: format!("{:?}", opt.implementation_difficulty),
677                        description: opt.explanation.clone(),
678                    })
679                })
680            })
681            .collect();
682        high_impact_optimizations.sort_by(|a, b| {
683            b.expected_speedup
684                .partial_cmp(&a.expected_speedup)
685                .unwrap_or(std::cmp::Ordering::Equal)
686        });
687
688        // Fusion opportunities and regression alerts are surfaced per kernel by
689        // `KernelOptimizationAnalyzer::get_optimization_report`, which needs a
690        // kernel name; count them across the kernels really analysed.
691        let mut fusion_opportunities = 0usize;
692        let mut regression_alerts = 0usize;
693        for name in &kernel_names {
694            if let Ok(report) = self.kernel_optimizer.get_optimization_report(name) {
695                fusion_opportunities += report.fusion_opportunities.len();
696                if report.regression_status.as_ref().is_some_and(|status| status.has_regression) {
697                    regression_alerts += 1;
698                }
699            }
700        }
701
702        // Score: start at 100 and dock the mean promised gain still on the
703        // table, so a kernel set with nothing left to improve scores 100.
704        let overall_optimization_score = if total_kernels_analyzed == 0 {
705            None
706        } else {
707            let total_gain: f64 = suggestions
708                .values()
709                .flat_map(|opts| opts.iter())
710                .map(|opt| opt.expected_improvement.performance_gain_percentage)
711                .sum();
712            Some((100.0 - total_gain / total_kernels_analyzed as f64).clamp(0.0, 100.0))
713        };
714
715        let top_recommendations: Vec<String> = high_impact_optimizations
716            .iter()
717            .take(5)
718            .map(|opt| format!("{}: {}", opt.kernel_name, opt.description))
719            .collect();
720
721        Ok(KernelOptimizationSummaryReport {
722            total_kernels_analyzed,
723            optimization_opportunities_found,
724            high_impact_optimizations,
725            fusion_opportunities,
726            regression_alerts,
727            overall_optimization_score,
728            top_recommendations,
729        })
730    }
731
732    /// Convenience method for debugging gradients (used by debug_gradient! macro)
733    pub fn debug_gradients<T>(&mut self, layer_name: &str, gradients: &[T]) -> Result<()>
734    where
735        T: Clone + Into<f64> + fmt::Debug + 'static,
736    {
737        // Convert gradients vector to ndarray
738        use scirs2_core::ndarray::Array; // SciRS2 Integration Policy
739        let gradient_array = Array::from_vec(gradients.to_vec()).into_dyn();
740
741        // Record the gradients as a real tracked tensor of their own, named
742        // after the layer, so their statistics survive the call.
743        //
744        // Previously this minted a fresh `Uuid::new_v4()` and passed it to
745        // `inspect_gradients`, which looks the id up in `tracked_tensors`. A
746        // freshly generated v4 uuid is never in that map, so the statistics
747        // were computed and then dropped on the floor while the call reported
748        // `Ok(())`, and `_layer_name` was ignored entirely.
749        let gradient_tensor_name = format!("{layer_name}.grad");
750        self.tensor_inspector.inspect_tensor(
751            &gradient_array,
752            &gradient_tensor_name,
753            Some(layer_name),
754            Some("gradient"),
755        )?;
756
757        // Additionally attach them as this layer's `gradient_stats` when the
758        // layer's own tensor has been inspected before.
759        if let Some(tensor_id) = self.tensor_inspector.tensor_id_for_name(layer_name) {
760            self.tensor_inspector.inspect_gradients(tensor_id, &gradient_array)?;
761        } else {
762            tracing::debug!(
763                layer = layer_name,
764                "no previously inspected tensor for this layer; gradients recorded standalone \
765                 as {gradient_tensor_name}"
766            );
767        }
768
769        Ok(())
770    }
771}
772
773/// Comprehensive debug report
774#[derive(Debug, Clone, Serialize, Deserialize)]
775pub struct DebugReport {
776    pub session_id: Uuid,
777    pub tensor_report: Option<TensorInspectionReport>,
778    pub gradient_report: Option<GradientDebugReport>,
779    pub diagnostics_report: Option<ModelDiagnosticsReport>,
780    pub profiler_report: ProfilerReport,
781    pub memory_profiler_report: Option<MemoryProfilingReport>,
782    pub interactive_debugger_report: InteractiveDebuggerReport,
783    pub anomaly_report: AnomalyDetectorReport,
784    pub computation_graph_report: Option<GraphAnalysisResult>,
785    pub architecture_analysis_report: Option<ArchitectureAnalysisReport>,
786    pub behavior_analysis_report: Option<BehaviorAnalysisReport>,
787    pub training_dynamics_report: Option<model_diagnostics::training::TrainingDynamicsReport>,
788    pub differential_debugging_report: Option<DifferentialDebuggingReport>,
789    pub interpretability_report: Option<InterpretabilityReport>,
790    pub advanced_ml_debugging_report: Option<AdvancedMLDebuggingReport>,
791    pub advanced_gpu_profiling_report: Option<MemoryAnalysisReport>,
792    pub kernel_optimization_report: Option<KernelOptimizationSummaryReport>,
793    pub config: DebugConfig,
794}
795
796impl DebugReport {
797    /// Get summary of key findings
798    pub fn summary(&self) -> DebugSummary {
799        let mut issues = Vec::new();
800        let mut recommendations = Vec::new();
801
802        // Analyze tensor issues
803        if let Some(ref tensor_report) = self.tensor_report {
804            if tensor_report.has_nan_values() {
805                issues.push("NaN values detected in tensors".to_string());
806                recommendations.push("Check input data and model initialization".to_string());
807            }
808
809            if tensor_report.has_inf_values() {
810                issues.push("Infinite values detected in tensors".to_string());
811                recommendations.push("Reduce learning rate or add gradient clipping".to_string());
812            }
813        }
814
815        // Analyze gradient issues
816        if let Some(ref gradient_report) = self.gradient_report {
817            if gradient_report.has_vanishing_gradients() {
818                issues.push("Vanishing gradients detected".to_string());
819                recommendations
820                    .push("Consider residual connections or gradient scaling".to_string());
821            }
822
823            if gradient_report.has_exploding_gradients() {
824                issues.push("Exploding gradients detected".to_string());
825                recommendations.push("Add gradient clipping".to_string());
826            }
827        }
828
829        DebugSummary {
830            session_id: self.session_id,
831            total_issues: issues.len(),
832            critical_issues: issues
833                .iter()
834                .filter(|i| i.contains("NaN") || i.contains("exploding"))
835                .count(),
836            issues,
837            recommendations,
838        }
839    }
840}
841
842/// High-level summary of debug findings
843#[derive(Debug, Serialize, Deserialize)]
844pub struct DebugSummary {
845    pub session_id: Uuid,
846    pub total_issues: usize,
847    pub critical_issues: usize,
848    pub issues: Vec<String>,
849    pub recommendations: Vec<String>,
850}
851
852/// Convenience function to create a debug session with default config
853pub fn debug_session() -> DebugSession {
854    DebugSession::new(DebugConfig::default())
855}
856
857/// Convenience function to create a debug session with custom config
858pub fn debug_session_with_config(config: DebugConfig) -> DebugSession {
859    DebugSession::new(config)
860}
861
862/// Convenience function to create a debug session with transformer debugging enabled
863pub fn debug_session_with_transformer() -> DebugSession {
864    let config = DebugConfig {
865        neural_network_debugging_config: Some(
866            neural_network_debugging::TransformerDebugConfig::default(),
867        ),
868        ..Default::default()
869    };
870    DebugSession::new(config)
871}
trustformers_debug/core/session.rs

trustformers_debug/core/
session.rs
