entrenar/distill/mod.rs
1//! Knowledge Distillation
2//!
3//! This module implements various knowledge distillation techniques for training
4//! smaller student models from larger teacher models.
5//!
6//! ## Features
7//!
8//! - **Temperature-scaled KL divergence**: Standard distillation loss with soft targets
9//! - **Multi-teacher ensemble**: Distill from multiple teachers simultaneously
10//! - **Progressive distillation**: Layer-wise distillation for intermediate representations
11//! - **Checkpoint saving**: Student model checkpoints with distillation metadata
12//!
13//! ## Example
14//!
15//! ```
16//! use entrenar::distill::DistillationLoss;
17//! use ndarray::array;
18//!
19//! let loss_fn = DistillationLoss::new(3.0, 0.5);
20//! let student_logits = array![[1.0, 2.0, 1.5]];
21//! let teacher_logits = array![[1.2, 1.8, 1.6]];
22//! let labels = vec![1];
23//! let loss = loss_fn.forward(&student_logits, &teacher_logits, &labels);
24//! assert!(loss > 0.0);
25//! ```
26
27pub mod checkpoint;
28mod ensemble;
29mod loss;
30mod progressive;
31
32#[cfg(test)]
33mod tests;
34
35pub use checkpoint::{save_student_checkpoint, DistillationCheckpoint};
36pub use ensemble::EnsembleDistiller;
37pub use loss::DistillationLoss;
38pub use progressive::ProgressiveDistiller;