burn_core/module/

base.rs

use crate::module::{Lora, ParamGroup, QLora};

use super::{ApplyReparameterization, Param, ParamId, Quantizer, Reparameterizer};
use alloc::{
    string::{String, ToString},
    vec::Vec,
};
pub use burn_derive::Module;
use burn_tensor::{Bool, Device, Int, Tensor};

/// Type alias to `Vec<Device>` which supports `no_std` environments, but automatically using
/// the `alloc` crate.
pub type Devices = Vec<Device>;

// At the moment, our plan is to continue experimenting with the macro internally and monitor its development.
// We may consider making it public in the future.
macro_rules! module {
    (map=$module:ident, ops=$item:expr) => {{
        struct Mapper;
        impl ModuleMapper for Mapper {
            fn map_float<const D: usize>(&mut self, param: Param<Tensor<D>>) -> Param<Tensor<D>> {
                let (id, tensor, mapper) = param.consume();
                let func = $item;
                let tensor = func(tensor);
                Param::from_mapped_value(id, tensor, mapper)
            }
        }
        let mut mapper = Mapper;
        $module.map(&mut mapper)
    }};
    (map=$module:ident, ops=$item:expr, group=$group:ident) => {{
        struct Mapper {
            pub path: Vec<String>,
            pub group: ParamGroup,
        }
        impl ModuleMapper for Mapper {
            fn enter_module(&mut self, name: &str, _container_type: &str) {
                self.path.push(name.to_string());
            }

            fn exit_module(&mut self, _name: &str, _container_type: &str) {
                self.path.pop();
            }

            fn map_float<const D: usize>(&mut self, param: Param<Tensor<D>>) -> Param<Tensor<D>> {
                let (id, tensor, mapper) = param.consume();
                let path = self.path.join(".");
                if self.group.matches(&id, Some(&path)) {
                    let func = $item;
                    let tensor = func(tensor);
                    return Param::from_mapped_value(id, tensor, mapper);
                }
                Param::from_mapped_value(id, tensor, mapper)
            }
        }
        let mut mapper = Mapper {
            path: alloc::vec![],
            group: $group,
        };
        $module.map(&mut mapper)
    }};
    (visit_float=$module:ident, ops=$item:expr, state=$state_ty:ty, init=$init:expr) => {{
        struct Visitor<'a> {
            state: &'a mut $state_ty,
        }
        impl<'a> ModuleVisitor for Visitor<'a> {
            fn visit_float<const D: usize>(&mut self, param: &Param<Tensor<D>>) {
                let func = $item;
                func(&param.val(), &mut self.state)
            }
        }
        #[allow(clippy::redundant_closure_call)]
        let mut state = $init();
        let mut visitor = Visitor { state: &mut state };
        $module.visit(&mut visitor);
        state
    }};
}

/// Trait for all neural network modules.
///
/// Modules should be created using the [derive](burn_derive::Module) attribute.
/// This will make your module trainable, savable and loadable via
/// `state` and `load`.
///
/// # Example
///
/// ```rust, ignore
/// // Not necessary when using the burn crate directly.
/// use burn_core as burn;
///
/// use burn::{
///     module::Module,
///     nn::Linear,
///     tensor::Tensor,
/// };
///
/// #[derive(Module, Debug)]
/// struct MyModule {
///   my_param: Linear,
///   my_other_field: usize,
/// }
/// ```
pub trait Module: Clone + Send + core::fmt::Debug {
    /// Return all the devices found in the underneath module tree added to the given vector
    /// without duplicates.
    fn collect_devices(&self, devices: Devices) -> Devices;

    /// Return all the devices found in the underneath module tree without duplicates.
    fn devices(&self) -> Devices {
        self.collect_devices(Devices::new())
    }

    /// Fork the module and all of its sub-modules to the given device.
    ///
    /// # Notes
    ///
    /// This is similar to [to_device](Module::to_device), but it ensures the output module on the
    /// new device will have its own autodiff graph.
    fn fork(self, device: &Device) -> Self;

    /// Move the module and all of its sub-modules to the given device.
    ///
    /// # Warnings
    ///
    /// The operation supports autodiff and it will be registered when activated. However, this may
    /// not be what you want. The output model will be an intermediary model, meaning that you
    /// can't optimize it with gradient descent. If you want to optimize the output network on the
    /// target device, use [fork](Module::fork) instead.
    fn to_device(self, device: &Device) -> Self;

    /// Each tensor in the module tree will not require grad.
    ///
    /// # Warnings
    ///
    /// This should not be used for inference, use [valid](AutodiffModule::valid) when using
    /// AD modules. This is mostly useful when performing partial finetuning, which is updating only
    /// a small fraction of the parameters instead of finetuning all of them.
    fn no_grad(self) -> Self {
        module!(
            map = self,
            ops = |tensor: Tensor<D>| tensor.set_require_grad(false)
        )
    }

    /// Set `require_grad` to `false` for every parameter in the given group, leaving the rest
    /// of the module untouched.
    ///
    /// This is the group-scoped counterpart to [no_grad](Module::no_grad): where `no_grad` freezes
    /// the whole module tree, `freeze_group` freezes only the parameters matched by `group`.  
    ///
    /// # Warnings
    ///
    /// Like [no_grad](Module::no_grad), this should not be used for inference; use
    /// [valid](AutodiffModule::valid) with AD modules instead.
    fn freeze_group(self, group: ParamGroup) -> Self {
        module!(
            map = self,
            ops = |tensor: Tensor<D>| tensor.set_require_grad(false),
            group = group
        )
    }

    /// Set `require_grad` to `true` for every parameter in the given group, leaving the rest
    /// of the module untouched.
    ///
    /// The inverse of [freeze_group](Module::freeze_group): it re-enables gradient tracking for the
    /// parameters matched by `group`, e.g. to unfreeze a previously frozen module.
    fn unfreeze_group(self, group: ParamGroup) -> Self {
        module!(
            map = self,
            ops = |tensor: Tensor<D>| tensor.set_require_grad(true),
            group = group
        )
    }

    /// Move the module and all of its sub-modules to the autodiff backend.
    ///
    /// # Notes
    ///
    /// * Only plain modules (not already on an autodiff backend) can be moved.
    /// * Calling `train()` on a module that is already on an autodiff backend
    ///   will result in a type error, because the module's inner backend does not match.
    fn train(self) -> Self
    where
        Self: AutodiffModule,
    {
        AutodiffModule::from_inner(self)
    }

    /// Get the number of parameters the module has, including all of its sub-modules.
    fn num_params(&self) -> usize {
        module!(
            visit_float = self,
            ops = |tensor: &Tensor<D>, state: &mut usize| {
                *state += tensor.shape().num_elements();
            },
            state = usize,
            init = || 0
        )
    }
    /// Visit each tensor parameter in the module with a [visitor](ModuleVisitor).
    fn visit<Visitor: ModuleVisitor>(&self, visitor: &mut Visitor);

    /// Map each tensor parameter in the module with a [mapper](ModuleMapper).
    fn map<Mapper: ModuleMapper>(self, mapper: &mut Mapper) -> Self;

    /// Quantize the weights of the module.
    fn quantize_weights(self, quantizer: &mut Quantizer) -> Self {
        self.map(quantizer)
    }

    /// Quantize the weights of the given parameter group.
    fn quantize_weights_group(self, quantizer: &mut Quantizer, group: ParamGroup) -> Self {
        quantizer.set_param_group(group);
        self.map(quantizer)
    }

    /// Attach reparameterizations using the given [`Reparameterizer`].
    ///
    /// Every floating-point parameter is passed to the reparameterizer along with its module path.
    /// The reparameterizer prepares its structural base and optionally creates the state used by
    /// [`Param::val`]. [`Lora`] is a built-in example.
    ///
    /// # Limitations
    ///
    /// Nested reparameterizations are not supported. This method should only be called on modules
    /// that don't already contain reparameterized parameters.
    fn apply_reparameterization<R>(self, reparameterizer: R) -> Self
    where
        Self: Sized,
        R: Reparameterizer,
    {
        self.map(&mut ApplyReparameterization::new(reparameterizer))
    }

    /// Attach LoRA adapters to the module's 2-D weights, freezing the base weights.
    ///
    /// The same module keeps working without any code changes; adapted weights now produce
    /// `base + scale * (a @ b)`, and only the adapter factors are trainable.
    fn apply_lora(self, lora: Lora) -> Self
    where
        Self: Sized,
    {
        self.apply_reparameterization(lora)
    }

    /// Apply QLoRA to the module: quantize the (frozen) base weights and attach trainable LoRA
    /// adapters to 2-D weights.
    fn apply_qlora(self, qlora: QLora) -> Self
    where
        Self: Sized,
    {
        self.apply_reparameterization(qlora)
    }

    /// Collect this module's parameters into a [`ModuleRecord`](crate::store::ModuleRecord).
    ///
    /// The record can be saved to a burnpack file or byte buffer and applied back with
    /// [`load_record`](Module::load_record).
    fn into_record(self) -> crate::store::ModuleRecord
    where
        Self: Sized,
    {
        crate::store::ModuleRecord::from_module(self, None)
    }

    /// Collect the parameters `group` names into a [`ModuleRecord`](crate::store::ModuleRecord).
    ///
    /// The record of a part of the module rather than all of it — what a run that trained a
    /// group writes when the rest of the module is the checkpoint it started from, and what
    /// [`load_record`](Module::load_record) applies back over that checkpoint (with
    /// [`allow_partial`](crate::store::ModuleRecord::allow_partial), since the record holds
    /// nothing for the parameters outside the group).
    ///
    /// A parameter the group does not match is skipped before its data is read, so this never
    /// materializes the rest of the module.
    fn into_record_group(self, group: ParamGroup) -> crate::store::ModuleRecord
    where
        Self: Sized,
    {
        crate::store::ModuleRecord::from_module(self, Some(group))
    }

    /// Apply a [`ModuleRecord`](crate::store::ModuleRecord) to this module, returning the loaded
    /// module.
    ///
    /// Honors the record's [`DTypePolicy`](crate::store::DTypePolicy), `validate`, and
    /// `allow_partial` settings.
    fn try_load_record(
        self,
        record: crate::store::ModuleRecord,
    ) -> Result<Self, crate::store::RecordError>
    where
        Self: Sized,
    {
        record.apply(self)
    }

    /// Apply a [`ModuleRecord`](crate::store::ModuleRecord) to this module, consuming and returning
    /// it.
    ///
    /// Panics if validation fails; use [`try_load_record`](Module::try_load_record) for the
    /// fallible variant.
    fn load_record(self, record: crate::store::ModuleRecord) -> Self
    where
        Self: Sized,
    {
        self.try_load_record(record).expect("Failed to load record")
    }

    /// Save this module's parameters to a burnpack file on disk.
    ///
    /// Convenience for [`into_record`](Module::into_record) followed by
    /// [`ModuleRecord::save`](crate::store::ModuleRecord::save). For non-default load behavior
    /// (dtype policy, partial loading, validation), go through the record directly.
    #[cfg(feature = "std")]
    fn save_file<P: AsRef<std::path::Path>>(self, path: P) -> Result<(), crate::store::RecordError>
    where
        Self: Sized,
    {
        self.into_record().save(path)
    }

    /// Load this module's parameters from a burnpack file on disk, returning the loaded module.
    ///
    /// Uses the default load behavior. Panics on I/O or validation errors; use
    /// [`try_load_file`](Module::try_load_file) for the fallible variant, or go through
    /// [`ModuleRecord`](crate::store::ModuleRecord) to configure dtype policy, partial loading or
    /// validation.
    #[cfg(feature = "std")]
    fn load_file<P: AsRef<std::path::Path>>(self, path: P) -> Self
    where
        Self: Sized,
    {
        self.try_load_file(path)
            .expect("Failed to load module from file")
    }

    /// Fallible variant of [`load_file`](Module::load_file).
    ///
    /// Reads the record from `path` with [`ModuleRecord::load`](crate::store::ModuleRecord::load)
    /// and applies it through [`try_load_record`](Module::try_load_record).
    #[cfg(feature = "std")]
    fn try_load_file<P: AsRef<std::path::Path>>(
        self,
        path: P,
    ) -> Result<Self, crate::store::RecordError>
    where
        Self: Sized,
    {
        let record = crate::store::ModuleRecord::load(path)?;
        self.try_load_record(record)
    }
}

/// Module visitor trait for traversing and inspecting module parameters.
pub trait ModuleVisitor {
    /// Visit a float parameter in the module.
    ///
    /// # Parameters
    /// - `param`: The float parameter to visit
    #[allow(unused_variables)]
    fn visit_float<const D: usize>(&mut self, param: &Param<Tensor<D>>) {}

    /// Visit an int parameter in the module.
    ///
    /// # Parameters
    /// - `param`: The integer parameter to visit
    #[allow(unused_variables)]
    fn visit_int<const D: usize>(&mut self, param: &Param<Tensor<D, Int>>) {}

    /// Visit a bool parameter in the module.
    ///
    /// # Parameters
    /// - `param`: The boolean parameter to visit
    #[allow(unused_variables)]
    fn visit_bool<const D: usize>(&mut self, param: &Param<Tensor<D, Bool>>) {}

    /// Called when entering a submodule.
    ///
    /// # Parameters
    /// - `name`: The name of the submodule being entered
    /// - `container_type`: The type of the container with format:
    ///   - For user-defined structs: "Struct:TypeName" (e.g., "Struct:Linear")
    ///   - For user-defined enums: "Enum:TypeName" (e.g., "Enum:MyEnum")
    ///   - For Vec containers: "Vec" (name is the index)
    ///   - For Tuple containers: "Tuple" (name is the index)
    ///   - For Array containers: "Array" (name is the index)
    ///
    /// Note: Option containers do not call enter_module/exit_module to preserve
    /// the field name in the path (e.g., "bias" instead of "bias.Some")
    #[allow(unused_variables)]
    fn enter_module(&mut self, name: &str, container_type: &str) {}

    /// Called when exiting a submodule.
    ///
    /// # Parameters
    /// - `name`: The name of the submodule being exited
    /// - `container_type`: The type of the container with format:
    ///   - For user-defined structs: "Struct:TypeName" (e.g., "Struct:Linear")
    ///   - For user-defined enums: "Enum:TypeName" (e.g., "Enum:MyEnum")
    ///   - For Vec containers: "Vec" (name is the index)
    ///   - For Tuple containers: "Tuple" (name is the index)
    ///   - For Array containers: "Array" (name is the index)
    ///
    /// Note: Option containers do not call enter_module/exit_module to preserve
    /// the field name in the path (e.g., "bias" instead of "bias.Some")
    #[allow(unused_variables)]
    fn exit_module(&mut self, name: &str, container_type: &str) {}

    /// Visit a float tensor with its full module path.
    ///
    /// # Parameters
    /// - `path`: The path components to the tensor as a slice (e.g., &["encoder", "layer1", "weight"]).
    ///   Each element represents a module name in the hierarchy, with the final element
    ///   being the parameter name. This allows efficient reuse of the path stack.
    /// - `id`: The unique identifier of the parameter
    /// - `tensor`: The float tensor to visit
    #[allow(unused_variables)]
    fn visit_float_with_path<const D: usize>(
        &mut self,
        path: &[String],
        id: ParamId,
        tensor: &Tensor<D>,
    ) {
    }

    /// Visit an int tensor with its full module path.
    ///
    /// # Parameters
    /// - `path`: The path components to the tensor as a slice (e.g., &["encoder", "layer1", "weight"]).
    ///   Each element represents a module name in the hierarchy, with the final element
    ///   being the parameter name. This allows efficient reuse of the path stack.
    /// - `id`: The unique identifier of the parameter
    /// - `tensor`: The integer tensor to visit
    #[allow(unused_variables)]
    fn visit_int_with_path<const D: usize>(
        &mut self,
        path: &[String],
        id: ParamId,
        tensor: &Tensor<D, Int>,
    ) {
    }

    /// Visit a bool tensor with its full module path.
    ///
    /// # Parameters
    /// - `path`: The path components to the tensor as a slice (e.g., &["encoder", "layer1", "weight"]).
    ///   Each element represents a module name in the hierarchy, with the final element
    ///   being the parameter name. This allows efficient reuse of the path stack.
    /// - `id`: The unique identifier of the parameter
    /// - `tensor`: The boolean tensor to visit
    #[allow(unused_variables)]
    fn visit_bool_with_path<const D: usize>(
        &mut self,
        path: &[String],
        id: ParamId,
        tensor: &Tensor<D, Bool>,
    ) {
    }
}

/// Module mapper trait for transforming module parameters.
pub trait ModuleMapper {
    /// Called when entering a submodule.
    ///
    /// # Parameters
    /// - `name`: The name of the submodule being entered
    /// - `container_type`: The type of the container with format:
    ///   - For user-defined structs: "Struct:TypeName" (e.g., "Struct:Linear")
    ///   - For user-defined enums: "Enum:TypeName" (e.g., "Enum:MyEnum")
    ///   - For Vec containers: "Vec" (name is the index)
    ///   - For Tuple containers: "Tuple" (name is the index)
    ///   - For Array containers: "Array" (name is the index)
    ///
    /// Note: Option containers do not call enter_module/exit_module to preserve
    /// the field name in the path (e.g., "bias" instead of "bias.Some")
    #[allow(unused_variables)]
    fn enter_module(&mut self, name: &str, container_type: &str) {}

    /// Called when exiting a submodule.
    ///
    /// # Parameters
    /// - `name`: The name of the submodule being exited
    /// - `container_type`: The type of the container with format:
    ///   - For user-defined structs: "Struct:TypeName" (e.g., "Struct:Linear")
    ///   - For user-defined enums: "Enum:TypeName" (e.g., "Enum:MyEnum")
    ///   - For Vec containers: "Vec" (name is the index)
    ///   - For Tuple containers: "Tuple" (name is the index)
    ///   - For Array containers: "Array" (name is the index)
    ///
    /// Note: Option containers do not call enter_module/exit_module to preserve
    /// the field name in the path (e.g., "bias" instead of "bias.Some")
    #[allow(unused_variables)]
    fn exit_module(&mut self, name: &str, container_type: &str) {}

    /// Map a float parameter in the module.
    ///
    /// # Parameters
    /// - `param`: The float parameter to transform
    ///
    /// # Returns
    /// The transformed parameter
    #[allow(unused_variables)]
    fn map_float<const D: usize>(&mut self, param: Param<Tensor<D>>) -> Param<Tensor<D>> {
        let (id, tensor, mapper) = param.consume();
        Param::from_mapped_value(id, tensor, mapper)
    }

    /// Map an int parameter in the module.
    ///
    /// # Parameters
    /// - `param`: The integer parameter to transform
    ///
    /// # Returns
    /// The transformed parameter
    #[allow(unused_variables)]
    fn map_int<const D: usize>(&mut self, param: Param<Tensor<D, Int>>) -> Param<Tensor<D, Int>> {
        let (id, tensor, mapper) = param.consume();
        Param::from_mapped_value(id, tensor, mapper)
    }

    /// Map a bool parameter in the module.
    ///
    /// # Parameters
    /// - `param`: The boolean parameter to transform
    ///
    /// # Returns
    /// The transformed parameter
    #[allow(unused_variables)]
    fn map_bool<const D: usize>(
        &mut self,
        param: Param<Tensor<D, Bool>>,
    ) -> Param<Tensor<D, Bool>> {
        let (id, tensor, mapper) = param.consume();
        Param::from_mapped_value(id, tensor, mapper)
    }
}

/// Module with auto-differentiation backend.
pub trait AutodiffModule: Module + Send + core::fmt::Debug {
    /// Returns the same module, but on the inner backend without auto-differentiation.
    fn valid(&self) -> Self;

    /// Wraps an inner module back into an auto-diff module.
    fn from_inner(module: Self) -> Self;
}

#[cfg(all(test, feature = "autodiff"))]
mod tests {
    use super::*;

    use crate::module::ParamGroup;
    use crate::{test_device, test_utils::SimpleLinear};

    #[test]
    fn test_module_val_train_stateful() {
        let device = test_device().autodiff();
        let module = SimpleLinear::new(4, 4, &device);

        assert!(module.weight.is_require_grad());
        assert!(module.weight.require_grad);

        let module = module.valid();
        assert!(!module.weight.is_require_grad());
        assert!(module.weight.require_grad); // stateful

        // Without `HasAutodiffModule`, we would need to specify the module type as well, which would be annoying
        // let module: SimpleLinear<TestAutodiffBackend> = module.train();
        let module = module.train();
        assert!(module.weight.is_require_grad());
        assert!(module.weight.require_grad); // stateful

        let module = module.no_grad();
        assert!(!module.weight.is_require_grad());
        assert!(!module.weight.require_grad); // stateful

        let module = module.valid();
        assert!(!module.weight.is_require_grad()); // always
        assert!(!module.weight.require_grad); // stateful

        let module = module.train();
        assert!(!module.weight.is_require_grad());
        assert!(!module.weight.require_grad); // stateful
    }

    #[test]
    fn freeze_group_freezes_only_selected_params() {
        let device = test_device().autodiff();
        let module = SimpleLinear::new(4, 4, &device);

        assert!(module.weight.is_require_grad());
        assert!(module.bias.as_ref().unwrap().is_require_grad());

        let module = module.freeze_group(ParamGroup::from_path("weight"));

        assert!(!module.weight.is_require_grad());
        assert!(!module.weight.require_grad);

        let bias = module.bias.as_ref().unwrap();
        assert!(bias.is_require_grad());
        assert!(bias.require_grad);
    }

    #[test]
    fn unfreeze_group_only_thaws_selected_params() {
        let device = test_device().autodiff();
        let module = SimpleLinear::new(4, 4, &device);

        let module = module.no_grad();
        assert!(!module.weight.is_require_grad());
        assert!(!module.bias.as_ref().unwrap().is_require_grad());

        let module = module.unfreeze_group(ParamGroup::from_path("weight"));

        assert!(module.weight.is_require_grad());
        assert!(module.weight.require_grad);
        assert!(!module.bias.as_ref().unwrap().is_require_grad());
        assert!(!module.bias.as_ref().unwrap().require_grad);
    }
}