Struct SensorLMModel 

pub struct SensorLMModel<B: Backend> {
    pub sensor_encoder: SensorEncoder<B>,
    pub text_encoder: TextEncoder<B>,
    pub log_temperature: Param<Tensor<B, 1>>,
    pub bias: Param<Tensor<B, 1>>,
}

The combined SensorLM two-tower model.

Fields

sensor_encoder: SensorEncoder<B>

ViT sensor encoder.

text_encoder: TextEncoder<B>

Text transformer encoder.

log_temperature: Param<Tensor<B, 1>>

Log-temperature scalar (temperature = exp(log_temperature) > 0).

bias: Param<Tensor<B, 1>>

SigLIP bias scalar.

Implementations

impl<B: Backend> SensorLMModel<B>

pub fn new(cfg: &SensorLMConfig, device: &B::Device) -> Self

Construct from a SensorLMConfig.

pub fn encode_sensor(&self, x: Tensor<B, 3>) -> Tensor<B, 2>

Encode sensor data → (B, D) L2-normalised.

pub fn encode_text( &self, input_ids: Tensor<B, 2, Int>, attention_mask: Tensor<B, 2, Int>, ) -> Tensor<B, 2>

Encode text → (B, D) L2-normalised.

pub fn similarity_matrix( &self, z_sensor: Tensor<B, 2>, z_text: Tensor<B, 2>, ) -> Tensor<B, 2>

Compute (B, B) similarity matrix.

pub fn forward( &self, sensor: Tensor<B, 3>, input_ids: Tensor<B, 2, Int>, attention_mask: Tensor<B, 2, Int>, ) -> SensorLMOutput<B>

Full forward pass computing the SigLIP loss.

Trait Implementations

impl<B> AutodiffModule<B> for SensorLMModel<B>

where B: AutodiffBackend + Backend, <B as AutodiffBackend>::InnerBackend: Backend,

type InnerModule = SensorLMModel<<B as AutodiffBackend>::InnerBackend>

Inner module without auto-differentiation.

fn valid(&self) -> Self::InnerModule

Get the same module, but on the inner backend without auto-differentiation.

impl<B: Backend> Clone for SensorLMModel<B>

fn clone(&self) -> Self

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<B: Debug + Backend> Debug for SensorLMModel<B>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<B: Backend> Display for SensorLMModel<B>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<B: Backend> Module<B> for SensorLMModel<B>

type Record = SensorLMModelRecord<B>

Type to save and load the module.

fn load_record(self, record: Self::Record) -> Self

Load the module state from a record.

fn into_record(self) -> Self::Record

Convert the module into a record containing the state.

fn num_params(&self) -> usize

Get the number of parameters the module has, including all of its sub-modules.

fn visit<Visitor: ModuleVisitor<B>>(&self, visitor: &mut Visitor)

Visit each tensor parameter in the module with a visitor.

fn map<Mapper: ModuleMapper<B>>(self, mapper: &mut Mapper) -> Self

Map each tensor parameter in the module with a mapper.

fn collect_devices(&self, devices: Devices<B>) -> Devices<B>

Return all the devices found in the underneath module tree added to the given vector without duplicates.

fn to_device(self, device: &B::Device) -> Self

Move the module and all of its sub-modules to the given device.

fn fork(self, device: &B::Device) -> Self

Fork the module and all of its sub-modules to the given device.

fn devices(&self) -> Vec<<B as Backend>::Device>

Return all the devices found in the underneath module tree without duplicates.

fn no_grad(self) -> Self

Each tensor in the module tree will not require grad.

fn save_file<FR, PB>( self, file_path: PB, recorder: &FR, ) -> Result<(), RecorderError>

where FR: FileRecorder<B>, PB: Into<PathBuf>,

Save the module to a file using the provided file recorder.

fn load_file<FR, PB>( self, file_path: PB, recorder: &FR, device: &<B as Backend>::Device, ) -> Result<Self, RecorderError>

where FR: FileRecorder<B>, PB: Into<PathBuf>,

Load the module from a file using the provided file recorder.

fn quantize_weights<C>(self, quantizer: &mut Quantizer<C>) -> Self

where C: Calibration,

Quantize the weights of the module.

impl<B: Backend> ModuleDisplay for SensorLMModel<B>

fn format(&self, passed_settings: DisplaySettings) -> String

Formats the module with provided display settings.

fn custom_settings(&self) -> Option<DisplaySettings>

Custom display settings for the module.

fn custom_content(&self, _content: Content) -> Option<Content>

Custom attributes for the module.

impl<B: Backend> ModuleDisplayDefault for SensorLMModel<B>

fn content(&self, content: Content) -> Option<Content>

Attributes of the module used for display purposes.

fn num_params(&self) -> usize

Gets the number of the parameters of the module.

impl<B: AutodiffBackend> TrainStep<SensorLMBatch<B>, SensorLMOutput<B>> for SensorLMModel<B>

fn step(&self, batch: SensorLMBatch<B>) -> TrainOutput<SensorLMOutput<B>>

Runs the training step, which executes the forward and backward passes.

fn optimize<B, O>(self, optim: &mut O, lr: f64, grads: GradientsParams) -> Self

where B: AutodiffBackend, O: Optimizer<Self, B>, Self: AutodiffModule<B>,

Optimize the current module with the provided gradients and learning rate.

impl<B: Backend> ValidStep<SensorLMBatch<B>, SensorLMOutput<B>> for SensorLMModel<B>

fn step(&self, batch: SensorLMBatch<B>) -> SensorLMOutput<B>

Runs a validation step.