#[cfg(feature = "gpu")]
use crate::gpu::{GpuContext, GpuError};
use crate::gpu::GpuQuantizedLinearWeight;
use crate::initialization::{normal_with_rng, xavier_uniform_with_rng};
use crate::{IndexTensor, Tensor};
use rand::prelude::*;
use std::cell::RefCell;
use std::rc::Rc;
pub use crate::act_quant::{
ActGranularity, ActQuantConfig, ActQuantMode, ActQuantScheme, ActivationQuantization,
InferenceQuantMode,
};
pub use crate::weight_quant::{
QuantGranularity, QuantScheme, ScaleMethod, WeightQuantization, NF4_CODEBOOK,
};
#[derive(Clone)]
pub struct QuantizedLinearWeight {
packed: crate::weight_quant::PackedLinearWeight,
gpu: Rc<RefCell<Option<GpuQuantizedLinearWeight>>>,
}
impl QuantizedLinearWeight {
pub fn format(&self) -> WeightQuantization {
self.packed.scheme.format
}
pub fn scheme(&self) -> QuantScheme {
self.packed.scheme
}
pub fn scales(&self) -> &[f32] {
&self.packed.scales
}
pub fn zero_points(&self) -> Option<&[f32]> {
self.packed.zero_points.as_deref()
}
pub fn packed_values(&self) -> &[u32] {
&self.packed.packed_values
}
pub fn in_features(&self) -> usize {
self.packed.in_features
}
pub fn out_features(&self) -> usize {
self.packed.out_features
}
pub fn storage_bytes(&self) -> usize {
self.packed.storage_bytes()
}
pub fn group_size(&self) -> Option<usize> {
self.packed.scheme.granularity.group_size()
}
pub fn from_parts(
format: WeightQuantization,
packed_values: Vec<u32>,
scales: Vec<f32>,
in_features: usize,
out_features: usize,
) -> Result<Self, String> {
Self::from_parts_with_scheme(
QuantScheme::default_for(format),
packed_values,
scales,
None,
in_features,
out_features,
)
}
pub fn from_parts_with_scheme(
scheme: QuantScheme,
packed_values: Vec<u32>,
scales: Vec<f32>,
zero_points: Option<Vec<f32>>,
in_features: usize,
out_features: usize,
) -> Result<Self, String> {
crate::weight_quant::validate_packed(
scheme,
&packed_values,
&scales,
zero_points.as_deref(),
in_features,
out_features,
)?;
Ok(Self {
packed: crate::weight_quant::PackedLinearWeight {
scheme,
packed_values,
scales,
zero_points,
in_features,
out_features,
},
gpu: Rc::new(RefCell::new(None)),
})
}
pub fn from_packed(packed: crate::weight_quant::PackedLinearWeight) -> Self {
Self {
packed,
gpu: Rc::new(RefCell::new(None)),
}
}
pub fn dequantized_values(&self, transpose_weight: bool) -> Vec<f32> {
let mut output = vec![0.0; self.in_features() * self.out_features()];
for input in 0..self.in_features() {
for output_channel in 0..self.out_features() {
let value = self.packed.dequantized_value(input, output_channel);
if transpose_weight {
output[output_channel * self.in_features() + input] = value;
} else {
output[input * self.out_features() + output_channel] = value;
}
}
}
output
}
fn quantized_numeric(&self, input: usize, output: usize) -> f32 {
self.packed.quantized_numeric(input, output)
}
fn dequantized_value(&self, input: usize, output: usize) -> f32 {
self.packed.dequantized_value(input, output)
}
fn scale_for(&self, input: usize, output: usize) -> f32 {
self.packed.scales[self.packed.scale_index(input, output)]
}
#[cfg(feature = "gpu")]
fn gpu_storage(&self, gpu: &GpuContext) -> Result<GpuQuantizedLinearWeight, GpuError> {
if let Some(storage) = self.gpu.borrow().as_ref() {
if storage.belongs_to(gpu) {
return Ok(storage.clone());
}
}
let storage = gpu.upload_quantized_linear_weight(
&self.packed.packed_values,
&self.packed.scales,
self.packed.zero_points.as_deref(),
self.packed.in_features,
self.packed.out_features,
self.packed.scheme,
)?;
*self.gpu.borrow_mut() = Some(storage.clone());
Ok(storage)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum InputCaptureMode {
Hessian,
AbsMax,
}
#[derive(Clone, Debug)]
struct InputCapture {
mode: InputCaptureMode,
hessian: Vec<f32>,
absmax: Vec<f32>,
rows: usize,
}
#[derive(Clone, Debug)]
pub struct CapturedHessian {
pub matrix: Vec<f32>,
pub rows: usize,
pub dim: usize,
}
#[derive(Clone)]
pub struct Linear {
pub weight: Tensor,
pub bias: Option<Tensor>,
pub in_features: usize,
pub out_features: usize,
pub transpose_weight: bool,
quantized_weight: Option<QuantizedLinearWeight>,
qat: Option<crate::qat::QatRuntime>,
pub lora: Option<crate::peft::LoraAdapter>,
capture: Rc<RefCell<Option<InputCapture>>>,
activation_quant: Option<ActQuantConfig>,
}
impl Linear {
pub fn new(in_features: usize, out_features: usize, seed: u64) -> Self {
let mut initializer = Initializer::new(seed);
Self::from_initializer(in_features, out_features, &mut initializer, false)
}
pub fn new_with_bias(in_features: usize, out_features: usize, seed: u64) -> Self {
let mut initializer = Initializer::new(seed);
Self::from_initializer(in_features, out_features, &mut initializer, true)
}
pub(crate) fn from_initializer(
in_features: usize,
out_features: usize,
initializer: &mut Initializer,
use_bias: bool,
) -> Self {
assert!(in_features > 0 && out_features > 0);
Self {
weight: initializer.tensor(vec![in_features, out_features], in_features, out_features),
bias: Self::zero_bias(out_features, use_bias),
in_features,
out_features,
transpose_weight: false,
quantized_weight: None,
qat: None,
lora: None,
capture: Rc::new(RefCell::new(None)),
activation_quant: None,
}
}
pub(crate) fn from_transposed_weight(
weight: Tensor,
in_features: usize,
out_features: usize,
) -> Self {
assert_eq!(weight.shape(), vec![out_features, in_features]);
Self {
weight,
bias: None,
in_features,
out_features,
transpose_weight: true,
quantized_weight: None,
qat: None,
lora: None,
capture: Rc::new(RefCell::new(None)),
activation_quant: None,
}
}
pub(crate) fn from_initializer_scaled(
in_features: usize,
out_features: usize,
initializer: &mut Initializer,
scale: f32,
use_bias: bool,
) -> Self {
assert!(in_features > 0 && out_features > 0);
assert!(scale.is_finite() && scale > 0.0);
Self {
weight: initializer.tensor_scaled(
vec![in_features, out_features],
in_features,
out_features,
scale,
),
bias: Self::zero_bias(out_features, use_bias),
in_features,
out_features,
transpose_weight: false,
quantized_weight: None,
qat: None,
lora: None,
capture: Rc::new(RefCell::new(None)),
activation_quant: None,
}
}
fn zero_bias(out_features: usize, use_bias: bool) -> Option<Tensor> {
if use_bias {
Some(Tensor::zeros(vec![out_features], true))
} else {
None
}
}
fn apply_bias(&self, projected: Tensor) -> Tensor {
match &self.bias {
Some(bias) => &projected + bias,
None => projected,
}
}
#[cfg(feature = "gpu")]
fn apply_bias_gpu(&self, projected: Tensor, gpu: &GpuContext) -> Result<Tensor, GpuError> {
match &self.bias {
Some(bias) => {
let bias = bias.to_gpu(gpu)?;
gpu.bias_add_tensor(&projected, &bias)
}
None => Ok(projected),
}
}
pub fn quantize_weight(&mut self, format: WeightQuantization) {
self.quantize_weight_with(QuantScheme::default_for(format));
}
pub fn quantize_weight_with(&mut self, scheme: QuantScheme) {
self.qat = None;
let values = self.weight.shared_data();
let packed = crate::weight_quant::pack_weights(
self.in_features,
self.out_features,
scheme,
|input, output| self.logical_weight_value(&values, input, output),
)
.expect("quantize scheme should be validated by callers");
self.quantized_weight = Some(QuantizedLinearWeight::from_packed(packed));
}
pub fn clear_quantized_weight(&mut self) {
self.quantized_weight = None
}
pub fn quantized_weight(&self) -> Option<&QuantizedLinearWeight> {
self.quantized_weight.as_ref()
}
pub fn enable_activation_quant(&mut self, config: ActQuantConfig) -> Result<(), String> {
config.validate()?;
self.activation_quant = Some(config);
Ok(())
}
pub fn clear_activation_quant(&mut self) {
self.activation_quant = None;
}
pub fn activation_quant(&self) -> Option<&ActQuantConfig> {
self.activation_quant.as_ref()
}
pub fn begin_input_capture(&self, mode: InputCaptureMode) {
let dim = self.in_features;
*self.capture.borrow_mut() = Some(InputCapture {
mode,
hessian: match mode {
InputCaptureMode::Hessian => vec![0.0; dim * dim],
InputCaptureMode::AbsMax => Vec::new(),
},
absmax: match mode {
InputCaptureMode::AbsMax => vec![0.0; dim],
InputCaptureMode::Hessian => Vec::new(),
},
rows: 0,
});
}
pub fn clear_input_capture(&self) {
*self.capture.borrow_mut() = None;
}
pub fn end_hessian_capture(&self) -> Option<CapturedHessian> {
let capture = self.capture.borrow_mut().take()?;
if capture.mode != InputCaptureMode::Hessian {
return None;
}
Some(CapturedHessian {
matrix: capture.hessian,
rows: capture.rows,
dim: self.in_features,
})
}
pub fn end_absmax_capture(&self) -> Option<Vec<f32>> {
let capture = self.capture.borrow_mut().take()?;
if capture.mode != InputCaptureMode::AbsMax {
return None;
}
Some(capture.absmax)
}
fn observe_input(&self, input: &Tensor) {
let mut slot = self.capture.borrow_mut();
let Some(capture) = slot.as_mut() else {
return;
};
let values = input.shared_data();
let rows = input.numel() / self.in_features;
let dim = self.in_features;
match capture.mode {
InputCaptureMode::Hessian => {
for row in 0..rows {
let offset = row * dim;
for i in 0..dim {
let xi = values[offset + i];
if xi == 0.0 {
continue;
}
let row_base = i * dim;
for j in i..dim {
let prod = xi * values[offset + j];
capture.hessian[row_base + j] += prod;
if i != j {
capture.hessian[j * dim + i] += prod;
}
}
}
}
capture.rows = capture.rows.saturating_add(rows);
}
InputCaptureMode::AbsMax => {
for row in 0..rows {
let offset = row * dim;
for i in 0..dim {
let abs = values[offset + i].abs();
if abs > capture.absmax[i] {
capture.absmax[i] = abs;
}
}
}
capture.rows = capture.rows.saturating_add(rows);
}
}
}
pub fn set_quantized_weight(&mut self, weight: QuantizedLinearWeight) -> Result<(), String> {
if weight.in_features() != self.in_features || weight.out_features() != self.out_features {
return Err(format!(
"quantized weight shape [{}→{}] does not match linear [{}→{}]",
weight.in_features(),
weight.out_features(),
self.in_features,
self.out_features
));
}
self.qat = None;
let values = weight.dequantized_values(self.transpose_weight);
self.weight.set_data(values);
self.weight.set_requires_grad(false);
self.quantized_weight = Some(weight);
Ok(())
}
pub fn materialize_dequantized_master(&mut self) {
if let Some(weight) = &self.quantized_weight {
let values = weight.dequantized_values(self.transpose_weight);
self.weight.set_data(values);
self.weight.set_requires_grad(false);
}
}
pub fn enable_qat(&mut self, format: WeightQuantization) {
self.enable_qat_with(crate::qat::QatConfig::from_format(format))
.expect("default QAT format is valid");
}
pub fn enable_qat_with(&mut self, config: crate::qat::QatConfig) -> Result<(), String> {
config.validate().map_err(|error| error.to_string())?;
self.quantized_weight = None;
let init_scales = if config.learnable_scales {
let values = self.weight.shared_data();
Some(crate::qat::init_absmax_scales(
self.in_features,
self.out_features,
config.scheme(),
|input, output| self.logical_weight_value(&values, input, output),
))
} else {
None
};
self.qat = Some(
crate::qat::QatRuntime::new(config, self.in_features, self.out_features, init_scales)
.map_err(|error| error.to_string())?,
);
Ok(())
}
pub fn clear_qat(&mut self) {
self.qat = None;
}
pub fn qat_format(&self) -> Option<WeightQuantization> {
self.qat.as_ref().map(|runtime| runtime.config.format)
}
pub fn qat_config(&self) -> Option<&crate::qat::QatConfig> {
self.qat.as_ref().map(|runtime| &runtime.config)
}
pub fn qat_scales(&self) -> Option<&Tensor> {
self.qat.as_ref().and_then(|runtime| runtime.scales.as_ref())
}
pub fn export_qat_packed(&mut self) -> Result<(), String> {
let Some(runtime) = self.qat.take() else {
return Err("QAT is not enabled on this linear".into());
};
let scheme = runtime.config.scheme();
let values = self.weight.shared_data();
let packed = if let Some(scales) = &runtime.scales {
crate::weight_quant::pack_weights_with_scales(
self.in_features,
self.out_features,
scheme,
&scales.shared_data(),
None,
|input, output| self.logical_weight_value(&values, input, output),
)?
} else {
crate::weight_quant::pack_weights(
self.in_features,
self.out_features,
scheme,
|input, output| self.logical_weight_value(&values, input, output),
)?
};
self.quantized_weight = Some(QuantizedLinearWeight::from_packed(packed));
Ok(())
}
fn fake_quantized_weight_values(&self, runtime: &crate::qat::QatRuntime) -> Vec<f32> {
let values = self.weight.shared_data();
let scheme = runtime.config.scheme();
if let Some(scales) = &runtime.scales {
crate::qat::fake_quantize_storage(
self.in_features,
self.out_features,
self.transpose_weight,
scheme,
&scales.shared_data(),
|input, output| self.logical_weight_value(&values, input, output),
)
} else {
let packed = crate::weight_quant::pack_weights(
self.in_features,
self.out_features,
scheme,
|input, output| self.logical_weight_value(&values, input, output),
)
.expect("QAT scheme validated at enable time");
QuantizedLinearWeight::from_packed(packed).dequantized_values(self.transpose_weight)
}
}
fn qat_weight(&self, runtime: &crate::qat::QatRuntime) -> Tensor {
if let Some(scales) = &runtime.scales {
let values = self.weight.shared_data();
let scheme = runtime.config.scheme();
let codes = crate::qat::quantized_codes_storage(
self.in_features,
self.out_features,
self.transpose_weight,
scheme,
&scales.shared_data(),
|input, output| self.logical_weight_value(&values, input, output),
);
let q = Tensor::new(codes, self.weight.shape(), false);
let scale_factor = if self.transpose_weight {
scales.reshape(vec![self.out_features, 1])
} else {
scales.clone()
};
let reconstructed = q.mul(&scale_factor);
let zeros = vec![0.0; self.weight.numel()];
reconstructed.add(&self.weight.ste_replace(zeros))
} else {
self.weight
.ste_replace(self.fake_quantized_weight_values(runtime))
}
}
pub fn forward(&self, input: &Tensor) -> Tensor {
self.observe_input(input);
let input_shape = input.shape();
assert_eq!(
input_shape.last(),
Some(&self.in_features),
"linear input width mismatch"
);
let expected_weight_shape = if self.transpose_weight {
vec![self.out_features, self.in_features]
} else {
vec![self.in_features, self.out_features]
};
assert_eq!(
self.weight.shape(),
expected_weight_shape,
"linear weight shape mismatch"
);
if self.qat.is_none() {
if let Some(weight) = &self.quantized_weight {
let output = self.forward_quantized(input, weight);
let output = match &self.lora {
Some(lora) => self.apply_lora(output, input, lora),
None => output,
};
return self.apply_bias(output);
}
}
let rows = input.numel() / self.in_features;
let flattened = input.reshape(vec![rows, self.in_features]);
let weight = if let Some(runtime) = &self.qat {
let fake = self.qat_weight(runtime);
if self.transpose_weight {
fake.permute(vec![1, 0])
} else {
fake
}
} else if self.transpose_weight {
self.weight.permute(vec![1, 0])
} else {
self.weight.clone()
};
let projected = flattened.matmul(&weight);
let mut output_shape = input_shape;
*output_shape.last_mut().unwrap() = self.out_features;
let output = projected.reshape(output_shape);
let output = match &self.lora {
Some(lora) => self.apply_lora(output, input, lora),
None => output,
};
self.apply_bias(output)
}
fn apply_lora(&self, base_output: Tensor, input: &Tensor, lora: &crate::peft::LoraAdapter) -> Tensor {
let delta = lora.forward_delta(input);
let combined = base_output.add(&delta);
match &lora.dora_magnitude {
Some(magnitude) => {
let norms = self.weight_column_norms(crate::peft::DORA_MAGNITUDE_EPS);
let norms_tensor = Tensor::new(norms, vec![self.out_features], false);
combined.mul(&magnitude.div(&norms_tensor))
}
None => combined,
}
}
fn weight_column_norms(&self, eps: f32) -> Vec<f32> {
let values = self.weight.shared_data();
(0..self.out_features)
.map(|output| {
let sum_squares: f32 = (0..self.in_features)
.map(|input| {
let value = self.logical_weight_value(&values, input, output);
value * value
})
.sum();
sum_squares.sqrt() + eps
})
.collect()
}
#[cfg(feature = "gpu")]
pub fn forward_gpu(&self, input: &Tensor, gpu: &GpuContext) -> Result<Tensor, GpuError> {
let input_shape = input.shape();
assert_eq!(
input_shape.last(),
Some(&self.in_features),
"linear input width mismatch"
);
let expected_weight_shape = if self.transpose_weight {
vec![self.out_features, self.in_features]
} else {
vec![self.in_features, self.out_features]
};
assert_eq!(
self.weight.shape(),
expected_weight_shape,
"linear weight shape mismatch"
);
let rows = input.numel() / self.in_features;
let input_gpu = if let Some(act_config) = &self.activation_quant {
let values = input.shared_data();
let acts = crate::act_quant::quantize_activations(
&values,
rows,
self.in_features,
act_config,
)
.map_err(|error| GpuError::InvalidShape(error))?;
let dequant = acts.dequantized();
Tensor::new(dequant, input.shape(), false).to_gpu(gpu)?
} else {
input.to_gpu(gpu)?
};
if self.qat.is_none() {
if let Some(weight) = &self.quantized_weight {
let flattened = input_gpu.reshape(vec![rows, self.in_features]);
let input_storage = flattened.gpu_storage(gpu)?;
let weight_storage = weight.gpu_storage(gpu)?;
let projected = gpu
.quantized_linear(&input_storage, &weight_storage)?
.to_tensor(false)?;
let mut output_shape = input_shape;
*output_shape.last_mut().unwrap() = self.out_features;
let output = self.apply_lora_gpu(projected.reshape(output_shape), input, gpu)?;
return self.apply_bias_gpu(output, gpu);
}
}
let weight_gpu = if let Some(runtime) = &self.qat {
self.qat_weight(runtime).to_gpu(gpu)?
} else {
self.weight.to_gpu(gpu)?
};
let weight_gpu = if self.transpose_weight {
weight_gpu.gpu_permute(vec![1, 0], gpu)?
} else {
weight_gpu
};
let flattened = input_gpu.reshape(vec![rows, self.in_features]);
let projected = flattened.gpu_matmul(&weight_gpu, gpu)?;
let mut output_shape = input_shape;
*output_shape.last_mut().unwrap() = self.out_features;
let output = self.apply_lora_gpu(projected.reshape(output_shape), input, gpu)?;
self.apply_bias_gpu(output, gpu)
}
#[cfg(feature = "gpu")]
fn apply_lora_gpu(
&self,
base_output: Tensor,
input: &Tensor,
gpu: &GpuContext,
) -> Result<Tensor, GpuError> {
match &self.lora {
Some(lora) => {
let delta = lora.forward_gpu(input, gpu)?;
base_output.gpu_add(&delta, gpu)
}
None => Ok(base_output),
}
}
pub fn parameters(&self) -> Vec<Tensor> {
let mut parameters = vec![self.weight.clone()];
if let Some(bias) = &self.bias {
parameters.push(bias.clone());
}
if let Some(runtime) = &self.qat {
if let Some(scales) = &runtime.scales {
parameters.push(scales.clone());
}
}
if let Some(lora) = &self.lora {
parameters.extend(lora.parameters());
}
parameters
}
#[cfg(feature = "gpu")]
pub(crate) fn prepare_gpu(&self, gpu: &GpuContext) -> Result<(), GpuError> {
if self.qat.is_some() {
let _ = self.weight.to_gpu(gpu)?;
} else if let Some(weight) = &self.quantized_weight {
let _ = weight.gpu_storage(gpu)?;
} else {
let _ = self.weight.to_gpu(gpu)?;
}
if let Some(bias) = &self.bias {
let _ = bias.to_gpu(gpu)?;
}
if let Some(lora) = &self.lora {
let _ = lora.a.to_gpu(gpu)?;
let _ = lora.b.to_gpu(gpu)?;
if let Some(magnitude) = &lora.dora_magnitude {
let _ = magnitude.to_gpu(gpu)?;
}
}
Ok(())
}
pub(crate) fn logical_weight_value(&self, values: &[f32], input: usize, output: usize) -> f32 {
if self.transpose_weight {
values[output * self.in_features + input]
} else {
values[input * self.out_features + output]
}
}
fn forward_quantized(&self, input: &Tensor, weight: &QuantizedLinearWeight) -> Tensor {
let input_shape = input.shape();
let input_values = input.shared_data();
let rows = input.numel() / self.in_features;
let mut output = vec![0.0; rows * self.out_features];
if let Some(act_config) = &self.activation_quant {
let acts = crate::act_quant::quantize_activations(
&input_values,
rows,
self.in_features,
act_config,
)
.expect("activation quant config validated at enable time");
let per_channel_sym = matches!(
weight.scheme().granularity,
QuantGranularity::PerChannel
) && weight.scheme().symmetric
&& weight.scheme().format.is_integer();
for row in 0..rows {
let act_scale = acts.scale_for_row(row);
let row_offset = row * self.in_features;
for output_channel in 0..self.out_features {
let mut accumulator = 0.0f32;
if per_channel_sym {
for input_channel in 0..self.in_features {
let a = acts.codes[row_offset + input_channel] as f32;
let w = weight.quantized_numeric(input_channel, output_channel);
accumulator += a * w;
}
output[row * self.out_features + output_channel] =
accumulator * act_scale * weight.scale_for(0, output_channel);
} else {
for input_channel in 0..self.in_features {
let a = acts.codes[row_offset + input_channel] as f32 * act_scale;
accumulator +=
a * weight.dequantized_value(input_channel, output_channel);
}
output[row * self.out_features + output_channel] = accumulator;
}
}
}
} else {
let per_channel_sym = matches!(
weight.scheme().granularity,
QuantGranularity::PerChannel
) && weight.scheme().symmetric
&& weight.scheme().format.is_integer();
for row in 0..rows {
for output_channel in 0..self.out_features {
let mut accumulator = 0.0;
if per_channel_sym {
for input_channel in 0..self.in_features {
accumulator += input_values[row * self.in_features + input_channel]
* weight.quantized_numeric(input_channel, output_channel);
}
output[row * self.out_features + output_channel] =
accumulator * weight.scale_for(0, output_channel);
} else {
for input_channel in 0..self.in_features {
accumulator += input_values[row * self.in_features + input_channel]
* weight.dequantized_value(input_channel, output_channel);
}
output[row * self.out_features + output_channel] = accumulator;
}
}
}
}
let mut output_shape = input_shape;
*output_shape.last_mut().unwrap() = self.out_features;
Tensor::new(output, output_shape, false)
}
}
#[derive(Clone)]
pub struct Embedding {
pub weight: Tensor,
pub vocab_size: usize,
pub hidden_size: usize,
}
impl Embedding {
pub fn new(vocab_size: usize, hidden_size: usize, seed: u64) -> Self {
let mut initializer = Initializer::new(seed);
Self::from_initializer(vocab_size, hidden_size, &mut initializer)
}
pub(crate) fn from_initializer(
vocab_size: usize,
hidden_size: usize,
initializer: &mut Initializer,
) -> Self {
assert!(vocab_size > 0 && hidden_size > 0);
Self {
weight: initializer.tensor(vec![vocab_size, hidden_size], vocab_size, hidden_size),
vocab_size,
hidden_size,
}
}
pub fn forward(&self, input_ids: &IndexTensor) -> Tensor {
self.weight.embedding(input_ids)
}
#[cfg(feature = "gpu")]
pub fn forward_gpu(
&self,
input_ids: &IndexTensor,
gpu: &GpuContext,
) -> Result<Tensor, GpuError> {
let weight = self.weight.to_gpu(gpu)?;
gpu.embedding_tensor(&weight, input_ids)
}
pub fn parameters(&self) -> Vec<Tensor> {
vec![self.weight.clone()]
}
}
pub(crate) struct Initializer {
rng: StdRng,
normal_standard_deviation: Option<f32>,
}
impl Initializer {
pub(crate) fn new(seed: u64) -> Self {
Self {
rng: StdRng::seed_from_u64(seed),
normal_standard_deviation: None,
}
}
pub(crate) fn normal(seed: u64, standard_deviation: f32) -> Self {
assert!(standard_deviation.is_finite() && standard_deviation > 0.0);
Self {
rng: StdRng::seed_from_u64(seed),
normal_standard_deviation: Some(standard_deviation),
}
}
pub(crate) fn tensor(&mut self, shape: Vec<usize>, fan_in: usize, fan_out: usize) -> Tensor {
self.tensor_scaled(shape, fan_in, fan_out, 1.0)
}
pub(crate) fn tensor_scaled(
&mut self,
shape: Vec<usize>,
fan_in: usize,
fan_out: usize,
scale: f32,
) -> Tensor {
let length: usize = shape.iter().product();
let data = match self.normal_standard_deviation {
Some(standard_deviation) => {
normal_with_rng(length, standard_deviation * scale, &mut self.rng)
}
None => xavier_uniform_with_rng(length, fan_in, fan_out, &mut self.rng)
.into_iter()
.map(|value| value * scale)
.collect(),
};
Tensor::new(data, shape, true)
}
}
#[cfg(test)]
mod tests {
use super::Linear;
use crate::Tensor;
#[test]
fn linear_bias_is_broadcast_and_trainable() {
let linear = Linear::new_with_bias(3, 2, 0);
linear.weight.set_data(vec![1.0, 0.0, 0.0, 1.0, 0.0, 0.0]);
linear.bias.as_ref().unwrap().set_data(vec![10.0, 20.0]);
let input = Tensor::new(vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0], vec![2, 3], true);
let output = linear.forward(&input);
assert_eq!(output.data(), vec![11.0, 22.0, 14.0, 25.0]);
output.sum().backward();
assert_eq!(
linear.bias.as_ref().unwrap().grad().unwrap(),
vec![2.0, 2.0]
);
}
}