Struct ColumnParallelLayer 

pub struct ColumnParallelLayer { /* private fields */ }

This layer has a weight that is parallelized along the output dimension, taking the “full” input dimension.

Implementations

impl ColumnParallelLayer

pub fn new_with_shard( in_dim: usize, out_dim: usize, config: &Option<QuantizedConfig>, bias: bool, comm: &Arc<Comm>, shard: Shard, vb: ShardedVarBuilder, ) -> Result<Arc<dyn QuantMethod>>

pub fn new( in_dim: usize, out_dim: usize, config: &Option<QuantizedConfig>, bias: bool, comm: &Arc<Comm>, vb: ShardedVarBuilder, ) -> Result<Arc<dyn QuantMethod>>

pub fn new_matformer( in_dim: usize, out_dim: usize, orig_intermediate_size: usize, config: &Option<QuantizedConfig>, bias: bool, comm: &Arc<Comm>, vb: ShardedVarBuilder, ) -> Result<Arc<dyn QuantMethod>>

pub fn new_merged( in_dim: usize, out_dim: usize, chunks: usize, config: &Option<QuantizedConfig>, bias: bool, comm: &Arc<Comm>, vb: ShardedVarBuilder, ) -> Result<Vec<Arc<dyn QuantMethod>>>

Trait Implementations

impl Debug for ColumnParallelLayer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl QuantMethod for ColumnParallelLayer

fn new(_method: QuantMethodConfig) -> Result<Self>

where Self: Sized,

fn forward(&self, a: &Tensor) -> Result<Tensor>

Compute matmul of self and a. self should contain the weights.

fn add_delta_w(&self, delta: &Tensor) -> Result<Arc<dyn QuantMethod>>

Add a delta weight from LoRA to the weights. This should be prescaled with alpha.

fn dequantize_w(&self) -> Result<Tensor>

fn dtype_and_device(&self) -> (DType, Device)

Weight dtype and device

fn begin_track_stats(&mut self) -> Result<()>

Begin tracking stats into an ImatrixLayerStats

fn end_track_stats(&self) -> Result<Tensor>

End tracking stats into an ImatrixLayerStats. Returns the computed imatrix.

fn quantized_act_type(&self) -> Option<DType>

If a quantized method, return the activation dtype.

fn unquant_weight_bias(&self) -> Option<(Tensor, Option<Tensor>)>

fn apply_isq( self: Arc<Self>, dtype: Option<IsqType>, device: Device, n_quantized: &AtomicUsize, imatrix_weight: Option<Vec<f32>>, guard: QuantizeOntoGuard, ) -> Result<Arc<dyn QuantMethod>>

If the quant is backed by a qmatmul.

fn is_distributed(&self) -> Option<DistributedKind>

fn forward_autocast(&self, a: &Tensor) -> Result<Tensor>

Compute matmul of self and a. self should contain the weights. Automatically cast to required quantization activation type and back

fn gather_forward_autocast( &self, a: &Tensor, indices: &Tensor, ) -> Result<Tensor>

Compute matmul of self and a. self should contain the weights. Automatically cast to required quantization activation type and back.

fn gather_forward(&self, _a: &Tensor, _indices: &Tensor) -> Result<Tensor>

Compute matmul of self and a. self should contain the weights.

impl QuantizedSerde for ColumnParallelLayer

fn isq_serde_supported(&self) -> bool

fn name(&self) -> &'static str

fn serialize(&self) -> Result<Cow<'_, [u8]>>

fn deserialize( data: Cow<'_, [u8]>, device: &Device, _comm: &Arc<Comm>, guard: QuantizeOntoGuard, ) -> Result<Arc<dyn QuantMethod>>

where Self: Sized,

fn deserialize_ext_bias( _data: Cow<'_, [u8]>, _device: &Device, _guard: QuantizeOntoGuard, ) -> Result<(Arc<dyn QuantMethod>, Option<Tensor>)>

where Self: Sized,

fn serialize_with_bias(&self, _bias: Option<Tensor>) -> Result<Cow<'_, [u8]>>

NOT meant for external calling