1use crate::backend_authoring::TensorBackend;
2use crate::err::{Error, Result};
3use crate::exec::catalog::op;
4use crate::nn::module::Module;
5use crate::nn::module::ShapeInfo;
6use crate::nn::param::{Frozen, Param, TrainState, Trainable};
7use crate::nn::stats::{ComputeStats, LayerStats};
8use crate::shapes::error::OperationKind;
9use crate::shapes::shape::shape_buf_from_dims;
10use crate::shapes::{
11 Dim, DimCons, Dyn, DynShape, Nil, ReplaceLastDim, Shape, ShapeError, ShapeValue,
12};
13use crate::tensor::backend::Execute;
14use crate::tensor::base::Tensor;
15use crate::tensor::device::Device;
16use crate::tensor::dtype::DType;
17use crate::tensor::grad::{GradJoin, JoinedGrad, RequiresGrad};
18use alloc::string::String;
19
20pub trait LinearShape: Shape + DynShape {
37 type InF: Dim;
39 type OutF: Dim;
41 type WeightArg: crate::tensor::arg_into::NotUnit;
43 type BiasArg: crate::tensor::arg_into::NotUnit;
45 type WeightShape: Shape<Arg = Self::WeightArg> + DynShape;
47 type BiasShape: Shape<Arg = Self::BiasArg> + DynShape;
49
50 fn build_args(
52 target: (<Self::InF as Dim>::Arg, <Self::OutF as Dim>::Arg),
53 ) -> core::result::Result<(usize, usize, Self::WeightArg, Self::BiasArg), ShapeError>;
54}
55
56impl<InF: Dim, OutF: Dim> LinearShape
57 for crate::shapes::shape::DimCons<
58 InF,
59 crate::shapes::shape::DimCons<OutF, crate::shapes::shape::Nil>,
60 >
61{
62 type InF = InF;
63 type OutF = OutF;
64 type WeightArg = (<OutF as Dim>::Arg, (<InF as Dim>::Arg, ()));
65 type BiasArg = (<OutF as Dim>::Arg, ());
66 type WeightShape = crate::shapes::shape::DimCons<
67 OutF,
68 crate::shapes::shape::DimCons<InF, crate::shapes::shape::Nil>,
69 >;
70 type BiasShape = crate::shapes::shape::DimCons<OutF, crate::shapes::shape::Nil>;
71
72 #[inline]
73 fn build_args(
74 target: (<Self::InF as Dim>::Arg, <Self::OutF as Dim>::Arg),
75 ) -> core::result::Result<(usize, usize, Self::WeightArg, Self::BiasArg), ShapeError> {
76 let in_f = InF::resolve_arg(target.0.clone())?;
77 let out_f = OutF::resolve_arg(target.1.clone())?;
78 Ok((
79 in_f,
80 out_f,
81 (target.1.clone(), (target.0, ())),
82 (target.1, ()),
83 ))
84 }
85}
86
87impl LinearShape for Dyn {
88 type InF = usize;
90 type OutF = usize;
92 type WeightArg = alloc::vec::Vec<usize>;
94 type BiasArg = alloc::vec::Vec<usize>;
96 type WeightShape = Dyn;
98 type BiasShape = Dyn;
100
101 #[inline]
102 fn build_args(
104 target: (<Self::InF as Dim>::Arg, <Self::OutF as Dim>::Arg),
105 ) -> core::result::Result<(usize, usize, Self::WeightArg, Self::BiasArg), ShapeError> {
106 let in_f = target.0;
107 let out_f = target.1;
108 Ok((in_f, out_f, alloc::vec![out_f, in_f], alloc::vec![out_f]))
109 }
110}
111
112#[cfg(feature = "distributed")]
119pub trait TwoWayColumnLinearShape: LinearShape {
120 const LOCAL_OUT_FEATURES: usize;
122}
123
124#[cfg(feature = "distributed")]
125impl<InF, OutF> TwoWayColumnLinearShape for DimCons<InF, DimCons<OutF, Nil>>
126where
127 InF: Dim,
128 OutF: Dim + crate::dist::ShardDivisible<typenum::U2>,
129{
130 const LOCAL_OUT_FEATURES: usize = <OutF as crate::dist::ShardDivisible<typenum::U2>>::LOCAL;
131}
132
133#[cfg(feature = "distributed")]
140pub trait TwoWayRowLinearShape: LinearShape {
141 const LOCAL_IN_FEATURES: usize;
143}
144
145#[cfg(feature = "distributed")]
146impl<InF, OutF> TwoWayRowLinearShape for DimCons<InF, DimCons<OutF, Nil>>
147where
148 InF: Dim + crate::dist::ShardDivisible<typenum::U2>,
149 OutF: Dim,
150{
151 const LOCAL_IN_FEATURES: usize = <InF as crate::dist::ShardDivisible<typenum::U2>>::LOCAL;
152}
153
154pub struct LinearBuilder<
179 S: LinearShape,
180 Bias: crate::nn::optional::OptionalField = crate::nn::optional::True,
181 Train: TrainState = Trainable,
182> {
183 pub shape: ShapeValue<S>,
185 pub weight_init: crate::nn::init::Init,
187 pub bias_init: crate::nn::init::Init,
189 pub _phantom: core::marker::PhantomData<(Bias, Train)>,
191}
192
193impl<S: LinearShape, Bias: crate::nn::optional::OptionalField, Train: TrainState>
194 LinearBuilder<S, Bias, Train>
195{
196 pub fn shape(&self) -> &ShapeValue<S> {
198 &self.shape
199 }
200
201 pub fn weight_init_policy(&self) -> crate::nn::init::Init {
203 self.weight_init
204 }
205
206 pub fn bias_init_policy(&self) -> crate::nn::init::Init {
208 self.bias_init
209 }
210
211 pub fn no_bias(self) -> LinearBuilder<S, crate::nn::optional::False, Train> {
213 LinearBuilder {
214 shape: self.shape,
215 weight_init: self.weight_init,
216 bias_init: self.bias_init,
217 _phantom: core::marker::PhantomData,
218 }
219 }
220
221 pub fn frozen(self) -> LinearBuilder<S, Bias, Frozen> {
223 LinearBuilder {
224 shape: self.shape,
225 weight_init: self.weight_init,
226 bias_init: self.bias_init,
227 _phantom: core::marker::PhantomData,
228 }
229 }
230
231 pub fn weight_init(mut self, init: crate::nn::init::Init) -> Self {
233 self.weight_init = init;
234 self
235 }
236
237 pub fn bias_init(mut self, init: crate::nn::init::Init) -> Self {
239 self.bias_init = init;
240 self
241 }
242}
243
244pub fn linear<S: LinearShape>(shape: ShapeValue<S>) -> LinearBuilder<S> {
246 LinearBuilder {
247 shape,
248 weight_init: crate::nn::init::kaiming_uniform(),
249 bias_init: crate::nn::init::kaiming_uniform(),
250 _phantom: core::marker::PhantomData,
251 }
252}
253
254#[derive(Debug, Clone)]
256#[incin_macros::module(internal, no_stats)]
257pub struct Linear<
258 S: LinearShape,
259 B: crate::tensor::backend::VariableBackend,
260 Bias: crate::nn::optional::OptionalField = crate::nn::optional::True,
261 K: DType = f32,
262 Train: TrainState = Trainable,
263> {
264 pub weight: Param<S::WeightShape, B, K, Train>,
266 pub bias: Option<Param<S::BiasShape, B, K, Train>>,
268 #[module(ignore)]
269 _phantom: core::marker::PhantomData<(S, B, Bias, K, Train)>,
270}
271
272impl<
273 S: LinearShape,
274 B: crate::tensor::backend::VariableBackend,
275 Bias: crate::nn::optional::OptionalField,
276 K: DType,
277 Train: TrainState,
278> ShapeInfo for Linear<S, B, Bias, K, Train>
279{
280 fn shape_info(&self) -> Option<String> {
281 None
282 }
283}
284
285impl<
286 S: LinearShape,
287 B: crate::tensor::backend::VariableBackend,
288 Bias: crate::nn::optional::OptionalField,
289 K: DType,
290 Train: TrainState,
291> Linear<S, B, Bias, K, Train>
292{
293 pub fn from_raw_parts(
295 weight: Param<S::WeightShape, B, K, Train>,
296 bias: Option<Param<S::BiasShape, B, K, Train>>,
297 ) -> Self {
298 Self {
299 weight,
300 bias,
301 _phantom: core::marker::PhantomData,
302 }
303 }
304
305 pub fn freeze(self) -> Linear<S, B, Bias, K, Frozen> {
307 Linear {
308 weight: self.weight.freeze(),
309 bias: self.bias.map(|b| b.freeze()),
310 _phantom: core::marker::PhantomData,
311 }
312 }
313
314 pub fn unfreeze(self) -> Linear<S, B, Bias, K, Trainable> {
316 Linear {
317 weight: self.weight.unfreeze(),
318 bias: self.bias.map(|b| b.unfreeze()),
319 _phantom: core::marker::PhantomData,
320 }
321 }
322}
323
324impl<
325 S: LinearShape,
326 B: crate::tensor::backend::VariableBackend,
327 Bias: crate::nn::optional::OptionalField,
328 K: DType,
329 Train: TrainState,
330> ComputeStats for Linear<S, B, Bias, K, Train>
331{
332 fn compute_stats(&self, batch: u64) -> LayerStats {
333 let dims = self.weight.shape_dims();
334 let (out_f, in_f) = (
335 u64::try_from(dims[0]).expect("validated dimension must fit u64"),
336 u64::try_from(dims[1]).expect("validated dimension must fit u64"),
337 );
338 let bias_params = self
339 .bias
340 .as_ref()
341 .map(|b| crate::nn::stats::validated_parameter_count(&b.shape_dims()))
342 .unwrap_or(0);
343 LayerStats {
344 params: out_f * in_f + bias_params,
345 macs: in_f * out_f * batch,
346 }
347 }
348}
349
350impl<S, B, Bias, K: DType, Train: TrainState> Linear<S, B, Bias, K, Train>
351where
352 S: LinearShape,
353 B: TensorBackend<K> + crate::nn::param::ParameterInit<K>,
354 Bias: crate::nn::optional::OptionalField,
355 <K as DType>::Arg: Clone,
356 <B::Device as Device>::Arg: Clone,
357{
358 pub fn build<A>(args: A) -> Result<Self>
360 where
361 A: crate::tensor::arg_into::LayerArgInto<(
362 <S::InF as Dim>::Arg,
363 <S::OutF as Dim>::Arg,
364 <K as DType>::Arg,
365 <B::Device as Device>::Arg,
366 <Bias as crate::nn::optional::OptionalField>::Arg,
367 )>,
368 {
369 let (in_arg, out_arg, dtype, device, bias) = args.into_layer_arg();
370 Self::build_full(in_arg, out_arg, dtype, device, bias)
371 }
372
373 pub fn build_full(
375 in_arg: <S::InF as Dim>::Arg,
376 out_arg: <S::OutF as Dim>::Arg,
377 dtype: <K as DType>::Arg,
378 device: <B::Device as Device>::Arg,
379 bias_arg: <Bias as crate::nn::optional::OptionalField>::Arg,
380 ) -> Result<Self> {
381 let (in_f, _out_f, w_args, b_args) =
382 S::build_args((in_arg, out_arg)).map_err(Error::Shape)?;
383 let w_field = <S::WeightShape as Shape>::resolve(w_args).map_err(Error::Shape)?;
384 let w_dims = w_field.clone();
385 let init = crate::nn::init::kaiming_uniform();
386 let context_w = crate::nn::init::InitContext::new(crate::nn::init::ParameterRole::Weight)
387 .with_fan(in_f, _out_f);
388 let plan_w = init.plan(context_w)?;
389 let raw_w = crate::nn::param::execute_plan_raw::<B, K>(
390 w_dims.as_ref(),
391 &K::init(dtype.clone()),
392 &<B::Device as Device>::init(device.clone()),
393 plan_w,
394 )?;
395 let weight = Param::<S::WeightShape, B, K, Train>::from_parts_checked(
396 raw_w,
397 w_field,
398 K::init(dtype.clone()),
399 <B::Device as Device>::init(device.clone()),
400 )?;
401
402 let bias = if Bias::init(bias_arg) {
403 let b_field = <S::BiasShape as Shape>::resolve(b_args).map_err(Error::Shape)?;
404 let b_dims = b_field.clone();
405 let context_b = crate::nn::init::InitContext::new(crate::nn::init::ParameterRole::Bias)
406 .with_fan(in_f, _out_f);
407 let plan_b = init.plan(context_b)?;
408 let raw_b = crate::nn::param::execute_plan_raw::<B, K>(
409 b_dims.as_ref(),
410 &K::init(dtype.clone()),
411 &<B::Device as Device>::init(device.clone()),
412 plan_b,
413 )?;
414 Some(Param::<S::BiasShape, B, K, Train>::from_parts_checked(
415 raw_b,
416 b_field,
417 K::init(dtype),
418 <B::Device as Device>::init(device),
419 )?)
420 } else {
421 None
422 };
423
424 Ok(Self {
425 weight,
426 bias,
427 _phantom: core::marker::PhantomData,
428 })
429 }
430}
431
432impl<
433 B: crate::tensor::backend::VariableBackend
434 + Execute<op::MatMulExact>
435 + Execute<op::TransposeExact>
436 + crate::exec::Capabilities
437 + Execute<op::Add>,
438 K: DType,
439 Train: TrainState,
440 G: RequiresGrad,
441> Module<Tensor<Dyn, B, K, G>> for Linear<Dyn, B, crate::nn::optional::True, K, Train>
442where
443 G: GradJoin<Train::TensorGrad>,
444 JoinedGrad<G, Train::TensorGrad>: GradJoin<Train::TensorGrad>,
445 <B as Execute<op::TransposeExact>>::Output: Into<B::Storage<K>>,
446 <B as Execute<op::MatMulExact>>::Output: Into<B::Storage<K>>,
447 <B as Execute<op::Add>>::Output: Into<B::Storage<K>>,
448{
449 type Output = Tensor<Dyn, B, K, JoinedGrad<G, Train::TensorGrad>>;
450 type Error = Error;
451
452 fn forward(&self, x: Tensor<Dyn, B, K, G>) -> core::result::Result<Self::Output, Error> {
453 let weight_t = self.weight.as_tensor()?.transpose(0isize, 1isize)?;
454 let out = x.matmul(&weight_t)?;
455 let bias_t = self.bias.as_ref().unwrap().as_tensor()?;
456 let out_final = out.broadcast_add::<Dyn, Train::TensorGrad>(&bias_t)?;
457 Tensor::from_shape_value(
458 out_final.inner,
459 out_final._shape,
460 out_final._dtype,
461 out_final._device,
462 out._grad,
463 )
464 }
465}
466
467impl<
468 B: crate::tensor::backend::VariableBackend
469 + Execute<op::MatMulExact>
470 + Execute<op::TransposeExact>
471 + crate::exec::Capabilities,
472 K: DType,
473 Train: TrainState,
474 G: RequiresGrad,
475> Module<Tensor<Dyn, B, K, G>> for Linear<Dyn, B, crate::nn::optional::False, K, Train>
476where
477 G: GradJoin<Train::TensorGrad>,
478 <B as Execute<op::TransposeExact>>::Output: Into<B::Storage<K>>,
479 <B as Execute<op::MatMulExact>>::Output: Into<B::Storage<K>>,
480{
481 type Output = Tensor<Dyn, B, K, JoinedGrad<G, Train::TensorGrad>>;
482 type Error = Error;
483
484 fn forward(&self, x: Tensor<Dyn, B, K, G>) -> core::result::Result<Self::Output, Error> {
485 let weight_t = self.weight.as_tensor()?.transpose(0isize, 1isize)?;
486 x.matmul(&weight_t)
487 }
488}
489
490impl<
491 B: crate::tensor::backend::VariableBackend
492 + Execute<op::MatMulExact>
493 + Execute<op::TransposeExact>
494 + crate::exec::Capabilities
495 + Execute<op::Add>,
496 K: DType,
497 Train: TrainState,
498> Module<Tensor<Dyn, B, K>> for Linear<Dyn, B, Dyn, K, Train>
499where
500 <B as Execute<op::TransposeExact>>::Output: Into<B::Storage<K>>,
501 <B as Execute<op::MatMulExact>>::Output: Into<B::Storage<K>>,
502 <B as Execute<op::Add>>::Output: Into<B::Storage<K>>,
503{
504 type Output = Tensor<Dyn, B, K>;
505 type Error = Error;
506
507 fn forward(&self, x: Tensor<Dyn, B, K>) -> core::result::Result<Tensor<Dyn, B, K>, Error> {
508 let weight_t = self.weight.as_tensor()?.transpose(0isize, 1isize)?;
509 let out = x.matmul(&weight_t)?;
510 if let Some(b) = &self.bias {
511 let bias_t = b.as_tensor()?;
512 let out_final = out.broadcast_add(&bias_t)?;
513 Tensor::from_shape_value(
514 out_final.inner,
515 out_final._shape,
516 out_final._dtype,
517 out_final._device,
518 core::marker::PhantomData,
519 )
520 } else {
521 Tensor::from_shape_value(
522 out.inner,
523 out._shape,
524 out._dtype,
525 out._device,
526 core::marker::PhantomData,
527 )
528 }
529 }
530}
531
532impl<
533 InF: Dim,
534 OutF: Dim,
535 InShape: Shape + DynShape + ReplaceLastDim<OutF> + crate::shapes::EndsWith<InF>,
536 B: crate::tensor::backend::VariableBackend
537 + Execute<op::MatMulExact>
538 + Execute<op::TransposeExact>
539 + crate::exec::Capabilities
540 + Execute<op::Add>,
541 K: DType,
542 Train: TrainState,
543 G: RequiresGrad,
544> Module<Tensor<InShape, B, K, G>>
545 for Linear<DimCons<InF, DimCons<OutF, Nil>>, B, crate::nn::optional::True, K, Train>
546where
547 InShape::Output: DynShape,
548 G: GradJoin<Train::TensorGrad>,
549 JoinedGrad<G, Train::TensorGrad>: GradJoin<Train::TensorGrad>,
550 <B as Execute<op::TransposeExact>>::Output: Into<B::Storage<K>>,
551 <B as Execute<op::MatMulExact>>::Output: Into<B::Storage<K>>,
552 <B as Execute<op::Add>>::Output: Into<B::Storage<K>>,
553{
554 type Output = Tensor<InShape::Output, B, K, JoinedGrad<G, Train::TensorGrad>>;
555 type Error = Error;
556
557 fn forward(&self, x: Tensor<InShape, B, K, G>) -> core::result::Result<Self::Output, Error> {
558 let dtype = x._dtype.clone();
559 let device = x._device.clone();
560
561 let mut dims = x.shape_buf().as_ref().to_vec();
562 let last_idx = dims.len().saturating_sub(1);
563 if last_idx < dims.len() {
564 dims[last_idx] = self.weight.shape_dims()[0];
565 }
566 let shape = shape_buf_from_dims::<InShape::Output>(OperationKind::MatMul, &dims)?;
567
568 let weight_dyn = self.weight.as_tensor()?.into_shape::<Dyn>()?;
569 let weight_t = weight_dyn.transpose(0isize, 1isize)?;
570 let x_dyn = x.into_shape::<Dyn>()?;
571 let out_dyn = x_dyn.matmul(&weight_t)?;
572
573 let bias_dyn = self
574 .bias
575 .as_ref()
576 .unwrap()
577 .as_tensor()?
578 .into_shape::<Dyn>()?;
579 let out_final = out_dyn.broadcast_add::<Dyn, Train::TensorGrad>(&bias_dyn)?;
580
581 let grad = out_dyn._grad.clone();
582 Tensor::from_parts(out_final.into_inner(), shape, dtype, device, grad)
583 }
584}
585
586impl<
587 InF: Dim,
588 OutF: Dim,
589 InShape: Shape + DynShape + ReplaceLastDim<OutF> + crate::shapes::EndsWith<InF>,
590 B: crate::tensor::backend::VariableBackend
591 + Execute<op::MatMulExact>
592 + Execute<op::TransposeExact>
593 + crate::exec::Capabilities,
594 K: DType,
595 Train: TrainState,
596 G: RequiresGrad,
597> Module<Tensor<InShape, B, K, G>>
598 for Linear<DimCons<InF, DimCons<OutF, Nil>>, B, crate::nn::optional::False, K, Train>
599where
600 InShape::Output: DynShape,
601 G: GradJoin<Train::TensorGrad>,
602 <B as Execute<op::TransposeExact>>::Output: Into<B::Storage<K>>,
603 <B as Execute<op::MatMulExact>>::Output: Into<B::Storage<K>>,
604{
605 type Output = Tensor<InShape::Output, B, K, JoinedGrad<G, Train::TensorGrad>>;
606 type Error = Error;
607
608 fn forward(&self, x: Tensor<InShape, B, K, G>) -> core::result::Result<Self::Output, Error> {
609 let dtype = x._dtype.clone();
610 let device = x._device.clone();
611
612 let mut dims = x.shape_buf().as_ref().to_vec();
613 let last_idx = dims.len().saturating_sub(1);
614 if last_idx < dims.len() {
615 dims[last_idx] = self.weight.shape_dims()[0];
616 }
617 let shape = shape_buf_from_dims::<InShape::Output>(OperationKind::MatMul, &dims)?;
618
619 let weight_dyn = self.weight.as_tensor()?.into_shape::<Dyn>()?;
620 let weight_t = weight_dyn.transpose(0isize, 1isize)?;
621 let x_dyn = x.into_shape::<Dyn>()?;
622 let out_final = x_dyn.matmul(&weight_t)?;
623
624 let grad = out_final._grad.clone();
625 Tensor::from_parts(out_final.into_inner(), shape, dtype, device, grad)
626 }
627}
628
629impl<
630 InF: Dim,
631 OutF: Dim,
632 InShape: Shape + DynShape + ReplaceLastDim<OutF> + crate::shapes::EndsWith<InF>,
633 B: crate::tensor::backend::VariableBackend
634 + Execute<op::MatMulExact>
635 + Execute<op::TransposeExact>
636 + crate::exec::Capabilities
637 + Execute<op::Add>,
638 K: DType,
639 Train: TrainState,
640> Module<Tensor<InShape, B, K>> for Linear<DimCons<InF, DimCons<OutF, Nil>>, B, Dyn, K, Train>
641where
642 InShape::Output: DynShape,
643 <B as Execute<op::TransposeExact>>::Output: Into<B::Storage<K>>,
644 <B as Execute<op::MatMulExact>>::Output: Into<B::Storage<K>>,
645 <B as Execute<op::Add>>::Output: Into<B::Storage<K>>,
646{
647 type Output = Tensor<InShape::Output, B, K>;
648 type Error = Error;
649
650 fn forward(&self, x: Tensor<InShape, B, K>) -> core::result::Result<Self::Output, Error> {
651 let dtype = x._dtype.clone();
652 let device = x._device.clone();
653
654 let mut dims = x.shape_buf().as_ref().to_vec();
655 let last_idx = dims.len().saturating_sub(1);
656 if last_idx < dims.len() {
657 dims[last_idx] = self.weight.shape_dims()[0];
658 }
659 let shape = shape_buf_from_dims::<InShape::Output>(OperationKind::MatMul, &dims)?;
660
661 let weight_dyn = self.weight.as_tensor()?.into_shape::<Dyn>()?;
662 let weight_t = weight_dyn.transpose(0isize, 1isize)?;
663 let x_dyn = x.into_shape::<Dyn>()?;
664 let out_dyn = x_dyn.matmul(&weight_t)?;
665
666 let out_final = if let Some(b) = &self.bias {
667 let bias_dyn = b.as_tensor()?.into_shape::<Dyn>()?;
668 out_dyn.broadcast_add(&bias_dyn)?
669 } else {
670 out_dyn
671 };
672
673 Tensor::from_parts(
674 out_final.into_inner(),
675 shape,
676 dtype,
677 device,
678 core::marker::PhantomData,
679 )
680 }
681}