tract_core/ops/matmul/
pack.rs

1use crate::axes::Axis;
2use crate::internal::*;
3use ndarray::*;
4use tract_linalg::frame::PackedFormat;
5
6use super::ModePicker;
7
8#[derive(Debug, Clone, PartialEq, Eq, Hash)]
9pub struct OptMatMulPack {
10    pub(crate) packers: Vec<PackedFormat>,
11    pub(crate) mode_picker: ModePicker,
12    pub(crate) k_axis: usize,
13    pub(crate) mn_axis: usize,
14}
15
16impl Op for OptMatMulPack {
17    fn name(&self) -> Cow<str> {
18        "OptMatMulPack".into()
19    }
20
21    fn info(&self) -> TractResult<Vec<String>> {
22        Ok(vec![format!("{:?}. k axis: {}, mn axis: {}", self.packers, self.k_axis, self.mn_axis)])
23    }
24
25    op_as_typed_op!();
26    impl_op_same_as!();
27}
28
29impl EvalOp for OptMatMulPack {
30    fn is_stateless(&self) -> bool {
31        true
32    }
33
34    fn eval_with_session(
35        &self,
36        session: &SessionState,
37        mut inputs: TVec<TValue>,
38    ) -> TractResult<TVec<TValue>> {
39        self.do_eval(session, inputs.remove(0))
40    }
41}
42
43impl TypedOp for OptMatMulPack {
44    fn output_facts(&self, inputs: &[&TypedFact]) -> TractResult<TVec<TypedFact>> {
45        let k = inputs[0].shape[self.k_axis].clone();
46        let mn = inputs[0].shape[self.mn_axis].clone();
47        let opaque_fact = DynPackedOpaqueFact { k, mn, packers: self.packers.clone() };
48        Ok(tvec!(Opaque::datum_type()
49            .fact(self.output_shape(&inputs[0].shape))
50            .with_opaque_fact(opaque_fact)))
51    }
52
53    fn axes_mapping(
54        &self,
55        inputs: &[&TypedFact],
56        outputs: &[&TypedFact],
57    ) -> TractResult<AxesMapping> {
58        let mut axes: Vec<Axis> = (0..inputs[0].rank())
59            .filter(|&ix| ix != self.k_axis && ix != self.mn_axis)
60            .enumerate()
61            .zip('a'..)
62            .map(|((o, i), repr)| Axis::new(repr, 1, 1).input(0, i).output(0, o))
63            .collect();
64        axes.push(Axis::new('K', 1, 1).input(0, self.k_axis));
65        axes.push(Axis::new('M', 1, 1).input(0, self.mn_axis));
66        axes.push(Axis::new('P', 1, 1).output(0, outputs[0].rank()));
67        AxesMapping::new(1, 1, axes)
68    }
69
70    as_op!();
71}
72
73impl OptMatMulPack {
74    fn do_eval(&self, _session: &SessionState, input: TValue) -> TractResult<TVec<TValue>> {
75        unsafe {
76            let mode = self.mode_picker.pick(input.shape()[self.mn_axis])?;
77            let packer = &self.packers[mode];
78            let output_shape: TVec<usize> = self.output_shape(input.shape());
79            let stores = if output_shape.iter().all(|d| *d == 1) {
80                tensor0::<Opaque>(
81                    packer.pack_tensor_view(&input.view(), self.k_axis, self.mn_axis)?.into(),
82                )
83                .into_shape(&output_shape)?
84            } else {
85                let mut stores = Tensor::uninitialized_dt(Opaque::datum_type(), &output_shape)?;
86                let mut stores_view = stores.to_array_view_mut::<Opaque>()?;
87                let mut bc_shape: TVec<usize> = input.shape().into();
88                bc_shape[self.k_axis] = 1;
89                bc_shape[self.mn_axis] = 1;
90
91                for coord in indices(&*bc_shape) {
92                    let offset = coord
93                        .as_array_view()
94                        .iter()
95                        .zip(input.strides())
96                        .map(|(x, s)| *x as isize * s)
97                        .sum::<isize>()
98                        * input.datum_type().size_of() as isize;
99                    let mut pack_coords: TVec<usize> = coord.slice().into();
100                    pack_coords.remove(self.k_axis.max(self.mn_axis));
101                    pack_coords.remove(self.k_axis.min(self.mn_axis));
102                    stores_view[&*pack_coords] = packer
103                        .pack_tensor_view(
104                            &TensorView::from_bytes(&input, offset, input.shape(), input.strides()),
105                            self.k_axis,
106                            self.mn_axis,
107                        )?
108                        .into();
109                }
110                stores
111            };
112            Ok(tvec!(stores.into_tvalue()))
113        }
114    }
115
116    pub fn output_shape<D: DimLike>(&self, input: &[D]) -> TVec<D> {
117        let mut packed_shape: TVec<D> = input.into();
118        packed_shape.remove(self.mn_axis.max(self.k_axis));
119        packed_shape.remove(self.mn_axis.min(self.k_axis));
120        packed_shape
121    }
122}
123
124#[derive(Hash, Clone, Debug, PartialEq, Eq)]
125pub struct DynPackedOpaqueFact {
126    pub k: TDim,
127    pub mn: TDim,
128    pub packers: Vec<PackedFormat>,
129}
130
131impl OpaqueFact for DynPackedOpaqueFact {
132    fn mem_size(&self) -> TDim {
133        self.k.clone() * &self.mn * self.packers[0].dt.size_of()
134    }
135
136    fn same_as(&self, other: &dyn OpaqueFact) -> bool {
137        other.downcast_ref::<Self>().is_some_and(|o| o == self)
138    }
139}
tract_core/ops/matmul/pack.rs

tract_core/ops/matmul/
pack.rs
