tract_core/ops/matmul/
pack.rs

1use crate::axes::Axis;
2use crate::internal::*;
3use ndarray::*;
4use tract_linalg::block_quant::{BlockQuantValue, PackedBlockQuantFact, PackedBlockQuantFormat};
5use tract_linalg::mmm::MMMInputValue;
6use tract_linalg::pack::PackedFormat;
7
8use super::ModePicker;
9
10#[derive(Debug, Clone, PartialEq, Eq, Hash)]
11pub struct OptMatMulPack {
12    pub(crate) packers: Vec<PackedFormat>,
13    pub(crate) mode_picker: ModePicker,
14    pub(crate) k_axis: usize,
15    pub(crate) mn_axis: usize,
16}
17
18impl Op for OptMatMulPack {
19    fn name(&self) -> StaticName {
20        "OptMatMulPack".into()
21    }
22
23    fn info(&self) -> TractResult<Vec<String>> {
24        Ok(vec![format!("{:?}. k axis: {}, mn axis: {}", self.packers, self.k_axis, self.mn_axis)])
25    }
26
27    op_as_typed_op!();
28    impl_op_same_as!();
29}
30
31impl EvalOp for OptMatMulPack {
32    fn is_stateless(&self) -> bool {
33        true
34    }
35
36    fn eval_with_session(
37        &self,
38        _node_id: usize,
39        session: &SessionState,
40        mut inputs: TVec<TValue>,
41    ) -> TractResult<TVec<TValue>> {
42        self.do_eval(session, inputs.remove(0))
43    }
44}
45
46impl TypedOp for OptMatMulPack {
47    fn output_facts(&self, inputs: &[&TypedFact]) -> TractResult<TVec<TypedFact>> {
48        match self.mode_picker {
49            ModePicker::Single => ensure!(self.packers.len() == 1),
50            ModePicker::VecVsMat => ensure!(self.packers.len() == 2),
51        }
52        let k = inputs[0].shape[self.k_axis].clone();
53        let mn = inputs[0].shape[self.mn_axis].clone();
54        let opaque_fact = DynPackedOpaqueFact { k, mn, packers: self.packers.clone() };
55        Ok(tvec!(Opaque::datum_type()
56            .fact(self.output_shape(&inputs[0].shape))
57            .with_opaque_fact(opaque_fact)))
58    }
59
60    fn axes_mapping(
61        &self,
62        inputs: &[&TypedFact],
63        outputs: &[&TypedFact],
64    ) -> TractResult<AxesMapping> {
65        let mut axes: Vec<Axis> = (0..inputs[0].rank())
66            .filter(|&ix| ix != self.k_axis && ix != self.mn_axis)
67            .enumerate()
68            .zip('a'..)
69            .map(|((o, i), repr)| Axis::new(repr, 1, 1).input(0, i).output(0, o))
70            .collect();
71        axes.push(Axis::new('K', 1, 1).input(0, self.k_axis));
72        axes.push(Axis::new('M', 1, 1).input(0, self.mn_axis));
73        axes.push(Axis::new('P', 1, 1).output(0, outputs[0].rank()));
74        AxesMapping::new(1, 1, axes)
75    }
76
77    as_op!();
78}
79
80impl OptMatMulPack {
81    fn do_eval(&self, _session: &SessionState, input: TValue) -> TractResult<TVec<TValue>> {
82        unsafe {
83            let mode = self.mode_picker.pick(input.shape()[self.mn_axis])?;
84            let packer = &self.packers[mode];
85            let output_shape: TVec<usize> = self.output_shape(input.shape());
86            let stores = if output_shape.iter().all(|d| *d == 1) {
87                tensor0::<Opaque>(
88                    packer.pack_tensor_view(&input.view(), self.k_axis, self.mn_axis)?.into(),
89                )
90                .into_shape(&output_shape)?
91            } else {
92                let mut stores = Tensor::uninitialized_dt(Opaque::datum_type(), &output_shape)?;
93                let mut stores_view = stores.to_array_view_mut::<Opaque>()?;
94                let mut bc_shape: TVec<usize> = input.shape().into();
95                bc_shape[self.k_axis] = 1;
96                bc_shape[self.mn_axis] = 1;
97
98                for coord in indices(&*bc_shape) {
99                    let offset = coord
100                        .as_array_view()
101                        .iter()
102                        .zip(input.strides())
103                        .map(|(x, s)| *x as isize * s)
104                        .sum::<isize>()
105                        * input.datum_type().size_of() as isize;
106                    let mut pack_coords: TVec<usize> = coord.slice().into();
107                    pack_coords.remove(self.k_axis.max(self.mn_axis));
108                    pack_coords.remove(self.k_axis.min(self.mn_axis));
109                    stores_view[&*pack_coords] = packer
110                        .pack_tensor_view(
111                            &TensorView::from_bytes(&input, offset, input.shape(), input.strides()),
112                            self.k_axis,
113                            self.mn_axis,
114                        )?
115                        .into();
116                }
117                stores
118            };
119            Ok(tvec!(stores.into_tvalue()))
120        }
121    }
122
123    pub fn output_shape<D: DimLike>(&self, input: &[D]) -> TVec<D> {
124        let mut packed_shape: TVec<D> = input.into();
125        packed_shape.remove(self.mn_axis.max(self.k_axis));
126        packed_shape.remove(self.mn_axis.min(self.k_axis));
127        packed_shape
128    }
129}
130
131#[derive(Hash, Clone, Debug, PartialEq, Eq)]
132pub struct DynPackedOpaqueFact {
133    pub k: TDim,
134    pub mn: TDim,
135    pub packers: Vec<PackedFormat>,
136}
137
138impl OpaqueFact for DynPackedOpaqueFact {
139    fn mem_size(&self) -> TDim {
140        self.k.clone() * &self.mn * self.packers[0].dt.size_of()
141    }
142
143    fn same_as(&self, other: &dyn OpaqueFact) -> bool {
144        other.downcast_ref::<Self>().is_some_and(|o| o == self)
145    }
146}
147
148#[derive(Debug, Clone, Hash, Eq, PartialEq)]
149pub struct OptSimpleMatMulPack {
150    pub(crate) packed_format: PackedBlockQuantFormat,
151    pub(crate) k: usize,
152    pub(crate) m: usize,
153}
154
155impl Op for OptSimpleMatMulPack {
156    fn name(&self) -> StaticName {
157        "OptSimpleMatMulPack".into()
158    }
159    op_as_typed_op!();
160    impl_op_same_as!();
161}
162
163impl EvalOp for OptSimpleMatMulPack {
164    fn is_stateless(&self) -> bool {
165        true
166    }
167
168    fn state(
169        &self,
170        _session: &mut SessionState,
171        _node_id: usize,
172    ) -> TractResult<Option<Box<dyn OpState>>> {
173        Ok(None)
174    }
175
176    fn eval(&self, inputs: TVec<TValue>) -> TractResult<TVec<TValue>> {
177        let input = args_1!(inputs);
178        let mut output = tensor1(
179            &input
180                .as_slice::<Opaque>()?
181                .iter()
182                .map(|i| {
183                    let i = i.downcast_ref::<BlockQuantValue>().unwrap();
184                    let iv: Box<dyn MMMInputValue> =
185                        Box::new(self.packed_format.pack(&i.value, i.fact.k())?);
186                    Ok(Opaque(Arc::new(iv)))
187                })
188                .collect::<TractResult<Vec<_>>>()?,
189        );
190        output.set_shape(input.shape())?;
191        Ok(tvec!(output.into_tvalue()))
192    }
193}
194
195impl TypedOp for OptSimpleMatMulPack {
196    fn output_facts(&self, inputs: &[&TypedFact]) -> TractResult<TVec<TypedFact>> {
197        let fact = Opaque::fact(inputs[0].shape.clone()).with_opaque_fact(PackedBlockQuantFact {
198            format: self.packed_format.clone(),
199            shape: tvec!(self.m, self.k),
200        });
201        Ok(tvec!(fact))
202    }
203
204    as_op!();
205}
tract_core/ops/matmul/pack.rs

tract_core/ops/matmul/
pack.rs
