tract_core/ops/matmul/
pack.rs

1use crate::axes::Axis;
2use crate::internal::*;
3use ndarray::*;
4use tract_linalg::block_quant::{BlockQuantFact, PackedBlockQuantFact, PackedBlockQuantFormat};
5use tract_linalg::mmm::MMMInputValue;
6use tract_linalg::pack::PackedFormat;
7
8use super::ModePicker;
9
10#[derive(Debug, Clone, PartialEq, Eq, Hash)]
11pub struct OptMatMulPack {
12    pub(crate) packers: Vec<PackedFormat>,
13    pub(crate) mode_picker: ModePicker,
14    pub(crate) k_axis: usize,
15    pub(crate) mn_axis: usize,
16}
17
18impl Op for OptMatMulPack {
19    fn name(&self) -> StaticName {
20        "OptMatMulPack".into()
21    }
22
23    fn info(&self) -> TractResult<Vec<String>> {
24        Ok(vec![format!("{:?}. k axis: {}, mn axis: {}", self.packers, self.k_axis, self.mn_axis)])
25    }
26
27    op_as_typed_op!();
28    impl_op_same_as!();
29}
30
31impl EvalOp for OptMatMulPack {
32    fn is_stateless(&self) -> bool {
33        true
34    }
35
36    fn eval_with_session(
37        &self,
38        _node_id: usize,
39        session: &TurnState,
40        mut inputs: TVec<TValue>,
41    ) -> TractResult<TVec<TValue>> {
42        self.do_eval(session, inputs.remove(0))
43    }
44}
45
46impl TypedOp for OptMatMulPack {
47    fn output_facts(&self, inputs: &[&TypedFact]) -> TractResult<TVec<TypedFact>> {
48        match self.mode_picker {
49            ModePicker::Single => ensure!(self.packers.len() == 1),
50            ModePicker::VecVsMat => ensure!(self.packers.len() == 2),
51        }
52        let k = inputs[0].shape[self.k_axis].clone();
53        let mn = inputs[0].shape[self.mn_axis].clone();
54        let opaque_fact = DynPackedOpaqueFact { k, mn, packers: self.packers.clone() };
55        Ok(tvec!(
56            Opaque::datum_type()
57                .fact(self.output_shape(&inputs[0].shape))
58                .with_opaque_fact(opaque_fact)
59        ))
60    }
61
62    fn axes_mapping(
63        &self,
64        inputs: &[&TypedFact],
65        outputs: &[&TypedFact],
66    ) -> TractResult<AxesMapping> {
67        let mut axes: Vec<Axis> = (0..inputs[0].rank())
68            .filter(|&ix| ix != self.k_axis && ix != self.mn_axis)
69            .enumerate()
70            .zip('a'..)
71            .map(|((o, i), repr)| Axis::new(repr, 1, 1).input(0, i).output(0, o))
72            .collect();
73        axes.push(Axis::new('K', 1, 1).input(0, self.k_axis));
74        axes.push(Axis::new('M', 1, 1).input(0, self.mn_axis));
75        axes.push(Axis::new('P', 1, 1).output(0, outputs[0].rank()));
76        AxesMapping::new(1, 1, axes)
77    }
78
79    as_op!();
80}
81
82impl OptMatMulPack {
83    fn do_eval(&self, _session: &TurnState, input: TValue) -> TractResult<TVec<TValue>> {
84        unsafe {
85            let mode = self.mode_picker.pick(input.shape()[self.mn_axis])?;
86            let packer = &self.packers[mode];
87            let output_shape: TVec<usize> = self.output_shape(input.shape());
88            let stores = if output_shape.iter().all(|d| *d == 1) {
89                tensor0::<Opaque>(
90                    packer.pack_tensor_view(&input.view(), self.k_axis, self.mn_axis)?.into(),
91                )
92                .into_shape(&output_shape)?
93            } else {
94                let mut stores = Tensor::uninitialized_dt(Opaque::datum_type(), &output_shape)?;
95                let mut stores_view = stores.to_array_view_mut::<Opaque>()?;
96                let mut bc_shape: TVec<usize> = input.shape().into();
97                bc_shape[self.k_axis] = 1;
98                bc_shape[self.mn_axis] = 1;
99
100                for coord in indices(&*bc_shape) {
101                    let offset = coord
102                        .as_array_view()
103                        .iter()
104                        .zip(input.strides())
105                        .map(|(x, s)| *x as isize * s)
106                        .sum::<isize>()
107                        * input.datum_type().size_of() as isize;
108                    let mut pack_coords: TVec<usize> = coord.slice().into();
109                    pack_coords.remove(self.k_axis.max(self.mn_axis));
110                    pack_coords.remove(self.k_axis.min(self.mn_axis));
111                    stores_view[&*pack_coords] = packer
112                        .pack_tensor_view(
113                            &TensorView::from_bytes(&input, offset, input.shape(), input.strides()),
114                            self.k_axis,
115                            self.mn_axis,
116                        )?
117                        .into();
118                }
119                stores
120            };
121            Ok(tvec!(stores.into_tvalue()))
122        }
123    }
124
125    pub fn output_shape<D: DimLike>(&self, input: &[D]) -> TVec<D> {
126        let mut packed_shape: TVec<D> = input.into();
127        packed_shape.remove(self.mn_axis.max(self.k_axis));
128        packed_shape.remove(self.mn_axis.min(self.k_axis));
129        packed_shape
130    }
131}
132
133#[derive(Hash, Clone, Debug, PartialEq, Eq)]
134pub struct DynPackedOpaqueFact {
135    pub k: TDim,
136    pub mn: TDim,
137    pub packers: Vec<PackedFormat>,
138}
139
140impl OpaqueFact for DynPackedOpaqueFact {
141    fn same_as(&self, other: &dyn OpaqueFact) -> bool {
142        other.downcast_ref::<Self>().is_some_and(|o| o == self)
143    }
144
145    fn buffer_sizes(&self) -> TVec<TDim> {
146        tvec!(self.k.clone() * &self.mn * self.packers[0].dt.size_of())
147    }
148}
149
150#[derive(Debug, Clone, Hash, Eq, PartialEq)]
151pub struct OptSimpleMatMulPack {
152    pub(crate) packed_format: PackedBlockQuantFormat,
153    pub(crate) k: usize,
154    pub(crate) m: usize,
155}
156
157impl Op for OptSimpleMatMulPack {
158    fn name(&self) -> StaticName {
159        "OptSimpleMatMulPack".into()
160    }
161    op_as_typed_op!();
162    impl_op_same_as!();
163}
164
165impl EvalOp for OptSimpleMatMulPack {
166    fn is_stateless(&self) -> bool {
167        true
168    }
169
170    fn state(
171        &self,
172        _session: &TurnState,
173        _node_id: usize,
174    ) -> TractResult<Option<Box<dyn OpState>>> {
175        Ok(None)
176    }
177
178    fn eval(&self, inputs: TVec<TValue>) -> TractResult<TVec<TValue>> {
179        let input = args_1!(inputs);
180        let mut output = tensor1(
181            &input
182                .as_slice::<Opaque>()?
183                .iter()
184                .map(|i| {
185                    let i = i.downcast_ref::<BlobWithFact>().context("Expected BlockWithFact")?;
186                    let i_bqf = i
187                        .fact
188                        .downcast_ref::<BlockQuantFact>()
189                        .context("Expected BlockQuantFact")?;
190                    let iv: Box<dyn MMMInputValue> =
191                        Box::new(self.packed_format.pack(&i.value, i_bqf.k())?);
192                    Ok(Opaque(Arc::new(iv)))
193                })
194                .collect::<TractResult<Vec<_>>>()?,
195        );
196        output.set_shape(input.shape())?;
197        Ok(tvec!(output.into_tvalue()))
198    }
199}
200
201impl TypedOp for OptSimpleMatMulPack {
202    fn output_facts(&self, inputs: &[&TypedFact]) -> TractResult<TVec<TypedFact>> {
203        let fact = Opaque::fact(inputs[0].shape.clone()).with_opaque_fact(PackedBlockQuantFact {
204            format: self.packed_format.clone(),
205            shape: tvec!(self.m, self.k),
206        });
207        Ok(tvec!(fact))
208    }
209
210    as_op!();
211}
tract_core/ops/matmul/pack.rs

tract_core/ops/matmul/
pack.rs
