tract_core/ops/matmul/
pack.rs

1use crate::axes::Axis;
2use crate::internal::*;
3use ndarray::*;
4use tract_linalg::block_quant::{BlockQuantValue, PackedBlockQuantFact, PackedBlockQuantFormat};
5use tract_linalg::mmm::MMMInputValue;
6use tract_linalg::pack::PackedFormat;
7
8use super::ModePicker;
9
10#[derive(Debug, Clone, PartialEq, Eq, Hash)]
11pub struct OptMatMulPack {
12    pub(crate) packers: Vec<PackedFormat>,
13    pub(crate) mode_picker: ModePicker,
14    pub(crate) k_axis: usize,
15    pub(crate) mn_axis: usize,
16}
17
18impl Op for OptMatMulPack {
19    fn name(&self) -> Cow<str> {
20        "OptMatMulPack".into()
21    }
22
23    fn info(&self) -> TractResult<Vec<String>> {
24        Ok(vec![format!("{:?}. k axis: {}, mn axis: {}", self.packers, self.k_axis, self.mn_axis)])
25    }
26
27    op_as_typed_op!();
28    impl_op_same_as!();
29}
30
31impl EvalOp for OptMatMulPack {
32    fn is_stateless(&self) -> bool {
33        true
34    }
35
36    fn eval_with_session(
37        &self,
38        session: &SessionState,
39        mut inputs: TVec<TValue>,
40    ) -> TractResult<TVec<TValue>> {
41        self.do_eval(session, inputs.remove(0))
42    }
43}
44
45impl TypedOp for OptMatMulPack {
46    fn output_facts(&self, inputs: &[&TypedFact]) -> TractResult<TVec<TypedFact>> {
47        match self.mode_picker {
48            ModePicker::Single => ensure!(self.packers.len() == 1),
49            ModePicker::VecVsMat => ensure!(self.packers.len() == 2),
50        }
51        let k = inputs[0].shape[self.k_axis].clone();
52        let mn = inputs[0].shape[self.mn_axis].clone();
53        let opaque_fact = DynPackedOpaqueFact { k, mn, packers: self.packers.clone() };
54        Ok(tvec!(Opaque::datum_type()
55            .fact(self.output_shape(&inputs[0].shape))
56            .with_opaque_fact(opaque_fact)))
57    }
58
59    fn axes_mapping(
60        &self,
61        inputs: &[&TypedFact],
62        outputs: &[&TypedFact],
63    ) -> TractResult<AxesMapping> {
64        let mut axes: Vec<Axis> = (0..inputs[0].rank())
65            .filter(|&ix| ix != self.k_axis && ix != self.mn_axis)
66            .enumerate()
67            .zip('a'..)
68            .map(|((o, i), repr)| Axis::new(repr, 1, 1).input(0, i).output(0, o))
69            .collect();
70        axes.push(Axis::new('K', 1, 1).input(0, self.k_axis));
71        axes.push(Axis::new('M', 1, 1).input(0, self.mn_axis));
72        axes.push(Axis::new('P', 1, 1).output(0, outputs[0].rank()));
73        AxesMapping::new(1, 1, axes)
74    }
75
76    as_op!();
77}
78
79impl OptMatMulPack {
80    fn do_eval(&self, _session: &SessionState, input: TValue) -> TractResult<TVec<TValue>> {
81        unsafe {
82            let mode = self.mode_picker.pick(input.shape()[self.mn_axis])?;
83            let packer = &self.packers[mode];
84            let output_shape: TVec<usize> = self.output_shape(input.shape());
85            let stores = if output_shape.iter().all(|d| *d == 1) {
86                tensor0::<Opaque>(
87                    packer.pack_tensor_view(&input.view(), self.k_axis, self.mn_axis)?.into(),
88                )
89                .into_shape(&output_shape)?
90            } else {
91                let mut stores = Tensor::uninitialized_dt(Opaque::datum_type(), &output_shape)?;
92                let mut stores_view = stores.to_array_view_mut::<Opaque>()?;
93                let mut bc_shape: TVec<usize> = input.shape().into();
94                bc_shape[self.k_axis] = 1;
95                bc_shape[self.mn_axis] = 1;
96
97                for coord in indices(&*bc_shape) {
98                    let offset = coord
99                        .as_array_view()
100                        .iter()
101                        .zip(input.strides())
102                        .map(|(x, s)| *x as isize * s)
103                        .sum::<isize>()
104                        * input.datum_type().size_of() as isize;
105                    let mut pack_coords: TVec<usize> = coord.slice().into();
106                    pack_coords.remove(self.k_axis.max(self.mn_axis));
107                    pack_coords.remove(self.k_axis.min(self.mn_axis));
108                    stores_view[&*pack_coords] = packer
109                        .pack_tensor_view(
110                            &TensorView::from_bytes(&input, offset, input.shape(), input.strides()),
111                            self.k_axis,
112                            self.mn_axis,
113                        )?
114                        .into();
115                }
116                stores
117            };
118            Ok(tvec!(stores.into_tvalue()))
119        }
120    }
121
122    pub fn output_shape<D: DimLike>(&self, input: &[D]) -> TVec<D> {
123        let mut packed_shape: TVec<D> = input.into();
124        packed_shape.remove(self.mn_axis.max(self.k_axis));
125        packed_shape.remove(self.mn_axis.min(self.k_axis));
126        packed_shape
127    }
128}
129
130#[derive(Hash, Clone, Debug, PartialEq, Eq)]
131pub struct DynPackedOpaqueFact {
132    pub k: TDim,
133    pub mn: TDim,
134    pub packers: Vec<PackedFormat>,
135}
136
137impl OpaqueFact for DynPackedOpaqueFact {
138    fn mem_size(&self) -> TDim {
139        self.k.clone() * &self.mn * self.packers[0].dt.size_of()
140    }
141
142    fn same_as(&self, other: &dyn OpaqueFact) -> bool {
143        other.downcast_ref::<Self>().is_some_and(|o| o == self)
144    }
145}
146
147#[derive(Debug, Clone, Hash, Eq, PartialEq)]
148pub struct OptSimpleMatMulPack {
149    pub(crate) packed_format: PackedBlockQuantFormat,
150    pub(crate) k: usize,
151    pub(crate) m: usize,
152}
153
154impl Op for OptSimpleMatMulPack {
155    fn name(&self) -> Cow<str> {
156        "OptSimpleMatMulPack".into()
157    }
158    op_as_typed_op!();
159    impl_op_same_as!();
160}
161
162impl EvalOp for OptSimpleMatMulPack {
163    fn is_stateless(&self) -> bool {
164        true
165    }
166
167    fn state(
168        &self,
169        _session: &mut SessionState,
170        _node_id: usize,
171    ) -> TractResult<Option<Box<dyn OpState>>> {
172        Ok(None)
173    }
174
175    fn eval(&self, inputs: TVec<TValue>) -> TractResult<TVec<TValue>> {
176        let input = args_1!(inputs);
177        let mut output = tensor1(
178            &input
179                .as_slice::<Opaque>()?
180                .iter()
181                .map(|i| {
182                    let i = i.downcast_ref::<BlockQuantValue>().unwrap();
183                    let iv: Box<dyn MMMInputValue> =
184                        Box::new(self.packed_format.pack(&i.value, i.fact.k())?);
185                    Ok(Opaque(Arc::new(iv)))
186                })
187                .collect::<TractResult<Vec<_>>>()?,
188        );
189        output.set_shape(input.shape())?;
190        Ok(tvec!(output.into_tvalue()))
191    }
192}
193
194impl TypedOp for OptSimpleMatMulPack {
195    fn output_facts(&self, inputs: &[&TypedFact]) -> TractResult<TVec<TypedFact>> {
196        let fact = Opaque::fact(inputs[0].shape.clone()).with_opaque_fact(PackedBlockQuantFact {
197            format: self.packed_format.clone(),
198            shape: tvec!(self.m, self.k),
199        });
200        Ok(tvec!(fact))
201    }
202
203    as_op!();
204}
tract_core/ops/matmul/pack.rs

tract_core/ops/matmul/
pack.rs
