cubecl_ir/
processing.rs

1use core::fmt::Display;
2
3use alloc::string::ToString;
4use alloc::vec::Vec;
5
6use crate::{Allocator, AtomicOp, Bitwise, Comparison, Operator};
7
8use super::{
9    Arithmetic, Branch, CoopMma, ElemType, Instruction, Metadata, Operation, UIntKind, Variable,
10    VariableKind,
11};
12
13pub trait Processor: core::fmt::Debug {
14    fn transform(&self, processing: ScopeProcessing, allocator: Allocator) -> ScopeProcessing;
15}
16
17/// Information necessary when compiling a scope.
18pub struct ScopeProcessing {
19    /// The variable declarations.
20    pub variables: Vec<Variable>,
21    /// The operations.
22    pub instructions: Vec<Instruction>,
23}
24
25impl Display for ScopeProcessing {
26    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
27        writeln!(f, "{{")?;
28        for instruction in self.instructions.iter() {
29            let instruction_str = instruction.to_string();
30            if !instruction_str.is_empty() {
31                writeln!(f, "    {instruction_str}")?;
32            }
33        }
34        write!(f, "}}")?;
35        Ok(())
36    }
37}
38
39impl ScopeProcessing {
40    /// Optimize the [variables](Variable) and [operations](Operation).
41    ///
42    /// ## Notes:
43    ///
44    /// This should be called once right after the creation of the type.
45    /// If you built this type from the [scope process function](super::Scope::process), you don't have to
46    /// call it again.
47    pub fn optimize(self) -> Self {
48        self.sanitize_constant_scalars()
49    }
50
51    /// Make sure constant scalars are of the correct type so compilers don't have to do conversion
52    /// and handle edge cases such as indexing with a signed integer.
53    fn sanitize_constant_scalars(mut self) -> Self {
54        self.instructions
55            .iter_mut()
56            .for_each(|inst| match &mut inst.operation {
57                Operation::Copy(op) => {
58                    sanitize_constant_scalar_ref_var(op, &inst.out.unwrap());
59                }
60                Operation::Arithmetic(op) => match op {
61                    Arithmetic::Add(op) => {
62                        sanitize_constant_scalar_ref_var(&mut op.lhs, &inst.out.unwrap());
63                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
64                    }
65                    Arithmetic::SaturatingAdd(op) => {
66                        sanitize_constant_scalar_ref_var(&mut op.lhs, &inst.out.unwrap());
67                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
68                    }
69                    Arithmetic::Fma(op) => {
70                        sanitize_constant_scalar_ref_var(&mut op.a, &inst.out.unwrap());
71                        sanitize_constant_scalar_ref_var(&mut op.b, &inst.out.unwrap());
72                        sanitize_constant_scalar_ref_var(&mut op.c, &inst.out.unwrap());
73                    }
74                    Arithmetic::Sub(op) => {
75                        sanitize_constant_scalar_ref_var(&mut op.lhs, &inst.out.unwrap());
76                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
77                    }
78                    Arithmetic::SaturatingSub(op) => {
79                        sanitize_constant_scalar_ref_var(&mut op.lhs, &inst.out.unwrap());
80                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
81                    }
82                    Arithmetic::Mul(op) => {
83                        sanitize_constant_scalar_ref_var(&mut op.lhs, &inst.out.unwrap());
84                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
85                    }
86                    Arithmetic::Div(op) => {
87                        sanitize_constant_scalar_ref_var(&mut op.lhs, &inst.out.unwrap());
88                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
89                    }
90                    Arithmetic::MulHi(op) => {
91                        sanitize_constant_scalar_ref_var(&mut op.lhs, &inst.out.unwrap());
92                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
93                    }
94                    Arithmetic::Abs(op) => {
95                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
96                    }
97                    Arithmetic::Exp(op) => {
98                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
99                    }
100                    Arithmetic::Log(op) => {
101                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
102                    }
103                    Arithmetic::Log1p(op) => {
104                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
105                    }
106                    Arithmetic::Cos(op) => {
107                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
108                    }
109                    Arithmetic::Sin(op) => {
110                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
111                    }
112                    Arithmetic::Tanh(op) => {
113                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
114                    }
115                    Arithmetic::Powf(op) => {
116                        sanitize_constant_scalar_ref_var(&mut op.lhs, &inst.out.unwrap());
117                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
118                    }
119                    Arithmetic::Powi(op) => {
120                        sanitize_constant_scalar_ref_var(&mut op.lhs, &inst.out.unwrap());
121                    }
122                    Arithmetic::Sqrt(op) => {
123                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
124                    }
125                    Arithmetic::Round(op) => {
126                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
127                    }
128                    Arithmetic::Floor(op) => {
129                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
130                    }
131                    Arithmetic::Ceil(op) => {
132                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
133                    }
134                    Arithmetic::Trunc(op) => {
135                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
136                    }
137                    Arithmetic::Erf(op) => {
138                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
139                    }
140                    Arithmetic::Recip(op) => {
141                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
142                    }
143                    Arithmetic::Clamp(op) => {
144                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
145                        sanitize_constant_scalar_ref_var(&mut op.min_value, &inst.out.unwrap());
146                        sanitize_constant_scalar_ref_var(&mut op.max_value, &inst.out.unwrap());
147                    }
148                    Arithmetic::Modulo(op) => {
149                        sanitize_constant_scalar_ref_var(&mut op.lhs, &inst.out.unwrap());
150                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
151                    }
152                    Arithmetic::Neg(op) => {
153                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap())
154                    }
155                    Arithmetic::Max(op) => {
156                        sanitize_constant_scalar_ref_var(&mut op.lhs, &inst.out.unwrap());
157                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
158                    }
159                    Arithmetic::Min(op) => {
160                        sanitize_constant_scalar_ref_var(&mut op.lhs, &inst.out.unwrap());
161                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
162                    }
163                    Arithmetic::Remainder(op) => {
164                        sanitize_constant_scalar_ref_var(&mut op.lhs, &inst.out.unwrap());
165                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
166                    }
167                    Arithmetic::Magnitude(op) => {
168                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
169                    }
170                    Arithmetic::Normalize(op) => {
171                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
172                    }
173                    Arithmetic::Dot(op) => {
174                        sanitize_constant_scalar_ref_var(&mut op.lhs, &inst.out.unwrap());
175                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
176                    }
177                },
178                Operation::Comparison(op) => match op {
179                    Comparison::Greater(op) => {
180                        sanitize_constant_scalar_ref_var(&mut op.lhs, &op.rhs);
181                        sanitize_constant_scalar_ref_var(&mut op.rhs, &op.lhs);
182                    }
183                    Comparison::LowerEqual(op) => {
184                        sanitize_constant_scalar_ref_var(&mut op.lhs, &op.rhs);
185                        sanitize_constant_scalar_ref_var(&mut op.rhs, &op.lhs);
186                    }
187                    Comparison::GreaterEqual(op) => {
188                        sanitize_constant_scalar_ref_var(&mut op.lhs, &op.rhs);
189                        sanitize_constant_scalar_ref_var(&mut op.rhs, &op.lhs);
190                    }
191                    Comparison::Equal(op) => {
192                        sanitize_constant_scalar_ref_var(&mut op.lhs, &op.rhs);
193                        sanitize_constant_scalar_ref_var(&mut op.rhs, &op.lhs);
194                    }
195                    Comparison::NotEqual(op) => {
196                        sanitize_constant_scalar_ref_var(&mut op.lhs, &op.rhs);
197                        sanitize_constant_scalar_ref_var(&mut op.rhs, &op.lhs);
198                    }
199                    Comparison::Lower(op) => {
200                        sanitize_constant_scalar_ref_var(&mut op.lhs, &op.rhs);
201                        sanitize_constant_scalar_ref_var(&mut op.rhs, &op.lhs);
202                    }
203                    Comparison::IsNan(_op) | Comparison::IsInf(_op) => {
204                        // Nothing to do
205                    }
206                },
207                Operation::Bitwise(op) => match op {
208                    Bitwise::BitwiseAnd(op) => {
209                        sanitize_constant_scalar_ref_var(&mut op.lhs, &inst.out.unwrap());
210                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
211                    }
212                    Bitwise::BitwiseOr(op) => {
213                        sanitize_constant_scalar_ref_var(&mut op.lhs, &inst.out.unwrap());
214                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
215                    }
216                    Bitwise::BitwiseXor(op) => {
217                        sanitize_constant_scalar_ref_var(&mut op.lhs, &inst.out.unwrap());
218                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
219                    }
220                    Bitwise::CountOnes(_) | Bitwise::LeadingZeros(_) | Bitwise::FindFirstSet(_) => {
221                        // Nothing to do
222                    }
223                    Bitwise::ReverseBits(op) => {
224                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
225                    }
226                    Bitwise::ShiftLeft(op) => {
227                        sanitize_constant_scalar_ref_var(&mut op.lhs, &inst.out.unwrap());
228                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
229                    }
230                    Bitwise::ShiftRight(op) => {
231                        sanitize_constant_scalar_ref_var(&mut op.lhs, &inst.out.unwrap());
232                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
233                    }
234                    Bitwise::BitwiseNot(op) => {
235                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
236                    }
237                },
238                Operation::Operator(op) => match op {
239                    Operator::Index(op) => {
240                        sanitize_constant_scalar_ref_var(&mut op.list, &inst.out.unwrap());
241                        sanitize_constant_scalar_ref_elem(
242                            &mut op.index,
243                            ElemType::UInt(UIntKind::U32),
244                        );
245                    }
246                    Operator::UncheckedIndex(op) => {
247                        sanitize_constant_scalar_ref_var(&mut op.list, &inst.out.unwrap());
248                        sanitize_constant_scalar_ref_elem(
249                            &mut op.index,
250                            ElemType::UInt(UIntKind::U32),
251                        );
252                    }
253                    Operator::IndexAssign(op) => {
254                        sanitize_constant_scalar_ref_elem(
255                            &mut op.index,
256                            ElemType::UInt(UIntKind::U32),
257                        );
258                        sanitize_constant_scalar_ref_var(&mut op.value, &inst.out.unwrap());
259                    }
260                    Operator::UncheckedIndexAssign(op) => {
261                        sanitize_constant_scalar_ref_elem(
262                            &mut op.index,
263                            ElemType::UInt(UIntKind::U32),
264                        );
265                        sanitize_constant_scalar_ref_var(&mut op.value, &inst.out.unwrap());
266                    }
267                    Operator::And(op) => {
268                        sanitize_constant_scalar_ref_var(&mut op.lhs, &op.rhs);
269                        sanitize_constant_scalar_ref_var(&mut op.rhs, &op.lhs);
270                    }
271                    Operator::Or(op) => {
272                        sanitize_constant_scalar_ref_var(&mut op.lhs, &op.rhs);
273                        sanitize_constant_scalar_ref_var(&mut op.rhs, &op.lhs);
274                    }
275                    Operator::Not(op) => {
276                        sanitize_constant_scalar_ref_elem(&mut op.input, ElemType::Bool);
277                    }
278                    Operator::InitLine(_) => {
279                        // TODO: Sanitize based on elem
280                    }
281                    Operator::CopyMemory(op) => {
282                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
283                        sanitize_constant_scalar_ref_elem(
284                            &mut op.in_index,
285                            ElemType::UInt(UIntKind::U32),
286                        );
287                        sanitize_constant_scalar_ref_elem(
288                            &mut op.out_index,
289                            ElemType::UInt(UIntKind::U32),
290                        );
291                    }
292                    Operator::CopyMemoryBulk(op) => {
293                        sanitize_constant_scalar_ref_var(&mut op.input, &inst.out.unwrap());
294                        sanitize_constant_scalar_ref_elem(
295                            &mut op.in_index,
296                            ElemType::UInt(UIntKind::U32),
297                        );
298                        sanitize_constant_scalar_ref_elem(
299                            &mut op.out_index,
300                            ElemType::UInt(UIntKind::U32),
301                        );
302                    }
303                    Operator::Select(op) => {
304                        sanitize_constant_scalar_ref_elem(&mut op.cond, ElemType::Bool);
305                        sanitize_constant_scalar_ref_var(&mut op.then, &inst.out.unwrap());
306                        sanitize_constant_scalar_ref_var(&mut op.or_else, &inst.out.unwrap());
307                    }
308                    Operator::Cast(_) => {}
309                    Operator::Reinterpret(_) => {}
310                },
311                Operation::Atomic(op) => match op {
312                    AtomicOp::Load(_) => {}
313                    AtomicOp::Store(_) => {}
314                    AtomicOp::Swap(op) => {
315                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
316                    }
317                    AtomicOp::CompareAndSwap(op) => {
318                        sanitize_constant_scalar_ref_var(&mut op.cmp, &inst.out.unwrap());
319                        sanitize_constant_scalar_ref_var(&mut op.val, &inst.out.unwrap());
320                    }
321                    AtomicOp::Add(op) => {
322                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
323                    }
324                    AtomicOp::Sub(op) => {
325                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
326                    }
327                    AtomicOp::Max(op) => {
328                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
329                    }
330                    AtomicOp::Min(op) => {
331                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
332                    }
333                    AtomicOp::And(op) => {
334                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
335                    }
336                    AtomicOp::Or(op) => {
337                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
338                    }
339                    AtomicOp::Xor(op) => {
340                        sanitize_constant_scalar_ref_var(&mut op.rhs, &inst.out.unwrap());
341                    }
342                },
343                Operation::Metadata(op) => match op {
344                    Metadata::Stride { dim, .. } => {
345                        sanitize_constant_scalar_ref_elem(dim, ElemType::UInt(UIntKind::U32));
346                    }
347                    Metadata::Shape { dim, .. } => {
348                        sanitize_constant_scalar_ref_elem(dim, ElemType::UInt(UIntKind::U32));
349                    }
350                    Metadata::Length { .. }
351                    | Metadata::BufferLength { .. }
352                    | Metadata::Rank { .. } => {
353                        // Nothing to do
354                    }
355                },
356                Operation::Branch(op) => match op {
357                    Branch::If(op) => {
358                        sanitize_constant_scalar_ref_elem(&mut op.cond, ElemType::Bool);
359                    }
360                    Branch::IfElse(op) => {
361                        sanitize_constant_scalar_ref_elem(&mut op.cond, ElemType::Bool);
362                    }
363                    Branch::RangeLoop(op) => {
364                        sanitize_constant_scalar_ref_var(&mut op.end, &op.start);
365                        sanitize_constant_scalar_ref_var(&mut op.i, &op.start);
366                        if let Some(step) = &mut op.step {
367                            sanitize_constant_scalar_ref_elem(step, ElemType::UInt(UIntKind::U32));
368                        }
369                    }
370                    _ => {
371                        // Nothing to do.
372                    }
373                },
374                Operation::Synchronization(_) => {
375                    // Nothing to do.
376                }
377                Operation::Plane(_) => {
378                    // Nothing to do since no constant is possible.
379                }
380                Operation::CoopMma(op) => match op {
381                    CoopMma::Fill { value } => {
382                        sanitize_constant_scalar_ref_var(value, &inst.out.unwrap());
383                    }
384                    CoopMma::Load { value, stride, .. } => {
385                        sanitize_constant_scalar_ref_var(value, &inst.out.unwrap());
386                        sanitize_constant_scalar_ref_elem(stride, ElemType::UInt(UIntKind::U32));
387                    }
388                    CoopMma::Execute { .. }
389                    | CoopMma::ExecuteManual { .. }
390                    | CoopMma::ExecuteScaled { .. } => {
391                        // Nothing to do.
392                    }
393                    CoopMma::Store { stride, .. } => {
394                        sanitize_constant_scalar_ref_elem(stride, ElemType::UInt(UIntKind::U32));
395                    }
396                    CoopMma::Cast { .. } => {
397                        // Nothing to do.
398                    }
399                    CoopMma::RowIndex { lane_id, i, .. } => {
400                        sanitize_constant_scalar_ref_elem(lane_id, ElemType::UInt(UIntKind::U32));
401                        sanitize_constant_scalar_ref_elem(i, ElemType::UInt(UIntKind::U32));
402                    }
403                    CoopMma::ColIndex { lane_id, i, .. } => {
404                        sanitize_constant_scalar_ref_elem(lane_id, ElemType::UInt(UIntKind::U32));
405                        sanitize_constant_scalar_ref_elem(i, ElemType::UInt(UIntKind::U32));
406                    }
407                },
408                Operation::NonSemantic(_) => {
409                    // Nothing to do.
410                }
411                Operation::Barrier(_) => {
412                    // Nothing to do
413                }
414                Operation::Tma(_) => {
415                    // Nothing to do
416                }
417                Operation::Free(_) => {
418                    // Nothing to do
419                }
420            });
421        self
422    }
423}
424
425fn sanitize_constant_scalar_ref_var(var: &mut Variable, reference: &Variable) {
426    let elem = reference.ty.elem_type();
427    sanitize_constant_scalar_ref_elem(var, elem);
428}
429
430fn sanitize_constant_scalar_ref_elem(var: &mut Variable, elem: ElemType) {
431    if let VariableKind::ConstantScalar(scalar) = var.kind
432        && scalar.elem_type() != elem
433    {
434        *var = match scalar {
435            super::ConstantScalarValue::Int(val, _) => elem.constant_from_i64(val),
436            super::ConstantScalarValue::Float(val, _) => elem.constant_from_f64(val),
437            super::ConstantScalarValue::UInt(val, _) => elem.constant_from_u64(val),
438            super::ConstantScalarValue::Bool(val) => elem.constant_from_bool(val),
439        };
440    }
441}
cubecl_ir/processing.rs

cubecl_ir/
processing.rs
