Skip to main content

hydroplane_macros/
lib.rs

1//! The `#[kernel]` attribute — write a `hydroplane` kernel as a plain generic function.
2//!
3//! From a function whose first parameter is the SIMD context, `#[kernel]` generates the kernel
4//! struct and `Kernel` impl (`MatrixKernel` with `#[kernel(matrix)]`, adding the `.tiles()`
5//! surface), a callable wrapper that runs `dispatch` for you, and a `<name>_on(ctx, …)` companion
6//! that runs the body on a caller-supplied context — call it from another kernel to skip dispatch.
7//!
8//! ```ignore
9//! #[hydroplane::kernel]
10//! fn any_overlap<'a, T: Scalar>(ctx: Gang, soa: &'a Soa<T>, q: [T; 4]) -> bool { /* body */ }
11//! let hit = any_overlap(&soa, q);   // call site — no struct, no impl, no `dispatch`
12//! ```
13//!
14//! The context is written bare (`ctx: Gang`) or concrete (`ctx: Gang<f32>`); every later parameter
15//! becomes a struct field carried across the dispatch boundary, so borrows need explicit
16//! lifetimes. The dispatch element is the generic bound by `Scalar`/`FloatScalar`/`IntScalar` (or
17//! the one named `T`), inferred for concrete kernels; attribute overrides: `scalar = U`, extra
18//! element names (OR'd into the dispatch type-combo), and `unroll = N`, which caps the ILP unroll
19//! factor for this kernel. `tiny` and `noalias` are still accepted but no
20//! longer change the generated code: the body always runs in an inlined `_on`.
21//!
22//! Inside the body, `a * b + c` and `c + a * b` on varyings compile to a fused multiply-add, as
23//! `.fma` would; on anything else they stay the two operators.
24
25use hydroplane_auto::analysis;
26
27use proc_macro::TokenStream;
28use proc_macro2::{Span, TokenStream as TokenStream2};
29use quote::{ToTokens, format_ident, quote};
30use syn::parse::Parser;
31use syn::spanned::Spanned;
32use syn::{
33    FnArg, GenericParam, Generics, Ident, ItemFn, Meta, Pat, ReturnType, Token, Type,
34    TypeParamBound, WherePredicate, parse_macro_input, parse_quote, punctuated::Punctuated,
35};
36
37/// Generate a [`Kernel`] (or, with `#[kernel(matrix)]`, a `MatrixKernel`) and a dispatching wrapper
38/// from one annotated function. See the [crate docs](crate) for the full shape.
39///
40/// [`Kernel`]: ../hydroplane/trait.Kernel.html
41#[proc_macro_attribute]
42pub fn kernel(attr: TokenStream, item: TokenStream) -> TokenStream {
43    let func = parse_macro_input!(item as ItemFn);
44    let opts = match KernelOpts::parse(attr.into()) {
45        Ok(o) => o,
46        Err(e) => return e.to_compile_error().into(),
47    };
48    match expand(func, opts) {
49        Ok(ts) => ts.into(),
50        Err(e) => e.to_compile_error().into(),
51    }
52}
53
54struct KernelOpts {
55    /// `#[kernel(matrix)]`: the context is a [`MatrixBackend`] (still a [`Backend`], so vector ops
56    /// work too).
57    matrix: bool,
58    scalar: Option<Ident>,
59    /// Whether the build-time analysis measures this kernel (its body is the `_on` companion);
60    /// `tiny` turns it off.
61    noalias: bool,
62    /// `tiny` or `noalias` was written explicitly; build-time analysis must not override it.
63    explicit_boundary: bool,
64    /// Explicit ILP unroll cap. The combinators already fit their chains to the accumulator's
65    /// registers; this is for steps that keep many other varyings live.
66    unroll: Option<usize>,
67    /// Element names listed in the attribute (`#[kernel(u32, f32)]`): OR'd into the type-combo
68    /// bitmask unconditionally, even over MIR-measured sets. The first listed doubles as the
69    /// dispatch element when nothing else names one.
70    types: Vec<Ident>,
71}
72
73fn element_bits(name: &str) -> Option<u8> {
74    Some(match name {
75        "f32" => 1,
76        "f64" => 2,
77        "f16" => 4,
78        "bf16" => 8,
79        "u32" => 16,
80        "i32" => 32,
81        _ => return None,
82    })
83}
84
85impl KernelOpts {
86    fn parse(tokens: TokenStream2) -> syn::Result<Self> {
87        let mut matrix = false;
88        let mut scalar = None;
89        let mut noalias = true;
90        let mut tiny_span = None;
91        let mut noalias_span = None;
92        let mut unroll = None;
93        let mut types = Vec::new();
94        let metas = Punctuated::<Meta, Token![,]>::parse_terminated.parse2(tokens)?;
95        for meta in metas {
96            match meta {
97                Meta::Path(p)
98                    if p.get_ident().is_some_and(|i| element_bits(&i.to_string()).is_some()) =>
99                {
100                    types.push(p.get_ident().unwrap().clone());
101                }
102                Meta::Path(p) if p.is_ident("matrix") => {
103                    if matrix {
104                        return Err(syn::Error::new(p.span(), "duplicate `matrix`"));
105                    }
106                    matrix = true;
107                }
108                Meta::Path(p) if p.is_ident("tiny") => {
109                    if tiny_span.is_some() {
110                        return Err(syn::Error::new(p.span(), "duplicate `tiny`"));
111                    }
112                    tiny_span = Some(p.span());
113                    noalias = false;
114                }
115                // Already the default; accepting it explicitly marks the boundary as authored.
116                Meta::Path(p) if p.is_ident("noalias") => {
117                    if noalias_span.is_some() {
118                        return Err(syn::Error::new(p.span(), "duplicate `noalias`"));
119                    }
120                    noalias_span = Some(p.span());
121                }
122                Meta::NameValue(nv) if nv.path.is_ident("scalar") => {
123                    let id = match &nv.value {
124                        syn::Expr::Path(ep) => ep.path.get_ident().cloned(),
125                        _ => None,
126                    };
127                    match id {
128                        Some(id) => scalar = Some(id),
129                        None => {
130                            return Err(syn::Error::new_spanned(
131                                &nv.value,
132                                "expected an identifier, e.g. `scalar = T`",
133                            ));
134                        }
135                    }
136                }
137                Meta::NameValue(nv) if nv.path.is_ident("unroll") => match &nv.value {
138                    syn::Expr::Lit(syn::ExprLit { lit: syn::Lit::Int(i), .. }) => {
139                        unroll = Some(i.base10_parse::<usize>()?);
140                    }
141                    _ => {
142                        return Err(syn::Error::new_spanned(
143                            &nv.value,
144                            "expected an integer, e.g. `unroll = 4`",
145                        ));
146                    }
147                },
148                other => {
149                    return Err(syn::Error::new_spanned(
150                        other,
151                        "unknown `#[kernel]` option; expected `matrix`, `scalar = Ident`, \
152                         `unroll = N`, `tiny`, or `noalias`",
153                    ));
154                }
155            }
156        }
157        if let (Some(ts), Some(ns)) = (tiny_span, noalias_span) {
158            let mut e = syn::Error::new(ns, "`noalias` and `tiny` are opposites — pick one");
159            e.combine(syn::Error::new(ts, "`tiny` set here"));
160            return Err(e);
161        }
162        let explicit_boundary = tiny_span.is_some() || noalias_span.is_some();
163        Ok(KernelOpts { matrix, scalar, noalias, explicit_boundary, unroll, types })
164    }
165}
166
167fn expand(func: ItemFn, opts: KernelOpts) -> syn::Result<TokenStream2> {
168    let func_tokens = func.to_token_stream();
169    let ItemFn {
170        attrs,
171        vis,
172        sig,
173        mut block,
174    } = func;
175    syn::visit_mut::VisitMut::visit_block_mut(&mut FuseMulAdd, &mut block);
176
177    if let Some(a) = sig.asyncness {
178        return Err(syn::Error::new_spanned(a, "a kernel cannot be `async`"));
179    }
180    if let Some(c) = sig.constness {
181        return Err(syn::Error::new_spanned(c, "a kernel cannot be `const`"));
182    }
183    if let Some(v) = sig.variadic {
184        return Err(syn::Error::new_spanned(v, "a kernel cannot be variadic"));
185    }
186
187    let mut inputs = sig.inputs.into_iter();
188    let mut ctx_idents = Vec::with_capacity(1);
189    let mut ctx_tys = Vec::with_capacity(1);
190    let mut ctx_scalar = None;
191    for _ in 0..1 {
192        match inputs.next() {
193            Some(FnArg::Typed(pt)) => {
194                match *pt.pat {
195                    Pat::Ident(pi) => ctx_idents.push(pi.ident),
196                    other => {
197                        return Err(syn::Error::new_spanned(
198                            other,
199                            "context parameters must be plain identifiers",
200                        ));
201                    }
202                }
203                let ty = *pt.ty;
204                if ctx_scalar.is_none() {
205                    ctx_scalar = ctx_scalar_arg(&ty);
206                }
207                ctx_tys.push(ctx_type_with_backend(ty)?);
208            }
209            Some(FnArg::Receiver(r)) => {
210                return Err(syn::Error::new_spanned(
211                    r,
212                    "a kernel takes no `self` receiver",
213                ));
214            }
215            None => {
216                return Err(syn::Error::new_spanned(
217                    &sig.ident,
218                    "a kernel needs a leading context parameter (e.g. `ctx: Gang<T>`) before its \
219                     data parameters",
220                ));
221            }
222        }
223    }
224
225    let mut field_idents = Vec::new();
226    let mut field_tys = Vec::new();
227    for arg in inputs {
228        match arg {
229            FnArg::Typed(pt) => {
230                // `#[hint_cnt(N)]` is a slice-length hint, parsed and discarded (reserved for
231                // count-driven codegen). Never re-emitted, so it can't reach rustc.
232                let _hint_cnt = parse_hint_cnt(&pt.attrs);
233                match *pt.pat {
234                    Pat::Ident(pi) => field_idents.push(pi.ident),
235                    other => {
236                        return Err(syn::Error::new_spanned(
237                            other,
238                            "kernel parameters must be plain identifiers",
239                        ));
240                    }
241                }
242                field_tys.push(*pt.ty);
243            }
244            FnArg::Receiver(r) => {
245                return Err(syn::Error::new_spanned(
246                    r,
247                    "a kernel takes no `self` receiver",
248                ));
249            }
250        }
251    }
252
253    let ret: Type = match sig.output {
254        ReturnType::Default => parse_quote!(()),
255        ReturnType::Type(_, t) => *t,
256    };
257
258    // Scalar precedence: explicit `scalar = …` > `Scalar`-bound or `T`-named generic > the
259    // context's element type > first attribute element > lowest scanned element > `f32`. The
260    // element only keys the `Kernel<T>` trait, so any element the kernel touches is correct.
261    let scanned_bits_early = {
262        let mut ts = TokenStream2::new();
263        func_tokens.to_tokens(&mut ts);
264        scan_type_bits(&ts)
265    };
266    let detected = detect_scalar(&sig.generics)?;
267    let scalar = if let Some(s) = &opts.scalar {
268        quote!(#s)
269    } else if let Some(s) = &detected {
270        quote!(#s)
271    } else if let Some(t) = &ctx_scalar {
272        quote!(#t)
273    } else if let Some(first) = opts.types.first() {
274        quote!(#first)
275    } else {
276        match 1u8 << scanned_bits_early.trailing_zeros().min(6) {
277            1 => quote!(f32),
278            2 => quote!(f64),
279            4 => quote!(::hydroplane::f16),
280            8 => quote!(::hydroplane::bf16),
281            16 => quote!(u32),
282            32 => quote!(i32),
283            _ => quote!(f32),
284        }
285    };
286
287    let generics = sig.generics;
288    let (impl_generics, ty_generics, where_clause) = generics.split_for_impl();
289
290    let mut lifetimes = Vec::new();
291    let mut type_params = Vec::new();
292    let mut turbofish_args = Vec::new();
293    for p in &generics.params {
294        match p {
295            GenericParam::Lifetime(l) => lifetimes.push(l.lifetime.clone()),
296            GenericParam::Type(t) => {
297                type_params.push(t.ident.clone());
298                turbofish_args.push(t.ident.clone());
299            }
300            GenericParam::Const(c) => turbofish_args.push(c.ident.clone()),
301        }
302    }
303
304    // PhantomData over every lifetime and type parameter keeps the struct well-formed when a
305    // parameter only appears in the return type or a const dimension; `*const T` adds no
306    // Send/Sync/Drop obligation. Unused const generics are allowed, so no marker for them.
307    let phantom_items: Vec<TokenStream2> = lifetimes
308        .iter()
309        .map(|lt| quote!(& #lt ()))
310        .chain(type_params.iter().map(|t| quote!(*const #t)))
311        .collect();
312    let (phantom_field, phantom_init) = if phantom_items.is_empty() {
313        (quote!(), quote!())
314    } else {
315        (
316            quote!(pub(super) __phantom: ::core::marker::PhantomData<(#(#phantom_items),*)>,),
317            quote!(__phantom: ::core::marker::PhantomData,),
318        )
319    };
320
321    let matrix_present = opts.matrix;
322    let (ktrait, bbound, dispatch_fn, dbound) = if matrix_present {
323        (
324            quote!(::hydroplane::MatrixKernel),
325            quote!(::hydroplane::MatrixBackend),
326            quote!(::hydroplane::dispatch_matrix),
327            quote!(::hydroplane::MatrixDispatch),
328        )
329    } else {
330        (
331            quote!(::hydroplane::Kernel),
332            quote!(::hydroplane::Backend),
333            quote!(::hydroplane::dispatch),
334            quote!(::hydroplane::SimdDispatch),
335        )
336    };
337
338    let run_bound = if matrix_present {
339        quote!(#bbound<#scalar>)
340    } else {
341        quote!(::hydroplane::BackendAll + ::hydroplane::Backend<#scalar>)
342    };
343
344    let mut wrapper_generics = generics.clone();
345    let pred: WherePredicate = parse_quote!(#scalar: #dbound);
346    wrapper_generics.make_where_clause().predicates.push(pred);
347    let (w_impl_generics, _, w_where) = wrapper_generics.split_for_impl();
348
349    let turbofish = if turbofish_args.is_empty() {
350        quote!()
351    } else {
352        quote!(::<#(#turbofish_args),*>)
353    };
354
355    let name = sig.ident;
356
357    // Fold build-time MIR analysis into the codegen knobs. Matrix kernels dispatch through a
358    // different path (no `Unroll` wrapper, no `run_scalar`), so they take none of it; an explicit
359    // `tiny`/`noalias` always wins over the analysis boundary choice.
360    let name_str = name.to_string();
361    let metrics = if matrix_present {
362        None
363    } else {
364        analysis::lookup(&name_str)
365    };
366    let use_noalias = match metrics {
367        Some(m) if !opts.explicit_boundary => m.noalias(),
368        _ => opts.noalias,
369    };
370    let k_cap = opts.unroll;
371
372    // Under the default `noalias` boundary the real body lives in `_on`, which is what the analysis
373    // driver measures; explicit-`tiny` and matrix kernels aren't measured. The metrics attribute is
374    // emitted only inside the driver's nested build (`HYDRO_ANALYZE_INNER` + `--cfg hp_analyze`):
375    // a normal consumer build must never see the `hp_analyze` token, because `unexpected_cfgs`
376    // fires before any generated `#[allow]` could apply, so keeping the `cfg_attr` out entirely is
377    // the only way consumers stay warning-free without registering the cfg themselves.
378    // Only the crate under analysis gets it: dependencies with kernels of their own compile with
379    // the same `--cfg hp_analyze` but through plain rustc, which rejects the unregistered tool.
380    let analyzed = std::env::var("HYDRO_ANALYZE_CRATE").ok().is_some_and(|want| {
381        std::env::var("CARGO_CRATE_NAME").is_ok_and(|name| name == want)
382    });
383    let metrics_attr = if use_noalias
384        && !matrix_present
385        && std::env::var_os("HYDRO_ANALYZE_INNER").is_some()
386        && analyzed
387    {
388        quote!(#[cfg_attr(hp_analyze, hp_analyze::metrics(kernel = #name_str))])
389    } else {
390        quote!()
391    };
392    let k_cap_item = match k_cap {
393        Some(k) => quote!(const K_CAP: usize = #k;),
394        None => quote!(),
395    };
396
397    // The trait's `run` takes one context; any extra contexts become `Copy` aliases of it so each
398    // keeps its declared name inside the body.
399    let primary_ctx = &ctx_idents[0];
400    let primary_ty = &ctx_tys[0];
401    let ctx_aliases: Vec<TokenStream2> = ctx_idents[1..]
402        .iter()
403        .map(|id| quote!(let #id = #primary_ctx;))
404        .collect();
405
406    // `<name>_on(ctx, …)` runs the body on a caller-supplied context, skipping dispatch, so nested
407    // kernel calls dispatch once at the outer boundary. Generic over the backend `__S`, no dispatch
408    // bound.
409    let on_name = format_ident!("{}_on", name);
410    let mut on_generics = generics.clone();
411    on_generics.params.push(syn::GenericParam::Type(parse_quote!(__S: #run_bound)));
412    let (on_impl_generics, _, on_where) = on_generics.split_for_impl();
413
414    let on_call_turbofish = if turbofish_args.is_empty() {
415        quote!(::<__S>)
416    } else {
417        quote!(::<#(#turbofish_args,)* __S>)
418    };
419    let run_body = quote! {
420        let Self { #( #field_idents, )* .. } = self;
421        #on_name #on_call_turbofish ( #primary_ctx, #( #field_idents, )* )
422    };
423
424    // The body is emitted once per ISA the build's architecture can dispatch to, each copy a
425    // `#[target_feature]` fn, plus a plain copy in `_on` for the baseline and scalar backends.
426    // `_on` picks the copy from the backend's `ISA` constant, which folds away per backend. Code
427    // compiled inside a copy (closures, `array::map`, helpers the body defines) inherits its
428    // target features, so every op inlines whether or not LLVM inlines the code around it. Matrix
429    // kernels keep a single body: their backends dispatch per tile.
430    let isa_copies: Vec<(&str, bool)> = if matrix_present {
431        Vec::new()
432    } else {
433        let generic = scalar_is_generic_early(&generics, &scalar);
434        let bits = metrics.and_then(|m| m.type_bits()).unwrap_or(scanned_bits_early);
435        vec![
436            ("SSE41", true),
437            ("AVX", generic || bits & 60 == 0),
438            ("AVX2", true),
439            ("AVX512", true),
440            ("AVX512FP16", generic || bits & 4 != 0),
441            ("AVX512BF16", generic || bits & 8 != 0),
442            ("SVE", generic || bits & 48 == 0),
443        ]
444    };
445    let mut isa_fns = Vec::new();
446    let mut isa_arms = Vec::new();
447    for (code, wanted) in isa_copies {
448        if !wanted {
449            continue;
450        }
451        let code_ident = format_ident!("{}", code);
452        let fn_name = format_ident!("__{}_{}", name, code.to_lowercase());
453        // `__hp_isa!` owns the tier table: each tier's feature bundle, its architecture, and
454        // which tiers this build of hydroplane dispatches to.
455        isa_fns.push(quote! {
456            ::hydroplane::__hp_isa! { kitem #code_ident
457                #[doc(hidden)]
458                #[inline]
459                #[allow(non_snake_case, clippy::too_many_arguments, clippy::multiple_bound_locations)]
460                fn #fn_name #on_impl_generics ( #primary_ctx: #primary_ty, #( #field_idents: #field_tys ),* ) -> #ret
461                #on_where
462                {
463                    #( #ctx_aliases )*
464                    #block
465                }
466            }
467        });
468        isa_arms.push(quote! {
469            ::hydroplane::__hp_isa! { kstmt #code_ident
470                if <__S as ::hydroplane::Backend<f32>>::ISA == ::hydroplane::isa::#code_ident {
471                    // SAFETY: a backend reports this tier only when its token was built on a CPU
472                    // with the tier's whole feature bundle.
473                    return unsafe { #fn_name #on_call_turbofish ( #primary_ctx, #( #field_idents, )* ) };
474                }
475            }
476        });
477    }
478    let on_body = quote! {
479        #( #isa_arms )*
480        #( #ctx_aliases )*
481        #block
482    };
483    let on_inline = quote!(#[inline(always)]);
484
485    // Type-combo: the scanned (or MIR-measured) element set plus the dispatch element's own bits,
486    // baked as a constant in the wrapper.
487    let attr_bits: u8 = opts
488        .types
489        .iter()
490        .filter_map(|i| element_bits(&i.to_string()))
491        .fold(0, |a, b| a | b);
492    let combo_bits = metrics.and_then(|m| m.type_bits()).unwrap_or(scanned_bits_early) | attr_bits;
493    // A generic scalar's tier arms cannot be pruned; the element is only known at monomorphization.
494    let scalar_str = quote!(#scalar).to_string();
495    let scalar_is_generic = generics.params.iter().any(|p| match p {
496        GenericParam::Type(t) => t.ident == scalar_str,
497        _ => false,
498    });
499    let may_half_f16 = combo_bits & 4 != 0;
500    let may_half_bf16 = combo_bits & 8 != 0;
501    let pure_float = combo_bits & 60 == 0;
502    let no_ints = combo_bits & 48 == 0;
503
504    let fp16_arm = may_half_f16.then(|| quote! {
505        #[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
506        // SAFETY (all arms): `combo_tier` returns a tier code only after feature detection (or a
507        // compile-time guarantee) confirmed the host supports it.
508        ::hydroplane::towers::AVX512FP16 => ::hydroplane::dispatch::run_kernel_on(
509            __kernel,
510            unsafe { ::hydroplane::towers::Avx512Fp16::new_unchecked() },
511        ),
512    });
513    let bf16_arm = may_half_bf16.then(|| quote! {
514        #[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
515        ::hydroplane::towers::AVX512BF16 => ::hydroplane::dispatch::run_kernel_on(
516            __kernel,
517            unsafe { ::hydroplane::towers::Avx512Bf16::new_unchecked() },
518        ),
519    });
520    let avx1_arm = pure_float.then(|| quote! {
521        #[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
522        ::hydroplane::towers::AVX1 => ::hydroplane::dispatch::run_kernel_on(
523            __kernel,
524            unsafe { ::hydroplane::towers::Avx1::new_unchecked() },
525        ),
526    });
527    let sve_arms = no_ints.then(|| quote! {
528        #[cfg(all(target_arch = "aarch64", not(target_vendor = "apple")))]
529        ::hydroplane::towers::SVE16 => ::hydroplane::dispatch::run_kernel_on(
530            __kernel,
531            unsafe { ::hydroplane::towers::Sve::<16>::new_unchecked() },
532        ),
533        #[cfg(all(target_arch = "aarch64", not(target_vendor = "apple")))]
534        ::hydroplane::towers::SVE32 => ::hydroplane::dispatch::run_kernel_on(
535            __kernel,
536            unsafe { ::hydroplane::towers::Sve::<32>::new_unchecked() },
537        ),
538        #[cfg(all(target_arch = "aarch64", not(target_vendor = "apple")))]
539        ::hydroplane::towers::SVE64 => ::hydroplane::dispatch::run_kernel_on(
540            __kernel,
541            unsafe { ::hydroplane::towers::Sve::<64>::new_unchecked() },
542        ),
543    });
544
545    // Combo dispatch needs `Token: Backend<T>` provable in the wrapper body, which only holds for
546    // concrete elements; generic-scalar kernels keep the element-keyed `dispatch` path.
547    let dispatch_call = if matrix_present || scalar_is_generic {
548        quote!(#dispatch_fn(#name::__Kernel #turbofish { #( #field_idents, )* #phantom_init }))
549    } else {
550        quote! {{
551            let __combo: u8 = #combo_bits | <#scalar as ::hydroplane::Scalar>::TYPE_BITS;
552            let __kernel = #name::__Kernel #turbofish { #( #field_idents, )* #phantom_init };
553            match ::hydroplane::dispatch::combo_tier(__combo) {
554                #[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
555                ::hydroplane::towers::SSE4 => ::hydroplane::dispatch::run_kernel_on(
556                    __kernel,
557                    unsafe { ::hydroplane::towers::Sse4::new_unchecked() },
558                ),
559                #avx1_arm
560                #[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
561                ::hydroplane::towers::AVX2 => ::hydroplane::dispatch::run_kernel_on(
562                    __kernel,
563                    unsafe { ::hydroplane::towers::Avx2::new_unchecked() },
564                ),
565                #[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
566                ::hydroplane::towers::AVX512 => ::hydroplane::dispatch::run_kernel_on(
567                    __kernel,
568                    unsafe { ::hydroplane::towers::Avx512::new_unchecked() },
569                ),
570                #fp16_arm
571                #bf16_arm
572                #[cfg(target_arch = "aarch64")]
573                ::hydroplane::towers::NEON => ::hydroplane::dispatch::run_kernel_on(
574                    __kernel,
575                    ::hydroplane::towers::Neon::new(),
576                ),
577                #sve_arms
578                #[cfg(all(target_arch = "wasm32", target_feature = "simd128", target_feature = "relaxed-simd"))]
579                ::hydroplane::towers::RELAXED => ::hydroplane::dispatch::run_kernel_on(
580                    __kernel,
581                    ::hydroplane::towers::RelaxedSimd::new(),
582                ),
583                #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))]
584                ::hydroplane::towers::SIMD128 => ::hydroplane::dispatch::run_kernel_on(
585                    __kernel,
586                    ::hydroplane::towers::Simd128::new(),
587                ),
588                _ => ::hydroplane::dispatch::run_kernel_on(__kernel, ::hydroplane::ScalarBackend),
589            }
590        }}
591    };
592
593    Ok(quote! {
594        #[doc(hidden)]
595        #[allow(non_snake_case)]
596        mod #name {
597            #[allow(unused_imports)]
598            use super::*;
599            #[allow(dead_code)]
600            pub(super) struct __Kernel #impl_generics #where_clause {
601                #( pub(super) #field_idents: #field_tys, )*
602                #phantom_field
603            }
604            impl #impl_generics #ktrait<#scalar> for __Kernel #ty_generics #where_clause {
605                type Output = #ret;
606                #k_cap_item
607                #[inline(always)]
608                fn run<__S: #run_bound>(self, #primary_ctx: #primary_ty) -> #ret {
609                    #run_body
610                }
611            }
612        }
613        #(#attrs)*
614        #[inline]
615        #[allow(clippy::multiple_bound_locations)]
616        // Splat-constant scalars arrive as plain params, so kernels routinely exceed clippy's arg
617        // limit; the author can't annotate a generated fn.
618        #[allow(clippy::too_many_arguments)]
619        #vis fn #name #w_impl_generics ( #( #field_idents: #field_tys ),* ) -> #ret
620        #w_where
621        {
622            #dispatch_call
623        }
624        #( #isa_fns )*
625        #[doc = concat!("Run [`", stringify!(#name), "`]'s body on an already-dispatched context, ")]
626        #[doc = "skipping a second dispatch — call this from inside another kernel to reuse its backend."]
627        #metrics_attr
628        #on_inline
629        #[allow(clippy::too_many_arguments)]
630        #vis fn #on_name #on_impl_generics ( #primary_ctx: #primary_ty, #( #field_idents: #field_tys ),* ) -> #ret
631        #on_where
632        {
633            #on_body
634        }
635    })
636}
637
638/// Whether the kernel's element is one of its own generic parameters, known only at
639/// monomorphization.
640fn scalar_is_generic_early(generics: &syn::Generics, scalar: &TokenStream2) -> bool {
641    let scalar_str = scalar.to_string();
642    generics.params.iter().any(|p| match p {
643        GenericParam::Type(t) => t.ident == scalar_str,
644        _ => false,
645    })
646}
647
648/// Element bits (`Scalar::TYPE_BITS` values) whose names appear anywhere in the kernel's tokens: a
649/// safe over-approximation of the elements the body touches, used to prune the combo-dispatch
650/// match. MIR analysis replaces this with the measured set when available; the generated combo
651/// also ORs the dispatch element's own `TYPE_BITS`, so the declared element is never missed.
652fn scan_type_bits(tokens: &TokenStream2) -> u8 {
653    fn walk(ts: TokenStream2, bits: &mut u8) {
654        for tt in ts {
655            match tt {
656                proc_macro2::TokenTree::Ident(id) => {
657                    *bits |= match id.to_string().as_str() {
658                        "f32" => 1,
659                        "f64" => 2,
660                        "f16" => 4,
661                        "bf16" => 8,
662                        "u32" => 16,
663                        "i32" => 32,
664                        _ => 0,
665                    };
666                }
667                proc_macro2::TokenTree::Group(g) => walk(g.stream(), bits),
668                _ => {}
669            }
670        }
671    }
672    let mut bits = 0;
673    walk(tokens.clone(), &mut bits);
674    bits
675}
676
677/// The `#[hint_cnt(N)]` value on a kernel parameter, if present. Currently discarded by the caller;
678/// malformed forms are leniently ignored.
679fn parse_hint_cnt(attrs: &[syn::Attribute]) -> Option<u64> {
680    attrs
681        .iter()
682        .filter(|a| a.path().is_ident("hint_cnt"))
683        .find_map(|a| a.parse_args::<syn::LitInt>().ok())
684        .and_then(|lit| lit.base10_parse::<u64>().ok())
685}
686
687/// Rewrite the context parameter's written type into the backend-carrying form, filling in `__S`.
688/// The author's path is preserved, so a plain `Gang` import and a fully-qualified
689/// `hydroplane::Gang<T>` both work.
690fn ctx_type_with_backend(ty: Type) -> syn::Result<Type> {
691    let bad = |t: &dyn quote::ToTokens| {
692        syn::Error::new_spanned(
693            t,
694            "the context parameter must be typed `Gang<T>` (the scalar); the backend type argument \
695             is filled in for you",
696        )
697    };
698    let Type::Path(mut tp) = ty else {
699        return Err(bad(&ty));
700    };
701    let Some(seg) = tp.path.segments.last_mut() else {
702        return Err(bad(&tp));
703    };
704    // Bare `ctx: Gang` is canonical (the gang is element-free); a legacy bracketed element
705    // (`Gang<f32>`) is accepted and names the dispatch element.
706    match &mut seg.arguments {
707        syn::PathArguments::None => {
708            seg.arguments = syn::PathArguments::AngleBracketed(parse_quote!(<__S>));
709        }
710        syn::PathArguments::AngleBracketed(args) => {
711            args.args.clear();
712            args.args.push(parse_quote!(__S));
713        }
714        _ => return Err(bad(&tp)),
715    }
716    Ok(Type::Path(tp))
717}
718
719/// First type argument of the context type (`Gang<f32>` → `f32`), used to infer the scalar of a
720/// kernel with no generic scalar parameter.
721fn ctx_scalar_arg(ty: &Type) -> Option<Type> {
722    let Type::Path(tp) = ty else { return None };
723    let seg = tp.path.segments.last()?;
724    let syn::PathArguments::AngleBracketed(ab) = &seg.arguments else {
725        return None;
726    };
727    ab.args.iter().find_map(|a| match a {
728        syn::GenericArgument::Type(t) => Some(t.clone()),
729        _ => None,
730    })
731}
732
733/// Find the scalar type parameter: the one bound by a trait named `Scalar` (inline or in the
734/// where-clause), falling back to a parameter literally named `T`. `Ok(None)` if there is no scalar
735/// type parameter at all (a concrete-scalar kernel); `Err` only when the choice is ambiguous.
736fn detect_scalar(generics: &Generics) -> syn::Result<Option<Ident>> {
737    let has_scalar_bound = |bounds: &Punctuated<TypeParamBound, Token![+]>| -> bool {
738        bounds.iter().any(|b| match b {
739            TypeParamBound::Trait(tb) => tb.path.segments.last().is_some_and(|s| {
740                s.ident == "Scalar" || s.ident == "FloatScalar" || s.ident == "IntScalar"
741            }),
742            _ => false,
743        })
744    };
745
746    let mut scalar_bound: Option<Ident> = None;
747    let mut named_t: Option<Ident> = None;
748    let mut record = |id: &Ident, span: Span| -> syn::Result<()> {
749        if let Some(prev) = &scalar_bound
750            && prev != id
751        {
752            return Err(syn::Error::new(
753                span,
754                "multiple type parameters are bound by `Scalar`; disambiguate with `#[kernel(scalar = ...)]`",
755            ));
756        }
757        scalar_bound = Some(id.clone());
758        Ok(())
759    };
760
761    for p in &generics.params {
762        if let GenericParam::Type(tp) = p {
763            if tp.ident == "T" {
764                named_t = Some(tp.ident.clone());
765            }
766            if has_scalar_bound(&tp.bounds) {
767                record(&tp.ident, tp.ident.span())?;
768            }
769        }
770    }
771    if let Some(wc) = &generics.where_clause {
772        for pred in &wc.predicates {
773            if let WherePredicate::Type(pt) = pred
774                && let Type::Path(typ) = &pt.bounded_ty
775                && let Some(id) = typ.path.get_ident()
776                && has_scalar_bound(&pt.bounds)
777            {
778                record(id, id.span())?;
779            }
780        }
781    }
782
783    Ok(scalar_bound.or(named_t))
784}
785
786/// Rewrites `a * b + c` and `c + a * b` in a kernel body into a `::hydroplane::__fuse` call,
787/// which is a fused multiply-add when the operands are varyings and the same two operators
788/// otherwise, so kernels written with plain operators get the FMA throughput of `.fma`.
789/// Operands keep their evaluation order. Constant contexts (types, array lengths, `const`
790/// items and blocks, generic arguments, patterns) are left alone, since the rewrite is a
791/// method call.
792struct FuseMulAdd;
793
794fn peel(e: &syn::Expr) -> &syn::Expr {
795    match e {
796        syn::Expr::Paren(p) => peel(&p.expr),
797        _ => e,
798    }
799}
800
801fn as_mul(e: &syn::Expr) -> Option<(syn::Expr, syn::Expr)> {
802    match peel(e) {
803        syn::Expr::Binary(b) if matches!(b.op, syn::BinOp::Mul(_)) => Some((peel(&b.left).clone(), peel(&b.right).clone())),
804        _ => None,
805    }
806}
807
808impl syn::visit_mut::VisitMut for FuseMulAdd {
809    fn visit_expr_mut(&mut self, e: &mut syn::Expr) {
810        match e {
811            syn::Expr::Const(_) => return,
812            syn::Expr::Repeat(r) => {
813                self.visit_expr_mut(&mut r.expr);
814                return;
815            }
816            _ => {}
817        }
818        syn::visit_mut::visit_expr_mut(self, e);
819        let syn::Expr::Binary(b) = e else { return };
820        if !matches!(b.op, syn::BinOp::Add(_)) {
821            return;
822        }
823        let (x, y, c, c_first) = if let Some((x, y)) = as_mul(&b.right) {
824            (x, y, peel(&b.left).clone(), true)
825        } else if let Some((x, y)) = as_mul(&b.left) {
826            (x, y, peel(&b.right).clone(), false)
827        } else {
828            return;
829        };
830        let bind = if c_first {
831            quote! { let __hp_c = #c; let __hp_a = #x; let __hp_b = #y; }
832        } else {
833            quote! { let __hp_a = #x; let __hp_b = #y; let __hp_c = #c; }
834        };
835        *e = syn::parse_quote!({
836            #[allow(unused_imports)]
837            use ::hydroplane::__fuse::{ViaFma as _, ViaOps as _};
838            #bind
839            (&::hydroplane::__fuse::Fuse::new(__hp_a, __hp_b, __hp_c)).__hp_fuse()
840        });
841    }
842    fn visit_type_mut(&mut self, _: &mut syn::Type) {}
843    fn visit_pat_mut(&mut self, _: &mut syn::Pat) {}
844    fn visit_generic_argument_mut(&mut self, _: &mut syn::GenericArgument) {}
845    fn visit_item_const_mut(&mut self, _: &mut syn::ItemConst) {}
846    fn visit_item_static_mut(&mut self, _: &mut syn::ItemStatic) {}
847}