1use hydroplane_auto::analysis;
26
27use proc_macro::TokenStream;
28use proc_macro2::{Span, TokenStream as TokenStream2};
29use quote::{ToTokens, format_ident, quote};
30use syn::parse::Parser;
31use syn::spanned::Spanned;
32use syn::{
33 FnArg, GenericParam, Generics, Ident, ItemFn, Meta, Pat, ReturnType, Token, Type,
34 TypeParamBound, WherePredicate, parse_macro_input, parse_quote, punctuated::Punctuated,
35};
36
37#[proc_macro_attribute]
42pub fn kernel(attr: TokenStream, item: TokenStream) -> TokenStream {
43 let func = parse_macro_input!(item as ItemFn);
44 let opts = match KernelOpts::parse(attr.into()) {
45 Ok(o) => o,
46 Err(e) => return e.to_compile_error().into(),
47 };
48 match expand(func, opts) {
49 Ok(ts) => ts.into(),
50 Err(e) => e.to_compile_error().into(),
51 }
52}
53
54struct KernelOpts {
55 matrix: bool,
58 scalar: Option<Ident>,
59 noalias: bool,
62 explicit_boundary: bool,
64 unroll: Option<usize>,
67 types: Vec<Ident>,
71}
72
73fn element_bits(name: &str) -> Option<u8> {
74 Some(match name {
75 "f32" => 1,
76 "f64" => 2,
77 "f16" => 4,
78 "bf16" => 8,
79 "u32" => 16,
80 "i32" => 32,
81 _ => return None,
82 })
83}
84
85impl KernelOpts {
86 fn parse(tokens: TokenStream2) -> syn::Result<Self> {
87 let mut matrix = false;
88 let mut scalar = None;
89 let mut noalias = true;
90 let mut tiny_span = None;
91 let mut noalias_span = None;
92 let mut unroll = None;
93 let mut types = Vec::new();
94 let metas = Punctuated::<Meta, Token![,]>::parse_terminated.parse2(tokens)?;
95 for meta in metas {
96 match meta {
97 Meta::Path(p)
98 if p.get_ident().is_some_and(|i| element_bits(&i.to_string()).is_some()) =>
99 {
100 types.push(p.get_ident().unwrap().clone());
101 }
102 Meta::Path(p) if p.is_ident("matrix") => {
103 if matrix {
104 return Err(syn::Error::new(p.span(), "duplicate `matrix`"));
105 }
106 matrix = true;
107 }
108 Meta::Path(p) if p.is_ident("tiny") => {
109 if tiny_span.is_some() {
110 return Err(syn::Error::new(p.span(), "duplicate `tiny`"));
111 }
112 tiny_span = Some(p.span());
113 noalias = false;
114 }
115 Meta::Path(p) if p.is_ident("noalias") => {
117 if noalias_span.is_some() {
118 return Err(syn::Error::new(p.span(), "duplicate `noalias`"));
119 }
120 noalias_span = Some(p.span());
121 }
122 Meta::NameValue(nv) if nv.path.is_ident("scalar") => {
123 let id = match &nv.value {
124 syn::Expr::Path(ep) => ep.path.get_ident().cloned(),
125 _ => None,
126 };
127 match id {
128 Some(id) => scalar = Some(id),
129 None => {
130 return Err(syn::Error::new_spanned(
131 &nv.value,
132 "expected an identifier, e.g. `scalar = T`",
133 ));
134 }
135 }
136 }
137 Meta::NameValue(nv) if nv.path.is_ident("unroll") => match &nv.value {
138 syn::Expr::Lit(syn::ExprLit { lit: syn::Lit::Int(i), .. }) => {
139 unroll = Some(i.base10_parse::<usize>()?);
140 }
141 _ => {
142 return Err(syn::Error::new_spanned(
143 &nv.value,
144 "expected an integer, e.g. `unroll = 4`",
145 ));
146 }
147 },
148 other => {
149 return Err(syn::Error::new_spanned(
150 other,
151 "unknown `#[kernel]` option; expected `matrix`, `scalar = Ident`, \
152 `unroll = N`, `tiny`, or `noalias`",
153 ));
154 }
155 }
156 }
157 if let (Some(ts), Some(ns)) = (tiny_span, noalias_span) {
158 let mut e = syn::Error::new(ns, "`noalias` and `tiny` are opposites — pick one");
159 e.combine(syn::Error::new(ts, "`tiny` set here"));
160 return Err(e);
161 }
162 let explicit_boundary = tiny_span.is_some() || noalias_span.is_some();
163 Ok(KernelOpts { matrix, scalar, noalias, explicit_boundary, unroll, types })
164 }
165}
166
167fn expand(func: ItemFn, opts: KernelOpts) -> syn::Result<TokenStream2> {
168 let func_tokens = func.to_token_stream();
169 let ItemFn {
170 attrs,
171 vis,
172 sig,
173 mut block,
174 } = func;
175 syn::visit_mut::VisitMut::visit_block_mut(&mut FuseMulAdd, &mut block);
176
177 if let Some(a) = sig.asyncness {
178 return Err(syn::Error::new_spanned(a, "a kernel cannot be `async`"));
179 }
180 if let Some(c) = sig.constness {
181 return Err(syn::Error::new_spanned(c, "a kernel cannot be `const`"));
182 }
183 if let Some(v) = sig.variadic {
184 return Err(syn::Error::new_spanned(v, "a kernel cannot be variadic"));
185 }
186
187 let mut inputs = sig.inputs.into_iter();
188 let mut ctx_idents = Vec::with_capacity(1);
189 let mut ctx_tys = Vec::with_capacity(1);
190 let mut ctx_scalar = None;
191 for _ in 0..1 {
192 match inputs.next() {
193 Some(FnArg::Typed(pt)) => {
194 match *pt.pat {
195 Pat::Ident(pi) => ctx_idents.push(pi.ident),
196 other => {
197 return Err(syn::Error::new_spanned(
198 other,
199 "context parameters must be plain identifiers",
200 ));
201 }
202 }
203 let ty = *pt.ty;
204 if ctx_scalar.is_none() {
205 ctx_scalar = ctx_scalar_arg(&ty);
206 }
207 ctx_tys.push(ctx_type_with_backend(ty)?);
208 }
209 Some(FnArg::Receiver(r)) => {
210 return Err(syn::Error::new_spanned(
211 r,
212 "a kernel takes no `self` receiver",
213 ));
214 }
215 None => {
216 return Err(syn::Error::new_spanned(
217 &sig.ident,
218 "a kernel needs a leading context parameter (e.g. `ctx: Gang<T>`) before its \
219 data parameters",
220 ));
221 }
222 }
223 }
224
225 let mut field_idents = Vec::new();
226 let mut field_tys = Vec::new();
227 for arg in inputs {
228 match arg {
229 FnArg::Typed(pt) => {
230 let _hint_cnt = parse_hint_cnt(&pt.attrs);
233 match *pt.pat {
234 Pat::Ident(pi) => field_idents.push(pi.ident),
235 other => {
236 return Err(syn::Error::new_spanned(
237 other,
238 "kernel parameters must be plain identifiers",
239 ));
240 }
241 }
242 field_tys.push(*pt.ty);
243 }
244 FnArg::Receiver(r) => {
245 return Err(syn::Error::new_spanned(
246 r,
247 "a kernel takes no `self` receiver",
248 ));
249 }
250 }
251 }
252
253 let ret: Type = match sig.output {
254 ReturnType::Default => parse_quote!(()),
255 ReturnType::Type(_, t) => *t,
256 };
257
258 let scanned_bits_early = {
262 let mut ts = TokenStream2::new();
263 func_tokens.to_tokens(&mut ts);
264 scan_type_bits(&ts)
265 };
266 let detected = detect_scalar(&sig.generics)?;
267 let scalar = if let Some(s) = &opts.scalar {
268 quote!(#s)
269 } else if let Some(s) = &detected {
270 quote!(#s)
271 } else if let Some(t) = &ctx_scalar {
272 quote!(#t)
273 } else if let Some(first) = opts.types.first() {
274 quote!(#first)
275 } else {
276 match 1u8 << scanned_bits_early.trailing_zeros().min(6) {
277 1 => quote!(f32),
278 2 => quote!(f64),
279 4 => quote!(::hydroplane::f16),
280 8 => quote!(::hydroplane::bf16),
281 16 => quote!(u32),
282 32 => quote!(i32),
283 _ => quote!(f32),
284 }
285 };
286
287 let generics = sig.generics;
288 let (impl_generics, ty_generics, where_clause) = generics.split_for_impl();
289
290 let mut lifetimes = Vec::new();
291 let mut type_params = Vec::new();
292 let mut turbofish_args = Vec::new();
293 for p in &generics.params {
294 match p {
295 GenericParam::Lifetime(l) => lifetimes.push(l.lifetime.clone()),
296 GenericParam::Type(t) => {
297 type_params.push(t.ident.clone());
298 turbofish_args.push(t.ident.clone());
299 }
300 GenericParam::Const(c) => turbofish_args.push(c.ident.clone()),
301 }
302 }
303
304 let phantom_items: Vec<TokenStream2> = lifetimes
308 .iter()
309 .map(|lt| quote!(& #lt ()))
310 .chain(type_params.iter().map(|t| quote!(*const #t)))
311 .collect();
312 let (phantom_field, phantom_init) = if phantom_items.is_empty() {
313 (quote!(), quote!())
314 } else {
315 (
316 quote!(pub(super) __phantom: ::core::marker::PhantomData<(#(#phantom_items),*)>,),
317 quote!(__phantom: ::core::marker::PhantomData,),
318 )
319 };
320
321 let matrix_present = opts.matrix;
322 let (ktrait, bbound, dispatch_fn, dbound) = if matrix_present {
323 (
324 quote!(::hydroplane::MatrixKernel),
325 quote!(::hydroplane::MatrixBackend),
326 quote!(::hydroplane::dispatch_matrix),
327 quote!(::hydroplane::MatrixDispatch),
328 )
329 } else {
330 (
331 quote!(::hydroplane::Kernel),
332 quote!(::hydroplane::Backend),
333 quote!(::hydroplane::dispatch),
334 quote!(::hydroplane::SimdDispatch),
335 )
336 };
337
338 let run_bound = if matrix_present {
339 quote!(#bbound<#scalar>)
340 } else {
341 quote!(::hydroplane::BackendAll + ::hydroplane::Backend<#scalar>)
342 };
343
344 let mut wrapper_generics = generics.clone();
345 let pred: WherePredicate = parse_quote!(#scalar: #dbound);
346 wrapper_generics.make_where_clause().predicates.push(pred);
347 let (w_impl_generics, _, w_where) = wrapper_generics.split_for_impl();
348
349 let turbofish = if turbofish_args.is_empty() {
350 quote!()
351 } else {
352 quote!(::<#(#turbofish_args),*>)
353 };
354
355 let name = sig.ident;
356
357 let name_str = name.to_string();
361 let metrics = if matrix_present {
362 None
363 } else {
364 analysis::lookup(&name_str)
365 };
366 let use_noalias = match metrics {
367 Some(m) if !opts.explicit_boundary => m.noalias(),
368 _ => opts.noalias,
369 };
370 let k_cap = opts.unroll;
371
372 let analyzed = std::env::var("HYDRO_ANALYZE_CRATE").ok().is_some_and(|want| {
381 std::env::var("CARGO_CRATE_NAME").is_ok_and(|name| name == want)
382 });
383 let metrics_attr = if use_noalias
384 && !matrix_present
385 && std::env::var_os("HYDRO_ANALYZE_INNER").is_some()
386 && analyzed
387 {
388 quote!(#[cfg_attr(hp_analyze, hp_analyze::metrics(kernel = #name_str))])
389 } else {
390 quote!()
391 };
392 let k_cap_item = match k_cap {
393 Some(k) => quote!(const K_CAP: usize = #k;),
394 None => quote!(),
395 };
396
397 let primary_ctx = &ctx_idents[0];
400 let primary_ty = &ctx_tys[0];
401 let ctx_aliases: Vec<TokenStream2> = ctx_idents[1..]
402 .iter()
403 .map(|id| quote!(let #id = #primary_ctx;))
404 .collect();
405
406 let on_name = format_ident!("{}_on", name);
410 let mut on_generics = generics.clone();
411 on_generics.params.push(syn::GenericParam::Type(parse_quote!(__S: #run_bound)));
412 let (on_impl_generics, _, on_where) = on_generics.split_for_impl();
413
414 let on_call_turbofish = if turbofish_args.is_empty() {
415 quote!(::<__S>)
416 } else {
417 quote!(::<#(#turbofish_args,)* __S>)
418 };
419 let run_body = quote! {
420 let Self { #( #field_idents, )* .. } = self;
421 #on_name #on_call_turbofish ( #primary_ctx, #( #field_idents, )* )
422 };
423
424 let isa_copies: Vec<(&str, bool)> = if matrix_present {
431 Vec::new()
432 } else {
433 let generic = scalar_is_generic_early(&generics, &scalar);
434 let bits = metrics.and_then(|m| m.type_bits()).unwrap_or(scanned_bits_early);
435 vec![
436 ("SSE41", true),
437 ("AVX", generic || bits & 60 == 0),
438 ("AVX2", true),
439 ("AVX512", true),
440 ("AVX512FP16", generic || bits & 4 != 0),
441 ("AVX512BF16", generic || bits & 8 != 0),
442 ("SVE", generic || bits & 48 == 0),
443 ]
444 };
445 let mut isa_fns = Vec::new();
446 let mut isa_arms = Vec::new();
447 for (code, wanted) in isa_copies {
448 if !wanted {
449 continue;
450 }
451 let code_ident = format_ident!("{}", code);
452 let fn_name = format_ident!("__{}_{}", name, code.to_lowercase());
453 isa_fns.push(quote! {
456 ::hydroplane::__hp_isa! { kitem #code_ident
457 #[doc(hidden)]
458 #[inline]
459 #[allow(non_snake_case, clippy::too_many_arguments, clippy::multiple_bound_locations)]
460 fn #fn_name #on_impl_generics ( #primary_ctx: #primary_ty, #( #field_idents: #field_tys ),* ) -> #ret
461 #on_where
462 {
463 #( #ctx_aliases )*
464 #block
465 }
466 }
467 });
468 isa_arms.push(quote! {
469 ::hydroplane::__hp_isa! { kstmt #code_ident
470 if <__S as ::hydroplane::Backend<f32>>::ISA == ::hydroplane::isa::#code_ident {
471 return unsafe { #fn_name #on_call_turbofish ( #primary_ctx, #( #field_idents, )* ) };
474 }
475 }
476 });
477 }
478 let on_body = quote! {
479 #( #isa_arms )*
480 #( #ctx_aliases )*
481 #block
482 };
483 let on_inline = quote!(#[inline(always)]);
484
485 let attr_bits: u8 = opts
488 .types
489 .iter()
490 .filter_map(|i| element_bits(&i.to_string()))
491 .fold(0, |a, b| a | b);
492 let combo_bits = metrics.and_then(|m| m.type_bits()).unwrap_or(scanned_bits_early) | attr_bits;
493 let scalar_str = quote!(#scalar).to_string();
495 let scalar_is_generic = generics.params.iter().any(|p| match p {
496 GenericParam::Type(t) => t.ident == scalar_str,
497 _ => false,
498 });
499 let may_half_f16 = combo_bits & 4 != 0;
500 let may_half_bf16 = combo_bits & 8 != 0;
501 let pure_float = combo_bits & 60 == 0;
502 let no_ints = combo_bits & 48 == 0;
503
504 let fp16_arm = may_half_f16.then(|| quote! {
505 #[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
506 ::hydroplane::towers::AVX512FP16 => ::hydroplane::dispatch::run_kernel_on(
509 __kernel,
510 unsafe { ::hydroplane::towers::Avx512Fp16::new_unchecked() },
511 ),
512 });
513 let bf16_arm = may_half_bf16.then(|| quote! {
514 #[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
515 ::hydroplane::towers::AVX512BF16 => ::hydroplane::dispatch::run_kernel_on(
516 __kernel,
517 unsafe { ::hydroplane::towers::Avx512Bf16::new_unchecked() },
518 ),
519 });
520 let avx1_arm = pure_float.then(|| quote! {
521 #[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
522 ::hydroplane::towers::AVX1 => ::hydroplane::dispatch::run_kernel_on(
523 __kernel,
524 unsafe { ::hydroplane::towers::Avx1::new_unchecked() },
525 ),
526 });
527 let sve_arms = no_ints.then(|| quote! {
528 #[cfg(all(target_arch = "aarch64", not(target_vendor = "apple")))]
529 ::hydroplane::towers::SVE16 => ::hydroplane::dispatch::run_kernel_on(
530 __kernel,
531 unsafe { ::hydroplane::towers::Sve::<16>::new_unchecked() },
532 ),
533 #[cfg(all(target_arch = "aarch64", not(target_vendor = "apple")))]
534 ::hydroplane::towers::SVE32 => ::hydroplane::dispatch::run_kernel_on(
535 __kernel,
536 unsafe { ::hydroplane::towers::Sve::<32>::new_unchecked() },
537 ),
538 #[cfg(all(target_arch = "aarch64", not(target_vendor = "apple")))]
539 ::hydroplane::towers::SVE64 => ::hydroplane::dispatch::run_kernel_on(
540 __kernel,
541 unsafe { ::hydroplane::towers::Sve::<64>::new_unchecked() },
542 ),
543 });
544
545 let dispatch_call = if matrix_present || scalar_is_generic {
548 quote!(#dispatch_fn(#name::__Kernel #turbofish { #( #field_idents, )* #phantom_init }))
549 } else {
550 quote! {{
551 let __combo: u8 = #combo_bits | <#scalar as ::hydroplane::Scalar>::TYPE_BITS;
552 let __kernel = #name::__Kernel #turbofish { #( #field_idents, )* #phantom_init };
553 match ::hydroplane::dispatch::combo_tier(__combo) {
554 #[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
555 ::hydroplane::towers::SSE4 => ::hydroplane::dispatch::run_kernel_on(
556 __kernel,
557 unsafe { ::hydroplane::towers::Sse4::new_unchecked() },
558 ),
559 #avx1_arm
560 #[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
561 ::hydroplane::towers::AVX2 => ::hydroplane::dispatch::run_kernel_on(
562 __kernel,
563 unsafe { ::hydroplane::towers::Avx2::new_unchecked() },
564 ),
565 #[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
566 ::hydroplane::towers::AVX512 => ::hydroplane::dispatch::run_kernel_on(
567 __kernel,
568 unsafe { ::hydroplane::towers::Avx512::new_unchecked() },
569 ),
570 #fp16_arm
571 #bf16_arm
572 #[cfg(target_arch = "aarch64")]
573 ::hydroplane::towers::NEON => ::hydroplane::dispatch::run_kernel_on(
574 __kernel,
575 ::hydroplane::towers::Neon::new(),
576 ),
577 #sve_arms
578 #[cfg(all(target_arch = "wasm32", target_feature = "simd128", target_feature = "relaxed-simd"))]
579 ::hydroplane::towers::RELAXED => ::hydroplane::dispatch::run_kernel_on(
580 __kernel,
581 ::hydroplane::towers::RelaxedSimd::new(),
582 ),
583 #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))]
584 ::hydroplane::towers::SIMD128 => ::hydroplane::dispatch::run_kernel_on(
585 __kernel,
586 ::hydroplane::towers::Simd128::new(),
587 ),
588 _ => ::hydroplane::dispatch::run_kernel_on(__kernel, ::hydroplane::ScalarBackend),
589 }
590 }}
591 };
592
593 Ok(quote! {
594 #[doc(hidden)]
595 #[allow(non_snake_case)]
596 mod #name {
597 #[allow(unused_imports)]
598 use super::*;
599 #[allow(dead_code)]
600 pub(super) struct __Kernel #impl_generics #where_clause {
601 #( pub(super) #field_idents: #field_tys, )*
602 #phantom_field
603 }
604 impl #impl_generics #ktrait<#scalar> for __Kernel #ty_generics #where_clause {
605 type Output = #ret;
606 #k_cap_item
607 #[inline(always)]
608 fn run<__S: #run_bound>(self, #primary_ctx: #primary_ty) -> #ret {
609 #run_body
610 }
611 }
612 }
613 #(#attrs)*
614 #[inline]
615 #[allow(clippy::multiple_bound_locations)]
616 #[allow(clippy::too_many_arguments)]
619 #vis fn #name #w_impl_generics ( #( #field_idents: #field_tys ),* ) -> #ret
620 #w_where
621 {
622 #dispatch_call
623 }
624 #( #isa_fns )*
625 #[doc = concat!("Run [`", stringify!(#name), "`]'s body on an already-dispatched context, ")]
626 #[doc = "skipping a second dispatch — call this from inside another kernel to reuse its backend."]
627 #metrics_attr
628 #on_inline
629 #[allow(clippy::too_many_arguments)]
630 #vis fn #on_name #on_impl_generics ( #primary_ctx: #primary_ty, #( #field_idents: #field_tys ),* ) -> #ret
631 #on_where
632 {
633 #on_body
634 }
635 })
636}
637
638fn scalar_is_generic_early(generics: &syn::Generics, scalar: &TokenStream2) -> bool {
641 let scalar_str = scalar.to_string();
642 generics.params.iter().any(|p| match p {
643 GenericParam::Type(t) => t.ident == scalar_str,
644 _ => false,
645 })
646}
647
648fn scan_type_bits(tokens: &TokenStream2) -> u8 {
653 fn walk(ts: TokenStream2, bits: &mut u8) {
654 for tt in ts {
655 match tt {
656 proc_macro2::TokenTree::Ident(id) => {
657 *bits |= match id.to_string().as_str() {
658 "f32" => 1,
659 "f64" => 2,
660 "f16" => 4,
661 "bf16" => 8,
662 "u32" => 16,
663 "i32" => 32,
664 _ => 0,
665 };
666 }
667 proc_macro2::TokenTree::Group(g) => walk(g.stream(), bits),
668 _ => {}
669 }
670 }
671 }
672 let mut bits = 0;
673 walk(tokens.clone(), &mut bits);
674 bits
675}
676
677fn parse_hint_cnt(attrs: &[syn::Attribute]) -> Option<u64> {
680 attrs
681 .iter()
682 .filter(|a| a.path().is_ident("hint_cnt"))
683 .find_map(|a| a.parse_args::<syn::LitInt>().ok())
684 .and_then(|lit| lit.base10_parse::<u64>().ok())
685}
686
687fn ctx_type_with_backend(ty: Type) -> syn::Result<Type> {
691 let bad = |t: &dyn quote::ToTokens| {
692 syn::Error::new_spanned(
693 t,
694 "the context parameter must be typed `Gang<T>` (the scalar); the backend type argument \
695 is filled in for you",
696 )
697 };
698 let Type::Path(mut tp) = ty else {
699 return Err(bad(&ty));
700 };
701 let Some(seg) = tp.path.segments.last_mut() else {
702 return Err(bad(&tp));
703 };
704 match &mut seg.arguments {
707 syn::PathArguments::None => {
708 seg.arguments = syn::PathArguments::AngleBracketed(parse_quote!(<__S>));
709 }
710 syn::PathArguments::AngleBracketed(args) => {
711 args.args.clear();
712 args.args.push(parse_quote!(__S));
713 }
714 _ => return Err(bad(&tp)),
715 }
716 Ok(Type::Path(tp))
717}
718
719fn ctx_scalar_arg(ty: &Type) -> Option<Type> {
722 let Type::Path(tp) = ty else { return None };
723 let seg = tp.path.segments.last()?;
724 let syn::PathArguments::AngleBracketed(ab) = &seg.arguments else {
725 return None;
726 };
727 ab.args.iter().find_map(|a| match a {
728 syn::GenericArgument::Type(t) => Some(t.clone()),
729 _ => None,
730 })
731}
732
733fn detect_scalar(generics: &Generics) -> syn::Result<Option<Ident>> {
737 let has_scalar_bound = |bounds: &Punctuated<TypeParamBound, Token![+]>| -> bool {
738 bounds.iter().any(|b| match b {
739 TypeParamBound::Trait(tb) => tb.path.segments.last().is_some_and(|s| {
740 s.ident == "Scalar" || s.ident == "FloatScalar" || s.ident == "IntScalar"
741 }),
742 _ => false,
743 })
744 };
745
746 let mut scalar_bound: Option<Ident> = None;
747 let mut named_t: Option<Ident> = None;
748 let mut record = |id: &Ident, span: Span| -> syn::Result<()> {
749 if let Some(prev) = &scalar_bound
750 && prev != id
751 {
752 return Err(syn::Error::new(
753 span,
754 "multiple type parameters are bound by `Scalar`; disambiguate with `#[kernel(scalar = ...)]`",
755 ));
756 }
757 scalar_bound = Some(id.clone());
758 Ok(())
759 };
760
761 for p in &generics.params {
762 if let GenericParam::Type(tp) = p {
763 if tp.ident == "T" {
764 named_t = Some(tp.ident.clone());
765 }
766 if has_scalar_bound(&tp.bounds) {
767 record(&tp.ident, tp.ident.span())?;
768 }
769 }
770 }
771 if let Some(wc) = &generics.where_clause {
772 for pred in &wc.predicates {
773 if let WherePredicate::Type(pt) = pred
774 && let Type::Path(typ) = &pt.bounded_ty
775 && let Some(id) = typ.path.get_ident()
776 && has_scalar_bound(&pt.bounds)
777 {
778 record(id, id.span())?;
779 }
780 }
781 }
782
783 Ok(scalar_bound.or(named_t))
784}
785
786struct FuseMulAdd;
793
794fn peel(e: &syn::Expr) -> &syn::Expr {
795 match e {
796 syn::Expr::Paren(p) => peel(&p.expr),
797 _ => e,
798 }
799}
800
801fn as_mul(e: &syn::Expr) -> Option<(syn::Expr, syn::Expr)> {
802 match peel(e) {
803 syn::Expr::Binary(b) if matches!(b.op, syn::BinOp::Mul(_)) => Some((peel(&b.left).clone(), peel(&b.right).clone())),
804 _ => None,
805 }
806}
807
808impl syn::visit_mut::VisitMut for FuseMulAdd {
809 fn visit_expr_mut(&mut self, e: &mut syn::Expr) {
810 match e {
811 syn::Expr::Const(_) => return,
812 syn::Expr::Repeat(r) => {
813 self.visit_expr_mut(&mut r.expr);
814 return;
815 }
816 _ => {}
817 }
818 syn::visit_mut::visit_expr_mut(self, e);
819 let syn::Expr::Binary(b) = e else { return };
820 if !matches!(b.op, syn::BinOp::Add(_)) {
821 return;
822 }
823 let (x, y, c, c_first) = if let Some((x, y)) = as_mul(&b.right) {
824 (x, y, peel(&b.left).clone(), true)
825 } else if let Some((x, y)) = as_mul(&b.left) {
826 (x, y, peel(&b.right).clone(), false)
827 } else {
828 return;
829 };
830 let bind = if c_first {
831 quote! { let __hp_c = #c; let __hp_a = #x; let __hp_b = #y; }
832 } else {
833 quote! { let __hp_a = #x; let __hp_b = #y; let __hp_c = #c; }
834 };
835 *e = syn::parse_quote!({
836 #[allow(unused_imports)]
837 use ::hydroplane::__fuse::{ViaFma as _, ViaOps as _};
838 #bind
839 (&::hydroplane::__fuse::Fuse::new(__hp_a, __hp_b, __hp_c)).__hp_fuse()
840 });
841 }
842 fn visit_type_mut(&mut self, _: &mut syn::Type) {}
843 fn visit_pat_mut(&mut self, _: &mut syn::Pat) {}
844 fn visit_generic_argument_mut(&mut self, _: &mut syn::GenericArgument) {}
845 fn visit_item_const_mut(&mut self, _: &mut syn::ItemConst) {}
846 fn visit_item_static_mut(&mut self, _: &mut syn::ItemStatic) {}
847}