Skip to main content

oxifft_codegen_impl/gen_simd/
runtime_dispatch.rs

1//! Centralized ISA runtime dispatch codegen for `OxiFFT` SIMD codelets.
2//!
3//! This module generates **cached** runtime ISA dispatchers that extend the
4//! inline dispatchers in [`super`] with an `AtomicU8`-based ISA level cache.
5//!
6//! # Motivation
7//!
8//! The basic dispatchers emitted by `super::gen_dispatcher` perform
9//! `is_x86_feature_detected!` / `is_aarch64_feature_detected!` on every call.
10//! While each call is cheap (typically one CPUID cache read), a hot codelet
11//! invoked millions of times per second may benefit from the cached path, which
12//! replaces repeated feature probes with a single `AtomicU8` load.
13//!
14//! # Priority order (high → low)
15//!
16//! ```text
17//! x86_64: AVX-512F > AVX2+FMA > AVX > SSE2 > scalar
18//! aarch64: NEON > scalar
19//! other: scalar
20//! ```
21//!
22//! # Generated code shape
23//!
24//! For each `(size, precision)` pair, the proc-macro emits:
25//! - ISA level constants (`ISA_SCALAR`, `ISA_SSE2`, … `ISA_UNDETECTED`)
26//! - A `static DETECTED_ISA_{size}_{TY}: AtomicU8` initialized to `ISA_UNDETECTED`
27//! - A private `detect_isa_{size}_{ty}() -> u8` function that probes the CPU once
28//! - A public `{fn_name}_cached(data, sign)` dispatcher that reads the cache first
29//!
30//! # Proc-macro entry
31//!
32//! ```ignore
33//! // Generates a cached dispatcher for size-4 f32.
34//! gen_dispatcher_codelet!(size = 4, ty = f32);
35//! ```
36
37use proc_macro2::TokenStream;
38use quote::{format_ident, quote};
39use syn::{
40    parse::{Parse, ParseStream},
41    LitInt, Token,
42};
43
44pub use super::multi_transform::Precision;
45
46// ============================================================================
47// Public types
48// ============================================================================
49
50/// Configuration for a cached runtime ISA dispatcher codelet.
51#[derive(Debug, Clone, Copy)]
52pub struct DispatcherConfig {
53    /// DFT size — must be one of 2, 4, 8, or 16.
54    pub size: usize,
55    /// Floating-point precision.
56    pub precision: Precision,
57}
58
59// ============================================================================
60// ISA level constants (used in generated code and in host-detection helper)
61// ============================================================================
62
63/// ISA level for scalar fallback.
64pub const ISA_SCALAR: u8 = 0;
65/// ISA level for SSE2.
66pub const ISA_SSE2: u8 = 1;
67/// ISA level for pure AVX (no FMA, no AVX2).
68pub const ISA_AVX: u8 = 2;
69/// ISA level for AVX2 + FMA.
70pub const ISA_AVX2_FMA: u8 = 3;
71/// ISA level for AVX-512F.
72pub const ISA_AVX512: u8 = 4;
73/// ISA level for NEON (aarch64).
74pub const ISA_NEON: u8 = 5;
75/// Sentinel: ISA not yet detected (stored in the `AtomicU8` before first call).
76pub const ISA_UNDETECTED: u8 = 255;
77
78// ============================================================================
79// Host-detection helper (used by tests and by the generated detection code)
80// ============================================================================
81
82/// Detect the best ISA available on the current host at runtime.
83///
84/// Returns one of the `ISA_*` constants.  Never returns `ISA_UNDETECTED`.
85///
86/// This function is also used in the unit tests to validate that we always
87/// detect a valid ISA on the host machine.
88#[must_use]
89pub fn detect_host_isa() -> u8 {
90    #[cfg(target_arch = "x86_64")]
91    {
92        if is_x86_feature_detected!("avx512f") {
93            return ISA_AVX512;
94        }
95        if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma") {
96            return ISA_AVX2_FMA;
97        }
98        if is_x86_feature_detected!("avx") {
99            return ISA_AVX;
100        }
101        if is_x86_feature_detected!("sse2") {
102            return ISA_SSE2;
103        }
104        return ISA_SCALAR;
105    }
106
107    #[cfg(target_arch = "aarch64")]
108    {
109        if std::arch::is_aarch64_feature_detected!("neon") {
110            return ISA_NEON;
111        }
112        return ISA_SCALAR;
113    }
114
115    // All other architectures (wasm32, riscv, etc.)
116    #[allow(unreachable_code)]
117    ISA_SCALAR
118}
119
120// ============================================================================
121// Code generation helpers
122// ============================================================================
123
124/// Build the `x86_64` ISA detection body emitted inside the detect function.
125fn build_detect_x86_body() -> TokenStream {
126    quote! {
127        #[cfg(target_arch = "x86_64")]
128        {
129            #[cfg(feature = "avx512")]
130            if is_x86_feature_detected!("avx512f") {
131                return ISA_AVX512_LEVEL;
132            }
133            if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma") {
134                return ISA_AVX2_FMA_LEVEL;
135            }
136            if is_x86_feature_detected!("avx") {
137                return ISA_AVX_LEVEL;
138            }
139            if is_x86_feature_detected!("sse2") {
140                return ISA_SSE2_LEVEL;
141            }
142            return ISA_SCALAR_LEVEL;
143        }
144    }
145}
146
147/// Build the aarch64 ISA detection body emitted inside the detect function.
148fn build_detect_aarch64_body() -> TokenStream {
149    quote! {
150        #[cfg(target_arch = "aarch64")]
151        {
152            if std::arch::is_aarch64_feature_detected!("neon") {
153                return ISA_NEON_LEVEL;
154            }
155            return ISA_SCALAR_LEVEL;
156        }
157    }
158}
159
160/// Build the `x86_64` dispatch branches for the cached dispatcher body.
161///
162/// For size-16 f32 only AVX-512 is available; for size-16 f64 no x86 SIMD
163/// path exists.  For all other sizes (2, 4, 8), all ISA levels are probed.
164///
165/// Each branch creates its own local `data_inner` reinterpretation so that
166/// the raw-pointer slice never aliases the original `data` borrow.
167fn build_x86_64_branches(config: DispatcherConfig) -> TokenStream {
168    let size = config.size;
169    let ty_str = config.precision.type_str();
170    let ty_tokens: TokenStream = ty_str
171        .parse()
172        .unwrap_or_else(|_| unreachable!("ty_str is always f32 or f64"));
173    let avx512_fn = format_ident!("codelet_simd_{}_avx512_{}", size, ty_str);
174    let avx2_fn = format_ident!("codelet_simd_{}_avx2_{}", size, ty_str);
175    let sse2_fn = format_ident!("codelet_simd_{}_sse2_{}", size, ty_str);
176
177    if size == 16 {
178        if config.precision == Precision::F32 {
179            return quote! {
180                #[cfg(all(target_arch = "x86_64", feature = "avx512"))]
181                {
182                    if cached_level == ISA_AVX512_LEVEL {
183                        // Safety: avx512f detected at runtime.
184                        // Layout: Complex<f32> is #[repr(C)] (re, im) — same as [f32; 2*N].
185                        let data_len = data.len() * 2;
186                        let data_ptr = data.as_mut_ptr().cast::<#ty_tokens>();
187                        let data_inner = unsafe { core::slice::from_raw_parts_mut(data_ptr, data_len) };
188                        unsafe { super::#avx512_fn(data_inner, sign); }
189                        return;
190                    }
191                }
192            };
193        }
194        // size-16 f64: no dedicated SIMD on x86_64
195        return quote! {};
196    }
197
198    // Pure-AVX path only exists for f64 (no pure-AVX f32 emitter)
199    let avx_branch = if config.precision == Precision::F64 {
200        let avx_f64_fn = format_ident!("codelet_simd_{}_avx_f64", size);
201        quote! {
202            if cached_level == ISA_AVX_LEVEL {
203                // Safety: avx detected at runtime; function has #[target_feature(enable = "avx")].
204                let data_len = data.len() * 2;
205                let data_ptr = data.as_mut_ptr().cast::<#ty_tokens>();
206                let data_inner = unsafe { core::slice::from_raw_parts_mut(data_ptr, data_len) };
207                unsafe { super::#avx_f64_fn(data_inner, sign); }
208                return;
209            }
210        }
211    } else {
212        quote! {}
213    };
214
215    quote! {
216        #[cfg(target_arch = "x86_64")]
217        {
218            #[cfg(feature = "avx512")]
219            if cached_level == ISA_AVX512_LEVEL {
220                // Safety: avx512f detected at runtime.
221                let data_len = data.len() * 2;
222                let data_ptr = data.as_mut_ptr().cast::<#ty_tokens>();
223                let data_inner = unsafe { core::slice::from_raw_parts_mut(data_ptr, data_len) };
224                unsafe { super::#avx512_fn(data_inner, sign); }
225                return;
226            }
227            if cached_level == ISA_AVX2_FMA_LEVEL {
228                // Safety: avx2+fma detected at runtime.
229                let data_len = data.len() * 2;
230                let data_ptr = data.as_mut_ptr().cast::<#ty_tokens>();
231                let data_inner = unsafe { core::slice::from_raw_parts_mut(data_ptr, data_len) };
232                unsafe { super::#avx2_fn(data_inner, sign); }
233                return;
234            }
235            #avx_branch
236            if cached_level == ISA_SSE2_LEVEL {
237                // Safety: sse2 detected at runtime.
238                let data_len = data.len() * 2;
239                let data_ptr = data.as_mut_ptr().cast::<#ty_tokens>();
240                let data_inner = unsafe { core::slice::from_raw_parts_mut(data_ptr, data_len) };
241                unsafe { super::#sse2_fn(data_inner, sign); }
242                return;
243            }
244        }
245    }
246}
247
248/// Build the aarch64 dispatch branch for the cached dispatcher body.
249///
250/// Size-16 has no NEON path.
251fn build_aarch64_branch(config: DispatcherConfig) -> TokenStream {
252    if config.size == 16 {
253        return quote! {};
254    }
255    let ty_str = config.precision.type_str();
256    let ty_tokens: TokenStream = ty_str
257        .parse()
258        .unwrap_or_else(|_| unreachable!("ty_str is always f32 or f64"));
259    let neon_fn = format_ident!("codelet_simd_{}_neon_{}", config.size, ty_str);
260    quote! {
261        #[cfg(target_arch = "aarch64")]
262        {
263            if cached_level == ISA_NEON_LEVEL {
264                // Safety: NEON detected at runtime; mandatory on aarch64.
265                let data_len = data.len() * 2;
266                let data_ptr = data.as_mut_ptr().cast::<#ty_tokens>();
267                let data_inner = unsafe { core::slice::from_raw_parts_mut(data_ptr, data_len) };
268                unsafe { super::#neon_fn(data_inner, sign); }
269                return;
270            }
271        }
272    }
273}
274
275// ============================================================================
276// Code generation
277// ============================================================================
278
279/// Generate a cached runtime ISA dispatcher `TokenStream`.
280///
281/// The emitted code:
282/// 1. Declares ISA constants (only once per invocation; the caller is
283///    responsible for deduplication if multiple sizes share a module).
284/// 2. Declares a `static DETECTED_ISA_{size}_{ty}: AtomicU8`.
285/// 3. Emits a private `detect_isa_{size}_{ty}() -> u8` probe function.
286/// 4. Emits a public `codelet_simd_{size}_cached_{ty}(data, sign)` dispatcher.
287///
288/// The dispatcher delegates to the same arch-specific inner functions that the
289/// basic (uncached) dispatcher in [`super`] uses, following the exact same
290/// naming convention: `codelet_simd_{size}_{isa}_{ty}`.
291///
292/// # Errors
293///
294/// Returns `syn::Error` when `config.size` is not one of 2, 4, 8, or 16.
295#[allow(clippy::too_many_lines)] // reason: token-stream assembly requires many local variables
296pub fn generate_dispatcher(config: DispatcherConfig) -> Result<TokenStream, syn::Error> {
297    let size = config.size;
298    if !matches!(size, 2 | 4 | 8 | 16) {
299        return Err(syn::Error::new(
300            proc_macro2::Span::call_site(),
301            format!(
302                "gen_dispatcher_codelet: unsupported size {size} (expected one of 2, 4, 8, 16)"
303            ),
304        ));
305    }
306
307    let ty_str = config.precision.type_str();
308    let ty_upper = ty_str.to_uppercase();
309    let size_str = size.to_string();
310
311    // AtomicU8 static name: DETECTED_ISA_4_F32
312    let static_name = format_ident!("DETECTED_ISA_{}_{}", size_str, ty_upper);
313    // Detect function name: detect_isa_4_f32
314    let detect_fn = format_ident!("detect_isa_{}_{}", size_str, ty_str);
315    // Cached dispatcher name: codelet_simd_4_cached_f32
316    let cached_fn = format_ident!("codelet_simd_{}_cached_{}", size_str, ty_str);
317    // Scalar fallback name: codelet_simd_4_scalar
318    let scalar_fn = format_ident!("codelet_simd_{}_scalar", size);
319
320    let detect_x86_body = build_detect_x86_body();
321    let detect_aarch64_body = build_detect_aarch64_body();
322    let x86_64_branches = build_x86_64_branches(config);
323    let aarch64_branch = build_aarch64_branch(config);
324
325    let ty_tokens: TokenStream = ty_str
326        .parse()
327        .unwrap_or_else(|_| unreachable!("ty_str is always f32 or f64"));
328
329    let fn_doc = format!(
330        "Cached runtime ISA dispatcher for size-{size} DFT ({ty_str}).\n\n\
331         On first call, probes CPU features and stores the ISA level in a\n\
332         thread-safe `AtomicU8` static.  Subsequent calls read the cache with\n\
333         `Relaxed` ordering (benign-racy: all threads converge on the same answer).\n\n\
334         Dispatch priority on `x86_64`: AVX-512F > AVX2+FMA > AVX > SSE2 > scalar.\n\
335         Dispatch priority on `aarch64`: NEON > scalar.\n\
336         Other architectures fall through to the scalar codelet."
337    );
338
339    let size_lit = size;
340
341    Ok(quote! {
342        // ISA level constants (private to the generated scope)
343        const ISA_SCALAR_LEVEL:     u8 = 0;
344        const ISA_SSE2_LEVEL:       u8 = 1;
345        const ISA_AVX_LEVEL:        u8 = 2;
346        const ISA_AVX2_FMA_LEVEL:   u8 = 3;
347        const ISA_AVX512_LEVEL:     u8 = 4;
348        const ISA_NEON_LEVEL:       u8 = 5;
349        const ISA_UNDETECTED_LEVEL: u8 = 255;
350
351        /// Cached ISA level for this (size, precision) pair.
352        ///
353        /// Initialized to `ISA_UNDETECTED_LEVEL`.  Written once on first dispatch call.
354        static #static_name: core::sync::atomic::AtomicU8 =
355            core::sync::atomic::AtomicU8::new(ISA_UNDETECTED_LEVEL);
356
357        /// Probe the CPU once and return the best ISA level for this target.
358        fn #detect_fn() -> u8 {
359            #detect_x86_body
360            #detect_aarch64_body
361            #[allow(unreachable_code)]
362            ISA_SCALAR_LEVEL
363        }
364
365        #[doc = #fn_doc]
366        #[inline]
367        pub fn #cached_fn(
368            data: &mut [crate::kernel::Complex<#ty_tokens>],
369            sign: i32,
370        ) {
371            debug_assert!(
372                data.len() >= #size_lit,
373                "codelet_simd_{}_cached_{}: need >= {} elements, got {}",
374                #size_lit,
375                stringify!(#ty_tokens),
376                #size_lit,
377                data.len(),
378            );
379
380            // Load cached ISA; detect on first call.
381            let cached_level = {
382                let level = #static_name.load(core::sync::atomic::Ordering::Relaxed);
383                if level == ISA_UNDETECTED_LEVEL {
384                    let detected = #detect_fn();
385                    // Relaxed store: benign-racy — all threads converge on the same value.
386                    #static_name.store(detected, core::sync::atomic::Ordering::Relaxed);
387                    detected
388                } else {
389                    level
390                }
391            };
392
393            // Architecture-specific SIMD paths.
394            //
395            // data_inner is created inside each cfg block so that the raw-pointer
396            // reinterpretation and the original `data` borrow never overlap.
397            // The scalar fallback uses `data` directly — no aliasing.
398            #x86_64_branches
399            #aarch64_branch
400
401            // Scalar fallback: use the original Complex slice directly.
402            // No reinterpretation needed — the scalar codelet accepts Complex<T>.
403            super::#scalar_fn(data, sign);
404        }
405    })
406}
407
408// ============================================================================
409// Proc-macro parse input
410// ============================================================================
411
412/// Parsed arguments from `gen_dispatcher_codelet!(size = 4, ty = f32)`.
413struct MacroArgs {
414    size: usize,
415    precision: Precision,
416}
417
418impl Parse for MacroArgs {
419    fn parse(input: ParseStream<'_>) -> syn::Result<Self> {
420        let mut size: Option<usize> = None;
421        let mut precision: Option<Precision> = None;
422
423        while !input.is_empty() {
424            let key: syn::Ident = input.parse()?;
425            let _eq: Token![=] = input.parse()?;
426            match key.to_string().as_str() {
427                "size" => {
428                    let lit: LitInt = input.parse()?;
429                    size = Some(lit.base10_parse::<usize>().map_err(|_| {
430                        syn::Error::new(lit.span(), "expected an integer literal for `size`")
431                    })?);
432                }
433                "ty" => {
434                    let ident: syn::Ident = input.parse()?;
435                    precision = Some(match ident.to_string().as_str() {
436                        "f32" => Precision::F32,
437                        "f64" => Precision::F64,
438                        other => {
439                            return Err(syn::Error::new(
440                                ident.span(),
441                                format!("unknown ty `{other}`, expected f32 or f64"),
442                            ));
443                        }
444                    });
445                }
446                other => {
447                    return Err(syn::Error::new(
448                        key.span(),
449                        format!("unknown key `{other}`, expected one of: size, ty"),
450                    ));
451                }
452            }
453            if input.peek(Token![,]) {
454                let _: Token![,] = input.parse()?;
455            }
456        }
457
458        let size = size.ok_or_else(|| {
459            syn::Error::new(proc_macro2::Span::call_site(), "missing `size` argument")
460        })?;
461        let precision = precision.ok_or_else(|| {
462            syn::Error::new(proc_macro2::Span::call_site(), "missing `ty` argument")
463        })?;
464
465        Ok(Self { size, precision })
466    }
467}
468
469/// Entry point for the `gen_dispatcher_codelet!` proc-macro.
470///
471/// Parses `size = N, ty = TY` and calls [`generate_dispatcher`].
472///
473/// # Errors
474///
475/// Returns a `syn::Error` when the input does not parse as valid key-value
476/// pairs, a required key is missing, or `size` / `ty` have unsupported values.
477pub fn generate_from_macro(input: TokenStream) -> Result<TokenStream, syn::Error> {
478    let args: MacroArgs = syn::parse2(input)?;
479    generate_dispatcher(DispatcherConfig {
480        size: args.size,
481        precision: args.precision,
482    })
483}
484
485// ============================================================================
486// Tests
487// ============================================================================
488
489#[cfg(test)]
490mod tests {
491    use super::*;
492
493    // ── DispatcherConfig construction ─────────────────────────────────────
494
495    #[test]
496    fn test_dispatcher_config_valid_f32() {
497        let config = DispatcherConfig {
498            size: 4,
499            precision: Precision::F32,
500        };
501        assert_eq!(config.size, 4);
502        assert_eq!(config.precision, Precision::F32);
503    }
504
505    #[test]
506    fn test_dispatcher_config_valid_f64() {
507        let config = DispatcherConfig {
508            size: 8,
509            precision: Precision::F64,
510        };
511        assert_eq!(config.size, 8);
512        assert_eq!(config.precision, Precision::F64);
513    }
514
515    // ── ISA constants ─────────────────────────────────────────────────────
516
517    #[test]
518    fn test_isa_constants_are_ordered() {
519        // Validate ordering as compile-time assertions embedded in a constant.
520        const _: () = {
521            assert!(ISA_SCALAR < ISA_SSE2);
522            assert!(ISA_SSE2 < ISA_AVX);
523            assert!(ISA_AVX < ISA_AVX2_FMA);
524            assert!(ISA_AVX2_FMA < ISA_AVX512);
525            assert!(ISA_NEON != ISA_SCALAR);
526            assert!(ISA_UNDETECTED == 255);
527        };
528    }
529
530    // ── generate_dispatcher: TokenStream checks ───────────────────────────
531
532    #[test]
533    fn test_generate_dispatcher_nonempty() {
534        let ts = generate_dispatcher(DispatcherConfig {
535            size: 4,
536            precision: Precision::F32,
537        })
538        .expect("should generate for size 4 f32");
539        assert!(!ts.is_empty(), "TokenStream must not be empty");
540    }
541
542    #[test]
543    fn test_generate_dispatcher_nonempty_f64() {
544        let ts = generate_dispatcher(DispatcherConfig {
545            size: 8,
546            precision: Precision::F64,
547        })
548        .expect("should generate for size 8 f64");
549        assert!(!ts.is_empty(), "TokenStream must not be empty");
550    }
551
552    #[test]
553    fn test_generate_dispatcher_contains_is_x86_feature_detected() {
554        let ts = generate_dispatcher(DispatcherConfig {
555            size: 4,
556            precision: Precision::F32,
557        })
558        .expect("should generate");
559        let s = ts.to_string();
560        assert!(
561            s.contains("is_x86_feature_detected"),
562            "generated code must contain is_x86_feature_detected! macro; got snippet: {}",
563            &s[..s.len().min(500)]
564        );
565    }
566
567    #[test]
568    fn test_generate_dispatcher_contains_atomic_u8() {
569        let ts = generate_dispatcher(DispatcherConfig {
570            size: 4,
571            precision: Precision::F32,
572        })
573        .expect("should generate");
574        let s = ts.to_string();
575        assert!(
576            s.contains("AtomicU8"),
577            "generated code must contain AtomicU8 static; got snippet: {}",
578            &s[..s.len().min(500)]
579        );
580    }
581
582    #[test]
583    fn test_generate_dispatcher_contains_isa_undetected() {
584        let ts = generate_dispatcher(DispatcherConfig {
585            size: 4,
586            precision: Precision::F32,
587        })
588        .expect("should generate");
589        let s = ts.to_string();
590        assert!(
591            s.contains("ISA_UNDETECTED_LEVEL") || s.contains("255"),
592            "generated code must reference ISA_UNDETECTED_LEVEL sentinel"
593        );
594    }
595
596    #[test]
597    fn test_generate_dispatcher_function_name_size4_f32() {
598        let ts = generate_dispatcher(DispatcherConfig {
599            size: 4,
600            precision: Precision::F32,
601        })
602        .expect("should generate");
603        let s = ts.to_string();
604        assert!(
605            s.contains("codelet_simd_4_cached_f32"),
606            "expected cached dispatcher name in output; snippet: {}",
607            &s[..s.len().min(400)]
608        );
609    }
610
611    #[test]
612    fn test_generate_dispatcher_function_name_size8_f64() {
613        let ts = generate_dispatcher(DispatcherConfig {
614            size: 8,
615            precision: Precision::F64,
616        })
617        .expect("should generate");
618        let s = ts.to_string();
619        assert!(
620            s.contains("codelet_simd_8_cached_f64"),
621            "expected cached dispatcher name in output"
622        );
623    }
624
625    #[test]
626    fn test_generate_dispatcher_all_valid_sizes() {
627        for &size in &[2_usize, 4, 8, 16] {
628            for &prec in &[Precision::F32, Precision::F64] {
629                let result = generate_dispatcher(DispatcherConfig {
630                    size,
631                    precision: prec,
632                });
633                assert!(
634                    result.is_ok(),
635                    "size={size} prec={prec:?} should succeed, got: {:?}",
636                    result.err()
637                );
638            }
639        }
640    }
641
642    #[test]
643    fn test_generate_dispatcher_unsupported_size_returns_error() {
644        let result = generate_dispatcher(DispatcherConfig {
645            size: 3,
646            precision: Precision::F32,
647        });
648        assert!(result.is_err(), "size 3 must return Err");
649    }
650
651    #[test]
652    fn test_generate_dispatcher_unsupported_size_6_returns_error() {
653        let result = generate_dispatcher(DispatcherConfig {
654            size: 6,
655            precision: Precision::F64,
656        });
657        assert!(result.is_err(), "size 6 must return Err");
658    }
659
660    // ── detect_host_isa ───────────────────────────────────────────────────
661
662    #[test]
663    fn test_dispatcher_isa_detection() {
664        // On the host machine, detect_host_isa() must always return a valid ISA level.
665        // On aarch64 macOS (Apple Silicon) this should be ISA_NEON.
666        // On x86_64 this should be ISA_SSE2 or higher.
667        let isa = detect_host_isa();
668        assert_ne!(
669            isa, ISA_UNDETECTED,
670            "detect_host_isa must never return ISA_UNDETECTED (255)"
671        );
672        // Must be one of the known constants
673        assert!(
674            matches!(
675                isa,
676                ISA_SCALAR | ISA_SSE2 | ISA_AVX | ISA_AVX2_FMA | ISA_AVX512 | ISA_NEON
677            ),
678            "detect_host_isa returned unknown level {isa}"
679        );
680    }
681
682    #[test]
683    fn test_detect_host_isa_is_deterministic() {
684        let first = detect_host_isa();
685        let second = detect_host_isa();
686        assert_eq!(first, second, "detect_host_isa must be deterministic");
687    }
688
689    // ── generate_from_macro ───────────────────────────────────────────────
690
691    #[test]
692    fn test_generate_from_macro_size4_f32() {
693        let input: TokenStream = "size = 4, ty = f32".parse().expect("valid token stream");
694        let result = generate_from_macro(input);
695        assert!(
696            result.is_ok(),
697            "size=4 ty=f32 must succeed: {:?}",
698            result.err()
699        );
700        let s = result.expect("TokenStream").to_string();
701        assert!(
702            s.contains("codelet_simd_4_cached_f32"),
703            "must contain cached dispatcher name"
704        );
705    }
706
707    #[test]
708    fn test_generate_from_macro_size8_f64() {
709        let input: TokenStream = "size = 8, ty = f64".parse().expect("valid token stream");
710        let result = generate_from_macro(input);
711        assert!(
712            result.is_ok(),
713            "size=8 ty=f64 must succeed: {:?}",
714            result.err()
715        );
716        let s = result.expect("TokenStream").to_string();
717        assert!(
718            s.contains("codelet_simd_8_cached_f64"),
719            "must contain cached dispatcher name"
720        );
721    }
722
723    #[test]
724    fn test_generate_from_macro_size2_f64() {
725        let input: TokenStream = "size = 2, ty = f64".parse().expect("valid token stream");
726        let result = generate_from_macro(input);
727        assert!(result.is_ok(), "size=2 ty=f64 must succeed");
728    }
729
730    #[test]
731    fn test_generate_from_macro_size16_f32() {
732        let input: TokenStream = "size = 16, ty = f32".parse().expect("valid token stream");
733        let result = generate_from_macro(input);
734        assert!(result.is_ok(), "size=16 ty=f32 must succeed");
735    }
736
737    #[test]
738    fn test_generate_from_macro_missing_size_returns_error() {
739        let input: TokenStream = "ty = f32".parse().expect("valid token stream");
740        let result = generate_from_macro(input);
741        assert!(result.is_err(), "missing size must return error");
742    }
743
744    #[test]
745    fn test_generate_from_macro_missing_ty_returns_error() {
746        let input: TokenStream = "size = 4".parse().expect("valid token stream");
747        let result = generate_from_macro(input);
748        assert!(result.is_err(), "missing ty must return error");
749    }
750
751    #[test]
752    fn test_generate_from_macro_unknown_ty_returns_error() {
753        let input: TokenStream = "size = 4, ty = f16".parse().expect("valid token stream");
754        let result = generate_from_macro(input);
755        assert!(result.is_err(), "unknown ty must return error");
756    }
757
758    #[test]
759    fn test_generate_from_macro_unknown_key_returns_error() {
760        let input: TokenStream = "size = 4, ty = f32, isa = avx2"
761            .parse()
762            .expect("valid token stream");
763        let result = generate_from_macro(input);
764        assert!(result.is_err(), "unknown key must return error");
765    }
766
767    #[test]
768    fn test_generate_from_macro_unsupported_size_returns_error() {
769        let input: TokenStream = "size = 5, ty = f32".parse().expect("valid token stream");
770        let result = generate_from_macro(input);
771        assert!(result.is_err(), "size=5 must return error");
772    }
773}