Skip to main content

vkfft_rs/
program_ir.rs

1//! Backend-neutral multi-pass program/resource IR.
2//!
3//! Kernel IR describes math inside one dispatch. `ProgramIr` describes how several
4//! dispatches share external buffers, scratch storage, immutable LUTs, and barriers.
5//! It is intentionally backend-neutral so Bluestein, FFT-convolution Rader, future
6//! mixed-radix programs, and Four-step transforms can share one execution model.
7
8use crate::bluestein_ir::BluesteinPipelineIr;
9use crate::complex::Complex64;
10use crate::convolution_ir::{ConvolutionIr, NdConvolutionIr, NdRealConvolutionIr};
11use crate::double_double::ComplexDoubleDouble;
12use crate::double_double_ir::{
13    DoubleDoubleBluesteinIr, DoubleDoubleDirectRaderIr, DoubleDoubleFftRaderIr,
14    DoubleDoubleNdFftIr, DoubleDoubleNdR2rIr, DoubleDoubleNdRealFftIr, DoubleDoubleOneDimIr,
15    DoubleDoubleR2rAlgorithm, DoubleDoubleR2rIr, DoubleDoubleRealFftIr, DoubleDoubleStockhamIr,
16};
17use crate::double_double_recursive_ir::{
18    DoubleDoubleCooleyTukeyOutputModifier, DoubleDoubleForcedRaderThreeUploadComponentIr,
19    DoubleDoubleRecursiveFftIr, DoubleDoubleRecursiveFftNodeIr,
20};
21use crate::error::{Result, VkFftError};
22use crate::kernel_ir::{BufferAccess, BufferRole, DispatchGeometry, KernelIr, ScalarType};
23use crate::mixed_ir::{MixedPrimeRaderIr, MixedRaderStockhamIr};
24use crate::nd_ir::NdFftIr;
25use crate::nd_real_ir::NdRealFftIr;
26use crate::one_dim_ir::OneDimFftIr;
27use crate::r2r_ir::{NdR2rIr, R2rIr};
28use crate::rader_ir::{RaderDirectIr, RaderFftInputStrategy, RaderFftPipelineIr};
29use crate::real_ir::{RealFftIr, RealFftKind};
30use crate::recursive_ir::{
31    CooleyTukeyInputModifier, CooleyTukeyOutputModifier, RecursiveFftIr, RecursiveFftNodeIr,
32};
33use crate::zero_pad_ir::ZeroPadPassIr;
34
35#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
36pub struct ProgramResourceId(pub usize);
37
38#[derive(Debug, Clone, Copy, PartialEq, Eq)]
39pub enum ProgramResourceKind {
40    Input,
41    Output,
42    Scratch,
43    LookupTable,
44}
45
46#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
47pub enum ProgramElementShape {
48    Complex,
49    Scalar,
50}
51
52impl ProgramElementShape {
53    pub const fn element_bytes(self, scalar: ScalarType) -> usize {
54        match self {
55            Self::Complex => scalar.complex_bytes(),
56            Self::Scalar => scalar.bytes(),
57        }
58    }
59}
60
61#[derive(Debug, Clone, Copy, PartialEq, Eq)]
62pub struct ExternalBufferLayout {
63    pub logical_len: usize,
64    pub physical_stride: usize,
65    pub batch_count: usize,
66    pub element_shape: ProgramElementShape,
67}
68
69impl ExternalBufferLayout {
70    pub fn logical_elements(self) -> Result<usize> {
71        self.logical_len
72            .checked_mul(self.batch_count)
73            .ok_or(VkFftError::ArithmeticOverflow {
74                operation: "program external logical element count",
75            })
76    }
77
78    pub fn physical_elements(self) -> Result<usize> {
79        self.physical_stride
80            .checked_mul(self.batch_count)
81            .ok_or(VkFftError::ArithmeticOverflow {
82                operation: "program external physical element count",
83            })
84    }
85
86    fn validate(self) -> Result<()> {
87        if self.logical_len == 0 || self.physical_stride == 0 || self.batch_count == 0 {
88            return Err(VkFftError::InvalidKernelIr(
89                "program external buffer layout must be non-zero",
90            ));
91        }
92        if self.logical_len > self.physical_stride {
93            return Err(VkFftError::InvalidKernelIr(
94                "program external logical length must fit inside physical stride",
95            ));
96        }
97        let _ = self.logical_elements()?;
98        let _ = self.physical_elements()?;
99        Ok(())
100    }
101}
102
103impl ProgramResource {
104    pub const fn element_shape(&self) -> ProgramElementShape {
105        match self.external_layout {
106            Some(layout) => layout.element_shape,
107            None => ProgramElementShape::Complex,
108        }
109    }
110
111    pub const fn element_bytes(&self) -> usize {
112        self.element_shape().element_bytes(self.scalar)
113    }
114}
115
116#[derive(Debug, Clone, PartialEq)]
117pub enum ProgramResourceInitialization {
118    ExternalInput,
119    Zeroed,
120    Complex64(Vec<Complex64>),
121    /// Immutable true double-double complex data. Each logical element contains
122    /// four binary64 words (`re.hi`, `re.lo`, `im.hi`, `im.lo`).
123    ComplexDoubleDouble(Vec<ComplexDoubleDouble>),
124    /// Lazily materialized direction-independent Stockham unit roots. Keeping only
125    /// the period in ProgramIr avoids embedding multi-megabyte Four-step tables in
126    /// the IR while preserving deterministic immutable-LUT identity.
127    StockhamUnitRoots {
128        len: usize,
129    },
130}
131
132#[derive(Debug, Clone, PartialEq)]
133pub struct ProgramResource {
134    pub id: ProgramResourceId,
135    pub name: String,
136    pub kind: ProgramResourceKind,
137    pub scalar: ScalarType,
138    pub elements: usize,
139    pub external_layout: Option<ExternalBufferLayout>,
140    pub initialization: ProgramResourceInitialization,
141}
142
143#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
144pub struct ProgramAllocationId(pub usize);
145
146#[derive(Debug, Clone, Copy, PartialEq, Eq)]
147pub enum ProgramAllocationKind {
148    ExternalInput,
149    ExternalOutput,
150    Scratch,
151    LookupTable,
152}
153
154#[derive(Debug, Clone, PartialEq, Eq)]
155pub struct ProgramAllocation {
156    pub id: ProgramAllocationId,
157    pub kind: ProgramAllocationKind,
158    pub scalar: ScalarType,
159    pub elements: usize,
160    pub resources: Vec<ProgramResourceId>,
161}
162
163#[derive(Debug, Clone, PartialEq, Eq)]
164pub struct ProgramMemoryPlan {
165    pub allocations: Vec<ProgramAllocation>,
166    pub resource_allocations: Vec<ProgramAllocationId>,
167}
168
169impl ProgramMemoryPlan {
170    pub fn allocation_for(&self, resource: ProgramResourceId) -> Result<ProgramAllocationId> {
171        self.resource_allocations
172            .get(resource.0)
173            .copied()
174            .ok_or(VkFftError::InvalidKernelIr(
175                "program memory plan references an unknown logical resource",
176            ))
177    }
178
179    pub fn allocated_elements(&self) -> Result<usize> {
180        self.allocations
181            .iter()
182            .try_fold(0usize, |total, allocation| {
183                total
184                    .checked_add(allocation.elements)
185                    .ok_or(VkFftError::ArithmeticOverflow {
186                        operation: "program physical allocation element count",
187                    })
188            })
189    }
190}
191
192#[derive(Debug, Clone, Copy, PartialEq, Eq)]
193pub struct ProgramPassBinding {
194    pub binding: u32,
195    pub resource: ProgramResourceId,
196    pub role: BufferRole,
197    pub access: BufferAccess,
198}
199
200#[derive(Debug, Clone, PartialEq, Eq)]
201pub struct ProgramPass {
202    pub name: String,
203    pub dispatch: DispatchGeometry,
204    pub bindings: Vec<ProgramPassBinding>,
205}
206
207#[derive(Debug, Clone, PartialEq)]
208pub struct ProgramIr {
209    pub name: String,
210    pub scalar: ScalarType,
211    pub resources: Vec<ProgramResource>,
212    pub passes: Vec<ProgramPass>,
213}
214
215impl ProgramIr {
216    pub fn double_double_stockham(ir: &DoubleDoubleStockhamIr) -> Result<Self> {
217        ir.validate()?;
218        if ir.sequence_len > 64 {
219            let core = Self::double_double_stockham_multi_pass(ir)?;
220            return Self::wrap_double_double_zero_pad(
221                ir.zero_pad_pass.as_ref(),
222                ir.sequence_len,
223                ir.batch_count,
224                "double_double_stockham",
225                core,
226            );
227        }
228        let elements =
229            ir.sequence_len
230                .checked_mul(ir.batch_count)
231                .ok_or(VkFftError::ArithmeticOverflow {
232                    operation: "double-double Stockham program element count",
233                })?;
234        let external_scalar = match ir.external_storage {
235            crate::PrecisionStorage::DoubleDouble => ScalarType::DoubleDouble,
236            crate::PrecisionStorage::F64 => ScalarType::F64,
237            _ => {
238                return Err(VkFftError::InvalidKernelIr(
239                    "double-double Stockham program requires DD or F64 external storage",
240                ));
241            }
242        };
243        let layout = ExternalBufferLayout {
244            logical_len: ir.sequence_len,
245            physical_stride: ir.sequence_len,
246            batch_count: ir.batch_count,
247            element_shape: ProgramElementShape::Complex,
248        };
249        let twiddles = ir.twiddles.packed_values();
250        let mut resources = vec![
251            ProgramResource {
252                id: ProgramResourceId(0),
253                name: "input".to_owned(),
254                kind: ProgramResourceKind::Input,
255                scalar: external_scalar,
256                elements,
257                external_layout: Some(layout),
258                initialization: ProgramResourceInitialization::ExternalInput,
259            },
260            ProgramResource {
261                id: ProgramResourceId(1),
262                name: "output".to_owned(),
263                kind: ProgramResourceKind::Output,
264                scalar: external_scalar,
265                elements,
266                external_layout: Some(layout),
267                initialization: ProgramResourceInitialization::Zeroed,
268            },
269        ];
270        let mut bindings = vec![
271            binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
272            binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
273        ];
274        if !twiddles.is_empty() {
275            let lut_id = ProgramResourceId(resources.len());
276            resources.push(ProgramResource {
277                id: lut_id,
278                name: "double_double_stockham_twiddles".to_owned(),
279                kind: ProgramResourceKind::LookupTable,
280                scalar: ScalarType::DoubleDouble,
281                elements: twiddles.len(),
282                external_layout: None,
283                initialization: ProgramResourceInitialization::ComplexDoubleDouble(twiddles),
284            });
285            bindings.push(ProgramPassBinding {
286                binding: 2,
287                resource: lut_id,
288                role: BufferRole::TwiddleLookupTable,
289                access: BufferAccess::ReadOnly,
290            });
291        }
292        let dispatch_x =
293            u32::try_from(ir.batch_group_count()).map_err(|_| VkFftError::ValueOutOfRange {
294                field: "double-double Stockham program dispatch count",
295            })?;
296        let program = Self {
297            name: format!("{}_program", ir.name),
298            scalar: ScalarType::DoubleDouble,
299            resources,
300            passes: vec![ProgramPass {
301                name: ir.name.clone(),
302                dispatch: DispatchGeometry {
303                    x: dispatch_x,
304                    y: 1,
305                    z: 1,
306                },
307                bindings,
308            }],
309        };
310        program.validate()?;
311        Self::wrap_double_double_zero_pad(
312            ir.zero_pad_pass.as_ref(),
313            ir.sequence_len,
314            ir.batch_count,
315            "double_double_stockham",
316            program,
317        )
318    }
319
320    fn double_double_stockham_internal(ir: &DoubleDoubleStockhamIr) -> Result<Self> {
321        if ir.external_storage != crate::PrecisionStorage::DoubleDouble
322            || ir.zero_pad_pass.is_some()
323        {
324            return Err(VkFftError::InvalidKernelIr(
325                "internal double-double Stockham child requires unmodified DD boundaries",
326            ));
327        }
328        let mut program = Self::double_double_stockham(ir)?;
329        if ir.sequence_len <= 64 {
330            return Ok(program);
331        }
332        if program.passes.len() != ir.stages.len() + 2 || ir.stages.is_empty() {
333            return Err(VkFftError::InvalidKernelIr(
334                "internal double-double Stockham child has an unexpected pass graph",
335            ));
336        }
337        program.passes.remove(0);
338        program.passes.pop();
339        let first_input = program
340            .passes
341            .first_mut()
342            .and_then(|pass| {
343                pass.bindings
344                    .iter_mut()
345                    .find(|binding| binding.binding == 0)
346            })
347            .ok_or(VkFftError::InvalidKernelIr(
348                "internal double-double Stockham first stage is missing input binding",
349            ))?;
350        first_input.resource = ProgramResourceId(0);
351        let final_output = program
352            .passes
353            .last_mut()
354            .and_then(|pass| {
355                pass.bindings
356                    .iter_mut()
357                    .find(|binding| binding.binding == 1)
358            })
359            .ok_or(VkFftError::InvalidKernelIr(
360                "internal double-double Stockham final stage is missing output binding",
361            ))?;
362        final_output.resource = ProgramResourceId(1);
363        program.name = format!("{}_internal_program", ir.name);
364        program.validate()?;
365        Ok(program)
366    }
367
368    fn double_double_stockham_multi_pass(ir: &DoubleDoubleStockhamIr) -> Result<Self> {
369        let elements =
370            ir.sequence_len
371                .checked_mul(ir.batch_count)
372                .ok_or(VkFftError::ArithmeticOverflow {
373                    operation: "double-double Stockham multi-pass element count",
374                })?;
375        let external_scalar = match ir.external_storage {
376            crate::PrecisionStorage::DoubleDouble => ScalarType::DoubleDouble,
377            crate::PrecisionStorage::F64 => ScalarType::F64,
378            _ => {
379                return Err(VkFftError::InvalidKernelIr(
380                    "double-double Stockham multi-pass requires DD or F64 external storage",
381                ));
382            }
383        };
384        let layout = ExternalBufferLayout {
385            logical_len: ir.sequence_len,
386            physical_stride: ir.sequence_len,
387            batch_count: ir.batch_count,
388            element_shape: ProgramElementShape::Complex,
389        };
390        let twiddles = ir.twiddles.packed_values();
391        let resources = vec![
392            ProgramResource {
393                id: ProgramResourceId(0),
394                name: "input".to_owned(),
395                kind: ProgramResourceKind::Input,
396                scalar: external_scalar,
397                elements,
398                external_layout: Some(layout),
399                initialization: ProgramResourceInitialization::ExternalInput,
400            },
401            ProgramResource {
402                id: ProgramResourceId(1),
403                name: "output".to_owned(),
404                kind: ProgramResourceKind::Output,
405                scalar: external_scalar,
406                elements,
407                external_layout: Some(layout),
408                initialization: ProgramResourceInitialization::Zeroed,
409            },
410            ProgramResource {
411                id: ProgramResourceId(2),
412                name: "double_double_stockham_scratch_a".to_owned(),
413                kind: ProgramResourceKind::Scratch,
414                scalar: ScalarType::DoubleDouble,
415                elements,
416                external_layout: None,
417                initialization: ProgramResourceInitialization::Zeroed,
418            },
419            ProgramResource {
420                id: ProgramResourceId(3),
421                name: "double_double_stockham_scratch_b".to_owned(),
422                kind: ProgramResourceKind::Scratch,
423                scalar: ScalarType::DoubleDouble,
424                elements,
425                external_layout: None,
426                initialization: ProgramResourceInitialization::Zeroed,
427            },
428            ProgramResource {
429                id: ProgramResourceId(4),
430                name: "double_double_stockham_twiddles".to_owned(),
431                kind: ProgramResourceKind::LookupTable,
432                scalar: ScalarType::DoubleDouble,
433                elements: twiddles.len(),
434                external_layout: None,
435                initialization: ProgramResourceInitialization::ComplexDoubleDouble(twiddles),
436            },
437        ];
438        let dispatch = DispatchGeometry {
439            x: u32::try_from(ir.batch_group_count()).map_err(|_| VkFftError::ValueOutOfRange {
440                field: "double-double Stockham multi-pass dispatch count",
441            })?,
442            y: 1,
443            z: 1,
444        };
445        let mut passes = Vec::with_capacity(ir.stages.len() + 2);
446        passes.push(ProgramPass {
447            name: format!("{}_promote", ir.name),
448            dispatch,
449            bindings: vec![
450                binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
451                binding(1, 2, BufferRole::Output, BufferAccess::WriteOnly),
452            ],
453        });
454        let mut source = 2usize;
455        let mut target = 3usize;
456        for stage in &ir.stages {
457            passes.push(ProgramPass {
458                name: format!("{}_stage_{}", ir.name, stage.index),
459                dispatch,
460                bindings: vec![
461                    binding(0, source, BufferRole::Input, BufferAccess::ReadOnly),
462                    binding(1, target, BufferRole::Output, BufferAccess::WriteOnly),
463                    binding(2, 4, BufferRole::TwiddleLookupTable, BufferAccess::ReadOnly),
464                ],
465            });
466            core::mem::swap(&mut source, &mut target);
467        }
468        passes.push(ProgramPass {
469            name: format!("{}_finalize", ir.name),
470            dispatch,
471            bindings: vec![
472                binding(0, source, BufferRole::Input, BufferAccess::ReadOnly),
473                binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
474            ],
475        });
476        let program = Self {
477            name: format!("{}_program", ir.name),
478            scalar: ScalarType::DoubleDouble,
479            resources,
480            passes,
481        };
482        program.validate()?;
483        Ok(program)
484    }
485
486    pub fn double_double_direct_rader(ir: &DoubleDoubleDirectRaderIr) -> Result<Self> {
487        ir.validate()?;
488        let elements =
489            ir.prime
490                .checked_mul(ir.batch_count)
491                .ok_or(VkFftError::ArithmeticOverflow {
492                    operation: "double-double direct Rader program element count",
493                })?;
494        let storage_scalar = |storage| match storage {
495            crate::PrecisionStorage::DoubleDouble => Ok(ScalarType::DoubleDouble),
496            crate::PrecisionStorage::F64 => Ok(ScalarType::F64),
497            _ => Err(VkFftError::InvalidKernelIr(
498                "double-double direct Rader program requires DD or F64 caller storage",
499            )),
500        };
501        let input_scalar = storage_scalar(ir.input_storage)?;
502        let output_scalar = storage_scalar(ir.output_storage)?;
503        let layout = ExternalBufferLayout {
504            logical_len: ir.prime,
505            physical_stride: ir.prime,
506            batch_count: ir.batch_count,
507            element_shape: ProgramElementShape::Complex,
508        };
509        let lut = ir.table.twiddles_by_generator_power.clone();
510        let mut resources = vec![
511            ProgramResource {
512                id: ProgramResourceId(0),
513                name: "input".to_owned(),
514                kind: ProgramResourceKind::Input,
515                scalar: input_scalar,
516                elements,
517                external_layout: Some(layout),
518                initialization: ProgramResourceInitialization::ExternalInput,
519            },
520            ProgramResource {
521                id: ProgramResourceId(1),
522                name: "output".to_owned(),
523                kind: ProgramResourceKind::Output,
524                scalar: output_scalar,
525                elements,
526                external_layout: Some(layout),
527                initialization: ProgramResourceInitialization::Zeroed,
528            },
529            ProgramResource {
530                id: ProgramResourceId(2),
531                name: "double_double_rader_twiddles".to_owned(),
532                kind: ProgramResourceKind::LookupTable,
533                scalar: ScalarType::DoubleDouble,
534                elements: lut.len(),
535                external_layout: None,
536                initialization: ProgramResourceInitialization::ComplexDoubleDouble(lut),
537            },
538        ];
539        let mut bindings = vec![
540            binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
541            binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
542            binding(2, 2, BufferRole::LookupTable, BufferAccess::ReadOnly),
543        ];
544        let parent_period = match ir.io_mapping {
545            crate::StockhamIoMapping::FourStepRight(mapping) => Some(mapping.logical_len),
546            crate::StockhamIoMapping::FourStepThreeUpload2(mapping) => Some(mapping.logical_len),
547            crate::StockhamIoMapping::FourStepThreeUpload1(mapping) => {
548                let [a, b, _] = mapping.axis_split;
549                Some(a.checked_mul(b).ok_or(VkFftError::ArithmeticOverflow {
550                    operation: "double-double direct Rader three-upload parent root period",
551                })?)
552            }
553            _ => None,
554        };
555        if let Some(period) = parent_period {
556            let roots = crate::lut::unit_root_table_double_double(period, ir.direction)?;
557            let root_id = ProgramResourceId(resources.len());
558            resources.push(ProgramResource {
559                id: root_id,
560                name: "double_double_rader_four_step_roots".to_owned(),
561                kind: ProgramResourceKind::LookupTable,
562                scalar: ScalarType::DoubleDouble,
563                elements: roots.len(),
564                external_layout: None,
565                initialization: ProgramResourceInitialization::ComplexDoubleDouble(roots),
566            });
567            bindings.push(ProgramPassBinding {
568                binding: 3,
569                resource: root_id,
570                role: BufferRole::TwiddleLookupTable,
571                access: BufferAccess::ReadOnly,
572            });
573        }
574        let program = Self {
575            name: format!("{}_program", ir.name),
576            scalar: ScalarType::DoubleDouble,
577            resources,
578            passes: vec![ProgramPass {
579                name: ir.name.clone(),
580                dispatch: DispatchGeometry {
581                    x: u32::try_from(ir.batch_group_count()).map_err(|_| {
582                        VkFftError::ValueOutOfRange {
583                            field: "double-double direct Rader program dispatch count",
584                        }
585                    })?,
586                    y: 1,
587                    z: 1,
588                },
589                bindings,
590            }],
591        };
592        program.validate()?;
593        Self::wrap_double_double_zero_pad(
594            ir.zero_pad_pass.as_ref(),
595            ir.prime,
596            ir.batch_count,
597            "double_double_direct_rader",
598            program,
599        )
600    }
601
602    pub fn double_double_one_dim(ir: &DoubleDoubleOneDimIr) -> Result<Self> {
603        match ir {
604            DoubleDoubleOneDimIr::Stockham(ir) => Self::double_double_stockham(ir),
605            DoubleDoubleOneDimIr::DirectRader(ir) => Self::double_double_direct_rader(ir),
606            DoubleDoubleOneDimIr::FftRader(ir) => Self::double_double_fft_rader(ir),
607            DoubleDoubleOneDimIr::Bluestein(ir) => Self::double_double_bluestein(ir),
608            DoubleDoubleOneDimIr::Recursive(ir) => Self::double_double_recursive(ir),
609        }
610    }
611
612    pub fn double_double_recursive(ir: &DoubleDoubleRecursiveFftIr) -> Result<Self> {
613        ir.validate()?;
614        let elements =
615            ir.logical_len
616                .checked_mul(ir.batch_count)
617                .ok_or(VkFftError::ArithmeticOverflow {
618                    operation: "double-double recursive program element count",
619                })?;
620        let external_scalar = match ir.external_storage {
621            crate::PrecisionStorage::DoubleDouble => ScalarType::DoubleDouble,
622            crate::PrecisionStorage::F64 => ScalarType::F64,
623            _ => {
624                return Err(VkFftError::InvalidKernelIr(
625                    "double-double recursive ProgramIr requires DD or F64 external storage",
626                ));
627            }
628        };
629        let layout = ExternalBufferLayout {
630            logical_len: ir.logical_len,
631            physical_stride: ir.logical_len,
632            batch_count: ir.batch_count,
633            element_shape: ProgramElementShape::Complex,
634        };
635        let mut resources = vec![
636            ProgramResource {
637                id: ProgramResourceId(0),
638                name: "input".to_owned(),
639                kind: ProgramResourceKind::Input,
640                scalar: external_scalar,
641                elements,
642                external_layout: Some(layout),
643                initialization: ProgramResourceInitialization::ExternalInput,
644            },
645            ProgramResource {
646                id: ProgramResourceId(1),
647                name: "output".to_owned(),
648                kind: ProgramResourceKind::Output,
649                scalar: external_scalar,
650                elements,
651                external_layout: Some(layout),
652                initialization: ProgramResourceInitialization::Zeroed,
653            },
654        ];
655        let core = if ir.two_upload_four_step_plan.is_some() {
656            Self::double_double_recursive_two_upload_four_step(ir, resources)?
657        } else if ir.three_upload_four_step_plan.is_some() {
658            Self::double_double_recursive_three_upload_four_step(ir, resources)?
659        } else if let Some(components) = ir.forced_rader_three_upload_mapped_components()? {
660            Self::double_double_recursive_forced_rader_three_upload(ir, resources, components)?
661        } else if let Some((high, _mapping)) = ir.forced_rader_two_upload_mapped_high_stockham()? {
662            Self::double_double_recursive_forced_rader_stockham_high_boundary(ir, resources, high)?
663        } else if let Some(mapped_high) = ir.forced_rader_two_upload_mapped_high_component()? {
664            Self::double_double_recursive_forced_rader_high_boundary(ir, resources, mapped_high)?
665        } else {
666            let mut passes = Vec::new();
667            let mut serial = 0usize;
668            flatten_double_double_recursive_node(
669                &ir.root,
670                ProgramResourceId(0),
671                ProgramResourceId(1),
672                &mut resources,
673                &mut passes,
674                &mut serial,
675            )?;
676            let program = Self {
677                name: format!("{}_program", ir.name),
678                scalar: ScalarType::DoubleDouble,
679                resources,
680                passes,
681            };
682            program.validate()?;
683            program
684        };
685        Self::wrap_double_double_zero_pad(
686            ir.zero_pad_pass.as_ref(),
687            ir.logical_len,
688            ir.batch_count,
689            "double_double_recursive",
690            core,
691        )
692    }
693
694    fn wrap_double_double_zero_pad(
695        pass: Option<&ZeroPadPassIr>,
696        logical_len: usize,
697        batch_count: usize,
698        stem: &str,
699        mut program: Self,
700    ) -> Result<Self> {
701        let Some(pass) = pass else {
702            return Ok(program);
703        };
704        pass.validate()?;
705        let input = program.input_resource()?.id;
706        let output = program.output_resource()?.id;
707        let elements =
708            logical_len
709                .checked_mul(batch_count)
710                .ok_or(VkFftError::ArithmeticOverflow {
711                    operation: "double-double zero-pad scratch element count",
712                })?;
713        let boundary = ProgramResourceId(program.resources.len());
714        let input_boundary = pass.operation.is_input_boundary();
715        program.resources.push(ProgramResource {
716            id: boundary,
717            name: format!(
718                "{stem}_zero_padded_{}",
719                if input_boundary { "input" } else { "output" }
720            ),
721            kind: ProgramResourceKind::Scratch,
722            scalar: if input_boundary {
723                pass.output_storage_scalar
724            } else {
725                pass.input_storage_scalar
726            },
727            elements,
728            external_layout: None,
729            initialization: ProgramResourceInitialization::Zeroed,
730        });
731        if input_boundary {
732            for program_pass in &mut program.passes {
733                for pass_binding in &mut program_pass.bindings {
734                    if pass_binding.resource == input {
735                        pass_binding.resource = boundary;
736                    }
737                }
738            }
739            program.passes.insert(
740                0,
741                ProgramPass {
742                    name: pass.name.clone(),
743                    dispatch: pass.dispatch,
744                    bindings: vec![
745                        ProgramPassBinding {
746                            binding: 0,
747                            resource: input,
748                            role: BufferRole::Input,
749                            access: BufferAccess::ReadOnly,
750                        },
751                        ProgramPassBinding {
752                            binding: 1,
753                            resource: boundary,
754                            role: BufferRole::Output,
755                            access: BufferAccess::WriteOnly,
756                        },
757                    ],
758                },
759            );
760        } else {
761            for program_pass in &mut program.passes {
762                for pass_binding in &mut program_pass.bindings {
763                    if pass_binding.resource == output {
764                        pass_binding.resource = boundary;
765                    }
766                }
767            }
768            program.passes.push(ProgramPass {
769                name: pass.name.clone(),
770                dispatch: pass.dispatch,
771                bindings: vec![
772                    ProgramPassBinding {
773                        binding: 0,
774                        resource: boundary,
775                        role: BufferRole::Input,
776                        access: BufferAccess::ReadOnly,
777                    },
778                    ProgramPassBinding {
779                        binding: 1,
780                        resource: output,
781                        role: BufferRole::Output,
782                        access: BufferAccess::WriteOnly,
783                    },
784                ],
785            });
786        }
787        program.validate()?;
788        Ok(program)
789    }
790
791    fn double_double_recursive_forced_rader_stockham_high_boundary(
792        ir: &DoubleDoubleRecursiveFftIr,
793        mut resources: Vec<ProgramResource>,
794        high: &DoubleDoubleStockhamIr,
795    ) -> Result<Self> {
796        let DoubleDoubleRecursiveFftNodeIr::CooleyTukey(root) = &ir.root else {
797            return Err(VkFftError::InvalidKernelIr(
798                "double-double Stockham-high forced-Rader boundary requires a Cooley-Tukey root",
799            ));
800        };
801        let elements =
802            ir.logical_len
803                .checked_mul(ir.batch_count)
804                .ok_or(VkFftError::ArithmeticOverflow {
805                    operation: "double-double Stockham-high forced-Rader scratch element count",
806                })?;
807        let exchange = ProgramResourceId(resources.len());
808        resources.push(ProgramResource {
809            id: exchange,
810            name: "double_double_rader_four_step_exchange".to_owned(),
811            kind: ProgramResourceKind::Scratch,
812            scalar: ScalarType::DoubleDouble,
813            elements,
814            external_layout: None,
815            initialization: ProgramResourceInitialization::Zeroed,
816        });
817        let high_twiddle_values = high.twiddles.packed_values();
818        let high_twiddles = ProgramResourceId(resources.len());
819        resources.push(ProgramResource {
820            id: high_twiddles,
821            name: "double_double_forced_rader_two_upload_1_stockham_twiddles".to_owned(),
822            kind: ProgramResourceKind::LookupTable,
823            scalar: ScalarType::DoubleDouble,
824            elements: high_twiddle_values.len(),
825            external_layout: None,
826            initialization: ProgramResourceInitialization::ComplexDoubleDouble(high_twiddle_values),
827        });
828        let parent_roots = ProgramResourceId(resources.len());
829        resources.push(ProgramResource {
830            id: parent_roots,
831            name: "double_double_forced_rader_two_upload_1_parent_roots".to_owned(),
832            kind: ProgramResourceKind::LookupTable,
833            scalar: ScalarType::DoubleDouble,
834            elements: root.twiddles.len(),
835            external_layout: None,
836            initialization: ProgramResourceInitialization::ComplexDoubleDouble(
837                root.twiddles.clone(),
838            ),
839        });
840        let passes = vec![ProgramPass {
841            name: format!("{}_forced_rader_two_upload_1", ir.name),
842            dispatch: DispatchGeometry {
843                x: u32::try_from(high.batch_group_count()).map_err(|_| {
844                    VkFftError::ValueOutOfRange {
845                        field: "double-double forced-Rader high Stockham dispatch count",
846                    }
847                })?,
848                y: 1,
849                z: 1,
850            },
851            bindings: vec![
852                binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
853                ProgramPassBinding {
854                    binding: 1,
855                    resource: exchange,
856                    role: BufferRole::Output,
857                    access: BufferAccess::WriteOnly,
858                },
859                ProgramPassBinding {
860                    binding: 2,
861                    resource: high_twiddles,
862                    role: BufferRole::TwiddleLookupTable,
863                    access: BufferAccess::ReadOnly,
864                },
865                ProgramPassBinding {
866                    binding: 3,
867                    resource: parent_roots,
868                    role: BufferRole::Auxiliary,
869                    access: BufferAccess::ReadOnly,
870                },
871            ],
872        }];
873        Self::finish_double_double_recursive_forced_rader_two_upload(
874            ir, resources, passes, 0, exchange, elements,
875        )
876    }
877
878    fn double_double_recursive_forced_rader_high_boundary(
879        ir: &DoubleDoubleRecursiveFftIr,
880        mut resources: Vec<ProgramResource>,
881        mapped_high: DoubleDoubleRecursiveFftNodeIr,
882    ) -> Result<Self> {
883        let DoubleDoubleRecursiveFftNodeIr::CooleyTukey(_) = &ir.root else {
884            return Err(VkFftError::InvalidKernelIr(
885                "double-double mapped forced-Rader boundary requires a Cooley-Tukey root",
886            ));
887        };
888        let elements =
889            ir.logical_len
890                .checked_mul(ir.batch_count)
891                .ok_or(VkFftError::ArithmeticOverflow {
892                    operation: "double-double mapped forced-Rader scratch element count",
893                })?;
894        let exchange = ProgramResourceId(resources.len());
895        resources.push(ProgramResource {
896            id: exchange,
897            name: "double_double_rader_four_step_exchange".to_owned(),
898            kind: ProgramResourceKind::Scratch,
899            scalar: ScalarType::DoubleDouble,
900            elements,
901            external_layout: None,
902            initialization: ProgramResourceInitialization::Zeroed,
903        });
904        let mut passes = Vec::new();
905        let mut serial = 0usize;
906        flatten_double_double_recursive_node(
907            &mapped_high,
908            ProgramResourceId(0),
909            exchange,
910            &mut resources,
911            &mut passes,
912            &mut serial,
913        )?;
914        Self::finish_double_double_recursive_forced_rader_two_upload(
915            ir, resources, passes, serial, exchange, elements,
916        )
917    }
918
919    fn finish_double_double_recursive_forced_rader_two_upload(
920        ir: &DoubleDoubleRecursiveFftIr,
921        mut resources: Vec<ProgramResource>,
922        mut passes: Vec<ProgramPass>,
923        mut serial: usize,
924        exchange: ProgramResourceId,
925        elements: usize,
926    ) -> Result<Self> {
927        let DoubleDoubleRecursiveFftNodeIr::CooleyTukey(root) = &ir.root else {
928            return Err(VkFftError::InvalidKernelIr(
929                "double-double mapped forced-Rader finalize requires a Cooley-Tukey root",
930            ));
931        };
932        if let Some(mapped_low) = ir.forced_rader_two_upload_mapped_low_component()? {
933            flatten_double_double_recursive_node(
934                &mapped_low,
935                exchange,
936                ProgramResourceId(1),
937                &mut resources,
938                &mut passes,
939                &mut serial,
940            )?;
941        } else if let Some((low, _mapping)) = ir.forced_rader_two_upload_mapped_low_stockham()? {
942            let child = ProgramIr::double_double_stockham(low)?;
943            serial += 1;
944            append_double_double_recursive_child(
945                &child,
946                &format!("vkfft_dd_recursive_leaf_{}", serial),
947                exchange,
948                ProgramResourceId(1),
949                &mut resources,
950                &mut passes,
951                true,
952            )?;
953            let final_pass = passes.last_mut().ok_or(VkFftError::InvalidKernelIr(
954                "double-double mapped forced-Rader low upload emitted no final pass",
955            ))?;
956            if low.sequence_len <= 64 {
957                if !final_pass.name.ends_with(&low.name) {
958                    return Err(VkFftError::InvalidKernelIr(
959                        "double-double mapped forced-Rader monolithic low upload does not end in its Stockham pass",
960                    ));
961                }
962                final_pass.name = format!("{}_four_step_left", low.name);
963            } else {
964                let expected_finalize = format!("{}_finalize", low.name);
965                if !final_pass.name.ends_with(&expected_finalize) {
966                    return Err(VkFftError::InvalidKernelIr(
967                        "double-double mapped forced-Rader low upload does not end in a Stockham finalize pass",
968                    ));
969                }
970                final_pass.name = format!("{}_four_step_left_finalize", low.name);
971            }
972        } else {
973            let left_output = ProgramResourceId(resources.len());
974            resources.push(ProgramResource {
975                id: left_output,
976                name: "double_double_rader_four_step_left_output".to_owned(),
977                kind: ProgramResourceKind::Scratch,
978                scalar: ScalarType::DoubleDouble,
979                elements,
980                external_layout: None,
981                initialization: ProgramResourceInitialization::Zeroed,
982            });
983            flatten_double_double_recursive_node(
984                &root.left,
985                exchange,
986                left_output,
987                &mut resources,
988                &mut passes,
989                &mut serial,
990            )?;
991            passes.push(two_buffer_pass(
992                &root.scatter_output.name,
993                root.scatter_output.dispatch,
994                left_output.0,
995                1,
996            ));
997        }
998        let program = Self {
999            name: format!("{}_mapped_rader_four_step_program", ir.name),
1000            scalar: ScalarType::DoubleDouble,
1001            resources,
1002            passes,
1003        };
1004        program.validate()?;
1005        Ok(program)
1006    }
1007
1008    fn double_double_recursive_forced_rader_three_upload(
1009        ir: &DoubleDoubleRecursiveFftIr,
1010        mut resources: Vec<ProgramResource>,
1011        components: Vec<DoubleDoubleForcedRaderThreeUploadComponentIr>,
1012    ) -> Result<Self> {
1013        if components.len() != 3
1014            || components
1015                .iter()
1016                .map(DoubleDoubleForcedRaderThreeUploadComponentIr::upload_id)
1017                .collect::<Vec<_>>()
1018                != vec![2, 1, 0]
1019        {
1020            return Err(VkFftError::InvalidKernelIr(
1021                "double-double forced-Rader three-upload components are out of execution order",
1022            ));
1023        }
1024        let schedule =
1025            ir.rader_forced_upload_schedule
1026                .as_ref()
1027                .ok_or(VkFftError::InvalidKernelIr(
1028                    "double-double forced-Rader three-upload ProgramIr requires schedule metadata",
1029                ))?;
1030        let [a, b, c]: [usize; 3] = schedule.axis_split.as_slice().try_into().map_err(|_| {
1031            VkFftError::InvalidKernelIr(
1032                "double-double forced-Rader three-upload ProgramIr requires three axis splits",
1033            )
1034        })?;
1035        let component_lengths = components
1036            .iter()
1037            .map(DoubleDoubleForcedRaderThreeUploadComponentIr::logical_len)
1038            .collect::<Vec<_>>();
1039        if component_lengths != vec![c, b, a] {
1040            return Err(VkFftError::InvalidKernelIr(
1041                "double-double forced-Rader three-upload component lengths do not match axisSplit",
1042            ));
1043        }
1044        let elements =
1045            ir.logical_len
1046                .checked_mul(ir.batch_count)
1047                .ok_or(VkFftError::ArithmeticOverflow {
1048                    operation: "double-double forced-Rader three-upload scratch element count",
1049                })?;
1050        let output_is_compute = resources
1051            .get(1)
1052            .ok_or(VkFftError::InvalidKernelIr(
1053                "double-double forced-Rader three-upload output resource is missing",
1054            ))?
1055            .scalar
1056            == ScalarType::DoubleDouble;
1057        let scratch0 = ProgramResourceId(resources.len());
1058        resources.push(ProgramResource {
1059            id: scratch0,
1060            name: "double_double_forced_rader_three_upload_scratch_0".to_owned(),
1061            kind: ProgramResourceKind::Scratch,
1062            scalar: ScalarType::DoubleDouble,
1063            elements,
1064            external_layout: None,
1065            initialization: ProgramResourceInitialization::Zeroed,
1066        });
1067        let scratch1 = if output_is_compute {
1068            None
1069        } else {
1070            let scratch = ProgramResourceId(resources.len());
1071            resources.push(ProgramResource {
1072                id: scratch,
1073                name: "double_double_forced_rader_three_upload_scratch_1".to_owned(),
1074                kind: ProgramResourceKind::Scratch,
1075                scalar: ScalarType::DoubleDouble,
1076                elements,
1077                external_layout: None,
1078                initialization: ProgramResourceInitialization::Zeroed,
1079            });
1080            Some(scratch)
1081        };
1082        let (exchange0, exchange1) = if output_is_compute {
1083            (ProgramResourceId(1), scratch0)
1084        } else {
1085            (
1086                scratch0,
1087                scratch1.expect("mixed-storage path allocates scratch1"),
1088            )
1089        };
1090        let mut passes = Vec::new();
1091        let mut serial = 0usize;
1092        for component in components {
1093            let upload_id = component.upload_id();
1094            let (input, output) = match upload_id {
1095                2 => (ProgramResourceId(0), exchange0),
1096                1 => (exchange0, exchange1),
1097                0 => (exchange1, ProgramResourceId(1)),
1098                _ => unreachable!(),
1099            };
1100            match component {
1101                DoubleDoubleForcedRaderThreeUploadComponentIr::Recursive { ir: node, .. } => {
1102                    flatten_double_double_recursive_node(
1103                        &node,
1104                        input,
1105                        output,
1106                        &mut resources,
1107                        &mut passes,
1108                        &mut serial,
1109                    )?;
1110                }
1111                DoubleDoubleForcedRaderThreeUploadComponentIr::Stockham {
1112                    ir: stockham, ..
1113                } => {
1114                    let twiddle_values = stockham.twiddles.packed_values();
1115                    let twiddle_id = ProgramResourceId(resources.len());
1116                    resources.push(ProgramResource {
1117                        id: twiddle_id,
1118                        name: format!(
1119                            "double_double_forced_rader_three_upload_{upload_id}_stockham_twiddles"
1120                        ),
1121                        kind: ProgramResourceKind::LookupTable,
1122                        scalar: ScalarType::DoubleDouble,
1123                        elements: twiddle_values.len(),
1124                        external_layout: None,
1125                        initialization: ProgramResourceInitialization::ComplexDoubleDouble(
1126                            twiddle_values,
1127                        ),
1128                    });
1129                    let mut bindings = vec![
1130                        ProgramPassBinding {
1131                            binding: 0,
1132                            resource: input,
1133                            role: BufferRole::Input,
1134                            access: BufferAccess::ReadOnly,
1135                        },
1136                        ProgramPassBinding {
1137                            binding: 1,
1138                            resource: output,
1139                            role: BufferRole::Output,
1140                            access: BufferAccess::WriteOnly,
1141                        },
1142                        ProgramPassBinding {
1143                            binding: 2,
1144                            resource: twiddle_id,
1145                            role: BufferRole::TwiddleLookupTable,
1146                            access: BufferAccess::ReadOnly,
1147                        },
1148                    ];
1149                    let parent_period = match upload_id {
1150                        2 => Some(ir.logical_len),
1151                        1 => Some(a.checked_mul(b).ok_or(VkFftError::ArithmeticOverflow {
1152                            operation: "double-double forced-Rader three-upload AB period",
1153                        })?),
1154                        0 => None,
1155                        _ => unreachable!(),
1156                    };
1157                    if let Some(period) = parent_period {
1158                        let roots =
1159                            crate::lut::unit_root_table_double_double(period, ir.direction)?;
1160                        let root_id = ProgramResourceId(resources.len());
1161                        resources.push(ProgramResource {
1162                            id: root_id,
1163                            name: format!(
1164                                "double_double_forced_rader_three_upload_{upload_id}_parent_roots"
1165                            ),
1166                            kind: ProgramResourceKind::LookupTable,
1167                            scalar: ScalarType::DoubleDouble,
1168                            elements: roots.len(),
1169                            external_layout: None,
1170                            initialization: ProgramResourceInitialization::ComplexDoubleDouble(
1171                                roots,
1172                            ),
1173                        });
1174                        bindings.push(ProgramPassBinding {
1175                            binding: 3,
1176                            resource: root_id,
1177                            role: BufferRole::Auxiliary,
1178                            access: BufferAccess::ReadOnly,
1179                        });
1180                    }
1181                    passes.push(ProgramPass {
1182                        name: format!("{}_forced_rader_three_upload_{upload_id}", ir.name),
1183                        dispatch: DispatchGeometry {
1184                            x: u32::try_from(stockham.batch_group_count()).map_err(|_| {
1185                                VkFftError::ValueOutOfRange {
1186                                    field: "double-double forced-Rader three-upload Stockham dispatch count",
1187                                }
1188                            })?,
1189                            y: 1,
1190                            z: 1,
1191                        },
1192                        bindings,
1193                    });
1194                }
1195            }
1196        }
1197        let program = Self {
1198            name: format!("{}_forced_rader_three_upload_program", ir.name),
1199            scalar: ScalarType::DoubleDouble,
1200            resources,
1201            passes,
1202        };
1203        program.validate()?;
1204        Ok(program)
1205    }
1206
1207    fn double_double_recursive_two_upload_four_step(
1208        ir: &DoubleDoubleRecursiveFftIr,
1209        mut resources: Vec<ProgramResource>,
1210    ) -> Result<Self> {
1211        let plan = ir
1212            .two_upload_four_step_plan
1213            .ok_or(VkFftError::InvalidKernelIr(
1214                "double-double Four-step ProgramIr requires plan metadata",
1215            ))?;
1216        plan.validate()?;
1217        let DoubleDoubleRecursiveFftNodeIr::CooleyTukey(root) = &ir.root else {
1218            return Err(VkFftError::InvalidKernelIr(
1219                "double-double Four-step ProgramIr requires a Cooley-Tukey root",
1220            ));
1221        };
1222        let (
1223            DoubleDoubleRecursiveFftNodeIr::Stockham(left),
1224            DoubleDoubleRecursiveFftNodeIr::Stockham(right),
1225        ) = (&root.left, &root.right)
1226        else {
1227            return Err(VkFftError::InvalidKernelIr(
1228                "double-double Four-step ProgramIr requires Stockham leaves",
1229            ));
1230        };
1231        let elements =
1232            ir.logical_len
1233                .checked_mul(ir.batch_count)
1234                .ok_or(VkFftError::ArithmeticOverflow {
1235                    operation: "double-double Four-step scratch element count",
1236                })?;
1237        let scratch = ProgramResourceId(resources.len());
1238        resources.push(ProgramResource {
1239            id: scratch,
1240            name: "double_double_four_step_scratch".to_owned(),
1241            kind: ProgramResourceKind::Scratch,
1242            scalar: ScalarType::DoubleDouble,
1243            elements,
1244            external_layout: None,
1245            initialization: ProgramResourceInitialization::Zeroed,
1246        });
1247        let right_twiddle_values = right.twiddles.packed_values();
1248        let right_twiddles = ProgramResourceId(resources.len());
1249        resources.push(ProgramResource {
1250            id: right_twiddles,
1251            name: "double_double_four_step_upload_1_twiddles".to_owned(),
1252            kind: ProgramResourceKind::LookupTable,
1253            scalar: ScalarType::DoubleDouble,
1254            elements: right_twiddle_values.len(),
1255            external_layout: None,
1256            initialization: ProgramResourceInitialization::ComplexDoubleDouble(
1257                right_twiddle_values,
1258            ),
1259        });
1260        let root_twiddles = ProgramResourceId(resources.len());
1261        resources.push(ProgramResource {
1262            id: root_twiddles,
1263            name: "double_double_four_step_root_twiddles".to_owned(),
1264            kind: ProgramResourceKind::LookupTable,
1265            scalar: ScalarType::DoubleDouble,
1266            elements: root.twiddles.len(),
1267            external_layout: None,
1268            initialization: ProgramResourceInitialization::ComplexDoubleDouble(
1269                root.twiddles.clone(),
1270            ),
1271        });
1272        let left_twiddle_values = left.twiddles.packed_values();
1273        let left_twiddles = ProgramResourceId(resources.len());
1274        resources.push(ProgramResource {
1275            id: left_twiddles,
1276            name: "double_double_four_step_upload_0_twiddles".to_owned(),
1277            kind: ProgramResourceKind::LookupTable,
1278            scalar: ScalarType::DoubleDouble,
1279            elements: left_twiddle_values.len(),
1280            external_layout: None,
1281            initialization: ProgramResourceInitialization::ComplexDoubleDouble(left_twiddle_values),
1282        });
1283        let right_dispatch = DispatchGeometry {
1284            x: u32::try_from(right.batch_group_count()).map_err(|_| {
1285                VkFftError::ValueOutOfRange {
1286                    field: "double-double Four-step upload 1 dispatch count",
1287                }
1288            })?,
1289            y: 1,
1290            z: 1,
1291        };
1292        let left_dispatch = DispatchGeometry {
1293            x: u32::try_from(left.batch_group_count()).map_err(|_| {
1294                VkFftError::ValueOutOfRange {
1295                    field: "double-double Four-step upload 0 dispatch count",
1296                }
1297            })?,
1298            y: 1,
1299            z: 1,
1300        };
1301        let passes = vec![
1302            ProgramPass {
1303                name: format!("{}_four_step_upload_1", ir.name),
1304                dispatch: right_dispatch,
1305                bindings: vec![
1306                    binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
1307                    ProgramPassBinding {
1308                        binding: 1,
1309                        resource: scratch,
1310                        role: BufferRole::Output,
1311                        access: BufferAccess::WriteOnly,
1312                    },
1313                    ProgramPassBinding {
1314                        binding: 2,
1315                        resource: right_twiddles,
1316                        role: BufferRole::TwiddleLookupTable,
1317                        access: BufferAccess::ReadOnly,
1318                    },
1319                    ProgramPassBinding {
1320                        binding: 3,
1321                        resource: root_twiddles,
1322                        role: BufferRole::Auxiliary,
1323                        access: BufferAccess::ReadOnly,
1324                    },
1325                ],
1326            },
1327            ProgramPass {
1328                name: format!("{}_four_step_upload_0", ir.name),
1329                dispatch: left_dispatch,
1330                bindings: vec![
1331                    ProgramPassBinding {
1332                        binding: 0,
1333                        resource: scratch,
1334                        role: BufferRole::Input,
1335                        access: BufferAccess::ReadOnly,
1336                    },
1337                    binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
1338                    ProgramPassBinding {
1339                        binding: 2,
1340                        resource: left_twiddles,
1341                        role: BufferRole::TwiddleLookupTable,
1342                        access: BufferAccess::ReadOnly,
1343                    },
1344                ],
1345            },
1346        ];
1347        let program = Self {
1348            name: format!("{}_four_step_program", ir.name),
1349            scalar: ScalarType::DoubleDouble,
1350            resources,
1351            passes,
1352        };
1353        program.validate()?;
1354        Ok(program)
1355    }
1356
1357    fn double_double_recursive_three_upload_four_step(
1358        ir: &DoubleDoubleRecursiveFftIr,
1359        mut resources: Vec<ProgramResource>,
1360    ) -> Result<Self> {
1361        let plan = ir
1362            .three_upload_four_step_plan
1363            .ok_or(VkFftError::InvalidKernelIr(
1364                "double-double three-upload Four-step ProgramIr requires plan metadata",
1365            ))?;
1366        plan.validate()?;
1367        let DoubleDoubleRecursiveFftNodeIr::CooleyTukey(root) = &ir.root else {
1368            return Err(VkFftError::InvalidKernelIr(
1369                "double-double three-upload Four-step ProgramIr requires a Cooley-Tukey root",
1370            ));
1371        };
1372        let DoubleDoubleRecursiveFftNodeIr::Stockham(low) = &root.left else {
1373            return Err(VkFftError::InvalidKernelIr(
1374                "double-double three-upload Four-step ProgramIr requires a low Stockham leaf",
1375            ));
1376        };
1377        let DoubleDoubleRecursiveFftNodeIr::CooleyTukey(upper) = &root.right else {
1378            return Err(VkFftError::InvalidKernelIr(
1379                "double-double three-upload Four-step ProgramIr requires an upper Cooley-Tukey node",
1380            ));
1381        };
1382        let (
1383            DoubleDoubleRecursiveFftNodeIr::Stockham(middle),
1384            DoubleDoubleRecursiveFftNodeIr::Stockham(high),
1385        ) = (&upper.left, &upper.right)
1386        else {
1387            return Err(VkFftError::InvalidKernelIr(
1388                "double-double three-upload Four-step ProgramIr requires middle/high Stockham leaves",
1389            ));
1390        };
1391        let [a, b, c] = plan.axis_split;
1392        if low.sequence_len != a || middle.sequence_len != b || high.sequence_len != c {
1393            return Err(VkFftError::InvalidKernelIr(
1394                "double-double three-upload Four-step ProgramIr leaf lengths do not match axisSplit",
1395            ));
1396        }
1397        let elements =
1398            ir.logical_len
1399                .checked_mul(ir.batch_count)
1400                .ok_or(VkFftError::ArithmeticOverflow {
1401                    operation: "double-double three-upload Four-step scratch element count",
1402                })?;
1403        let output_is_compute = resources
1404            .get(1)
1405            .ok_or(VkFftError::InvalidKernelIr(
1406                "double-double three-upload Four-step output resource is missing",
1407            ))?
1408            .scalar
1409            == ScalarType::DoubleDouble;
1410        let scratch0 = ProgramResourceId(resources.len());
1411        resources.push(ProgramResource {
1412            id: scratch0,
1413            name: "double_double_three_upload_four_step_scratch_0".to_owned(),
1414            kind: ProgramResourceKind::Scratch,
1415            scalar: ScalarType::DoubleDouble,
1416            elements,
1417            external_layout: None,
1418            initialization: ProgramResourceInitialization::Zeroed,
1419        });
1420        let scratch1 = if output_is_compute {
1421            None
1422        } else {
1423            let scratch = ProgramResourceId(resources.len());
1424            resources.push(ProgramResource {
1425                id: scratch,
1426                name: "double_double_three_upload_four_step_scratch_1".to_owned(),
1427                kind: ProgramResourceKind::Scratch,
1428                scalar: ScalarType::DoubleDouble,
1429                elements,
1430                external_layout: None,
1431                initialization: ProgramResourceInitialization::Zeroed,
1432            });
1433            Some(scratch)
1434        };
1435        let (exchange0, exchange1) = if output_is_compute {
1436            (ProgramResourceId(1), scratch0)
1437        } else {
1438            (
1439                scratch0,
1440                scratch1.expect("mixed-storage path allocates scratch1"),
1441            )
1442        };
1443
1444        let high_twiddle_values = high.twiddles.packed_values();
1445        let high_twiddles = ProgramResourceId(resources.len());
1446        resources.push(ProgramResource {
1447            id: high_twiddles,
1448            name: "double_double_three_upload_four_step_upload_2_twiddles".to_owned(),
1449            kind: ProgramResourceKind::LookupTable,
1450            scalar: ScalarType::DoubleDouble,
1451            elements: high_twiddle_values.len(),
1452            external_layout: None,
1453            initialization: ProgramResourceInitialization::ComplexDoubleDouble(high_twiddle_values),
1454        });
1455        let root_twiddles = ProgramResourceId(resources.len());
1456        resources.push(ProgramResource {
1457            id: root_twiddles,
1458            name: "double_double_three_upload_four_step_root_twiddles".to_owned(),
1459            kind: ProgramResourceKind::LookupTable,
1460            scalar: ScalarType::DoubleDouble,
1461            elements: root.twiddles.len(),
1462            external_layout: None,
1463            initialization: ProgramResourceInitialization::ComplexDoubleDouble(
1464                root.twiddles.clone(),
1465            ),
1466        });
1467
1468        let middle_twiddle_values = middle.twiddles.packed_values();
1469        let middle_twiddles = ProgramResourceId(resources.len());
1470        resources.push(ProgramResource {
1471            id: middle_twiddles,
1472            name: "double_double_three_upload_four_step_upload_1_twiddles".to_owned(),
1473            kind: ProgramResourceKind::LookupTable,
1474            scalar: ScalarType::DoubleDouble,
1475            elements: middle_twiddle_values.len(),
1476            external_layout: None,
1477            initialization: ProgramResourceInitialization::ComplexDoubleDouble(
1478                middle_twiddle_values,
1479            ),
1480        });
1481        let ab = a.checked_mul(b).ok_or(VkFftError::ArithmeticOverflow {
1482            operation: "double-double three-upload Four-step AB twiddle period",
1483        })?;
1484        let ab_twiddle_values = crate::lut::unit_root_table_double_double(ab, ir.direction)?;
1485        let ab_twiddles = ProgramResourceId(resources.len());
1486        resources.push(ProgramResource {
1487            id: ab_twiddles,
1488            name: "double_double_three_upload_four_step_ab_twiddles".to_owned(),
1489            kind: ProgramResourceKind::LookupTable,
1490            scalar: ScalarType::DoubleDouble,
1491            elements: ab_twiddle_values.len(),
1492            external_layout: None,
1493            initialization: ProgramResourceInitialization::ComplexDoubleDouble(ab_twiddle_values),
1494        });
1495
1496        let low_twiddle_values = low.twiddles.packed_values();
1497        let low_twiddles = ProgramResourceId(resources.len());
1498        resources.push(ProgramResource {
1499            id: low_twiddles,
1500            name: "double_double_three_upload_four_step_upload_0_twiddles".to_owned(),
1501            kind: ProgramResourceKind::LookupTable,
1502            scalar: ScalarType::DoubleDouble,
1503            elements: low_twiddle_values.len(),
1504            external_layout: None,
1505            initialization: ProgramResourceInitialization::ComplexDoubleDouble(low_twiddle_values),
1506        });
1507
1508        let dispatch = |leaf: &DoubleDoubleStockhamIr, field| {
1509            Ok(DispatchGeometry {
1510                x: u32::try_from(leaf.batch_group_count())
1511                    .map_err(|_| VkFftError::ValueOutOfRange { field })?,
1512                y: 1,
1513                z: 1,
1514            })
1515        };
1516        let passes = vec![
1517            ProgramPass {
1518                name: format!("{}_four_step_upload_2", ir.name),
1519                dispatch: dispatch(high, "double-double Four-step upload 2 dispatch count")?,
1520                bindings: vec![
1521                    binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
1522                    ProgramPassBinding {
1523                        binding: 1,
1524                        resource: exchange0,
1525                        role: BufferRole::Output,
1526                        access: BufferAccess::WriteOnly,
1527                    },
1528                    ProgramPassBinding {
1529                        binding: 2,
1530                        resource: high_twiddles,
1531                        role: BufferRole::TwiddleLookupTable,
1532                        access: BufferAccess::ReadOnly,
1533                    },
1534                    ProgramPassBinding {
1535                        binding: 3,
1536                        resource: root_twiddles,
1537                        role: BufferRole::Auxiliary,
1538                        access: BufferAccess::ReadOnly,
1539                    },
1540                ],
1541            },
1542            ProgramPass {
1543                name: format!("{}_four_step_upload_1", ir.name),
1544                dispatch: dispatch(middle, "double-double Four-step upload 1 dispatch count")?,
1545                bindings: vec![
1546                    ProgramPassBinding {
1547                        binding: 0,
1548                        resource: exchange0,
1549                        role: BufferRole::Input,
1550                        access: BufferAccess::ReadOnly,
1551                    },
1552                    ProgramPassBinding {
1553                        binding: 1,
1554                        resource: exchange1,
1555                        role: BufferRole::Output,
1556                        access: BufferAccess::WriteOnly,
1557                    },
1558                    ProgramPassBinding {
1559                        binding: 2,
1560                        resource: middle_twiddles,
1561                        role: BufferRole::TwiddleLookupTable,
1562                        access: BufferAccess::ReadOnly,
1563                    },
1564                    ProgramPassBinding {
1565                        binding: 3,
1566                        resource: ab_twiddles,
1567                        role: BufferRole::Auxiliary,
1568                        access: BufferAccess::ReadOnly,
1569                    },
1570                ],
1571            },
1572            ProgramPass {
1573                name: format!("{}_four_step_upload_0", ir.name),
1574                dispatch: dispatch(low, "double-double Four-step upload 0 dispatch count")?,
1575                bindings: vec![
1576                    ProgramPassBinding {
1577                        binding: 0,
1578                        resource: exchange1,
1579                        role: BufferRole::Input,
1580                        access: BufferAccess::ReadOnly,
1581                    },
1582                    binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
1583                    ProgramPassBinding {
1584                        binding: 2,
1585                        resource: low_twiddles,
1586                        role: BufferRole::TwiddleLookupTable,
1587                        access: BufferAccess::ReadOnly,
1588                    },
1589                ],
1590            },
1591        ];
1592        let program = Self {
1593            name: format!("{}_three_upload_four_step_program", ir.name),
1594            scalar: ScalarType::DoubleDouble,
1595            resources,
1596            passes,
1597        };
1598        program.validate()?;
1599        Ok(program)
1600    }
1601
1602    pub fn double_double_real(ir: &DoubleDoubleRealFftIr) -> Result<Self> {
1603        ir.validate()?;
1604        let full_elements =
1605            ir.length
1606                .checked_mul(ir.batch_count)
1607                .ok_or(VkFftError::ArithmeticOverflow {
1608                    operation: "double-double real full element count",
1609                })?;
1610        let compact_elements = ir.half_spectrum_len.checked_mul(ir.batch_count).ok_or(
1611            VkFftError::ArithmeticOverflow {
1612                operation: "double-double real compact element count",
1613            },
1614        )?;
1615        let child_elements = ir
1616            .transform
1617            .sequence_len()
1618            .checked_mul(ir.batch_count)
1619            .ok_or(VkFftError::ArithmeticOverflow {
1620                operation: "double-double real child element count",
1621            })?;
1622        let external_scalar = match ir.external_storage {
1623            crate::PrecisionStorage::DoubleDouble => ScalarType::DoubleDouble,
1624            crate::PrecisionStorage::F64 => ScalarType::F64,
1625            _ => {
1626                return Err(VkFftError::InvalidKernelIr(
1627                    "double-double real ProgramIr requires DD or F64 external storage",
1628                ));
1629            }
1630        };
1631        let (input_elements, output_elements, input_len, output_len) = match ir.kind {
1632            crate::RealFftKind::RealToComplex => (
1633                full_elements,
1634                compact_elements,
1635                ir.length,
1636                ir.half_spectrum_len,
1637            ),
1638            crate::RealFftKind::ComplexToReal => (
1639                compact_elements,
1640                full_elements,
1641                ir.half_spectrum_len,
1642                ir.length,
1643            ),
1644        };
1645        let mut resources = vec![
1646            ProgramResource {
1647                id: ProgramResourceId(0),
1648                name: "input".to_owned(),
1649                kind: ProgramResourceKind::Input,
1650                scalar: external_scalar,
1651                elements: input_elements,
1652                external_layout: Some(ExternalBufferLayout {
1653                    logical_len: input_len,
1654                    physical_stride: input_len,
1655                    batch_count: ir.batch_count,
1656                    element_shape: match ir.kind {
1657                        crate::RealFftKind::RealToComplex => ProgramElementShape::Scalar,
1658                        crate::RealFftKind::ComplexToReal => ProgramElementShape::Complex,
1659                    },
1660                }),
1661                initialization: ProgramResourceInitialization::ExternalInput,
1662            },
1663            ProgramResource {
1664                id: ProgramResourceId(1),
1665                name: "output".to_owned(),
1666                kind: ProgramResourceKind::Output,
1667                scalar: external_scalar,
1668                elements: output_elements,
1669                external_layout: Some(ExternalBufferLayout {
1670                    logical_len: output_len,
1671                    physical_stride: output_len,
1672                    batch_count: ir.batch_count,
1673                    element_shape: match ir.kind {
1674                        crate::RealFftKind::RealToComplex => ProgramElementShape::Complex,
1675                        crate::RealFftKind::ComplexToReal => ProgramElementShape::Scalar,
1676                    },
1677                }),
1678                initialization: ProgramResourceInitialization::Zeroed,
1679            },
1680            ProgramResource {
1681                id: ProgramResourceId(2),
1682                name: "double_double_real_child_input".to_owned(),
1683                kind: ProgramResourceKind::Scratch,
1684                scalar: ScalarType::DoubleDouble,
1685                elements: child_elements,
1686                external_layout: None,
1687                initialization: ProgramResourceInitialization::Zeroed,
1688            },
1689            ProgramResource {
1690                id: ProgramResourceId(3),
1691                name: "double_double_real_child_output".to_owned(),
1692                kind: ProgramResourceKind::Scratch,
1693                scalar: ScalarType::DoubleDouble,
1694                elements: child_elements,
1695                external_layout: None,
1696                initialization: ProgramResourceInitialization::Zeroed,
1697            },
1698        ];
1699        let child = Self::double_double_one_dim(&ir.transform)?;
1700        if child.scalar != ScalarType::DoubleDouble
1701            || child.input_resource()?.scalar != ScalarType::DoubleDouble
1702            || child.output_resource()?.scalar != ScalarType::DoubleDouble
1703            || child.input_resource()?.elements != child_elements
1704            || child.output_resource()?.elements != child_elements
1705        {
1706            return Err(VkFftError::InvalidKernelIr(
1707                "double-double real child ProgramIr must expose child-sized DD boundaries",
1708            ));
1709        }
1710        let mut resource_map = vec![None; child.resources.len()];
1711        for resource in &child.resources {
1712            let mapped = match resource.kind {
1713                ProgramResourceKind::Input => ProgramResourceId(2),
1714                ProgramResourceKind::Output => ProgramResourceId(3),
1715                _ => {
1716                    let new_id = ProgramResourceId(resources.len());
1717                    let mut cloned = resource.clone();
1718                    cloned.id = new_id;
1719                    cloned.name = format!("double_double_real_{}", resource.name);
1720                    cloned.external_layout = None;
1721                    resources.push(cloned);
1722                    new_id
1723                }
1724            };
1725            resource_map[resource.id.0] = Some(mapped);
1726        }
1727        let even_roots_resource = if ir.even_half_size {
1728            let id = ProgramResourceId(resources.len());
1729            resources.push(ProgramResource {
1730                id,
1731                name: "double_double_real_even_roots".to_owned(),
1732                kind: ProgramResourceKind::LookupTable,
1733                scalar: ScalarType::DoubleDouble,
1734                elements: ir.even_roots.len(),
1735                external_layout: None,
1736                initialization: ProgramResourceInitialization::ComplexDoubleDouble(
1737                    ir.even_roots.clone(),
1738                ),
1739            });
1740            Some(id)
1741        } else {
1742            None
1743        };
1744        let boundary_dispatch = DispatchGeometry {
1745            x: u32::try_from(ir.batch_group_count()).map_err(|_| VkFftError::ValueOutOfRange {
1746                field: "double-double real boundary dispatch count",
1747            })?,
1748            y: 1,
1749            z: 1,
1750        };
1751        let mut first_bindings = vec![
1752            binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
1753            binding(1, 2, BufferRole::Output, BufferAccess::WriteOnly),
1754        ];
1755        if ir.kind == crate::RealFftKind::ComplexToReal
1756            && let Some(roots) = even_roots_resource
1757        {
1758            first_bindings.push(ProgramPassBinding {
1759                binding: 2,
1760                resource: roots,
1761                role: BufferRole::LookupTable,
1762                access: BufferAccess::ReadOnly,
1763            });
1764        }
1765        let mut passes = Vec::with_capacity(child.passes.len() + 2);
1766        passes.push(ProgramPass {
1767            name: match (ir.kind, ir.even_half_size) {
1768                (crate::RealFftKind::RealToComplex, true) => {
1769                    "vkfft_dd_real_r2c_half_size_pack".to_owned()
1770                }
1771                (crate::RealFftKind::ComplexToReal, true) => {
1772                    "vkfft_dd_real_c2r_half_size_preprocess".to_owned()
1773                }
1774                (crate::RealFftKind::RealToComplex, false) => {
1775                    "vkfft_dd_real_r2c_promote".to_owned()
1776                }
1777                (crate::RealFftKind::ComplexToReal, false) => "vkfft_dd_real_c2r_expand".to_owned(),
1778            },
1779            dispatch: boundary_dispatch,
1780            bindings: first_bindings,
1781        });
1782        for child_pass in &child.passes {
1783            let mut pass = child_pass.clone();
1784            pass.name = format!("vkfft_dd_real_{}", child_pass.name);
1785            for binding in &mut pass.bindings {
1786                binding.resource = resource_map
1787                    .get(binding.resource.0)
1788                    .and_then(|mapped| *mapped)
1789                    .ok_or(VkFftError::InvalidKernelIr(
1790                        "double-double real child resource mapping is incomplete",
1791                    ))?;
1792            }
1793            passes.push(pass);
1794        }
1795        let mut last_bindings = vec![
1796            binding(0, 3, BufferRole::Input, BufferAccess::ReadOnly),
1797            binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
1798        ];
1799        if ir.kind == crate::RealFftKind::RealToComplex
1800            && let Some(roots) = even_roots_resource
1801        {
1802            last_bindings.push(ProgramPassBinding {
1803                binding: 2,
1804                resource: roots,
1805                role: BufferRole::LookupTable,
1806                access: BufferAccess::ReadOnly,
1807            });
1808        }
1809        passes.push(ProgramPass {
1810            name: match (ir.kind, ir.even_half_size) {
1811                (crate::RealFftKind::RealToComplex, true) => {
1812                    "vkfft_dd_real_r2c_half_size_postprocess".to_owned()
1813                }
1814                (crate::RealFftKind::ComplexToReal, true) => {
1815                    "vkfft_dd_real_c2r_half_size_unpack".to_owned()
1816                }
1817                (crate::RealFftKind::RealToComplex, false) => {
1818                    "vkfft_dd_real_r2c_compact".to_owned()
1819                }
1820                (crate::RealFftKind::ComplexToReal, false) => {
1821                    "vkfft_dd_real_c2r_finalize".to_owned()
1822                }
1823            },
1824            dispatch: boundary_dispatch,
1825            bindings: last_bindings,
1826        });
1827        let program = Self {
1828            name: format!(
1829                "vkfft_dd_real_{}_{}",
1830                match ir.kind {
1831                    crate::RealFftKind::RealToComplex => "r2c",
1832                    crate::RealFftKind::ComplexToReal => "c2r",
1833                },
1834                ir.length
1835            ),
1836            scalar: ScalarType::DoubleDouble,
1837            resources,
1838            passes,
1839        };
1840        program.validate()?;
1841        Ok(program)
1842    }
1843
1844    pub fn double_double_nd_real(ir: &DoubleDoubleNdRealFftIr) -> Result<Self> {
1845        ir.validate()?;
1846        let full_elements = ir.full_tensor_len.checked_mul(ir.batch_count).ok_or(
1847            VkFftError::ArithmeticOverflow {
1848                operation: "double-double ND real full element count",
1849            },
1850        )?;
1851        let compact_elements = ir.compact_tensor_len.checked_mul(ir.batch_count).ok_or(
1852            VkFftError::ArithmeticOverflow {
1853                operation: "double-double ND real compact element count",
1854            },
1855        )?;
1856        let external_scalar = match ir.external_storage {
1857            crate::PrecisionStorage::DoubleDouble => ScalarType::DoubleDouble,
1858            crate::PrecisionStorage::F64 => ScalarType::F64,
1859            _ => {
1860                return Err(VkFftError::InvalidKernelIr(
1861                    "double-double ND real ProgramIr requires DD or F64 external storage",
1862                ));
1863            }
1864        };
1865        let formatted_io = ir.formatted_io.as_ref();
1866        let input_elements = formatted_io
1867            .input_external_layout
1868            .batch_stride
1869            .checked_mul(ir.batch_count)
1870            .ok_or(VkFftError::ArithmeticOverflow {
1871                operation: "double-double ND real physical input element count",
1872            })?;
1873        let output_elements = formatted_io
1874            .output_external_layout
1875            .batch_stride
1876            .checked_mul(ir.batch_count)
1877            .ok_or(VkFftError::ArithmeticOverflow {
1878                operation: "double-double ND real physical output element count",
1879            })?;
1880        let input_layout = ExternalBufferLayout {
1881            logical_len: if formatted_io.input_formatted_copy.is_some() {
1882                formatted_io.input_external_layout.batch_stride
1883            } else {
1884                ir.input_tensor_len()
1885            },
1886            physical_stride: formatted_io.input_external_layout.batch_stride,
1887            batch_count: ir.batch_count,
1888            element_shape: match ir.kind {
1889                crate::RealFftKind::RealToComplex => ProgramElementShape::Scalar,
1890                crate::RealFftKind::ComplexToReal => ProgramElementShape::Complex,
1891            },
1892        };
1893        let output_layout = ExternalBufferLayout {
1894            logical_len: if formatted_io.output_formatted_copy.is_some() {
1895                formatted_io.output_external_layout.batch_stride
1896            } else {
1897                ir.output_tensor_len()
1898            },
1899            physical_stride: formatted_io.output_external_layout.batch_stride,
1900            batch_count: ir.batch_count,
1901            element_shape: match ir.kind {
1902                crate::RealFftKind::RealToComplex => ProgramElementShape::Complex,
1903                crate::RealFftKind::ComplexToReal => ProgramElementShape::Scalar,
1904            },
1905        };
1906        let full_scalar_layout = ExternalBufferLayout {
1907            logical_len: full_elements,
1908            physical_stride: full_elements,
1909            batch_count: 1,
1910            element_shape: ProgramElementShape::Scalar,
1911        };
1912        let mut resources = vec![
1913            ProgramResource {
1914                id: ProgramResourceId(0),
1915                name: "input".to_owned(),
1916                kind: ProgramResourceKind::Input,
1917                scalar: external_scalar,
1918                elements: input_elements,
1919                external_layout: Some(input_layout),
1920                initialization: ProgramResourceInitialization::ExternalInput,
1921            },
1922            ProgramResource {
1923                id: ProgramResourceId(1),
1924                name: "output".to_owned(),
1925                kind: ProgramResourceKind::Output,
1926                scalar: external_scalar,
1927                elements: output_elements,
1928                external_layout: Some(output_layout),
1929                initialization: ProgramResourceInitialization::Zeroed,
1930            },
1931            ProgramResource {
1932                id: ProgramResourceId(2),
1933                name: "double_double_nd_real_full_scalar".to_owned(),
1934                kind: ProgramResourceKind::Scratch,
1935                scalar: ScalarType::DoubleDouble,
1936                elements: full_elements,
1937                external_layout: Some(full_scalar_layout),
1938                initialization: ProgramResourceInitialization::Zeroed,
1939            },
1940            ProgramResource {
1941                id: ProgramResourceId(3),
1942                name: "double_double_nd_real_compact_a".to_owned(),
1943                kind: ProgramResourceKind::Scratch,
1944                scalar: ScalarType::DoubleDouble,
1945                elements: compact_elements,
1946                external_layout: None,
1947                initialization: ProgramResourceInitialization::Zeroed,
1948            },
1949            ProgramResource {
1950                id: ProgramResourceId(4),
1951                name: "double_double_nd_real_compact_b".to_owned(),
1952                kind: ProgramResourceKind::Scratch,
1953                scalar: ScalarType::DoubleDouble,
1954                elements: compact_elements,
1955                external_layout: None,
1956                initialization: ProgramResourceInitialization::Zeroed,
1957            },
1958            ProgramResource {
1959                id: ProgramResourceId(5),
1960                name: "double_double_nd_real_lines_a".to_owned(),
1961                kind: ProgramResourceKind::Scratch,
1962                scalar: ScalarType::DoubleDouble,
1963                elements: compact_elements,
1964                external_layout: None,
1965                initialization: ProgramResourceInitialization::Zeroed,
1966            },
1967            ProgramResource {
1968                id: ProgramResourceId(6),
1969                name: "double_double_nd_real_lines_b".to_owned(),
1970                kind: ProgramResourceKind::Scratch,
1971                scalar: ScalarType::DoubleDouble,
1972                elements: compact_elements,
1973                external_layout: None,
1974                initialization: ProgramResourceInitialization::Zeroed,
1975            },
1976        ];
1977        let boundary_dispatch = DispatchGeometry {
1978            x: u32::try_from(ir.batch_group_count()).map_err(|_| VkFftError::ValueOutOfRange {
1979                field: "double-double ND real boundary dispatch count",
1980            })?,
1981            y: 1,
1982            z: 1,
1983        };
1984        let mut passes = Vec::new();
1985        let input_internal = match ir.kind {
1986            crate::RealFftKind::RealToComplex => ProgramResourceId(2),
1987            crate::RealFftKind::ComplexToReal => ProgramResourceId(3),
1988        };
1989        if let Some(copy) = &formatted_io.input_formatted_copy {
1990            passes.push(ProgramPass {
1991                name: copy.name.clone(),
1992                dispatch: copy.dispatch,
1993                bindings: vec![
1994                    binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
1995                    ProgramPassBinding {
1996                        binding: 1,
1997                        resource: input_internal,
1998                        role: BufferRole::Output,
1999                        access: BufferAccess::WriteOnly,
2000                    },
2001                ],
2002            });
2003        } else {
2004            passes.push(ProgramPass {
2005                name: "vkfft_dd_nd_real_external_input_promote".to_owned(),
2006                dispatch: boundary_dispatch,
2007                bindings: vec![
2008                    binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
2009                    ProgramPassBinding {
2010                        binding: 1,
2011                        resource: input_internal,
2012                        role: BufferRole::Output,
2013                        access: BufferAccess::WriteOnly,
2014                    },
2015                ],
2016            });
2017        }
2018
2019        let append_child = |child: &ProgramIr,
2020                            prefix: &str,
2021                            input: ProgramResourceId,
2022                            output: ProgramResourceId,
2023                            resources: &mut Vec<ProgramResource>,
2024                            passes: &mut Vec<ProgramPass>|
2025         -> Result<()> {
2026            let mut resource_map = vec![None; child.resources.len()];
2027            for resource in &child.resources {
2028                let mapped = match resource.kind {
2029                    ProgramResourceKind::Input => input,
2030                    ProgramResourceKind::Output => output,
2031                    _ => {
2032                        let new_id = ProgramResourceId(resources.len());
2033                        let mut cloned = resource.clone();
2034                        cloned.id = new_id;
2035                        cloned.name = format!("{prefix}_{}", resource.name);
2036                        cloned.external_layout = None;
2037                        resources.push(cloned);
2038                        new_id
2039                    }
2040                };
2041                resource_map[resource.id.0] = Some(mapped);
2042            }
2043            for child_pass in &child.passes {
2044                let mut pass = child_pass.clone();
2045                pass.name = format!("{prefix}_{}", child_pass.name);
2046                for binding in &mut pass.bindings {
2047                    binding.resource = resource_map
2048                        .get(binding.resource.0)
2049                        .and_then(|mapped| *mapped)
2050                        .ok_or(VkFftError::InvalidKernelIr(
2051                            "double-double ND real child resource mapping is incomplete",
2052                        ))?;
2053                }
2054                passes.push(pass);
2055            }
2056            Ok(())
2057        };
2058
2059        let real_child = Self::double_double_real(&ir.real_axis)?;
2060        let expected_real_input = match ir.kind {
2061            crate::RealFftKind::RealToComplex => full_elements,
2062            crate::RealFftKind::ComplexToReal => compact_elements,
2063        };
2064        let expected_real_output = match ir.kind {
2065            crate::RealFftKind::RealToComplex => compact_elements,
2066            crate::RealFftKind::ComplexToReal => full_elements,
2067        };
2068        if real_child.input_resource()?.scalar != ScalarType::DoubleDouble
2069            || real_child.output_resource()?.scalar != ScalarType::DoubleDouble
2070            || real_child.input_resource()?.elements != expected_real_input
2071            || real_child.output_resource()?.elements != expected_real_output
2072        {
2073            return Err(VkFftError::InvalidKernelIr(
2074                "double-double ND real last-axis child has incompatible boundaries",
2075            ));
2076        }
2077
2078        let mut compact_source = ProgramResourceId(3);
2079        let mut compact_target = ProgramResourceId(4);
2080        if ir.kind == crate::RealFftKind::RealToComplex {
2081            append_child(
2082                &real_child,
2083                "vkfft_dd_nd_real_last_axis",
2084                ProgramResourceId(2),
2085                compact_source,
2086                &mut resources,
2087                &mut passes,
2088            )?;
2089        }
2090
2091        for axis in &ir.complex_axes {
2092            let dispatch = DispatchGeometry {
2093                x: u32::try_from(ir.batch_count.div_ceil(axis.grouped_batch)).map_err(|_| {
2094                    VkFftError::ValueOutOfRange {
2095                        field: "double-double ND real complex-axis dispatch count",
2096                    }
2097                })?,
2098                y: 1,
2099                z: 1,
2100            };
2101            passes.push(ProgramPass {
2102                name: format!("vkfft_dd_nd_real_axis_{}_pack", axis.axis),
2103                dispatch,
2104                bindings: vec![
2105                    ProgramPassBinding {
2106                        binding: 0,
2107                        resource: compact_source,
2108                        role: BufferRole::Input,
2109                        access: BufferAccess::ReadOnly,
2110                    },
2111                    binding(1, 5, BufferRole::Output, BufferAccess::WriteOnly),
2112                ],
2113            });
2114            let child = Self::double_double_one_dim(&axis.transform)?;
2115            if child.input_resource()?.scalar != ScalarType::DoubleDouble
2116                || child.output_resource()?.scalar != ScalarType::DoubleDouble
2117                || child.input_resource()?.elements != compact_elements
2118                || child.output_resource()?.elements != compact_elements
2119            {
2120                return Err(VkFftError::InvalidKernelIr(
2121                    "double-double ND real complex-axis child has incompatible boundaries",
2122                ));
2123            }
2124            append_child(
2125                &child,
2126                &format!("vkfft_dd_nd_real_axis_{}", axis.axis),
2127                ProgramResourceId(5),
2128                ProgramResourceId(6),
2129                &mut resources,
2130                &mut passes,
2131            )?;
2132            passes.push(ProgramPass {
2133                name: format!("vkfft_dd_nd_real_axis_{}_scatter", axis.axis),
2134                dispatch,
2135                bindings: vec![
2136                    binding(0, 6, BufferRole::Input, BufferAccess::ReadOnly),
2137                    ProgramPassBinding {
2138                        binding: 1,
2139                        resource: compact_target,
2140                        role: BufferRole::Output,
2141                        access: BufferAccess::WriteOnly,
2142                    },
2143                ],
2144            });
2145            compact_source = compact_target;
2146            compact_target = if compact_target == ProgramResourceId(3) {
2147                ProgramResourceId(4)
2148            } else {
2149                ProgramResourceId(3)
2150            };
2151        }
2152
2153        if ir.kind == crate::RealFftKind::ComplexToReal {
2154            append_child(
2155                &real_child,
2156                "vkfft_dd_nd_real_last_axis",
2157                compact_source,
2158                ProgramResourceId(2),
2159                &mut resources,
2160                &mut passes,
2161            )?;
2162        }
2163        let output_internal = match ir.kind {
2164            crate::RealFftKind::RealToComplex => compact_source,
2165            crate::RealFftKind::ComplexToReal => ProgramResourceId(2),
2166        };
2167        if let Some(copy) = &formatted_io.output_formatted_copy {
2168            passes.push(ProgramPass {
2169                name: copy.name.clone(),
2170                dispatch: copy.dispatch,
2171                bindings: vec![
2172                    ProgramPassBinding {
2173                        binding: 0,
2174                        resource: output_internal,
2175                        role: BufferRole::Input,
2176                        access: BufferAccess::ReadOnly,
2177                    },
2178                    binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
2179                ],
2180            });
2181        } else {
2182            passes.push(ProgramPass {
2183                name: "vkfft_dd_nd_real_external_output_finalize".to_owned(),
2184                dispatch: boundary_dispatch,
2185                bindings: vec![
2186                    ProgramPassBinding {
2187                        binding: 0,
2188                        resource: output_internal,
2189                        role: BufferRole::Input,
2190                        access: BufferAccess::ReadOnly,
2191                    },
2192                    binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
2193                ],
2194            });
2195        }
2196        let program = Self {
2197            name: format!("vkfft_dd_nd_real_{:?}_{:?}", ir.dimensions, ir.kind),
2198            scalar: ScalarType::DoubleDouble,
2199            resources,
2200            passes,
2201        };
2202        program.validate()?;
2203        Ok(program)
2204    }
2205
2206    pub fn double_double_nd(ir: &DoubleDoubleNdFftIr) -> Result<Self> {
2207        ir.validate()?;
2208        let elements =
2209            ir.tensor_len
2210                .checked_mul(ir.batch_count)
2211                .ok_or(VkFftError::ArithmeticOverflow {
2212                    operation: "double-double ND program element count",
2213                })?;
2214        let external_scalar = match ir.external_storage {
2215            crate::PrecisionStorage::DoubleDouble => ScalarType::DoubleDouble,
2216            crate::PrecisionStorage::F64 => ScalarType::F64,
2217            _ => {
2218                return Err(VkFftError::InvalidKernelIr(
2219                    "double-double ND program requires DD or F64 external storage",
2220                ));
2221            }
2222        };
2223        let input_layout = ExternalBufferLayout {
2224            logical_len: if ir.input_formatted_copy.is_some() {
2225                ir.input_external_layout.batch_stride
2226            } else {
2227                ir.tensor_len
2228            },
2229            physical_stride: ir.input_external_layout.batch_stride,
2230            batch_count: ir.batch_count,
2231            element_shape: ProgramElementShape::Complex,
2232        };
2233        let output_layout = ExternalBufferLayout {
2234            logical_len: if ir.output_formatted_copy.is_some() {
2235                ir.output_external_layout.batch_stride
2236            } else {
2237                ir.tensor_len
2238            },
2239            physical_stride: ir.output_external_layout.batch_stride,
2240            batch_count: ir.batch_count,
2241            element_shape: ProgramElementShape::Complex,
2242        };
2243        let input_elements = input_layout.physical_elements()?;
2244        let output_elements = output_layout.physical_elements()?;
2245        let mut resources = vec![
2246            ProgramResource {
2247                id: ProgramResourceId(0),
2248                name: "input".to_owned(),
2249                kind: ProgramResourceKind::Input,
2250                scalar: external_scalar,
2251                elements: input_elements,
2252                external_layout: Some(input_layout),
2253                initialization: ProgramResourceInitialization::ExternalInput,
2254            },
2255            ProgramResource {
2256                id: ProgramResourceId(1),
2257                name: "output".to_owned(),
2258                kind: ProgramResourceKind::Output,
2259                scalar: external_scalar,
2260                elements: output_elements,
2261                external_layout: Some(output_layout),
2262                initialization: ProgramResourceInitialization::Zeroed,
2263            },
2264            ProgramResource {
2265                id: ProgramResourceId(2),
2266                name: "double_double_nd_tensor_a".to_owned(),
2267                kind: ProgramResourceKind::Scratch,
2268                scalar: ScalarType::DoubleDouble,
2269                elements,
2270                external_layout: None,
2271                initialization: ProgramResourceInitialization::Zeroed,
2272            },
2273            ProgramResource {
2274                id: ProgramResourceId(3),
2275                name: "double_double_nd_tensor_b".to_owned(),
2276                kind: ProgramResourceKind::Scratch,
2277                scalar: ScalarType::DoubleDouble,
2278                elements,
2279                external_layout: None,
2280                initialization: ProgramResourceInitialization::Zeroed,
2281            },
2282            ProgramResource {
2283                id: ProgramResourceId(4),
2284                name: "double_double_nd_lines_a".to_owned(),
2285                kind: ProgramResourceKind::Scratch,
2286                scalar: ScalarType::DoubleDouble,
2287                elements,
2288                external_layout: None,
2289                initialization: ProgramResourceInitialization::Zeroed,
2290            },
2291            ProgramResource {
2292                id: ProgramResourceId(5),
2293                name: "double_double_nd_lines_b".to_owned(),
2294                kind: ProgramResourceKind::Scratch,
2295                scalar: ScalarType::DoubleDouble,
2296                elements,
2297                external_layout: None,
2298                initialization: ProgramResourceInitialization::Zeroed,
2299            },
2300        ];
2301        let mut passes = Vec::new();
2302        let mut tensor_source = if let Some(copy) = &ir.input_formatted_copy {
2303            passes.push(two_buffer_pass(&copy.name, copy.dispatch, 0, 2));
2304            ProgramResourceId(2)
2305        } else {
2306            ProgramResourceId(0)
2307        };
2308        let mut tensor_target = if tensor_source == ProgramResourceId(2) {
2309            ProgramResourceId(3)
2310        } else {
2311            ProgramResourceId(2)
2312        };
2313        let mut formatted_output_source = None;
2314        for (axis_index, axis) in ir.axes.iter().enumerate() {
2315            let dispatch = DispatchGeometry {
2316                x: u32::try_from(ir.batch_count.div_ceil(axis.grouped_batch)).map_err(|_| {
2317                    VkFftError::ValueOutOfRange {
2318                        field: "double-double ND grouped dispatch count",
2319                    }
2320                })?,
2321                y: 1,
2322                z: 1,
2323            };
2324            passes.push(ProgramPass {
2325                name: format!("vkfft_dd_nd_axis_{}_pack", axis.axis),
2326                dispatch,
2327                bindings: vec![
2328                    ProgramPassBinding {
2329                        binding: 0,
2330                        resource: tensor_source,
2331                        role: BufferRole::Input,
2332                        access: BufferAccess::ReadOnly,
2333                    },
2334                    binding(1, 4, BufferRole::Output, BufferAccess::WriteOnly),
2335                ],
2336            });
2337
2338            let child = Self::double_double_one_dim(&axis.transform)?;
2339            if child.scalar != ScalarType::DoubleDouble
2340                || child.input_resource()?.scalar != ScalarType::DoubleDouble
2341                || child.output_resource()?.scalar != ScalarType::DoubleDouble
2342                || child.input_resource()?.elements != elements
2343                || child.output_resource()?.elements != elements
2344            {
2345                return Err(VkFftError::InvalidKernelIr(
2346                    "double-double ND child ProgramIr must expose DD tensor-sized boundaries",
2347                ));
2348            }
2349            let mut resource_map = vec![None; child.resources.len()];
2350            for resource in &child.resources {
2351                let mapped = match resource.kind {
2352                    ProgramResourceKind::Input => ProgramResourceId(4),
2353                    ProgramResourceKind::Output => ProgramResourceId(5),
2354                    _ => {
2355                        let new_id = ProgramResourceId(resources.len());
2356                        let mut cloned = resource.clone();
2357                        cloned.id = new_id;
2358                        cloned.name =
2359                            format!("double_double_nd_axis_{}_{}", axis.axis, resource.name);
2360                        cloned.external_layout = None;
2361                        resources.push(cloned);
2362                        new_id
2363                    }
2364                };
2365                resource_map[resource.id.0] = Some(mapped);
2366            }
2367            for child_pass in &child.passes {
2368                let mut pass = child_pass.clone();
2369                pass.name = format!("vkfft_dd_nd_axis_{}_{}", axis.axis, child_pass.name);
2370                for binding in &mut pass.bindings {
2371                    binding.resource = resource_map
2372                        .get(binding.resource.0)
2373                        .and_then(|mapped| *mapped)
2374                        .ok_or(VkFftError::InvalidKernelIr(
2375                            "double-double ND child resource mapping is incomplete",
2376                        ))?;
2377                }
2378                passes.push(pass);
2379            }
2380
2381            let last_axis = axis_index + 1 == ir.axes.len();
2382            let scatter_target = if last_axis && ir.output_formatted_copy.is_none() {
2383                ProgramResourceId(1)
2384            } else {
2385                tensor_target
2386            };
2387            passes.push(ProgramPass {
2388                name: format!("vkfft_dd_nd_axis_{}_scatter", axis.axis),
2389                dispatch,
2390                bindings: vec![
2391                    binding(0, 5, BufferRole::Input, BufferAccess::ReadOnly),
2392                    ProgramPassBinding {
2393                        binding: 1,
2394                        resource: scatter_target,
2395                        role: BufferRole::Output,
2396                        access: BufferAccess::WriteOnly,
2397                    },
2398                ],
2399            });
2400            if last_axis {
2401                formatted_output_source = ir.output_formatted_copy.as_ref().map(|_| scatter_target);
2402            } else {
2403                tensor_source = tensor_target;
2404                tensor_target = if tensor_target == ProgramResourceId(2) {
2405                    ProgramResourceId(3)
2406                } else {
2407                    ProgramResourceId(2)
2408                };
2409            }
2410        }
2411        if let (Some(copy), Some(source)) = (&ir.output_formatted_copy, formatted_output_source) {
2412            passes.push(ProgramPass {
2413                name: copy.name.clone(),
2414                dispatch: copy.dispatch,
2415                bindings: vec![
2416                    ProgramPassBinding {
2417                        binding: 0,
2418                        resource: source,
2419                        role: BufferRole::Input,
2420                        access: BufferAccess::ReadOnly,
2421                    },
2422                    binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
2423                ],
2424            });
2425        }
2426        let program = Self {
2427            name: format!("vkfft_dd_nd_program_{}d", ir.dimensions.len()),
2428            scalar: ScalarType::DoubleDouble,
2429            resources,
2430            passes,
2431        };
2432        program.validate()?;
2433        Ok(program)
2434    }
2435
2436    pub fn double_double_nd_r2r(ir: &DoubleDoubleNdR2rIr) -> Result<Self> {
2437        ir.validate()?;
2438        let elements =
2439            ir.tensor_len
2440                .checked_mul(ir.batch_count)
2441                .ok_or(VkFftError::ArithmeticOverflow {
2442                    operation: "double-double ND R2R program element count",
2443                })?;
2444        let external_scalar = match ir.external_storage {
2445            crate::PrecisionStorage::DoubleDouble => ScalarType::DoubleDouble,
2446            crate::PrecisionStorage::F64 => ScalarType::F64,
2447            _ => {
2448                return Err(VkFftError::InvalidKernelIr(
2449                    "double-double ND R2R program requires DD or F64 external storage",
2450                ));
2451            }
2452        };
2453        let input_external_stride = if ir.input_formatted_copy.is_some() {
2454            ir.input_external_layout.batch_stride
2455        } else {
2456            ir.tensor_len
2457        };
2458        let output_external_stride = if ir.output_formatted_copy.is_some() {
2459            ir.output_external_layout.batch_stride
2460        } else {
2461            ir.tensor_len
2462        };
2463        let input_external_layout = ExternalBufferLayout {
2464            logical_len: input_external_stride,
2465            physical_stride: input_external_stride,
2466            batch_count: ir.batch_count,
2467            element_shape: ProgramElementShape::Scalar,
2468        };
2469        let output_external_layout = ExternalBufferLayout {
2470            logical_len: output_external_stride,
2471            physical_stride: output_external_stride,
2472            batch_count: ir.batch_count,
2473            element_shape: ProgramElementShape::Scalar,
2474        };
2475        let input_external_elements = input_external_layout.physical_elements()?;
2476        let output_external_elements = output_external_layout.physical_elements()?;
2477        let scalar_scratch_layout = ExternalBufferLayout {
2478            logical_len: elements,
2479            physical_stride: elements,
2480            batch_count: 1,
2481            element_shape: ProgramElementShape::Scalar,
2482        };
2483        let mut resources = vec![
2484            ProgramResource {
2485                id: ProgramResourceId(0),
2486                name: "input".to_owned(),
2487                kind: ProgramResourceKind::Input,
2488                scalar: external_scalar,
2489                elements: input_external_elements,
2490                external_layout: Some(input_external_layout),
2491                initialization: ProgramResourceInitialization::ExternalInput,
2492            },
2493            ProgramResource {
2494                id: ProgramResourceId(1),
2495                name: "output".to_owned(),
2496                kind: ProgramResourceKind::Output,
2497                scalar: external_scalar,
2498                elements: output_external_elements,
2499                external_layout: Some(output_external_layout),
2500                initialization: ProgramResourceInitialization::Zeroed,
2501            },
2502            ProgramResource {
2503                id: ProgramResourceId(2),
2504                name: "double_double_nd_r2r_tensor_a".to_owned(),
2505                kind: ProgramResourceKind::Scratch,
2506                scalar: ScalarType::DoubleDouble,
2507                elements,
2508                external_layout: Some(scalar_scratch_layout),
2509                initialization: ProgramResourceInitialization::Zeroed,
2510            },
2511            ProgramResource {
2512                id: ProgramResourceId(3),
2513                name: "double_double_nd_r2r_tensor_b".to_owned(),
2514                kind: ProgramResourceKind::Scratch,
2515                scalar: ScalarType::DoubleDouble,
2516                elements,
2517                external_layout: Some(scalar_scratch_layout),
2518                initialization: ProgramResourceInitialization::Zeroed,
2519            },
2520            ProgramResource {
2521                id: ProgramResourceId(4),
2522                name: "double_double_nd_r2r_lines_a".to_owned(),
2523                kind: ProgramResourceKind::Scratch,
2524                scalar: ScalarType::DoubleDouble,
2525                elements,
2526                external_layout: Some(scalar_scratch_layout),
2527                initialization: ProgramResourceInitialization::Zeroed,
2528            },
2529            ProgramResource {
2530                id: ProgramResourceId(5),
2531                name: "double_double_nd_r2r_lines_b".to_owned(),
2532                kind: ProgramResourceKind::Scratch,
2533                scalar: ScalarType::DoubleDouble,
2534                elements,
2535                external_layout: Some(scalar_scratch_layout),
2536                initialization: ProgramResourceInitialization::Zeroed,
2537            },
2538        ];
2539        let mut passes = Vec::new();
2540        let mut tensor_source = if let Some(copy) = &ir.input_formatted_copy {
2541            let dense = ProgramResourceId(resources.len());
2542            resources.push(ProgramResource {
2543                id: dense,
2544                name: "double_double_nd_r2r_formatted_input".to_owned(),
2545                kind: ProgramResourceKind::Scratch,
2546                scalar: ScalarType::DoubleDouble,
2547                elements,
2548                external_layout: Some(scalar_scratch_layout),
2549                initialization: ProgramResourceInitialization::Zeroed,
2550            });
2551            passes.push(ProgramPass {
2552                name: copy.name.clone(),
2553                dispatch: copy.dispatch,
2554                bindings: vec![
2555                    binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
2556                    ProgramPassBinding {
2557                        binding: 1,
2558                        resource: dense,
2559                        role: BufferRole::Output,
2560                        access: BufferAccess::WriteOnly,
2561                    },
2562                ],
2563            });
2564            dense
2565        } else {
2566            ProgramResourceId(0)
2567        };
2568        let formatted_output = if ir.output_formatted_copy.is_some() {
2569            let dense = ProgramResourceId(resources.len());
2570            resources.push(ProgramResource {
2571                id: dense,
2572                name: "double_double_nd_r2r_formatted_output".to_owned(),
2573                kind: ProgramResourceKind::Scratch,
2574                scalar: ScalarType::DoubleDouble,
2575                elements,
2576                external_layout: Some(scalar_scratch_layout),
2577                initialization: ProgramResourceInitialization::Zeroed,
2578            });
2579            Some(dense)
2580        } else {
2581            None
2582        };
2583        let mut tensor_target = ProgramResourceId(2);
2584        for (axis_index, axis) in ir.axes.iter().enumerate() {
2585            let dispatch = DispatchGeometry {
2586                x: u32::try_from(ir.batch_count.div_ceil(axis.grouped_batch)).map_err(|_| {
2587                    VkFftError::ValueOutOfRange {
2588                        field: "double-double ND R2R grouped dispatch count",
2589                    }
2590                })?,
2591                y: 1,
2592                z: 1,
2593            };
2594            passes.push(ProgramPass {
2595                name: format!("vkfft_dd_nd_r2r_axis_{}_pack", axis.axis),
2596                dispatch,
2597                bindings: vec![
2598                    ProgramPassBinding {
2599                        binding: 0,
2600                        resource: tensor_source,
2601                        role: BufferRole::Input,
2602                        access: BufferAccess::ReadOnly,
2603                    },
2604                    binding(1, 4, BufferRole::Output, BufferAccess::WriteOnly),
2605                ],
2606            });
2607            let child = Self::double_double_r2r(&axis.transform)?;
2608            if child.scalar != ScalarType::DoubleDouble
2609                || child.input_resource()?.scalar != ScalarType::DoubleDouble
2610                || child.output_resource()?.scalar != ScalarType::DoubleDouble
2611                || child.input_resource()?.element_shape() != ProgramElementShape::Scalar
2612                || child.output_resource()?.element_shape() != ProgramElementShape::Scalar
2613                || child.input_resource()?.elements != elements
2614                || child.output_resource()?.elements != elements
2615            {
2616                return Err(VkFftError::InvalidKernelIr(
2617                    "double-double ND R2R child ProgramIr must expose tensor-sized scalar DD boundaries",
2618                ));
2619            }
2620            let mut resource_map = vec![None; child.resources.len()];
2621            for resource in &child.resources {
2622                let mapped = match resource.kind {
2623                    ProgramResourceKind::Input => ProgramResourceId(4),
2624                    ProgramResourceKind::Output => ProgramResourceId(5),
2625                    _ => {
2626                        let new_id = ProgramResourceId(resources.len());
2627                        let mut cloned = resource.clone();
2628                        cloned.id = new_id;
2629                        cloned.name =
2630                            format!("double_double_nd_r2r_axis_{}_{}", axis.axis, resource.name);
2631                        cloned.external_layout = None;
2632                        resources.push(cloned);
2633                        new_id
2634                    }
2635                };
2636                resource_map[resource.id.0] = Some(mapped);
2637            }
2638            for child_pass in &child.passes {
2639                let mut pass = child_pass.clone();
2640                pass.name = format!("vkfft_dd_nd_r2r_axis_{}_{}", axis.axis, child_pass.name);
2641                for binding in &mut pass.bindings {
2642                    binding.resource = resource_map
2643                        .get(binding.resource.0)
2644                        .and_then(|mapped| *mapped)
2645                        .ok_or(VkFftError::InvalidKernelIr(
2646                            "double-double ND R2R child resource mapping is incomplete",
2647                        ))?;
2648                }
2649                passes.push(pass);
2650            }
2651            let last_axis = axis_index + 1 == ir.axes.len();
2652            let scatter_target = if last_axis {
2653                formatted_output.unwrap_or(ProgramResourceId(1))
2654            } else {
2655                tensor_target
2656            };
2657            passes.push(ProgramPass {
2658                name: format!("vkfft_dd_nd_r2r_axis_{}_scatter", axis.axis),
2659                dispatch,
2660                bindings: vec![
2661                    binding(0, 5, BufferRole::Input, BufferAccess::ReadOnly),
2662                    ProgramPassBinding {
2663                        binding: 1,
2664                        resource: scatter_target,
2665                        role: BufferRole::Output,
2666                        access: BufferAccess::WriteOnly,
2667                    },
2668                ],
2669            });
2670            if !last_axis {
2671                tensor_source = tensor_target;
2672                tensor_target = if tensor_target == ProgramResourceId(2) {
2673                    ProgramResourceId(3)
2674                } else {
2675                    ProgramResourceId(2)
2676                };
2677            }
2678        }
2679        if let Some(copy) = &ir.output_formatted_copy {
2680            passes.push(ProgramPass {
2681                name: copy.name.clone(),
2682                dispatch: copy.dispatch,
2683                bindings: vec![
2684                    ProgramPassBinding {
2685                        binding: 0,
2686                        resource: formatted_output.expect("formatted DD R2R output scratch"),
2687                        role: BufferRole::Input,
2688                        access: BufferAccess::ReadOnly,
2689                    },
2690                    binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
2691                ],
2692            });
2693        }
2694        let program = Self {
2695            name: format!("vkfft_dd_nd_r2r_program_{}d", ir.dimensions.len()),
2696            scalar: ScalarType::DoubleDouble,
2697            resources,
2698            passes,
2699        };
2700        program.validate()?;
2701        Ok(program)
2702    }
2703
2704    pub fn double_double_bluestein(ir: &DoubleDoubleBluesteinIr) -> Result<Self> {
2705        ir.validate()?;
2706        let logical_elements =
2707            ir.logical_len
2708                .checked_mul(ir.batch_count)
2709                .ok_or(VkFftError::ArithmeticOverflow {
2710                    operation: "double-double Bluestein program logical element count",
2711                })?;
2712        let convolution_elements = ir.convolution_len.checked_mul(ir.batch_count).ok_or(
2713            VkFftError::ArithmeticOverflow {
2714                operation: "double-double Bluestein program convolution element count",
2715            },
2716        )?;
2717        let external_scalar = match ir.external_storage {
2718            crate::PrecisionStorage::DoubleDouble => ScalarType::DoubleDouble,
2719            crate::PrecisionStorage::F64 => ScalarType::F64,
2720            _ => {
2721                return Err(VkFftError::InvalidKernelIr(
2722                    "double-double Bluestein program requires DD or F64 external storage",
2723                ));
2724            }
2725        };
2726        let fused_inverse_stockham = matches!(
2727            &ir.inverse_fft,
2728            crate::DoubleDoubleBluesteinConvolutionIr::Stockham(child)
2729                if double_double_stockham_input_lookup_eligible(child)
2730        );
2731        let layout = ExternalBufferLayout {
2732            logical_len: ir.logical_len,
2733            physical_stride: ir.logical_len,
2734            batch_count: ir.batch_count,
2735            element_shape: ProgramElementShape::Complex,
2736        };
2737        let mut resources = vec![
2738            ProgramResource {
2739                id: ProgramResourceId(0),
2740                name: "input".to_owned(),
2741                kind: ProgramResourceKind::Input,
2742                scalar: external_scalar,
2743                elements: logical_elements,
2744                external_layout: Some(layout),
2745                initialization: ProgramResourceInitialization::ExternalInput,
2746            },
2747            ProgramResource {
2748                id: ProgramResourceId(1),
2749                name: "output".to_owned(),
2750                kind: ProgramResourceKind::Output,
2751                scalar: external_scalar,
2752                elements: logical_elements,
2753                external_layout: Some(layout),
2754                initialization: ProgramResourceInitialization::Zeroed,
2755            },
2756        ];
2757        let push_convolution_scratch =
2758            |resources: &mut Vec<ProgramResource>, name: &str| -> ProgramResourceId {
2759                let id = ProgramResourceId(resources.len());
2760                resources.push(ProgramResource {
2761                    id,
2762                    name: name.to_owned(),
2763                    kind: ProgramResourceKind::Scratch,
2764                    scalar: ScalarType::DoubleDouble,
2765                    elements: convolution_elements,
2766                    external_layout: None,
2767                    initialization: ProgramResourceInitialization::Zeroed,
2768                });
2769                id
2770            };
2771        let forward_input_id =
2772            push_convolution_scratch(&mut resources, "double_double_bluestein_forward_input");
2773        let forward_output_id =
2774            push_convolution_scratch(&mut resources, "double_double_bluestein_forward_output");
2775        let inverse_input_id = if fused_inverse_stockham {
2776            None
2777        } else {
2778            Some(push_convolution_scratch(
2779                &mut resources,
2780                "double_double_bluestein_inverse_input",
2781            ))
2782        };
2783        let inverse_output_id =
2784            push_convolution_scratch(&mut resources, "double_double_bluestein_inverse_output");
2785        let chirp_id = ProgramResourceId(resources.len());
2786        resources.push(ProgramResource {
2787            id: chirp_id,
2788            name: "double_double_bluestein_chirp".to_owned(),
2789            kind: ProgramResourceKind::LookupTable,
2790            scalar: ScalarType::DoubleDouble,
2791            elements: ir.table.chirp.len(),
2792            external_layout: None,
2793            initialization: ProgramResourceInitialization::ComplexDoubleDouble(
2794                ir.table.chirp.clone(),
2795            ),
2796        });
2797        let kernel_id = ProgramResourceId(resources.len());
2798        resources.push(ProgramResource {
2799            id: kernel_id,
2800            name: "double_double_bluestein_kernel_spectrum".to_owned(),
2801            kind: ProgramResourceKind::LookupTable,
2802            scalar: ScalarType::DoubleDouble,
2803            elements: ir.kernel_spectrum.len(),
2804            external_layout: None,
2805            initialization: ProgramResourceInitialization::ComplexDoubleDouble(
2806                ir.kernel_spectrum.clone(),
2807            ),
2808        });
2809        let dispatch = DispatchGeometry {
2810            x: u32::try_from(ir.batch_group_count()).map_err(|_| VkFftError::ValueOutOfRange {
2811                field: "double-double Bluestein program dispatch count",
2812            })?,
2813            y: 1,
2814            z: 1,
2815        };
2816        let mut passes = Vec::new();
2817        passes.push(ProgramPass {
2818            name: format!("{}_preprocess", ir.name),
2819            dispatch,
2820            bindings: vec![
2821                binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
2822                binding(
2823                    1,
2824                    forward_input_id.0,
2825                    BufferRole::Output,
2826                    BufferAccess::WriteOnly,
2827                ),
2828                ProgramPassBinding {
2829                    binding: 2,
2830                    resource: chirp_id,
2831                    role: BufferRole::LookupTable,
2832                    access: BufferAccess::ReadOnly,
2833                },
2834            ],
2835        });
2836
2837        let forward_child = match &ir.forward_fft {
2838            crate::DoubleDoubleBluesteinConvolutionIr::Stockham(child) => {
2839                ProgramIr::double_double_stockham_internal(child)?
2840            }
2841            crate::DoubleDoubleBluesteinConvolutionIr::Recursive(child) => {
2842                ProgramIr::double_double_recursive(child)?
2843            }
2844            crate::DoubleDoubleBluesteinConvolutionIr::Bluestein(child) => {
2845                ProgramIr::double_double_bluestein(child)?
2846            }
2847        };
2848        append_double_double_recursive_child(
2849            &forward_child,
2850            "vkfft_dd_bluestein_forward",
2851            forward_input_id,
2852            forward_output_id,
2853            &mut resources,
2854            &mut passes,
2855            false,
2856        )?;
2857
2858        if !fused_inverse_stockham {
2859            passes.push(ProgramPass {
2860                name: format!("{}_multiply", ir.name),
2861                dispatch,
2862                bindings: vec![
2863                    binding(
2864                        0,
2865                        forward_output_id.0,
2866                        BufferRole::Input,
2867                        BufferAccess::ReadOnly,
2868                    ),
2869                    binding(
2870                        1,
2871                        inverse_input_id
2872                            .expect("explicit DD Bluestein multiply requires inverse input scratch")
2873                            .0,
2874                        BufferRole::Output,
2875                        BufferAccess::WriteOnly,
2876                    ),
2877                    ProgramPassBinding {
2878                        binding: 2,
2879                        resource: kernel_id,
2880                        role: BufferRole::LookupTable,
2881                        access: BufferAccess::ReadOnly,
2882                    },
2883                ],
2884            });
2885        }
2886
2887        let inverse_child = match &ir.inverse_fft {
2888            crate::DoubleDoubleBluesteinConvolutionIr::Stockham(child) => {
2889                ProgramIr::double_double_stockham_internal(child)?
2890            }
2891            crate::DoubleDoubleBluesteinConvolutionIr::Recursive(child) => {
2892                ProgramIr::double_double_recursive(child)?
2893            }
2894            crate::DoubleDoubleBluesteinConvolutionIr::Bluestein(child) => {
2895                ProgramIr::double_double_bluestein(child)?
2896            }
2897        };
2898        let inverse_pass_start = passes.len();
2899        append_double_double_recursive_child(
2900            &inverse_child,
2901            "vkfft_dd_bluestein_inverse",
2902            if fused_inverse_stockham {
2903                forward_output_id
2904            } else {
2905                inverse_input_id.expect("explicit DD Bluestein inverse requires input scratch")
2906            },
2907            inverse_output_id,
2908            &mut resources,
2909            &mut passes,
2910            false,
2911        )?;
2912        if fused_inverse_stockham {
2913            attach_double_double_stockham_input_lookup(
2914                passes
2915                    .get_mut(inverse_pass_start)
2916                    .ok_or(VkFftError::InvalidKernelIr(
2917                        "fused DD Bluestein inverse Stockham child emitted no pass",
2918                    ))?,
2919                kernel_id,
2920            )?;
2921        }
2922
2923        passes.push(ProgramPass {
2924            name: format!("{}_postprocess", ir.name),
2925            dispatch,
2926            bindings: vec![
2927                binding(
2928                    0,
2929                    inverse_output_id.0,
2930                    BufferRole::Input,
2931                    BufferAccess::ReadOnly,
2932                ),
2933                binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
2934                ProgramPassBinding {
2935                    binding: 2,
2936                    resource: chirp_id,
2937                    role: BufferRole::LookupTable,
2938                    access: BufferAccess::ReadOnly,
2939                },
2940            ],
2941        });
2942        let program = Self {
2943            name: format!("{}_program", ir.name),
2944            scalar: ScalarType::DoubleDouble,
2945            resources,
2946            passes,
2947        };
2948        program.validate()?;
2949        Ok(program)
2950    }
2951
2952    pub fn double_double_fft_rader(ir: &DoubleDoubleFftRaderIr) -> Result<Self> {
2953        ir.validate()?;
2954        let external_elements =
2955            ir.prime
2956                .checked_mul(ir.batch_count)
2957                .ok_or(VkFftError::ArithmeticOverflow {
2958                    operation: "double-double FFT Rader program external element count",
2959                })?;
2960        let scratch_elements = ir.convolution_len.checked_mul(ir.batch_count).ok_or(
2961            VkFftError::ArithmeticOverflow {
2962                operation: "double-double FFT Rader program scratch element count",
2963            },
2964        )?;
2965        let storage_scalar = |storage| match storage {
2966            crate::PrecisionStorage::DoubleDouble => Ok(ScalarType::DoubleDouble),
2967            crate::PrecisionStorage::F64 => Ok(ScalarType::F64),
2968            _ => Err(VkFftError::InvalidKernelIr(
2969                "double-double FFT Rader program requires DD or F64 caller storage",
2970            )),
2971        };
2972        let input_scalar = storage_scalar(ir.input_storage)?;
2973        let output_scalar = storage_scalar(ir.output_storage)?;
2974        let layout = ExternalBufferLayout {
2975            logical_len: ir.prime,
2976            physical_stride: ir.prime,
2977            batch_count: ir.batch_count,
2978            element_shape: ProgramElementShape::Complex,
2979        };
2980        let mut resources = vec![
2981            ProgramResource {
2982                id: ProgramResourceId(0),
2983                name: "input".to_owned(),
2984                kind: ProgramResourceKind::Input,
2985                scalar: input_scalar,
2986                elements: external_elements,
2987                external_layout: Some(layout),
2988                initialization: ProgramResourceInitialization::ExternalInput,
2989            },
2990            ProgramResource {
2991                id: ProgramResourceId(1),
2992                name: "output".to_owned(),
2993                kind: ProgramResourceKind::Output,
2994                scalar: output_scalar,
2995                elements: external_elements,
2996                external_layout: Some(layout),
2997                initialization: ProgramResourceInitialization::Zeroed,
2998            },
2999            ProgramResource {
3000                id: ProgramResourceId(2),
3001                name: "double_double_rader_scratch_a".to_owned(),
3002                kind: ProgramResourceKind::Scratch,
3003                scalar: ScalarType::DoubleDouble,
3004                elements: scratch_elements,
3005                external_layout: None,
3006                initialization: ProgramResourceInitialization::Zeroed,
3007            },
3008            ProgramResource {
3009                id: ProgramResourceId(3),
3010                name: "double_double_rader_scratch_b".to_owned(),
3011                kind: ProgramResourceKind::Scratch,
3012                scalar: ScalarType::DoubleDouble,
3013                elements: scratch_elements,
3014                external_layout: None,
3015                initialization: ProgramResourceInitialization::Zeroed,
3016            },
3017        ];
3018        let kernel_id = ProgramResourceId(resources.len());
3019        resources.push(ProgramResource {
3020            id: kernel_id,
3021            name: "double_double_rader_kernel_spectrum".to_owned(),
3022            kind: ProgramResourceKind::LookupTable,
3023            scalar: ScalarType::DoubleDouble,
3024            elements: ir.kernel_spectrum.len(),
3025            external_layout: None,
3026            initialization: ProgramResourceInitialization::ComplexDoubleDouble(
3027                ir.kernel_spectrum.clone(),
3028            ),
3029        });
3030        let dispatch = DispatchGeometry {
3031            x: u32::try_from(ir.batch_group_count()).map_err(|_| VkFftError::ValueOutOfRange {
3032                field: "double-double FFT Rader program dispatch count",
3033            })?,
3034            y: 1,
3035            z: 1,
3036        };
3037        let convolution_grouped_batch = ir
3038            .forward_fft
3039            .stockham_axis_batch_block()
3040            .map_or(ir.grouped_batch, |block| block.grouped_batch);
3041        let convolution_dispatch = DispatchGeometry {
3042            x: u32::try_from(ir.batch_count.div_ceil(convolution_grouped_batch)).map_err(|_| {
3043                VkFftError::ValueOutOfRange {
3044                    field: "double-double FFT Rader convolution dispatch count",
3045                }
3046            })?,
3047            y: 1,
3048            z: 1,
3049        };
3050        let mut passes = Vec::new();
3051        passes.push(ProgramPass {
3052            name: format!("{}_gather", ir.name),
3053            dispatch,
3054            bindings: vec![
3055                binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
3056                binding(1, 2, BufferRole::Output, BufferAccess::WriteOnly),
3057            ],
3058        });
3059
3060        let forward_child = match &ir.forward_fft {
3061            crate::DoubleDoubleBluesteinConvolutionIr::Stockham(child) => {
3062                ProgramIr::double_double_stockham_internal(child)?
3063            }
3064            crate::DoubleDoubleBluesteinConvolutionIr::Recursive(child) => {
3065                ProgramIr::double_double_recursive(child)?
3066            }
3067            crate::DoubleDoubleBluesteinConvolutionIr::Bluestein(child) => {
3068                ProgramIr::double_double_bluestein(child)?
3069            }
3070        };
3071        append_double_double_recursive_child(
3072            &forward_child,
3073            "vkfft_dd_fft_rader_forward",
3074            ProgramResourceId(2),
3075            ProgramResourceId(3),
3076            &mut resources,
3077            &mut passes,
3078            false,
3079        )?;
3080        let fused_inverse_stockham = matches!(
3081            &ir.inverse_fft,
3082            crate::DoubleDoubleBluesteinConvolutionIr::Stockham(child)
3083                if double_double_stockham_input_lookup_eligible(child)
3084        );
3085        if !fused_inverse_stockham {
3086            passes.push(ProgramPass {
3087                name: format!("{}_multiply", ir.name),
3088                dispatch: convolution_dispatch,
3089                bindings: vec![
3090                    binding(0, 3, BufferRole::Input, BufferAccess::ReadOnly),
3091                    binding(1, 2, BufferRole::Output, BufferAccess::WriteOnly),
3092                    ProgramPassBinding {
3093                        binding: 2,
3094                        resource: kernel_id,
3095                        role: BufferRole::LookupTable,
3096                        access: BufferAccess::ReadOnly,
3097                    },
3098                ],
3099            });
3100        }
3101        let inverse_child = match &ir.inverse_fft {
3102            crate::DoubleDoubleBluesteinConvolutionIr::Stockham(child) => {
3103                ProgramIr::double_double_stockham_internal(child)?
3104            }
3105            crate::DoubleDoubleBluesteinConvolutionIr::Recursive(child) => {
3106                ProgramIr::double_double_recursive(child)?
3107            }
3108            crate::DoubleDoubleBluesteinConvolutionIr::Bluestein(child) => {
3109                ProgramIr::double_double_bluestein(child)?
3110            }
3111        };
3112        let inverse_pass_start = passes.len();
3113        append_double_double_recursive_child(
3114            &inverse_child,
3115            "vkfft_dd_fft_rader_inverse",
3116            if fused_inverse_stockham {
3117                ProgramResourceId(3)
3118            } else {
3119                ProgramResourceId(2)
3120            },
3121            if fused_inverse_stockham {
3122                ProgramResourceId(2)
3123            } else {
3124                ProgramResourceId(3)
3125            },
3126            &mut resources,
3127            &mut passes,
3128            false,
3129        )?;
3130        if fused_inverse_stockham {
3131            attach_double_double_stockham_input_lookup(
3132                passes
3133                    .get_mut(inverse_pass_start)
3134                    .ok_or(VkFftError::InvalidKernelIr(
3135                        "fused DD FFT-Rader inverse Stockham child emitted no pass",
3136                    ))?,
3137                kernel_id,
3138            )?;
3139        }
3140        let inverse_output = if fused_inverse_stockham { 2 } else { 3 };
3141        let mut scatter_bindings = vec![
3142            binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
3143            binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
3144            binding(
3145                2,
3146                inverse_output,
3147                BufferRole::Auxiliary,
3148                BufferAccess::ReadOnly,
3149            ),
3150        ];
3151        let parent_period = match ir.io_mapping {
3152            crate::StockhamIoMapping::FourStepRight(mapping) => Some(mapping.logical_len),
3153            crate::StockhamIoMapping::FourStepThreeUpload2(mapping) => Some(mapping.logical_len),
3154            crate::StockhamIoMapping::FourStepThreeUpload1(mapping) => {
3155                let [a, b, _] = mapping.axis_split;
3156                Some(a.checked_mul(b).ok_or(VkFftError::ArithmeticOverflow {
3157                    operation: "double-double FFT Rader three-upload parent root period",
3158                })?)
3159            }
3160            _ => None,
3161        };
3162        if let Some(period) = parent_period {
3163            let roots = crate::lut::unit_root_table_double_double(period, ir.direction)?;
3164            let root_id = ProgramResourceId(resources.len());
3165            resources.push(ProgramResource {
3166                id: root_id,
3167                name: "double_double_fft_rader_four_step_roots".to_owned(),
3168                kind: ProgramResourceKind::LookupTable,
3169                scalar: ScalarType::DoubleDouble,
3170                elements: roots.len(),
3171                external_layout: None,
3172                initialization: ProgramResourceInitialization::ComplexDoubleDouble(roots),
3173            });
3174            scatter_bindings.push(ProgramPassBinding {
3175                binding: 3,
3176                resource: root_id,
3177                role: BufferRole::TwiddleLookupTable,
3178                access: BufferAccess::ReadOnly,
3179            });
3180        }
3181        passes.push(ProgramPass {
3182            name: format!("{}_scatter", ir.name),
3183            dispatch,
3184            bindings: scatter_bindings,
3185        });
3186        let program = Self {
3187            name: format!("{}_program", ir.name),
3188            scalar: ScalarType::DoubleDouble,
3189            resources,
3190            passes,
3191        };
3192        program.validate()?;
3193        Self::wrap_double_double_zero_pad(
3194            ir.zero_pad_pass.as_ref(),
3195            ir.prime,
3196            ir.batch_count,
3197            "double_double_fft_rader",
3198            program,
3199        )
3200    }
3201
3202    pub fn stockham(kernel: &KernelIr) -> Result<Self> {
3203        kernel.validate()?;
3204        let elements = kernel.sequence_len.checked_mul(kernel.batch_count).ok_or(
3205            VkFftError::ArithmeticOverflow {
3206                operation: "Stockham program element count",
3207            },
3208        )?;
3209        let layout = ExternalBufferLayout {
3210            logical_len: kernel.sequence_len,
3211            physical_stride: kernel.sequence_len,
3212            batch_count: kernel.batch_count,
3213            element_shape: ProgramElementShape::Complex,
3214        };
3215        let mut builder = RecursiveProgramBuilder {
3216            scalar: kernel.scalar,
3217            resources: vec![
3218                ProgramResource {
3219                    id: ProgramResourceId(0),
3220                    name: "input".to_owned(),
3221                    kind: ProgramResourceKind::Input,
3222                    scalar: kernel.bindings[0].scalar,
3223                    elements,
3224                    external_layout: Some(layout),
3225                    initialization: ProgramResourceInitialization::ExternalInput,
3226                },
3227                ProgramResource {
3228                    id: ProgramResourceId(1),
3229                    name: "output".to_owned(),
3230                    kind: ProgramResourceKind::Output,
3231                    scalar: kernel.bindings[1].scalar,
3232                    elements,
3233                    external_layout: Some(layout),
3234                    initialization: ProgramResourceInitialization::Zeroed,
3235                },
3236            ],
3237            passes: Vec::new(),
3238            serial: 0,
3239        };
3240        builder.push_stockham_pass(
3241            kernel,
3242            ProgramResourceId(0),
3243            ProgramResourceId(1),
3244            None,
3245            None,
3246        )?;
3247        let program = Self {
3248            name: format!("{}_program", kernel.name),
3249            scalar: kernel.scalar,
3250            resources: builder.resources,
3251            passes: builder.passes,
3252        };
3253        program.validate()?;
3254        Ok(program)
3255    }
3256
3257    fn double_double_r2r_fused_physical_stockham(
3258        ir: &DoubleDoubleR2rIr,
3259        elements: usize,
3260        external_scalar: ScalarType,
3261        external: ExternalBufferLayout,
3262    ) -> Result<Option<Self>> {
3263        let DoubleDoubleR2rAlgorithm::FftReduction { fft, phases, .. } = &ir.algorithm else {
3264            return Ok(None);
3265        };
3266        if !matches!(
3267            ir.effective_transform,
3268            crate::R2rTransform::Dct(crate::DctType::II | crate::DctType::III)
3269                | crate::R2rTransform::Dst(crate::DstType::II | crate::DstType::III)
3270        ) {
3271            return Ok(None);
3272        }
3273        let DoubleDoubleOneDimIr::Stockham(stockham) = fft.as_ref() else {
3274            return Ok(None);
3275        };
3276        if stockham.axis_batch_block.is_none()
3277            || stockham.external_storage != crate::PrecisionStorage::DoubleDouble
3278            || stockham.zero_pad_pass.is_some()
3279            || stockham.normalize
3280            || stockham.sequence_len != ir.length
3281            || phases.len() != ir.length
3282        {
3283            return Ok(None);
3284        }
3285
3286        let mut resources = vec![
3287            ProgramResource {
3288                id: ProgramResourceId(0),
3289                name: "input".to_owned(),
3290                kind: ProgramResourceKind::Input,
3291                scalar: external_scalar,
3292                elements,
3293                external_layout: Some(external),
3294                initialization: ProgramResourceInitialization::ExternalInput,
3295            },
3296            ProgramResource {
3297                id: ProgramResourceId(1),
3298                name: "output".to_owned(),
3299                kind: ProgramResourceKind::Output,
3300                scalar: external_scalar,
3301                elements,
3302                external_layout: Some(external),
3303                initialization: ProgramResourceInitialization::Zeroed,
3304            },
3305        ];
3306        let mut bindings = vec![
3307            binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
3308            binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
3309        ];
3310        let twiddles = stockham.twiddles.packed_values();
3311        if !twiddles.is_empty() {
3312            let twiddle_id = ProgramResourceId(resources.len());
3313            resources.push(ProgramResource {
3314                id: twiddle_id,
3315                name: "double_double_r2r_fused_stockham_twiddles".to_owned(),
3316                kind: ProgramResourceKind::LookupTable,
3317                scalar: ScalarType::DoubleDouble,
3318                elements: twiddles.len(),
3319                external_layout: None,
3320                initialization: ProgramResourceInitialization::ComplexDoubleDouble(twiddles),
3321            });
3322            bindings.push(ProgramPassBinding {
3323                binding: 2,
3324                resource: twiddle_id,
3325                role: BufferRole::TwiddleLookupTable,
3326                access: BufferAccess::ReadOnly,
3327            });
3328        }
3329        let phases_id = ProgramResourceId(resources.len());
3330        resources.push(ProgramResource {
3331            id: phases_id,
3332            name: "double_double_r2r_phases".to_owned(),
3333            kind: ProgramResourceKind::LookupTable,
3334            scalar: ScalarType::DoubleDouble,
3335            elements: phases.len(),
3336            external_layout: None,
3337            initialization: ProgramResourceInitialization::ComplexDoubleDouble(phases.clone()),
3338        });
3339        bindings.push(ProgramPassBinding {
3340            binding: 3,
3341            resource: phases_id,
3342            role: BufferRole::LookupTable,
3343            access: BufferAccess::ReadOnly,
3344        });
3345        let label = match ir.effective_transform {
3346            crate::R2rTransform::Dct(crate::DctType::II) => "dct2",
3347            crate::R2rTransform::Dct(crate::DctType::III) => "dct3",
3348            crate::R2rTransform::Dst(crate::DstType::II) => "dst2",
3349            crate::R2rTransform::Dst(crate::DstType::III) => "dst3",
3350            _ => unreachable!("validated fused DD Type-II/III transform"),
3351        };
3352        let program = Self {
3353            name: format!("vkfft_dd_r2r_{label}_stockham_fused_{}_program", ir.length),
3354            scalar: ScalarType::DoubleDouble,
3355            resources,
3356            passes: vec![ProgramPass {
3357                name: format!("vkfft_dd_r2r_{label}_stockham_fused_{}", ir.length),
3358                dispatch: DispatchGeometry {
3359                    x: u32::try_from(stockham.batch_group_count()).map_err(|_| {
3360                        VkFftError::ValueOutOfRange {
3361                            field: "fused double-double R2R Stockham dispatch count",
3362                        }
3363                    })?,
3364                    y: 1,
3365                    z: 1,
3366                },
3367                bindings,
3368            }],
3369        };
3370        program.validate()?;
3371        Ok(Some(program))
3372    }
3373
3374    fn double_double_r2r_fused_two_upload_four_step(
3375        ir: &DoubleDoubleR2rIr,
3376        elements: usize,
3377        external_scalar: ScalarType,
3378        external: ExternalBufferLayout,
3379    ) -> Result<Option<Self>> {
3380        let DoubleDoubleR2rAlgorithm::FftReduction { fft, phases, .. } = &ir.algorithm else {
3381            return Ok(None);
3382        };
3383        if !matches!(
3384            ir.effective_transform,
3385            crate::R2rTransform::Dct(crate::DctType::II | crate::DctType::III)
3386                | crate::R2rTransform::Dst(crate::DstType::II | crate::DstType::III)
3387        ) {
3388            return Ok(None);
3389        }
3390        let DoubleDoubleOneDimIr::Recursive(recursive) = fft.as_ref() else {
3391            return Ok(None);
3392        };
3393        if recursive.two_upload_four_step_plan.is_none()
3394            || recursive.three_upload_four_step_plan.is_some()
3395            || recursive.rader_forced_upload_schedule.is_some()
3396            || recursive.external_storage != crate::PrecisionStorage::DoubleDouble
3397            || recursive.zero_pad_pass.is_some()
3398            || recursive.normalize
3399            || recursive.logical_len != ir.length
3400            || recursive.batch_count != ir.batch_count
3401            || recursive.grouped_batch != ir.grouped_batch
3402            || phases.len() != ir.length
3403        {
3404            return Ok(None);
3405        }
3406
3407        let mut program = Self::double_double_recursive(recursive)?;
3408        if program.passes.len() != 2
3409            || program.input_resource()?.scalar != ScalarType::DoubleDouble
3410            || program.output_resource()?.scalar != ScalarType::DoubleDouble
3411        {
3412            return Ok(None);
3413        }
3414        for resource in &mut program.resources {
3415            match resource.kind {
3416                ProgramResourceKind::Input => {
3417                    resource.scalar = external_scalar;
3418                    resource.elements = elements;
3419                    resource.external_layout = Some(external);
3420                }
3421                ProgramResourceKind::Output => {
3422                    resource.scalar = external_scalar;
3423                    resource.elements = elements;
3424                    resource.external_layout = Some(external);
3425                }
3426                _ => {}
3427            }
3428        }
3429        let phases_id = ProgramResourceId(program.resources.len());
3430        program.resources.push(ProgramResource {
3431            id: phases_id,
3432            name: "double_double_r2r_phases".to_owned(),
3433            kind: ProgramResourceKind::LookupTable,
3434            scalar: ScalarType::DoubleDouble,
3435            elements: phases.len(),
3436            external_layout: None,
3437            initialization: ProgramResourceInitialization::ComplexDoubleDouble(phases.clone()),
3438        });
3439        let label = match ir.effective_transform {
3440            crate::R2rTransform::Dct(crate::DctType::II) => "dct2",
3441            crate::R2rTransform::Dct(crate::DctType::III) => "dct3",
3442            crate::R2rTransform::Dst(crate::DstType::II) => "dst2",
3443            crate::R2rTransform::Dst(crate::DstType::III) => "dst3",
3444            _ => unreachable!("validated two-upload DD Type-II/III fusion"),
3445        };
3446        let phase_binding = ProgramPassBinding {
3447            binding: if matches!(
3448                ir.effective_transform,
3449                crate::R2rTransform::Dct(crate::DctType::III)
3450                    | crate::R2rTransform::Dst(crate::DstType::III)
3451            ) {
3452                4
3453            } else {
3454                3
3455            },
3456            resource: phases_id,
3457            role: BufferRole::LookupTable,
3458            access: BufferAccess::ReadOnly,
3459        };
3460        if matches!(
3461            ir.effective_transform,
3462            crate::R2rTransform::Dct(crate::DctType::III)
3463                | crate::R2rTransform::Dst(crate::DstType::III)
3464        ) {
3465            program.passes[0].bindings.push(phase_binding);
3466        } else {
3467            program.passes[1].bindings.push(phase_binding);
3468        }
3469        program.passes[0].name = format!("vkfft_dd_r2r_{label}_four_step_upload_1");
3470        program.passes[1].name = format!("vkfft_dd_r2r_{label}_four_step_upload_0");
3471        program.name = format!("vkfft_dd_r2r_{label}_four_step_fused_{}_program", ir.length);
3472        program.validate()?;
3473        Ok(Some(program))
3474    }
3475
3476    fn double_double_r2r_fused_forced_rader_two_upload(
3477        ir: &DoubleDoubleR2rIr,
3478        elements: usize,
3479        external_scalar: ScalarType,
3480        external: ExternalBufferLayout,
3481    ) -> Result<Option<Self>> {
3482        let DoubleDoubleR2rAlgorithm::FftReduction { fft, phases, .. } = &ir.algorithm else {
3483            return Ok(None);
3484        };
3485        if !matches!(
3486            ir.effective_transform,
3487            crate::R2rTransform::Dct(crate::DctType::II | crate::DctType::III)
3488                | crate::R2rTransform::Dst(crate::DstType::II | crate::DstType::III)
3489        ) {
3490            return Ok(None);
3491        }
3492        let DoubleDoubleOneDimIr::Recursive(recursive) = fft.as_ref() else {
3493            return Ok(None);
3494        };
3495        if recursive.stockham_upload_schedule.is_some()
3496            || recursive.two_upload_four_step_plan.is_some()
3497            || recursive.three_upload_four_step_plan.is_some()
3498            || recursive.external_storage != crate::PrecisionStorage::DoubleDouble
3499            || recursive.zero_pad_pass.is_some()
3500            || recursive.normalize
3501            || recursive.logical_len != ir.length
3502            || recursive.batch_count != ir.batch_count
3503            || recursive.grouped_batch != ir.grouped_batch
3504            || phases.len() != ir.length
3505        {
3506            return Ok(None);
3507        }
3508        let Some(mapped_high) = recursive.forced_rader_two_upload_mapped_high_component()? else {
3509            return Ok(None);
3510        };
3511        let crate::DoubleDoubleRecursiveFftNodeIr::CooleyTukey(high) = mapped_high else {
3512            return Ok(None);
3513        };
3514        if !matches!(
3515            high.pack_right.input_modifier,
3516            crate::double_double_recursive_ir::DoubleDoubleCooleyTukeyInputModifier::FourStepRight(
3517                _
3518            )
3519        ) || high.pack_right.input_storage != crate::PrecisionStorage::DoubleDouble
3520            || high.pack_right.output_storage != crate::PrecisionStorage::DoubleDouble
3521        {
3522            return Ok(None);
3523        }
3524        let Some((low, mapping)) = recursive.forced_rader_two_upload_mapped_low_stockham()? else {
3525            return Ok(None);
3526        };
3527        if low.sequence_len > 64 || low.axis_batch_block.is_none() {
3528            return Ok(None);
3529        }
3530        let crate::double_double_recursive_ir::DoubleDoubleCooleyTukeyInputModifier::FourStepRight(
3531            high_mapping,
3532        ) = high.pack_right.input_modifier
3533        else {
3534            unreachable!("validated forced-two recursive high mapping");
3535        };
3536        if high_mapping != mapping {
3537            return Ok(None);
3538        }
3539
3540        let mut program = Self::double_double_recursive(recursive)?;
3541        let expected_last = format!("{}_four_step_left", low.name);
3542        if program.passes.len() < 2
3543            || program.input_resource()?.scalar != ScalarType::DoubleDouble
3544            || program.output_resource()?.scalar != ScalarType::DoubleDouble
3545            || program
3546                .passes
3547                .first()
3548                .is_none_or(|pass| pass.name != high.pack_right.name)
3549            || program
3550                .passes
3551                .last()
3552                .is_none_or(|pass| !pass.name.ends_with(&expected_last))
3553        {
3554            return Ok(None);
3555        }
3556        for resource in &mut program.resources {
3557            match resource.kind {
3558                ProgramResourceKind::Input => {
3559                    resource.scalar = external_scalar;
3560                    resource.elements = elements;
3561                    resource.external_layout = Some(external);
3562                }
3563                ProgramResourceKind::Output => {
3564                    resource.scalar = external_scalar;
3565                    resource.elements = elements;
3566                    resource.external_layout = Some(external);
3567                }
3568                _ => {}
3569            }
3570        }
3571        let phases_id = ProgramResourceId(program.resources.len());
3572        program.resources.push(ProgramResource {
3573            id: phases_id,
3574            name: "double_double_r2r_phases".to_owned(),
3575            kind: ProgramResourceKind::LookupTable,
3576            scalar: ScalarType::DoubleDouble,
3577            elements: phases.len(),
3578            external_layout: None,
3579            initialization: ProgramResourceInitialization::ComplexDoubleDouble(phases.clone()),
3580        });
3581        let label = match ir.effective_transform {
3582            crate::R2rTransform::Dct(crate::DctType::II) => "dct2",
3583            crate::R2rTransform::Dct(crate::DctType::III) => "dct3",
3584            crate::R2rTransform::Dst(crate::DstType::II) => "dst2",
3585            crate::R2rTransform::Dst(crate::DstType::III) => "dst3",
3586            _ => unreachable!("validated forced-two-upload DD Type-II/III fusion"),
3587        };
3588        let phase_binding = ProgramPassBinding {
3589            binding: if matches!(
3590                ir.effective_transform,
3591                crate::R2rTransform::Dct(crate::DctType::III)
3592                    | crate::R2rTransform::Dst(crate::DstType::III)
3593            ) {
3594                2
3595            } else {
3596                3
3597            },
3598            resource: phases_id,
3599            role: BufferRole::LookupTable,
3600            access: BufferAccess::ReadOnly,
3601        };
3602        let last_pass = program.passes.len() - 1;
3603        if matches!(
3604            ir.effective_transform,
3605            crate::R2rTransform::Dct(crate::DctType::III)
3606                | crate::R2rTransform::Dst(crate::DstType::III)
3607        ) {
3608            program.passes[0].bindings.push(phase_binding);
3609        } else {
3610            program.passes[last_pass].bindings.push(phase_binding);
3611        }
3612        program.passes[0].name = format!("vkfft_dd_r2r_{label}_forced_rader_two_upload_1");
3613        program.passes[last_pass].name = format!("vkfft_dd_r2r_{label}_forced_rader_two_upload_0");
3614        program.name = format!(
3615            "vkfft_dd_r2r_{label}_forced_rader_two_upload_fused_{}_program",
3616            ir.length
3617        );
3618        program.validate()?;
3619        Ok(Some(program))
3620    }
3621
3622    fn double_double_r2r_fused_forced_rader_two_upload_recursive_output(
3623        ir: &DoubleDoubleR2rIr,
3624        elements: usize,
3625        external_scalar: ScalarType,
3626        external: ExternalBufferLayout,
3627    ) -> Result<Option<Self>> {
3628        let DoubleDoubleR2rAlgorithm::FftReduction { fft, phases, .. } = &ir.algorithm else {
3629            return Ok(None);
3630        };
3631        if !matches!(
3632            ir.effective_transform,
3633            crate::R2rTransform::Dct(crate::DctType::II | crate::DctType::III)
3634                | crate::R2rTransform::Dst(crate::DstType::II | crate::DstType::III)
3635        ) {
3636            return Ok(None);
3637        }
3638        let DoubleDoubleOneDimIr::Recursive(recursive) = fft.as_ref() else {
3639            return Ok(None);
3640        };
3641        if recursive.stockham_upload_schedule.is_some()
3642            || recursive.two_upload_four_step_plan.is_some()
3643            || recursive.three_upload_four_step_plan.is_some()
3644            || recursive.external_storage != crate::PrecisionStorage::DoubleDouble
3645            || recursive.zero_pad_pass.is_some()
3646            || recursive.normalize
3647            || recursive.logical_len != ir.length
3648            || recursive.batch_count != ir.batch_count
3649            || recursive.grouped_batch != ir.grouped_batch
3650            || phases.len() != ir.length
3651        {
3652            return Ok(None);
3653        }
3654        let (mapping, first_pass_suffix, fft_rader_scatter_suffix) = if let Some((high, mapping)) =
3655            recursive.forced_rader_two_upload_mapped_high_stockham()?
3656        {
3657            if high.axis_batch_block.is_none() {
3658                return Ok(None);
3659            }
3660            (mapping, "forced_rader_two_upload_1".to_owned(), None)
3661        } else {
3662            let Some(mapped_high) = recursive.forced_rader_two_upload_mapped_high_component()?
3663            else {
3664                return Ok(None);
3665            };
3666            match mapped_high {
3667                crate::DoubleDoubleRecursiveFftNodeIr::DirectRader(high) => {
3668                    let crate::StockhamIoMapping::FourStepRight(mapping) = high.io_mapping else {
3669                        return Ok(None);
3670                    };
3671                    if high.prime != mapping.right_len
3672                        || high.axis_batch_block.is_none()
3673                        || high.input_storage != crate::PrecisionStorage::DoubleDouble
3674                        || high.output_storage != crate::PrecisionStorage::DoubleDouble
3675                        || high.zero_pad_pass.is_some()
3676                    {
3677                        return Ok(None);
3678                    }
3679                    (mapping, high.name.clone(), None)
3680                }
3681                crate::DoubleDoubleRecursiveFftNodeIr::FftRader(high) => {
3682                    let crate::StockhamIoMapping::FourStepRight(mapping) = high.io_mapping else {
3683                        return Ok(None);
3684                    };
3685                    if high.prime != mapping.right_len
3686                        || high.caller_axis_batch_block.is_none()
3687                        || high.external_storage != crate::PrecisionStorage::DoubleDouble
3688                        || high.input_storage != crate::PrecisionStorage::DoubleDouble
3689                        || high.output_storage != crate::PrecisionStorage::DoubleDouble
3690                        || high.zero_pad_pass.is_some()
3691                    {
3692                        return Ok(None);
3693                    }
3694                    (
3695                        mapping,
3696                        format!("{}_gather", high.name),
3697                        Some(format!("{}_scatter", high.name)),
3698                    )
3699                }
3700                _ => return Ok(None),
3701            }
3702        };
3703        let Some(mapped_low) = recursive.forced_rader_two_upload_mapped_low_component()? else {
3704            return Ok(None);
3705        };
3706        let (low_pass_name, low_pass_exact, low_phase_binding) = match mapped_low {
3707            crate::DoubleDoubleRecursiveFftNodeIr::CooleyTukey(low) => {
3708                if !matches!(
3709                    low.scatter_output.output_modifier,
3710                    crate::double_double_recursive_ir::DoubleDoubleCooleyTukeyOutputModifier::FourStepLeft(low_mapping)
3711                        if low_mapping == mapping
3712                ) || low.scatter_output.input_storage != crate::PrecisionStorage::DoubleDouble
3713                    || low.scatter_output.output_storage != crate::PrecisionStorage::DoubleDouble
3714                {
3715                    return Ok(None);
3716                }
3717                (low.scatter_output.name.clone(), true, 2u32)
3718            }
3719            crate::DoubleDoubleRecursiveFftNodeIr::DirectRader(low) => {
3720                let crate::StockhamIoMapping::FourStepLeft(low_mapping) = low.io_mapping else {
3721                    return Ok(None);
3722                };
3723                if low_mapping != mapping
3724                    || low.prime != mapping.left_len
3725                    || low.axis_batch_block.is_none()
3726                    || low.input_storage != crate::PrecisionStorage::DoubleDouble
3727                    || low.output_storage != crate::PrecisionStorage::DoubleDouble
3728                    || low.zero_pad_pass.is_some()
3729                {
3730                    return Ok(None);
3731                }
3732                (low.name.clone(), false, 3u32)
3733            }
3734            crate::DoubleDoubleRecursiveFftNodeIr::FftRader(low) => {
3735                let crate::StockhamIoMapping::FourStepLeft(low_mapping) = low.io_mapping else {
3736                    return Ok(None);
3737                };
3738                if low_mapping != mapping
3739                    || low.prime != mapping.left_len
3740                    || low.caller_axis_batch_block.is_none()
3741                    || low.external_storage != crate::PrecisionStorage::DoubleDouble
3742                    || low.input_storage != crate::PrecisionStorage::DoubleDouble
3743                    || low.output_storage != crate::PrecisionStorage::DoubleDouble
3744                    || low.zero_pad_pass.is_some()
3745                {
3746                    return Ok(None);
3747                }
3748                (format!("{}_scatter", low.name), false, 3u32)
3749            }
3750            _ => return Ok(None),
3751        };
3752
3753        let mut program = Self::double_double_recursive(recursive)?;
3754        if program.passes.len() < 2
3755            || program.input_resource()?.scalar != ScalarType::DoubleDouble
3756            || program.output_resource()?.scalar != ScalarType::DoubleDouble
3757            || !program
3758                .passes
3759                .first()
3760                .is_some_and(|pass| pass.name.ends_with(&first_pass_suffix))
3761            || !program.passes.last().is_some_and(|pass| {
3762                if low_pass_exact {
3763                    pass.name == low_pass_name
3764                } else {
3765                    pass.name.ends_with(&low_pass_name)
3766                }
3767            })
3768        {
3769            return Ok(None);
3770        }
3771        for resource in &mut program.resources {
3772            match resource.kind {
3773                ProgramResourceKind::Input => {
3774                    resource.scalar = external_scalar;
3775                    resource.elements = elements;
3776                    resource.external_layout = Some(external);
3777                }
3778                ProgramResourceKind::Output => {
3779                    resource.scalar = external_scalar;
3780                    resource.elements = elements;
3781                    resource.external_layout = Some(external);
3782                }
3783                _ => {}
3784            }
3785        }
3786        let phases_id = ProgramResourceId(program.resources.len());
3787        program.resources.push(ProgramResource {
3788            id: phases_id,
3789            name: "double_double_r2r_phases".to_owned(),
3790            kind: ProgramResourceKind::LookupTable,
3791            scalar: ScalarType::DoubleDouble,
3792            elements: phases.len(),
3793            external_layout: None,
3794            initialization: ProgramResourceInitialization::ComplexDoubleDouble(phases.clone()),
3795        });
3796        let label = match ir.effective_transform {
3797            crate::R2rTransform::Dct(crate::DctType::II) => "dct2",
3798            crate::R2rTransform::Dct(crate::DctType::III) => "dct3",
3799            crate::R2rTransform::Dst(crate::DstType::II) => "dst2",
3800            crate::R2rTransform::Dst(crate::DstType::III) => "dst3",
3801            _ => unreachable!("validated forced-two recursive-output DD Type-II/III fusion"),
3802        };
3803        let last_pass = program.passes.len() - 1;
3804        if matches!(
3805            ir.effective_transform,
3806            crate::R2rTransform::Dct(crate::DctType::III)
3807                | crate::R2rTransform::Dst(crate::DstType::III)
3808        ) {
3809            if let Some(scatter_suffix) = fft_rader_scatter_suffix.as_deref() {
3810                program.passes[0].bindings.push(ProgramPassBinding {
3811                    binding: 2,
3812                    resource: phases_id,
3813                    role: BufferRole::LookupTable,
3814                    access: BufferAccess::ReadOnly,
3815                });
3816                let Some(scatter) = program
3817                    .passes
3818                    .iter_mut()
3819                    .take(last_pass)
3820                    .find(|pass| pass.name.ends_with(scatter_suffix))
3821                else {
3822                    return Ok(None);
3823                };
3824                scatter.bindings.push(ProgramPassBinding {
3825                    binding: 4,
3826                    resource: phases_id,
3827                    role: BufferRole::LookupTable,
3828                    access: BufferAccess::ReadOnly,
3829                });
3830            } else {
3831                program.passes[0].bindings.push(ProgramPassBinding {
3832                    binding: 4,
3833                    resource: phases_id,
3834                    role: BufferRole::LookupTable,
3835                    access: BufferAccess::ReadOnly,
3836                });
3837            }
3838        } else {
3839            program.passes[last_pass].bindings.push(ProgramPassBinding {
3840                binding: low_phase_binding,
3841                resource: phases_id,
3842                role: BufferRole::LookupTable,
3843                access: BufferAccess::ReadOnly,
3844            });
3845        }
3846        program.passes[0].name = format!("vkfft_dd_r2r_{label}_forced_rader_two_upload_1");
3847        program.passes[last_pass].name =
3848            format!("vkfft_dd_r2r_{label}_forced_rader_two_upload_0_recursive");
3849        program.name = format!(
3850            "vkfft_dd_r2r_{label}_forced_rader_two_upload_recursive_output_fused_{}_program",
3851            ir.length
3852        );
3853        program.validate()?;
3854        Ok(Some(program))
3855    }
3856
3857    fn double_double_r2r_fused_forced_rader_three_upload(
3858        ir: &DoubleDoubleR2rIr,
3859        elements: usize,
3860        external_scalar: ScalarType,
3861        external: ExternalBufferLayout,
3862    ) -> Result<Option<Self>> {
3863        let DoubleDoubleR2rAlgorithm::FftReduction { fft, phases, .. } = &ir.algorithm else {
3864            return Ok(None);
3865        };
3866        if !matches!(
3867            ir.effective_transform,
3868            crate::R2rTransform::Dct(crate::DctType::II | crate::DctType::III)
3869                | crate::R2rTransform::Dst(crate::DstType::II | crate::DstType::III)
3870        ) {
3871            return Ok(None);
3872        }
3873        let DoubleDoubleOneDimIr::Recursive(recursive) = fft.as_ref() else {
3874            return Ok(None);
3875        };
3876        if recursive.stockham_upload_schedule.is_some()
3877            || recursive.two_upload_four_step_plan.is_some()
3878            || recursive.three_upload_four_step_plan.is_some()
3879            || recursive.external_storage != crate::PrecisionStorage::DoubleDouble
3880            || recursive.zero_pad_pass.is_some()
3881            || recursive.normalize
3882            || recursive.logical_len != ir.length
3883            || recursive.batch_count != ir.batch_count
3884            || recursive.grouped_batch != ir.grouped_batch
3885            || phases.len() != ir.length
3886        {
3887            return Ok(None);
3888        }
3889        let Some(components) = recursive.forced_rader_three_upload_mapped_components()? else {
3890            return Ok(None);
3891        };
3892        if components.len() != 3 {
3893            return Ok(None);
3894        }
3895        let recursive_u2_pack_name = match components.first() {
3896            Some(
3897                crate::double_double_recursive_ir::DoubleDoubleForcedRaderThreeUploadComponentIr::Stockham {
3898                    upload_id: 2,
3899                    ..
3900                },
3901            ) => None,
3902            Some(
3903                crate::double_double_recursive_ir::DoubleDoubleForcedRaderThreeUploadComponentIr::Recursive {
3904                    upload_id: 2,
3905                    ir: crate::DoubleDoubleRecursiveFftNodeIr::CooleyTukey(cooley),
3906                },
3907            ) if matches!(
3908                cooley.pack_right.input_modifier,
3909                crate::double_double_recursive_ir::DoubleDoubleCooleyTukeyInputModifier::FourStepThreeUpload2(_)
3910            ) && cooley.pack_right.input_storage == crate::PrecisionStorage::DoubleDouble
3911                && cooley.pack_right.output_storage == crate::PrecisionStorage::DoubleDouble =>
3912            {
3913                Some(cooley.pack_right.name.clone())
3914            }
3915            _ => return Ok(None),
3916        };
3917        if !matches!(
3918            components.last(),
3919            Some(
3920                crate::double_double_recursive_ir::DoubleDoubleForcedRaderThreeUploadComponentIr::Stockham {
3921                    upload_id: 0,
3922                    ..
3923                }
3924            )
3925        ) {
3926            return Ok(None);
3927        }
3928
3929        let mut program = Self::double_double_recursive(recursive)?;
3930        let first_pass_matches = match recursive_u2_pack_name.as_deref() {
3931            Some(name) => program.passes.first().is_some_and(|pass| pass.name == name),
3932            None => program
3933                .passes
3934                .first()
3935                .is_some_and(|pass| pass.name.ends_with("forced_rader_three_upload_2")),
3936        };
3937        if program.passes.len() < 3
3938            || program.input_resource()?.scalar != ScalarType::DoubleDouble
3939            || program.output_resource()?.scalar != ScalarType::DoubleDouble
3940            || !first_pass_matches
3941            || !program
3942                .passes
3943                .last()
3944                .is_some_and(|pass| pass.name.ends_with("forced_rader_three_upload_0"))
3945        {
3946            return Ok(None);
3947        }
3948
3949        // The complex-DD forced-three-upload builder intentionally reuses the external
3950        // output resource as the u2->u1 exchange when caller storage is compute-sized.
3951        // R2R needs that resource to be a scalar caller boundary, so split the exchange
3952        // into a dedicated full-DD scratch before changing the external output ABI.
3953        let external_output = ProgramResourceId(1);
3954        let exchange0 = ProgramResourceId(program.resources.len());
3955        program.resources.push(ProgramResource {
3956            id: exchange0,
3957            name: "double_double_r2r_forced_three_upload_exchange_0".to_owned(),
3958            kind: ProgramResourceKind::Scratch,
3959            scalar: ScalarType::DoubleDouble,
3960            elements,
3961            external_layout: None,
3962            initialization: ProgramResourceInitialization::Zeroed,
3963        });
3964        let last_pass = program.passes.len() - 1;
3965        for (index, pass) in program.passes.iter_mut().enumerate() {
3966            if index == last_pass {
3967                continue;
3968            }
3969            for binding in &mut pass.bindings {
3970                if binding.resource == external_output {
3971                    binding.resource = exchange0;
3972                }
3973            }
3974        }
3975
3976        for resource in &mut program.resources {
3977            match resource.kind {
3978                ProgramResourceKind::Input => {
3979                    resource.scalar = external_scalar;
3980                    resource.elements = elements;
3981                    resource.external_layout = Some(external);
3982                }
3983                ProgramResourceKind::Output => {
3984                    resource.scalar = external_scalar;
3985                    resource.elements = elements;
3986                    resource.external_layout = Some(external);
3987                }
3988                _ => {}
3989            }
3990        }
3991        let phases_id = ProgramResourceId(program.resources.len());
3992        program.resources.push(ProgramResource {
3993            id: phases_id,
3994            name: "double_double_r2r_phases".to_owned(),
3995            kind: ProgramResourceKind::LookupTable,
3996            scalar: ScalarType::DoubleDouble,
3997            elements: phases.len(),
3998            external_layout: None,
3999            initialization: ProgramResourceInitialization::ComplexDoubleDouble(phases.clone()),
4000        });
4001        let label = match ir.effective_transform {
4002            crate::R2rTransform::Dct(crate::DctType::II) => "dct2",
4003            crate::R2rTransform::Dct(crate::DctType::III) => "dct3",
4004            crate::R2rTransform::Dst(crate::DstType::II) => "dst2",
4005            crate::R2rTransform::Dst(crate::DstType::III) => "dst3",
4006            _ => unreachable!("validated forced-three-upload DD Type-II/III fusion"),
4007        };
4008        let phase_binding = ProgramPassBinding {
4009            binding: if matches!(
4010                ir.effective_transform,
4011                crate::R2rTransform::Dct(crate::DctType::III)
4012                    | crate::R2rTransform::Dst(crate::DstType::III)
4013            ) {
4014                if recursive_u2_pack_name.is_some() {
4015                    2
4016                } else {
4017                    4
4018                }
4019            } else {
4020                3
4021            },
4022            resource: phases_id,
4023            role: BufferRole::LookupTable,
4024            access: BufferAccess::ReadOnly,
4025        };
4026        if matches!(
4027            ir.effective_transform,
4028            crate::R2rTransform::Dct(crate::DctType::III)
4029                | crate::R2rTransform::Dst(crate::DstType::III)
4030        ) {
4031            program.passes[0].bindings.push(phase_binding);
4032        } else {
4033            program.passes[last_pass].bindings.push(phase_binding);
4034        }
4035        program.passes[0].name = format!("vkfft_dd_r2r_{label}_forced_rader_three_upload_2");
4036        program.passes[last_pass].name =
4037            format!("vkfft_dd_r2r_{label}_forced_rader_three_upload_0");
4038        program.name = format!(
4039            "vkfft_dd_r2r_{label}_forced_rader_three_upload_fused_{}_program",
4040            ir.length
4041        );
4042        program.validate()?;
4043        Ok(Some(program))
4044    }
4045
4046    fn double_double_r2r_fused_bluestein(
4047        ir: &DoubleDoubleR2rIr,
4048        elements: usize,
4049        external_scalar: ScalarType,
4050        external: ExternalBufferLayout,
4051    ) -> Result<Option<Self>> {
4052        let DoubleDoubleR2rAlgorithm::FftReduction { fft, phases, .. } = &ir.algorithm else {
4053            return Ok(None);
4054        };
4055        if !matches!(
4056            ir.effective_transform,
4057            crate::R2rTransform::Dct(crate::DctType::II | crate::DctType::III)
4058                | crate::R2rTransform::Dst(crate::DstType::II | crate::DstType::III)
4059        ) {
4060            return Ok(None);
4061        }
4062        let DoubleDoubleOneDimIr::Bluestein(bluestein) = fft.as_ref() else {
4063            return Ok(None);
4064        };
4065        if bluestein.logical_len != ir.length
4066            || bluestein.batch_count != ir.batch_count
4067            || bluestein.grouped_batch != ir.grouped_batch
4068            || bluestein.external_storage != crate::PrecisionStorage::DoubleDouble
4069            || bluestein.zero_padding.is_some()
4070            || phases.len() != ir.length
4071        {
4072            return Ok(None);
4073        }
4074
4075        let mut program = Self::double_double_bluestein(bluestein)?;
4076        if program.passes.len() < 2
4077            || program.input_resource()?.scalar != ScalarType::DoubleDouble
4078            || program.output_resource()?.scalar != ScalarType::DoubleDouble
4079            || !program
4080                .passes
4081                .first()
4082                .is_some_and(|pass| pass.name == format!("{}_preprocess", bluestein.name))
4083            || !program
4084                .passes
4085                .last()
4086                .is_some_and(|pass| pass.name == format!("{}_postprocess", bluestein.name))
4087        {
4088            return Ok(None);
4089        }
4090        for resource in &mut program.resources {
4091            match resource.kind {
4092                ProgramResourceKind::Input => {
4093                    resource.scalar = external_scalar;
4094                    resource.elements = elements;
4095                    resource.external_layout = Some(external);
4096                }
4097                ProgramResourceKind::Output => {
4098                    resource.scalar = external_scalar;
4099                    resource.elements = elements;
4100                    resource.external_layout = Some(external);
4101                }
4102                _ => {}
4103            }
4104        }
4105        let phases_id = ProgramResourceId(program.resources.len());
4106        program.resources.push(ProgramResource {
4107            id: phases_id,
4108            name: "double_double_r2r_phases".to_owned(),
4109            kind: ProgramResourceKind::LookupTable,
4110            scalar: ScalarType::DoubleDouble,
4111            elements: phases.len(),
4112            external_layout: None,
4113            initialization: ProgramResourceInitialization::ComplexDoubleDouble(phases.clone()),
4114        });
4115        let label = match ir.effective_transform {
4116            crate::R2rTransform::Dct(crate::DctType::II) => "dct2",
4117            crate::R2rTransform::Dct(crate::DctType::III) => "dct3",
4118            crate::R2rTransform::Dst(crate::DstType::II) => "dst2",
4119            crate::R2rTransform::Dst(crate::DstType::III) => "dst3",
4120            _ => unreachable!("validated DD Bluestein Type-II/III fusion"),
4121        };
4122        let phase_binding = ProgramPassBinding {
4123            binding: 3,
4124            resource: phases_id,
4125            role: BufferRole::LookupTable,
4126            access: BufferAccess::ReadOnly,
4127        };
4128        let last_pass = program.passes.len() - 1;
4129        if matches!(
4130            ir.effective_transform,
4131            crate::R2rTransform::Dct(crate::DctType::III)
4132                | crate::R2rTransform::Dst(crate::DstType::III)
4133        ) {
4134            program.passes[0].bindings.push(phase_binding);
4135        } else {
4136            program.passes[last_pass].bindings.push(phase_binding);
4137        }
4138        program.passes[0].name = format!("vkfft_dd_r2r_{label}_bluestein_preprocess");
4139        program.passes[last_pass].name = format!("vkfft_dd_r2r_{label}_bluestein_postprocess");
4140        program.name = format!("vkfft_dd_r2r_{label}_bluestein_fused_{}_program", ir.length);
4141        program.validate()?;
4142        Ok(Some(program))
4143    }
4144
4145    pub fn double_double_r2r(ir: &DoubleDoubleR2rIr) -> Result<Self> {
4146        ir.validate()?;
4147        let elements =
4148            ir.length
4149                .checked_mul(ir.batch_count)
4150                .ok_or(VkFftError::ArithmeticOverflow {
4151                    operation: "double-double DCT/DST program element count",
4152                })?;
4153        let external_scalar = match ir.external_storage {
4154            crate::PrecisionStorage::DoubleDouble => ScalarType::DoubleDouble,
4155            crate::PrecisionStorage::F64 => ScalarType::F64,
4156            _ => {
4157                return Err(VkFftError::InvalidKernelIr(
4158                    "double-double R2R ProgramIr requires DD or F64 external storage",
4159                ));
4160            }
4161        };
4162        let external = ExternalBufferLayout {
4163            logical_len: ir.length,
4164            physical_stride: ir.length,
4165            batch_count: ir.batch_count,
4166            element_shape: ProgramElementShape::Scalar,
4167        };
4168        if let Some(program) =
4169            Self::double_double_r2r_fused_bluestein(ir, elements, external_scalar, external)?
4170        {
4171            return Ok(program);
4172        }
4173        if let Some(program) = Self::double_double_r2r_fused_physical_stockham(
4174            ir,
4175            elements,
4176            external_scalar,
4177            external,
4178        )? {
4179            return Ok(program);
4180        }
4181        if let Some(program) = Self::double_double_r2r_fused_two_upload_four_step(
4182            ir,
4183            elements,
4184            external_scalar,
4185            external,
4186        )? {
4187            return Ok(program);
4188        }
4189        if let Some(program) = Self::double_double_r2r_fused_forced_rader_two_upload(
4190            ir,
4191            elements,
4192            external_scalar,
4193            external,
4194        )? {
4195            return Ok(program);
4196        }
4197        if let Some(program) =
4198            Self::double_double_r2r_fused_forced_rader_two_upload_recursive_output(
4199                ir,
4200                elements,
4201                external_scalar,
4202                external,
4203            )?
4204        {
4205            return Ok(program);
4206        }
4207        if let Some(program) = Self::double_double_r2r_fused_forced_rader_three_upload(
4208            ir,
4209            elements,
4210            external_scalar,
4211            external,
4212        )? {
4213            return Ok(program);
4214        }
4215        let mut resources = vec![
4216            ProgramResource {
4217                id: ProgramResourceId(0),
4218                name: "input".to_owned(),
4219                kind: ProgramResourceKind::Input,
4220                scalar: external_scalar,
4221                elements,
4222                external_layout: Some(external),
4223                initialization: ProgramResourceInitialization::ExternalInput,
4224            },
4225            ProgramResource {
4226                id: ProgramResourceId(1),
4227                name: "output".to_owned(),
4228                kind: ProgramResourceKind::Output,
4229                scalar: external_scalar,
4230                elements,
4231                external_layout: Some(external),
4232                initialization: ProgramResourceInitialization::Zeroed,
4233            },
4234        ];
4235        let mut passes = Vec::new();
4236        let dispatch = DispatchGeometry {
4237            x: u32::try_from(ir.batch_group_count()).map_err(|_| VkFftError::ValueOutOfRange {
4238                field: "double-double R2R boundary dispatch count",
4239            })?,
4240            y: 1,
4241            z: 1,
4242        };
4243        match &ir.algorithm {
4244            DoubleDoubleR2rAlgorithm::Direct { coefficients } => {
4245                let coefficients_id = ProgramResourceId(resources.len());
4246                resources.push(ProgramResource {
4247                    id: coefficients_id,
4248                    name: "double_double_r2r_coefficients".to_owned(),
4249                    kind: ProgramResourceKind::LookupTable,
4250                    scalar: ScalarType::DoubleDouble,
4251                    elements: coefficients.len(),
4252                    external_layout: None,
4253                    initialization: ProgramResourceInitialization::ComplexDoubleDouble(
4254                        coefficients
4255                            .iter()
4256                            .copied()
4257                            .map(|value| ComplexDoubleDouble::new(value, crate::DoubleDouble::ZERO))
4258                            .collect(),
4259                    ),
4260                });
4261                passes.push(ProgramPass {
4262                    name: format!("vkfft_dd_r2r_direct_{}", ir.length),
4263                    dispatch,
4264                    bindings: vec![
4265                        binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
4266                        binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
4267                        ProgramPassBinding {
4268                            binding: 2,
4269                            resource: coefficients_id,
4270                            role: BufferRole::LookupTable,
4271                            access: BufferAccess::ReadOnly,
4272                        },
4273                    ],
4274                });
4275            }
4276            DoubleDoubleR2rAlgorithm::FftReduction {
4277                fft_len,
4278                fft,
4279                phases,
4280            } => {
4281                let fft_elements =
4282                    fft_len
4283                        .checked_mul(ir.batch_count)
4284                        .ok_or(VkFftError::ArithmeticOverflow {
4285                            operation: "double-double R2R FFT program element count",
4286                        })?;
4287                let fft_input = ProgramResourceId(resources.len());
4288                resources.push(ProgramResource {
4289                    id: fft_input,
4290                    name: "double_double_r2r_fft_input".to_owned(),
4291                    kind: ProgramResourceKind::Scratch,
4292                    scalar: ScalarType::DoubleDouble,
4293                    elements: fft_elements,
4294                    external_layout: None,
4295                    initialization: ProgramResourceInitialization::Zeroed,
4296                });
4297                let fft_output = ProgramResourceId(resources.len());
4298                resources.push(ProgramResource {
4299                    id: fft_output,
4300                    name: "double_double_r2r_fft_output".to_owned(),
4301                    kind: ProgramResourceKind::Scratch,
4302                    scalar: ScalarType::DoubleDouble,
4303                    elements: fft_elements,
4304                    external_layout: None,
4305                    initialization: ProgramResourceInitialization::Zeroed,
4306                });
4307                let phases_id = if phases.is_empty() {
4308                    None
4309                } else {
4310                    let phases_id = ProgramResourceId(resources.len());
4311                    resources.push(ProgramResource {
4312                        id: phases_id,
4313                        name: "double_double_r2r_phases".to_owned(),
4314                        kind: ProgramResourceKind::LookupTable,
4315                        scalar: ScalarType::DoubleDouble,
4316                        elements: phases.len(),
4317                        external_layout: None,
4318                        initialization: ProgramResourceInitialization::ComplexDoubleDouble(
4319                            phases.clone(),
4320                        ),
4321                    });
4322                    Some(phases_id)
4323                };
4324                let child = Self::double_double_one_dim(fft)?;
4325                if child.scalar != ScalarType::DoubleDouble
4326                    || child.input_resource()?.elements != fft_elements
4327                    || child.output_resource()?.elements != fft_elements
4328                    || child.input_resource()?.scalar != ScalarType::DoubleDouble
4329                    || child.output_resource()?.scalar != ScalarType::DoubleDouble
4330                {
4331                    return Err(VkFftError::InvalidKernelIr(
4332                        "double-double R2R FFT child must expose full-DD boundaries",
4333                    ));
4334                }
4335                let mut resource_map = vec![None; child.resources.len()];
4336                for resource in &child.resources {
4337                    let mapped = match resource.kind {
4338                        ProgramResourceKind::Input => fft_input,
4339                        ProgramResourceKind::Output => fft_output,
4340                        _ => {
4341                            let new_id = ProgramResourceId(resources.len());
4342                            let mut cloned = resource.clone();
4343                            cloned.id = new_id;
4344                            cloned.name = format!("double_double_r2r_{}", resource.name);
4345                            cloned.external_layout = None;
4346                            resources.push(cloned);
4347                            new_id
4348                        }
4349                    };
4350                    resource_map[resource.id.0] = Some(mapped);
4351                }
4352                let mut pre_bindings = vec![
4353                    binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
4354                    ProgramPassBinding {
4355                        binding: 1,
4356                        resource: fft_input,
4357                        role: BufferRole::Output,
4358                        access: BufferAccess::WriteOnly,
4359                    },
4360                ];
4361                if matches!(
4362                    ir.effective_transform,
4363                    crate::R2rTransform::Dct(crate::DctType::III | crate::DctType::IV)
4364                        | crate::R2rTransform::Dst(crate::DstType::III | crate::DstType::IV)
4365                ) {
4366                    pre_bindings.push(ProgramPassBinding {
4367                        binding: 2,
4368                        resource: phases_id.ok_or(VkFftError::InvalidKernelIr(
4369                            "double-double R2R FFT preprocess is missing its phase LUT",
4370                        ))?,
4371                        role: BufferRole::LookupTable,
4372                        access: BufferAccess::ReadOnly,
4373                    });
4374                }
4375                passes.push(ProgramPass {
4376                    name: match ir.effective_transform {
4377                        crate::R2rTransform::Dct(crate::DctType::I) => {
4378                            "vkfft_dd_r2r_fft_dct1_pre".to_owned()
4379                        }
4380                        crate::R2rTransform::Dst(crate::DstType::I) => {
4381                            "vkfft_dd_r2r_fft_dst1_pre".to_owned()
4382                        }
4383                        crate::R2rTransform::Dct(crate::DctType::II) => {
4384                            "vkfft_dd_r2r_fft_dct2_pre".to_owned()
4385                        }
4386                        crate::R2rTransform::Dct(crate::DctType::III) => {
4387                            "vkfft_dd_r2r_fft_dct3_pre".to_owned()
4388                        }
4389                        crate::R2rTransform::Dst(crate::DstType::II) => {
4390                            "vkfft_dd_r2r_fft_dst2_pre".to_owned()
4391                        }
4392                        crate::R2rTransform::Dst(crate::DstType::III) => {
4393                            "vkfft_dd_r2r_fft_dst3_pre".to_owned()
4394                        }
4395                        crate::R2rTransform::Dct(crate::DctType::IV) => {
4396                            "vkfft_dd_r2r_fft_dct4_pre".to_owned()
4397                        }
4398                        crate::R2rTransform::Dst(crate::DstType::IV) => {
4399                            "vkfft_dd_r2r_fft_dst4_pre".to_owned()
4400                        }
4401                    },
4402                    dispatch,
4403                    bindings: pre_bindings,
4404                });
4405                for child_pass in &child.passes {
4406                    let mut pass = child_pass.clone();
4407                    pass.name = format!("vkfft_dd_r2r_{}", child_pass.name);
4408                    for child_binding in &mut pass.bindings {
4409                        child_binding.resource = resource_map
4410                            .get(child_binding.resource.0)
4411                            .and_then(|mapped| *mapped)
4412                            .ok_or(VkFftError::InvalidKernelIr(
4413                                "double-double R2R child resource mapping is incomplete",
4414                            ))?;
4415                    }
4416                    passes.push(pass);
4417                }
4418                let mut post_bindings = vec![
4419                    ProgramPassBinding {
4420                        binding: 0,
4421                        resource: fft_output,
4422                        role: BufferRole::Input,
4423                        access: BufferAccess::ReadOnly,
4424                    },
4425                    binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
4426                ];
4427                if matches!(
4428                    ir.effective_transform,
4429                    crate::R2rTransform::Dct(crate::DctType::II | crate::DctType::IV)
4430                        | crate::R2rTransform::Dst(crate::DstType::II | crate::DstType::IV)
4431                ) {
4432                    post_bindings.push(ProgramPassBinding {
4433                        binding: 2,
4434                        resource: phases_id.ok_or(VkFftError::InvalidKernelIr(
4435                            "double-double R2R FFT postprocess is missing its phase LUT",
4436                        ))?,
4437                        role: BufferRole::LookupTable,
4438                        access: BufferAccess::ReadOnly,
4439                    });
4440                }
4441                passes.push(ProgramPass {
4442                    name: match ir.effective_transform {
4443                        crate::R2rTransform::Dct(crate::DctType::I) => {
4444                            "vkfft_dd_r2r_fft_dct1_post".to_owned()
4445                        }
4446                        crate::R2rTransform::Dst(crate::DstType::I) => {
4447                            "vkfft_dd_r2r_fft_dst1_post".to_owned()
4448                        }
4449                        crate::R2rTransform::Dct(crate::DctType::II) => {
4450                            "vkfft_dd_r2r_fft_dct2_post".to_owned()
4451                        }
4452                        crate::R2rTransform::Dct(crate::DctType::III) => {
4453                            "vkfft_dd_r2r_fft_dct3_post".to_owned()
4454                        }
4455                        crate::R2rTransform::Dst(crate::DstType::II) => {
4456                            "vkfft_dd_r2r_fft_dst2_post".to_owned()
4457                        }
4458                        crate::R2rTransform::Dst(crate::DstType::III) => {
4459                            "vkfft_dd_r2r_fft_dst3_post".to_owned()
4460                        }
4461                        crate::R2rTransform::Dct(crate::DctType::IV) => {
4462                            "vkfft_dd_r2r_fft_dct4_post".to_owned()
4463                        }
4464                        crate::R2rTransform::Dst(crate::DstType::IV) => {
4465                            "vkfft_dd_r2r_fft_dst4_post".to_owned()
4466                        }
4467                    },
4468                    dispatch,
4469                    bindings: post_bindings,
4470                });
4471            }
4472            DoubleDoubleR2rAlgorithm::EvenTypeIvHalfSize {
4473                fft_len,
4474                fft,
4475                pack_phases,
4476                extract_phases,
4477            } => {
4478                let fft_elements =
4479                    fft_len
4480                        .checked_mul(ir.batch_count)
4481                        .ok_or(VkFftError::ArithmeticOverflow {
4482                            operation: "double-double even-IV half-size program element count",
4483                        })?;
4484                let fft_input = ProgramResourceId(resources.len());
4485                resources.push(ProgramResource {
4486                    id: fft_input,
4487                    name: "double_double_r2r_even_iv_fft_input".to_owned(),
4488                    kind: ProgramResourceKind::Scratch,
4489                    scalar: ScalarType::DoubleDouble,
4490                    elements: fft_elements,
4491                    external_layout: None,
4492                    initialization: ProgramResourceInitialization::Zeroed,
4493                });
4494                let fft_output = ProgramResourceId(resources.len());
4495                resources.push(ProgramResource {
4496                    id: fft_output,
4497                    name: "double_double_r2r_even_iv_fft_output".to_owned(),
4498                    kind: ProgramResourceKind::Scratch,
4499                    scalar: ScalarType::DoubleDouble,
4500                    elements: fft_elements,
4501                    external_layout: None,
4502                    initialization: ProgramResourceInitialization::Zeroed,
4503                });
4504                let phases_id = ProgramResourceId(resources.len());
4505                let mut phases = Vec::with_capacity(pack_phases.len() + extract_phases.len());
4506                phases.extend_from_slice(pack_phases);
4507                phases.extend_from_slice(extract_phases);
4508                resources.push(ProgramResource {
4509                    id: phases_id,
4510                    name: "double_double_r2r_even_iv_phases".to_owned(),
4511                    kind: ProgramResourceKind::LookupTable,
4512                    scalar: ScalarType::DoubleDouble,
4513                    elements: phases.len(),
4514                    external_layout: None,
4515                    initialization: ProgramResourceInitialization::ComplexDoubleDouble(phases),
4516                });
4517
4518                let child = Self::double_double_one_dim(fft)?;
4519                if child.scalar != ScalarType::DoubleDouble
4520                    || child.input_resource()?.elements != fft_elements
4521                    || child.output_resource()?.elements != fft_elements
4522                    || child.input_resource()?.scalar != ScalarType::DoubleDouble
4523                    || child.output_resource()?.scalar != ScalarType::DoubleDouble
4524                {
4525                    return Err(VkFftError::InvalidKernelIr(
4526                        "double-double even-IV child must expose full-DD boundaries",
4527                    ));
4528                }
4529                let mut resource_map = vec![None; child.resources.len()];
4530                for resource in &child.resources {
4531                    let mapped = match resource.kind {
4532                        ProgramResourceKind::Input => fft_input,
4533                        ProgramResourceKind::Output => fft_output,
4534                        _ => {
4535                            let new_id = ProgramResourceId(resources.len());
4536                            let mut cloned = resource.clone();
4537                            cloned.id = new_id;
4538                            cloned.name = format!("double_double_r2r_even_iv_{}", resource.name);
4539                            cloned.external_layout = None;
4540                            resources.push(cloned);
4541                            new_id
4542                        }
4543                    };
4544                    resource_map[resource.id.0] = Some(mapped);
4545                }
4546
4547                passes.push(ProgramPass {
4548                    name: "vkfft_dd_r2r_even_iv_pre".to_owned(),
4549                    dispatch,
4550                    bindings: vec![
4551                        binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
4552                        ProgramPassBinding {
4553                            binding: 1,
4554                            resource: fft_input,
4555                            role: BufferRole::Output,
4556                            access: BufferAccess::WriteOnly,
4557                        },
4558                        ProgramPassBinding {
4559                            binding: 2,
4560                            resource: phases_id,
4561                            role: BufferRole::LookupTable,
4562                            access: BufferAccess::ReadOnly,
4563                        },
4564                    ],
4565                });
4566                for child_pass in &child.passes {
4567                    let mut pass = child_pass.clone();
4568                    pass.name = format!("vkfft_dd_r2r_even_iv_{}", child_pass.name);
4569                    for child_binding in &mut pass.bindings {
4570                        child_binding.resource = resource_map
4571                            .get(child_binding.resource.0)
4572                            .and_then(|mapped| *mapped)
4573                            .ok_or(VkFftError::InvalidKernelIr(
4574                                "double-double even-IV child resource mapping is incomplete",
4575                            ))?;
4576                    }
4577                    passes.push(pass);
4578                }
4579                passes.push(ProgramPass {
4580                    name: "vkfft_dd_r2r_even_iv_post".to_owned(),
4581                    dispatch,
4582                    bindings: vec![
4583                        ProgramPassBinding {
4584                            binding: 0,
4585                            resource: fft_output,
4586                            role: BufferRole::Input,
4587                            access: BufferAccess::ReadOnly,
4588                        },
4589                        binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
4590                        ProgramPassBinding {
4591                            binding: 2,
4592                            resource: phases_id,
4593                            role: BufferRole::LookupTable,
4594                            access: BufferAccess::ReadOnly,
4595                        },
4596                    ],
4597                });
4598            }
4599        }
4600        let program = Self {
4601            name: format!("vkfft_dd_r2r_program_{}", ir.length),
4602            scalar: ScalarType::DoubleDouble,
4603            resources,
4604            passes,
4605        };
4606        program.validate()?;
4607        Ok(program)
4608    }
4609
4610    pub fn r2r(ir: &R2rIr) -> Result<Self> {
4611        ir.validate()?;
4612        let elements =
4613            ir.length
4614                .checked_mul(ir.batch_count)
4615                .ok_or(VkFftError::ArithmeticOverflow {
4616                    operation: "DCT/DST program element count",
4617                })?;
4618        let external = ExternalBufferLayout {
4619            logical_len: ir.length,
4620            physical_stride: ir.length,
4621            batch_count: ir.batch_count,
4622            element_shape: ProgramElementShape::Complex,
4623        };
4624        let mut builder = RecursiveProgramBuilder {
4625            scalar: ir.scalar,
4626            resources: vec![
4627                ProgramResource {
4628                    id: ProgramResourceId(0),
4629                    name: "input".to_owned(),
4630                    kind: ProgramResourceKind::Input,
4631                    scalar: ir.external_scalar,
4632                    elements,
4633                    external_layout: Some(external),
4634                    initialization: ProgramResourceInitialization::ExternalInput,
4635                },
4636                ProgramResource {
4637                    id: ProgramResourceId(1),
4638                    name: "output".to_owned(),
4639                    kind: ProgramResourceKind::Output,
4640                    scalar: ir.external_scalar,
4641                    elements,
4642                    external_layout: Some(external),
4643                    initialization: ProgramResourceInitialization::Zeroed,
4644                },
4645            ],
4646            passes: Vec::new(),
4647            serial: 0,
4648        };
4649        builder.flatten_r2r(ir, ProgramResourceId(0), ProgramResourceId(1))?;
4650        let program = Self {
4651            name: format!("vkfft_r2r_program_{}", ir.length),
4652            scalar: ir.scalar,
4653            resources: builder.resources,
4654            passes: builder.passes,
4655        };
4656        program.validate()?;
4657        Ok(program)
4658    }
4659
4660    pub fn nd_r2r(ir: &NdR2rIr) -> Result<Self> {
4661        ir.validate()?;
4662        let dense_elements =
4663            ir.tensor_len
4664                .checked_mul(ir.batch_count)
4665                .ok_or(VkFftError::ArithmeticOverflow {
4666                    operation: "multidimensional DCT/DST program element count",
4667                })?;
4668        let input_external_stride = if ir.input_formatted_copy.is_some() {
4669            ir.input_external_layout.batch_stride
4670        } else {
4671            ir.tensor_len
4672        };
4673        let output_external_stride = if ir.output_formatted_copy.is_some() {
4674            ir.output_external_layout.batch_stride
4675        } else {
4676            ir.tensor_len
4677        };
4678        let input_external = ExternalBufferLayout {
4679            logical_len: input_external_stride,
4680            physical_stride: input_external_stride,
4681            batch_count: ir.batch_count,
4682            element_shape: ProgramElementShape::Complex,
4683        };
4684        let output_external = ExternalBufferLayout {
4685            logical_len: output_external_stride,
4686            physical_stride: output_external_stride,
4687            batch_count: ir.batch_count,
4688            element_shape: ProgramElementShape::Complex,
4689        };
4690        let mut builder = RecursiveProgramBuilder {
4691            scalar: ir.scalar,
4692            resources: vec![
4693                ProgramResource {
4694                    id: ProgramResourceId(0),
4695                    name: "input".to_owned(),
4696                    kind: ProgramResourceKind::Input,
4697                    scalar: ir.external_scalar,
4698                    elements: input_external.physical_elements()?,
4699                    external_layout: Some(input_external),
4700                    initialization: ProgramResourceInitialization::ExternalInput,
4701                },
4702                ProgramResource {
4703                    id: ProgramResourceId(1),
4704                    name: "output".to_owned(),
4705                    kind: ProgramResourceKind::Output,
4706                    scalar: ir.external_scalar,
4707                    elements: output_external.physical_elements()?,
4708                    external_layout: Some(output_external),
4709                    initialization: ProgramResourceInitialization::Zeroed,
4710                },
4711            ],
4712            passes: Vec::new(),
4713            serial: 0,
4714        };
4715        let mut natural_input = if let Some(copy) = &ir.input_formatted_copy {
4716            let dense = builder.add_scratch("nd_r2r_formatted_input", dense_elements);
4717            builder.passes.push(two_buffer_pass(
4718                &copy.name,
4719                copy.dispatch,
4720                ProgramResourceId(0).0,
4721                dense.0,
4722            ));
4723            dense
4724        } else {
4725            ProgramResourceId(0)
4726        };
4727        let formatted_output = if ir.output_formatted_copy.is_some() {
4728            Some(builder.add_scratch("nd_r2r_formatted_output", dense_elements))
4729        } else {
4730            None
4731        };
4732        for (axis_index, axis) in ir.axes.iter().enumerate() {
4733            let packed_input = builder.add_scratch("nd_r2r_axis_input", dense_elements);
4734            let packed_output = builder.add_scratch("nd_r2r_axis_output", dense_elements);
4735            let natural_output = if axis_index + 1 == ir.axes.len() {
4736                formatted_output.unwrap_or(ProgramResourceId(1))
4737            } else {
4738                builder.add_scratch("nd_r2r_natural_output", dense_elements)
4739            };
4740            builder.passes.push(two_buffer_pass(
4741                &axis.pack.name,
4742                axis.pack.dispatch,
4743                natural_input.0,
4744                packed_input.0,
4745            ));
4746            builder.flatten_r2r(&axis.transform, packed_input, packed_output)?;
4747            builder.passes.push(two_buffer_pass(
4748                &axis.scatter.name,
4749                axis.scatter.dispatch,
4750                packed_output.0,
4751                natural_output.0,
4752            ));
4753            natural_input = natural_output;
4754        }
4755        if let Some(copy) = &ir.output_formatted_copy {
4756            builder.passes.push(two_buffer_pass(
4757                &copy.name,
4758                copy.dispatch,
4759                formatted_output.expect("formatted R2R output scratch").0,
4760                ProgramResourceId(1).0,
4761            ));
4762        }
4763        let program = Self {
4764            name: format!("vkfft_nd_r2r_program_{:?}", ir.dimensions),
4765            scalar: ir.scalar,
4766            resources: builder.resources,
4767            passes: builder.passes,
4768        };
4769        program.validate()?;
4770        Ok(program)
4771    }
4772
4773    pub fn nd_convolution(ir: &NdConvolutionIr) -> Result<Self> {
4774        ir.validate()?;
4775        let input_transform_batch_count = ir.batch_count.checked_mul(ir.coordinate_count).ok_or(
4776            VkFftError::ArithmeticOverflow {
4777                operation: "multidimensional performConvolution input coordinate-expanded batch count",
4778            },
4779        )?;
4780        let output_transform_batch_count = ir
4781            .output_batch_count()
4782            .checked_mul(ir.coordinate_count)
4783            .ok_or(VkFftError::ArithmeticOverflow {
4784                operation: "multidimensional performConvolution output coordinate-expanded batch count",
4785            })?;
4786        let input_elements = ir
4787            .tensor_len
4788            .checked_mul(input_transform_batch_count)
4789            .ok_or(VkFftError::ArithmeticOverflow {
4790                operation: "multidimensional performConvolution input program element count",
4791            })?;
4792        let output_elements = ir
4793            .tensor_len
4794            .checked_mul(output_transform_batch_count)
4795            .ok_or(VkFftError::ArithmeticOverflow {
4796                operation: "multidimensional performConvolution output program element count",
4797            })?;
4798        let input_external = ExternalBufferLayout {
4799            logical_len: ir.tensor_len,
4800            physical_stride: ir.tensor_len,
4801            batch_count: input_transform_batch_count,
4802            element_shape: ProgramElementShape::Complex,
4803        };
4804        let output_external = ExternalBufferLayout {
4805            logical_len: ir.tensor_len,
4806            physical_stride: ir.tensor_len,
4807            batch_count: output_transform_batch_count,
4808            element_shape: ProgramElementShape::Complex,
4809        };
4810        let mut resources = vec![
4811            ProgramResource {
4812                id: ProgramResourceId(0),
4813                name: "input".to_owned(),
4814                kind: ProgramResourceKind::Input,
4815                scalar: ir.scalar,
4816                elements: input_elements,
4817                external_layout: Some(input_external),
4818                initialization: ProgramResourceInitialization::ExternalInput,
4819            },
4820            ProgramResource {
4821                id: ProgramResourceId(1),
4822                name: "output".to_owned(),
4823                kind: ProgramResourceKind::Output,
4824                scalar: ir.scalar,
4825                elements: output_elements,
4826                external_layout: Some(output_external),
4827                initialization: ProgramResourceInitialization::Zeroed,
4828            },
4829            ProgramResource {
4830                id: ProgramResourceId(2),
4831                name: "nd_convolution_kernel_spectrum".to_owned(),
4832                kind: ProgramResourceKind::LookupTable,
4833                scalar: ir.scalar,
4834                elements: ir.kernel_spectrum().len(),
4835                external_layout: None,
4836                initialization: ProgramResourceInitialization::Complex64(
4837                    ir.kernel_spectrum().to_vec(),
4838                ),
4839            },
4840            ProgramResource {
4841                id: ProgramResourceId(3),
4842                name: "nd_convolution_forward_spectrum".to_owned(),
4843                kind: ProgramResourceKind::Scratch,
4844                scalar: ir.scalar,
4845                elements: input_elements,
4846                external_layout: None,
4847                initialization: ProgramResourceInitialization::Zeroed,
4848            },
4849            ProgramResource {
4850                id: ProgramResourceId(4),
4851                name: "nd_convolution_multiplied_spectrum".to_owned(),
4852                kind: ProgramResourceKind::Scratch,
4853                scalar: ir.scalar,
4854                elements: output_elements,
4855                external_layout: None,
4856                initialization: ProgramResourceInitialization::Zeroed,
4857            },
4858        ];
4859        let mut passes = Vec::new();
4860        let append_child = |child: &NdFftIr,
4861                            prefix: &str,
4862                            input: ProgramResourceId,
4863                            output: ProgramResourceId,
4864                            resources: &mut Vec<ProgramResource>,
4865                            passes: &mut Vec<ProgramPass>|
4866         -> Result<()> {
4867            let child = Self::nd_fft(child)?;
4868            let mut resource_map = vec![None; child.resources.len()];
4869            for resource in &child.resources {
4870                let mapped = match resource.kind {
4871                    ProgramResourceKind::Input => input,
4872                    ProgramResourceKind::Output => output,
4873                    _ => {
4874                        let id = ProgramResourceId(resources.len());
4875                        let mut cloned = resource.clone();
4876                        cloned.id = id;
4877                        cloned.name = format!("{prefix}_{}", resource.name);
4878                        cloned.external_layout = None;
4879                        resources.push(cloned);
4880                        id
4881                    }
4882                };
4883                resource_map[resource.id.0] = Some(mapped);
4884            }
4885            for child_pass in &child.passes {
4886                let mut pass = child_pass.clone();
4887                pass.name = format!("{prefix}_{}", child_pass.name);
4888                for binding in &mut pass.bindings {
4889                    binding.resource = resource_map
4890                        .get(binding.resource.0)
4891                        .and_then(|mapped| *mapped)
4892                        .ok_or(VkFftError::InvalidKernelIr(
4893                            "multidimensional convolution child resource mapping is incomplete",
4894                        ))?;
4895                }
4896                passes.push(pass);
4897            }
4898            Ok(())
4899        };
4900        append_child(
4901            &ir.forward_fft,
4902            "vkfft_nd_convolution_forward",
4903            ProgramResourceId(0),
4904            ProgramResourceId(3),
4905            &mut resources,
4906            &mut passes,
4907        )?;
4908        passes.push(ProgramPass {
4909            name: ir.multiply.name.clone(),
4910            dispatch: ir.multiply.dispatch,
4911            bindings: vec![
4912                binding(0, 3, BufferRole::Input, BufferAccess::ReadOnly),
4913                binding(1, 4, BufferRole::Output, BufferAccess::WriteOnly),
4914                binding(2, 2, BufferRole::LookupTable, BufferAccess::ReadOnly),
4915            ],
4916        });
4917        append_child(
4918            &ir.inverse_fft,
4919            "vkfft_nd_convolution_inverse",
4920            ProgramResourceId(4),
4921            ProgramResourceId(1),
4922            &mut resources,
4923            &mut passes,
4924        )?;
4925        let program = Self {
4926            name: format!("vkfft_nd_convolution_program_{:?}", ir.dimensions),
4927            scalar: ir.scalar,
4928            resources,
4929            passes,
4930        };
4931        program.validate()?;
4932        Ok(program)
4933    }
4934
4935    pub fn nd_real_convolution(ir: &NdRealConvolutionIr) -> Result<Self> {
4936        ir.validate()?;
4937        let forward = Self::nd_real_fft(&ir.forward_r2c)?;
4938        let inverse = Self::nd_real_fft(&ir.inverse_c2r)?;
4939
4940        let child_resource =
4941            |program: &ProgramIr, kind: ProgramResourceKind| -> Result<ProgramResource> {
4942                program
4943                    .resources
4944                    .iter()
4945                    .find(|resource| resource.kind == kind)
4946                    .cloned()
4947                    .ok_or(VkFftError::InvalidKernelIr(
4948                        "multidimensional real convolution child is missing an external resource",
4949                    ))
4950            };
4951        let mut input = child_resource(&forward, ProgramResourceKind::Input)?;
4952        input.id = ProgramResourceId(0);
4953        input.name = "input".to_owned();
4954        let mut output = child_resource(&inverse, ProgramResourceKind::Output)?;
4955        output.id = ProgramResourceId(1);
4956        output.name = "output".to_owned();
4957        let mut forward_spectrum = child_resource(&forward, ProgramResourceKind::Output)?;
4958        forward_spectrum.id = ProgramResourceId(3);
4959        forward_spectrum.name = "nd_real_convolution_forward_spectrum".to_owned();
4960        forward_spectrum.kind = ProgramResourceKind::Scratch;
4961        forward_spectrum.external_layout = None;
4962        forward_spectrum.initialization = ProgramResourceInitialization::Zeroed;
4963        let mut multiplied_spectrum = child_resource(&inverse, ProgramResourceKind::Input)?;
4964        multiplied_spectrum.id = ProgramResourceId(4);
4965        multiplied_spectrum.name = "nd_real_convolution_multiplied_spectrum".to_owned();
4966        multiplied_spectrum.kind = ProgramResourceKind::Scratch;
4967        multiplied_spectrum.external_layout = None;
4968        multiplied_spectrum.initialization = ProgramResourceInitialization::Zeroed;
4969
4970        let mut resources = vec![
4971            input,
4972            output,
4973            ProgramResource {
4974                id: ProgramResourceId(2),
4975                name: "nd_real_convolution_kernel_spectrum".to_owned(),
4976                kind: ProgramResourceKind::LookupTable,
4977                scalar: ir.scalar,
4978                elements: ir.kernel_spectrum().len(),
4979                external_layout: None,
4980                initialization: ProgramResourceInitialization::Complex64(
4981                    ir.kernel_spectrum().to_vec(),
4982                ),
4983            },
4984            forward_spectrum,
4985            multiplied_spectrum,
4986        ];
4987        let mut passes = Vec::new();
4988        let append_child = |child: &ProgramIr,
4989                            prefix: &str,
4990                            input: ProgramResourceId,
4991                            output: ProgramResourceId,
4992                            resources: &mut Vec<ProgramResource>,
4993                            passes: &mut Vec<ProgramPass>|
4994         -> Result<()> {
4995            let mut resource_map = vec![None; child.resources.len()];
4996            for resource in &child.resources {
4997                let mapped = match resource.kind {
4998                    ProgramResourceKind::Input => input,
4999                    ProgramResourceKind::Output => output,
5000                    _ => {
5001                        let id = ProgramResourceId(resources.len());
5002                        let mut cloned = resource.clone();
5003                        cloned.id = id;
5004                        cloned.name = format!("{prefix}_{}", resource.name);
5005                        cloned.external_layout = None;
5006                        resources.push(cloned);
5007                        id
5008                    }
5009                };
5010                resource_map[resource.id.0] = Some(mapped);
5011            }
5012            for child_pass in &child.passes {
5013                let mut pass = child_pass.clone();
5014                pass.name = format!("{prefix}_{}", child_pass.name);
5015                for binding in &mut pass.bindings {
5016                    binding.resource = resource_map
5017                        .get(binding.resource.0)
5018                        .and_then(|mapped| *mapped)
5019                        .ok_or(VkFftError::InvalidKernelIr(
5020                            "multidimensional real convolution child resource mapping is incomplete",
5021                        ))?;
5022                }
5023                passes.push(pass);
5024            }
5025            Ok(())
5026        };
5027        append_child(
5028            &forward,
5029            "vkfft_nd_real_convolution_forward",
5030            ProgramResourceId(0),
5031            ProgramResourceId(3),
5032            &mut resources,
5033            &mut passes,
5034        )?;
5035        passes.push(ProgramPass {
5036            name: ir.multiply.name.clone(),
5037            dispatch: ir.multiply.dispatch,
5038            bindings: vec![
5039                binding(0, 3, BufferRole::Input, BufferAccess::ReadOnly),
5040                binding(1, 4, BufferRole::Output, BufferAccess::WriteOnly),
5041                binding(2, 2, BufferRole::LookupTable, BufferAccess::ReadOnly),
5042            ],
5043        });
5044        append_child(
5045            &inverse,
5046            "vkfft_nd_real_convolution_inverse",
5047            ProgramResourceId(4),
5048            ProgramResourceId(1),
5049            &mut resources,
5050            &mut passes,
5051        )?;
5052        let program = Self {
5053            name: format!("vkfft_nd_real_convolution_program_{:?}", ir.dimensions),
5054            scalar: ir.scalar,
5055            resources,
5056            passes,
5057        };
5058        program.validate()?;
5059        Ok(program)
5060    }
5061
5062    pub fn convolution(ir: &ConvolutionIr) -> Result<Self> {
5063        ir.validate()?;
5064        let input_transform_batch_count = ir.batch_count.checked_mul(ir.coordinate_count).ok_or(
5065            VkFftError::ArithmeticOverflow {
5066                operation: "performConvolution program input coordinate-expanded batch count",
5067            },
5068        )?;
5069        let output_transform_batch_count = ir
5070            .output_batch_count()
5071            .checked_mul(ir.coordinate_count)
5072            .ok_or(VkFftError::ArithmeticOverflow {
5073                operation: "performConvolution program output coordinate-expanded batch count",
5074            })?;
5075        let input_elements = ir
5076            .sequence_len
5077            .checked_mul(input_transform_batch_count)
5078            .ok_or(VkFftError::ArithmeticOverflow {
5079                operation: "performConvolution program input element count",
5080            })?;
5081        let output_elements = ir
5082            .sequence_len
5083            .checked_mul(output_transform_batch_count)
5084            .ok_or(VkFftError::ArithmeticOverflow {
5085                operation: "performConvolution program output element count",
5086            })?;
5087        let input_external = ExternalBufferLayout {
5088            logical_len: ir.sequence_len,
5089            physical_stride: ir.sequence_len,
5090            batch_count: input_transform_batch_count,
5091            element_shape: ProgramElementShape::Complex,
5092        };
5093        let output_external = ExternalBufferLayout {
5094            logical_len: ir.sequence_len,
5095            physical_stride: ir.sequence_len,
5096            batch_count: output_transform_batch_count,
5097            element_shape: ProgramElementShape::Complex,
5098        };
5099        let mut builder = RecursiveProgramBuilder {
5100            scalar: ir.scalar,
5101            resources: vec![
5102                ProgramResource {
5103                    id: ProgramResourceId(0),
5104                    name: "input".to_owned(),
5105                    kind: ProgramResourceKind::Input,
5106                    scalar: ir.external_scalar,
5107                    elements: input_elements,
5108                    external_layout: Some(input_external),
5109                    initialization: ProgramResourceInitialization::ExternalInput,
5110                },
5111                ProgramResource {
5112                    id: ProgramResourceId(1),
5113                    name: "output".to_owned(),
5114                    kind: ProgramResourceKind::Output,
5115                    scalar: ir.external_scalar,
5116                    elements: output_elements,
5117                    external_layout: Some(output_external),
5118                    initialization: ProgramResourceInitialization::Zeroed,
5119                },
5120            ],
5121            passes: Vec::new(),
5122            serial: 0,
5123        };
5124        let kernel = builder.add_lut("convolution_kernel_spectrum", ir.kernel_spectrum().to_vec());
5125        if let (Some(input_copy), Some(output_copy)) =
5126            (&ir.input_storage_copy, &ir.output_storage_copy)
5127        {
5128            let input_compute =
5129                builder.add_scratch("convolution_caller_input_compute", input_elements);
5130            let output_compute =
5131                builder.add_scratch("convolution_caller_output_compute", output_elements);
5132            let forward_spectrum =
5133                builder.add_scratch("convolution_forward_spectrum", input_elements);
5134            let multiplied_spectrum =
5135                builder.add_scratch("convolution_multiplied_spectrum", output_elements);
5136            builder.passes.push(two_buffer_pass(
5137                &input_copy.name,
5138                input_copy.dispatch,
5139                0,
5140                input_compute.0,
5141            ));
5142            builder.flatten_one_dim(&ir.forward_fft, input_compute, forward_spectrum)?;
5143            builder.passes.push(ProgramPass {
5144                name: ir.multiply.name.clone(),
5145                dispatch: ir.multiply.dispatch,
5146                bindings: vec![
5147                    binding(
5148                        0,
5149                        forward_spectrum.0,
5150                        BufferRole::Input,
5151                        BufferAccess::ReadOnly,
5152                    ),
5153                    binding(
5154                        1,
5155                        multiplied_spectrum.0,
5156                        BufferRole::Output,
5157                        BufferAccess::WriteOnly,
5158                    ),
5159                    binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5160                ],
5161            });
5162            builder.flatten_one_dim(&ir.inverse_fft, multiplied_spectrum, output_compute)?;
5163            builder.passes.push(two_buffer_pass(
5164                &output_copy.name,
5165                output_copy.dispatch,
5166                output_compute.0,
5167                1,
5168            ));
5169            let program = Self {
5170                name: format!("vkfft_convolution_program_{}", ir.sequence_len),
5171                scalar: ir.scalar,
5172                resources: builder.resources,
5173                passes: builder.passes,
5174            };
5175            program.validate()?;
5176            return Ok(program);
5177        }
5178        if let Some(step) = &ir.fused_stockham_step {
5179            builder.passes.push(ProgramPass {
5180                name: step.name.clone(),
5181                dispatch: step.dispatch,
5182                bindings: vec![
5183                    binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
5184                    binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
5185                    binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5186                ],
5187            });
5188        } else if let Some(step) = &ir.fused_multi_kernel_stockham_step {
5189            let stockham_twiddles = if step.requires_twiddle_lut()? {
5190                Some(builder.add_stockham_root_lut(step.sequence_len)?)
5191            } else {
5192                None
5193            };
5194            let mut bindings = vec![
5195                binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
5196                binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
5197                binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5198            ];
5199            if let Some(twiddles) = stockham_twiddles {
5200                bindings.push(binding(
5201                    3,
5202                    twiddles.0,
5203                    BufferRole::TwiddleLookupTable,
5204                    BufferAccess::ReadOnly,
5205                ));
5206            }
5207            builder.passes.push(ProgramPass {
5208                name: step.name.clone(),
5209                dispatch: step.dispatch,
5210                bindings,
5211            });
5212        } else if let Some(step) = &ir.fused_matrix_stockham_step {
5213            let stockham_twiddles = if step.requires_twiddle_lut()? {
5214                Some(builder.add_stockham_root_lut(step.sequence_len)?)
5215            } else {
5216                None
5217            };
5218            let mut bindings = vec![
5219                binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
5220                binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
5221                binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5222            ];
5223            if let Some(twiddles) = stockham_twiddles {
5224                bindings.push(binding(
5225                    3,
5226                    twiddles.0,
5227                    BufferRole::TwiddleLookupTable,
5228                    BufferAccess::ReadOnly,
5229                ));
5230            }
5231            builder.passes.push(ProgramPass {
5232                name: step.name.clone(),
5233                dispatch: step.dispatch,
5234                bindings,
5235            });
5236        } else if let Some(step) = &ir.fused_direct_rader_step {
5237            let rader_twiddles = builder.add_lut(
5238                "convolution_direct_rader_twiddles",
5239                step.forward.table.twiddles_by_generator_power.clone(),
5240            );
5241            builder.passes.push(ProgramPass {
5242                name: step.name.clone(),
5243                dispatch: step.dispatch,
5244                bindings: vec![
5245                    binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
5246                    binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
5247                    binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5248                    binding(
5249                        3,
5250                        rader_twiddles.0,
5251                        BufferRole::TwiddleLookupTable,
5252                        BufferAccess::ReadOnly,
5253                    ),
5254                ],
5255            });
5256        } else if let Some(step) = &ir.fused_direct_rader_multi_kernel_step {
5257            let rader_twiddles = builder.add_lut(
5258                "convolution_direct_rader_multi_kernel_twiddles",
5259                step.forward.table.twiddles_by_generator_power.clone(),
5260            );
5261            builder.passes.push(ProgramPass {
5262                name: step.name.clone(),
5263                dispatch: step.dispatch,
5264                bindings: vec![
5265                    binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
5266                    binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
5267                    binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5268                    binding(
5269                        3,
5270                        rader_twiddles.0,
5271                        BufferRole::TwiddleLookupTable,
5272                        BufferAccess::ReadOnly,
5273                    ),
5274                ],
5275            });
5276        } else if let Some(step) = &ir.fused_fft_rader_step {
5277            let forward_rader_spectrum = builder.add_lut(
5278                "convolution_fft_rader_forward_spectrum",
5279                step.forward.kernel_spectrum()?.to_vec(),
5280            );
5281            let inverse_rader_spectrum = builder.add_lut(
5282                "convolution_fft_rader_inverse_spectrum",
5283                step.inverse.kernel_spectrum()?.to_vec(),
5284            );
5285            let stockham_twiddles = if step.requires_twiddle_lut()? {
5286                Some(builder.add_stockham_root_lut(step.convolution_len)?)
5287            } else {
5288                None
5289            };
5290            let mut bindings = vec![
5291                binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
5292                binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
5293                binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5294                binding(
5295                    3,
5296                    forward_rader_spectrum.0,
5297                    BufferRole::LookupTable,
5298                    BufferAccess::ReadOnly,
5299                ),
5300                binding(
5301                    4,
5302                    inverse_rader_spectrum.0,
5303                    BufferRole::LookupTable,
5304                    BufferAccess::ReadOnly,
5305                ),
5306            ];
5307            if let Some(twiddles) = stockham_twiddles {
5308                bindings.push(binding(
5309                    5,
5310                    twiddles.0,
5311                    BufferRole::TwiddleLookupTable,
5312                    BufferAccess::ReadOnly,
5313                ));
5314            }
5315            builder.passes.push(ProgramPass {
5316                name: step.name.clone(),
5317                dispatch: step.dispatch,
5318                bindings,
5319            });
5320        } else if let Some(step) = &ir.fused_fft_rader_multi_kernel_step {
5321            let forward_rader_spectrum = builder.add_lut(
5322                "convolution_fft_rader_multi_kernel_forward_spectrum",
5323                step.forward.kernel_spectrum()?.to_vec(),
5324            );
5325            let inverse_rader_spectrum = builder.add_lut(
5326                "convolution_fft_rader_multi_kernel_inverse_spectrum",
5327                step.inverse.kernel_spectrum()?.to_vec(),
5328            );
5329            let stockham_twiddles = if step.requires_twiddle_lut()? {
5330                Some(builder.add_stockham_root_lut(step.convolution_len)?)
5331            } else {
5332                None
5333            };
5334            let mut bindings = vec![
5335                binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
5336                binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
5337                binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5338                binding(
5339                    3,
5340                    forward_rader_spectrum.0,
5341                    BufferRole::LookupTable,
5342                    BufferAccess::ReadOnly,
5343                ),
5344                binding(
5345                    4,
5346                    inverse_rader_spectrum.0,
5347                    BufferRole::LookupTable,
5348                    BufferAccess::ReadOnly,
5349                ),
5350            ];
5351            if let Some(twiddles) = stockham_twiddles {
5352                bindings.push(binding(
5353                    5,
5354                    twiddles.0,
5355                    BufferRole::TwiddleLookupTable,
5356                    BufferAccess::ReadOnly,
5357                ));
5358            }
5359            builder.passes.push(ProgramPass {
5360                name: step.name.clone(),
5361                dispatch: step.dispatch,
5362                bindings,
5363            });
5364        } else if let Some(two_upload) = &ir.fused_two_upload_multi_kernel_stockham {
5365            let forward_mid = builder.add_scratch(
5366                "convolution_two_upload_multi_kernel_forward_mid",
5367                input_elements,
5368            );
5369            let inverse_mid = builder.add_scratch(
5370                "convolution_two_upload_multi_kernel_inverse_mid",
5371                output_elements,
5372            );
5373            builder.push_stockham_pass(
5374                &two_upload.forward_high,
5375                ProgramResourceId(0),
5376                forward_mid,
5377                None,
5378                None,
5379            )?;
5380            let special_twiddle = match two_upload.special_twiddle_lut_len()? {
5381                Some(len) => Some(builder.add_stockham_root_lut(len)?),
5382                None => None,
5383            };
5384            let mut special_bindings = vec![
5385                binding(0, forward_mid.0, BufferRole::Input, BufferAccess::ReadOnly),
5386                binding(
5387                    1,
5388                    inverse_mid.0,
5389                    BufferRole::Output,
5390                    BufferAccess::WriteOnly,
5391                ),
5392                binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5393            ];
5394            if let Some(twiddle) = special_twiddle {
5395                special_bindings.push(binding(
5396                    3,
5397                    twiddle.0,
5398                    BufferRole::TwiddleLookupTable,
5399                    BufferAccess::ReadOnly,
5400                ));
5401            }
5402            builder.passes.push(ProgramPass {
5403                name: two_upload.name.clone(),
5404                dispatch: two_upload.special_dispatch,
5405                bindings: special_bindings,
5406            });
5407            builder.push_stockham_pass(
5408                &two_upload.inverse_high,
5409                inverse_mid,
5410                ProgramResourceId(1),
5411                None,
5412                None,
5413            )?;
5414        } else if let Some(two_upload) = &ir.fused_two_upload_stockham {
5415            let forward_mid =
5416                builder.add_scratch("convolution_two_upload_forward_mid", input_elements);
5417            let inverse_mid =
5418                builder.add_scratch("convolution_two_upload_inverse_mid", input_elements);
5419            builder.push_stockham_pass(
5420                &two_upload.forward_high,
5421                ProgramResourceId(0),
5422                forward_mid,
5423                None,
5424                None,
5425            )?;
5426            let special_twiddle = match two_upload.special_twiddle_lut_len()? {
5427                Some(len) => Some(builder.add_stockham_root_lut(len)?),
5428                None => None,
5429            };
5430            let mut special_bindings = vec![
5431                binding(0, forward_mid.0, BufferRole::Input, BufferAccess::ReadOnly),
5432                binding(
5433                    1,
5434                    inverse_mid.0,
5435                    BufferRole::Output,
5436                    BufferAccess::WriteOnly,
5437                ),
5438                binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5439            ];
5440            if let Some(twiddle) = special_twiddle {
5441                special_bindings.push(binding(
5442                    3,
5443                    twiddle.0,
5444                    BufferRole::TwiddleLookupTable,
5445                    BufferAccess::ReadOnly,
5446                ));
5447            }
5448            builder.passes.push(ProgramPass {
5449                name: two_upload.name.clone(),
5450                dispatch: two_upload.special_dispatch,
5451                bindings: special_bindings,
5452            });
5453            builder.push_stockham_pass(
5454                &two_upload.inverse_high,
5455                inverse_mid,
5456                ProgramResourceId(1),
5457                None,
5458                None,
5459            )?;
5460        } else if let Some(three_upload) = &ir.fused_three_upload_multi_kernel_stockham {
5461            let scratch_a = builder.add_scratch(
5462                "convolution_three_upload_multi_kernel_scratch_a",
5463                output_elements,
5464            );
5465            let scratch_b = builder.add_scratch(
5466                "convolution_three_upload_multi_kernel_scratch_b",
5467                output_elements,
5468            );
5469            builder.push_stockham_pass(
5470                &three_upload.forward_high,
5471                ProgramResourceId(0),
5472                scratch_a,
5473                None,
5474                None,
5475            )?;
5476            builder.push_stockham_pass(
5477                &three_upload.forward_middle,
5478                scratch_a,
5479                scratch_b,
5480                None,
5481                None,
5482            )?;
5483            builder.passes.push(ProgramPass {
5484                name: three_upload.name.clone(),
5485                dispatch: three_upload.special_dispatch,
5486                bindings: vec![
5487                    binding(0, scratch_b.0, BufferRole::Input, BufferAccess::ReadOnly),
5488                    binding(1, scratch_a.0, BufferRole::Output, BufferAccess::WriteOnly),
5489                    binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5490                ],
5491            });
5492            builder.push_stockham_pass(
5493                &three_upload.inverse_middle,
5494                scratch_a,
5495                scratch_b,
5496                None,
5497                None,
5498            )?;
5499            builder.push_stockham_pass(
5500                &three_upload.inverse_high,
5501                scratch_b,
5502                ProgramResourceId(1),
5503                None,
5504                None,
5505            )?;
5506        } else if let Some(three_upload) = &ir.fused_three_upload_stockham {
5507            let scratch_a =
5508                builder.add_scratch("convolution_three_upload_scratch_a", input_elements);
5509            let scratch_b =
5510                builder.add_scratch("convolution_three_upload_scratch_b", input_elements);
5511            builder.push_stockham_pass(
5512                &three_upload.forward_high,
5513                ProgramResourceId(0),
5514                scratch_a,
5515                None,
5516                None,
5517            )?;
5518            builder.push_stockham_pass(
5519                &three_upload.forward_middle,
5520                scratch_a,
5521                scratch_b,
5522                None,
5523                None,
5524            )?;
5525            let special_twiddle = match three_upload.special_twiddle_lut_len()? {
5526                Some(len) => Some(builder.add_stockham_root_lut(len)?),
5527                None => None,
5528            };
5529            let mut special_bindings = vec![
5530                binding(0, scratch_b.0, BufferRole::Input, BufferAccess::ReadOnly),
5531                binding(1, scratch_a.0, BufferRole::Output, BufferAccess::WriteOnly),
5532                binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5533            ];
5534            if let Some(twiddle) = special_twiddle {
5535                special_bindings.push(binding(
5536                    3,
5537                    twiddle.0,
5538                    BufferRole::TwiddleLookupTable,
5539                    BufferAccess::ReadOnly,
5540                ));
5541            }
5542            builder.passes.push(ProgramPass {
5543                name: three_upload.name.clone(),
5544                dispatch: three_upload.special_dispatch,
5545                bindings: special_bindings,
5546            });
5547            builder.push_stockham_pass(
5548                &three_upload.inverse_middle,
5549                scratch_a,
5550                scratch_b,
5551                None,
5552                None,
5553            )?;
5554            builder.push_stockham_pass(
5555                &three_upload.inverse_high,
5556                scratch_b,
5557                ProgramResourceId(1),
5558                None,
5559                None,
5560            )?;
5561        } else {
5562            let forward_spectrum =
5563                builder.add_scratch("convolution_forward_spectrum", input_elements);
5564            builder.flatten_one_dim(&ir.forward_fft, ProgramResourceId(0), forward_spectrum)?;
5565            if let Some(fused_inverse) = &ir.fused_inverse_stockham {
5566                builder.push_stockham_pass(
5567                    fused_inverse,
5568                    forward_spectrum,
5569                    ProgramResourceId(1),
5570                    Some(kernel),
5571                    None,
5572                )?;
5573            } else {
5574                let multiplied_spectrum =
5575                    builder.add_scratch("convolution_multiplied_spectrum", output_elements);
5576                builder.passes.push(ProgramPass {
5577                    name: ir.multiply.name.clone(),
5578                    dispatch: ir.multiply.dispatch,
5579                    bindings: vec![
5580                        binding(
5581                            0,
5582                            forward_spectrum.0,
5583                            BufferRole::Input,
5584                            BufferAccess::ReadOnly,
5585                        ),
5586                        binding(
5587                            1,
5588                            multiplied_spectrum.0,
5589                            BufferRole::Output,
5590                            BufferAccess::WriteOnly,
5591                        ),
5592                        binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5593                    ],
5594                });
5595                builder.flatten_one_dim(
5596                    &ir.inverse_fft,
5597                    multiplied_spectrum,
5598                    ProgramResourceId(1),
5599                )?;
5600            }
5601        }
5602        let program = Self {
5603            name: format!("vkfft_perform_convolution_program_{}", ir.sequence_len),
5604            scalar: ir.scalar,
5605            resources: builder.resources,
5606            passes: builder.passes,
5607        };
5608        program.validate()?;
5609        Ok(program)
5610    }
5611
5612    pub fn bluestein(pipeline: &BluesteinPipelineIr) -> Result<Self> {
5613        pipeline.validate()?;
5614        let logical_elements = pipeline
5615            .logical_len
5616            .checked_mul(pipeline.batch_count)
5617            .ok_or(VkFftError::ArithmeticOverflow {
5618                operation: "Bluestein program logical element count",
5619            })?;
5620        let convolution_elements = pipeline
5621            .convolution_len
5622            .checked_mul(pipeline.batch_count)
5623            .ok_or(VkFftError::ArithmeticOverflow {
5624                operation: "Bluestein program convolution element count",
5625            })?;
5626        let external = ExternalBufferLayout {
5627            logical_len: pipeline.logical_len,
5628            physical_stride: pipeline.logical_len,
5629            batch_count: pipeline.batch_count,
5630            element_shape: ProgramElementShape::Complex,
5631        };
5632        let lut = pipeline.kernel_spectrum()?.to_vec();
5633        let mut builder = RecursiveProgramBuilder {
5634            scalar: pipeline.scalar,
5635            resources: vec![
5636                ProgramResource {
5637                    id: ProgramResourceId(0),
5638                    name: "input".to_owned(),
5639                    kind: ProgramResourceKind::Input,
5640                    scalar: pipeline.external_storage_scalar(),
5641                    elements: logical_elements,
5642                    external_layout: Some(external),
5643                    initialization: ProgramResourceInitialization::ExternalInput,
5644                },
5645                ProgramResource {
5646                    id: ProgramResourceId(1),
5647                    name: "output".to_owned(),
5648                    kind: ProgramResourceKind::Output,
5649                    scalar: pipeline.external_storage_scalar(),
5650                    elements: logical_elements,
5651                    external_layout: Some(external),
5652                    initialization: ProgramResourceInitialization::Zeroed,
5653                },
5654                ProgramResource {
5655                    id: ProgramResourceId(2),
5656                    name: "kernel_spectrum".to_owned(),
5657                    kind: ProgramResourceKind::LookupTable,
5658                    scalar: pipeline.scalar,
5659                    elements: pipeline.convolution_len,
5660                    external_layout: None,
5661                    initialization: ProgramResourceInitialization::Complex64(lut),
5662                },
5663                ProgramResource {
5664                    id: ProgramResourceId(3),
5665                    name: "scratch_a".to_owned(),
5666                    kind: ProgramResourceKind::Scratch,
5667                    scalar: pipeline.scalar,
5668                    elements: convolution_elements,
5669                    external_layout: None,
5670                    initialization: ProgramResourceInitialization::Zeroed,
5671                },
5672                ProgramResource {
5673                    id: ProgramResourceId(4),
5674                    name: "scratch_b".to_owned(),
5675                    kind: ProgramResourceKind::Scratch,
5676                    scalar: pipeline.scalar,
5677                    elements: convolution_elements,
5678                    external_layout: None,
5679                    initialization: ProgramResourceInitialization::Zeroed,
5680                },
5681            ],
5682            passes: vec![two_buffer_pass(
5683                &pipeline.preprocess.name,
5684                pipeline.preprocess.dispatch,
5685                0,
5686                3,
5687            )],
5688            serial: 0,
5689        };
5690        builder.flatten_recursive_ir(
5691            &pipeline.forward_fft,
5692            ProgramResourceId(3),
5693            ProgramResourceId(4),
5694        )?;
5695        let convolution_output =
5696            if let Some(fused_inverse) = pipeline.fused_inverse_stockham_kernel()? {
5697                builder.push_stockham_pass(
5698                    &fused_inverse,
5699                    ProgramResourceId(4),
5700                    ProgramResourceId(3),
5701                    Some(ProgramResourceId(2)),
5702                    None,
5703                )?;
5704                ProgramResourceId(3)
5705            } else if pipeline.has_fused_recursive_inverse() {
5706                builder.flatten_node_with_boundary_resources(
5707                    &pipeline.inverse_fft.root,
5708                    ProgramResourceId(4),
5709                    ProgramResourceId(3),
5710                    Some(ProgramResourceId(2)),
5711                    None,
5712                )?;
5713                ProgramResourceId(3)
5714            } else {
5715                builder.passes.push(ProgramPass {
5716                    name: pipeline.multiply.name.clone(),
5717                    dispatch: pipeline.multiply.dispatch,
5718                    bindings: vec![
5719                        binding(0, 4, BufferRole::Input, BufferAccess::ReadOnly),
5720                        binding(1, 3, BufferRole::Output, BufferAccess::WriteOnly),
5721                        binding(2, 2, BufferRole::LookupTable, BufferAccess::ReadOnly),
5722                    ],
5723                });
5724                builder.flatten_recursive_ir(
5725                    &pipeline.inverse_fft,
5726                    ProgramResourceId(3),
5727                    ProgramResourceId(4),
5728                )?;
5729                ProgramResourceId(4)
5730            };
5731        builder.passes.push(two_buffer_pass(
5732            &pipeline.postprocess.name,
5733            pipeline.postprocess.dispatch,
5734            convolution_output.0,
5735            1,
5736        ));
5737        let program = Self {
5738            name: format!("vkfft_bluestein_program_{}", pipeline.logical_len),
5739            scalar: pipeline.scalar,
5740            resources: builder.resources,
5741            passes: builder.passes,
5742        };
5743        program.validate()?;
5744        Ok(program)
5745    }
5746
5747    pub fn rader_fft(pipeline: &RaderFftPipelineIr) -> Result<Self> {
5748        pipeline.validate()?;
5749        let logical_elements = pipeline.prime.checked_mul(pipeline.batch_count).ok_or(
5750            VkFftError::ArithmeticOverflow {
5751                operation: "FFT Rader program logical element count",
5752            },
5753        )?;
5754        let convolution_elements = pipeline
5755            .convolution_len
5756            .checked_mul(pipeline.batch_count)
5757            .ok_or(VkFftError::ArithmeticOverflow {
5758                operation: "FFT Rader program convolution element count",
5759            })?;
5760        let external = ExternalBufferLayout {
5761            logical_len: pipeline.prime,
5762            physical_stride: pipeline.prime,
5763            batch_count: pipeline.batch_count,
5764            element_shape: ProgramElementShape::Complex,
5765        };
5766        let mut builder = RecursiveProgramBuilder {
5767            scalar: pipeline.scalar,
5768            resources: vec![
5769                ProgramResource {
5770                    id: ProgramResourceId(0),
5771                    name: "input".to_owned(),
5772                    kind: ProgramResourceKind::Input,
5773                    scalar: pipeline.input_storage_scalar,
5774                    elements: logical_elements,
5775                    external_layout: Some(external),
5776                    initialization: ProgramResourceInitialization::ExternalInput,
5777                },
5778                ProgramResource {
5779                    id: ProgramResourceId(1),
5780                    name: "scratch_a".to_owned(),
5781                    kind: ProgramResourceKind::Scratch,
5782                    scalar: pipeline.scalar,
5783                    elements: convolution_elements,
5784                    external_layout: None,
5785                    initialization: ProgramResourceInitialization::Zeroed,
5786                },
5787                ProgramResource {
5788                    id: ProgramResourceId(2),
5789                    name: "scratch_b".to_owned(),
5790                    kind: ProgramResourceKind::Scratch,
5791                    scalar: pipeline.scalar,
5792                    elements: convolution_elements,
5793                    external_layout: None,
5794                    initialization: ProgramResourceInitialization::Zeroed,
5795                },
5796                ProgramResource {
5797                    id: ProgramResourceId(3),
5798                    name: "output".to_owned(),
5799                    kind: ProgramResourceKind::Output,
5800                    scalar: pipeline.output_storage_scalar,
5801                    elements: logical_elements,
5802                    external_layout: Some(external),
5803                    initialization: ProgramResourceInitialization::Zeroed,
5804                },
5805                ProgramResource {
5806                    id: ProgramResourceId(4),
5807                    name: "kernel_spectrum".to_owned(),
5808                    kind: ProgramResourceKind::LookupTable,
5809                    scalar: pipeline.scalar,
5810                    elements: pipeline.convolution_len,
5811                    external_layout: None,
5812                    initialization: ProgramResourceInitialization::Complex64(
5813                        pipeline.kernel_spectrum()?.to_vec(),
5814                    ),
5815                },
5816            ],
5817            passes: Vec::new(),
5818            serial: 0,
5819        };
5820        let forward_input = match pipeline.input_strategy {
5821            RaderFftInputStrategy::GatherReversePass => {
5822                builder.passes.push(two_buffer_pass(
5823                    &pipeline.gather.name,
5824                    pipeline.gather.dispatch,
5825                    0,
5826                    1,
5827                ));
5828                ProgramResourceId(1)
5829            }
5830            RaderFftInputStrategy::GeneratorOrderStockham
5831            | RaderFftInputStrategy::GeneratorOrderRecursive => ProgramResourceId(0),
5832        };
5833        builder.flatten_one_dim(&pipeline.forward_fft, forward_input, ProgramResourceId(2))?;
5834        if let Some(fused_inverse) = pipeline.fused_inverse_rader_kernel()? {
5835            builder.push_stockham_pass(
5836                &fused_inverse,
5837                ProgramResourceId(2),
5838                ProgramResourceId(3),
5839                Some(ProgramResourceId(4)),
5840                Some(ProgramResourceId(0)),
5841            )?;
5842        } else if pipeline.has_fused_recursive_inverse() {
5843            let inverse = pipeline
5844                .inverse_recursive()
5845                .ok_or(VkFftError::InvalidKernelIr(
5846                    "fused recursive Rader program lost its recursive inverse child",
5847                ))?;
5848            builder.flatten_node_with_boundary_resources(
5849                &inverse.root,
5850                ProgramResourceId(2),
5851                ProgramResourceId(3),
5852                Some(ProgramResourceId(4)),
5853                Some(ProgramResourceId(0)),
5854            )?;
5855        } else {
5856            builder.passes.push(ProgramPass {
5857                name: pipeline.multiply.name.clone(),
5858                dispatch: pipeline.multiply.dispatch,
5859                bindings: vec![
5860                    binding(0, 2, BufferRole::Input, BufferAccess::ReadOnly),
5861                    binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
5862                    binding(2, 4, BufferRole::LookupTable, BufferAccess::ReadOnly),
5863                ],
5864            });
5865            builder.flatten_one_dim(
5866                &pipeline.inverse_fft,
5867                ProgramResourceId(1),
5868                ProgramResourceId(2),
5869            )?;
5870            builder.passes.push(ProgramPass {
5871                name: pipeline.scatter.name.clone(),
5872                dispatch: pipeline.scatter.dispatch,
5873                bindings: vec![
5874                    binding(0, 2, BufferRole::Input, BufferAccess::ReadOnly),
5875                    binding(1, 3, BufferRole::Output, BufferAccess::WriteOnly),
5876                    binding(2, 0, BufferRole::Auxiliary, BufferAccess::ReadOnly),
5877                ],
5878            });
5879        }
5880        let program = Self {
5881            name: format!("vkfft_rader_fft_program_{}", pipeline.prime),
5882            scalar: pipeline.scalar,
5883            resources: builder.resources,
5884            passes: builder.passes,
5885        };
5886        program.validate()?;
5887        Ok(program)
5888    }
5889
5890    pub fn mixed_rader_stockham(ir: &MixedRaderStockhamIr) -> Result<Self> {
5891        ir.validate()?;
5892        match &ir.prime_stage {
5893            MixedPrimeRaderIr::FftConvolution(rader) => Self::mixed_rader_stockham_fft(ir, rader),
5894            MixedPrimeRaderIr::Direct(direct) => Self::mixed_rader_stockham_direct(ir, direct),
5895        }
5896    }
5897
5898    fn mixed_rader_stockham_fft(
5899        ir: &MixedRaderStockhamIr,
5900        rader: &RaderFftPipelineIr,
5901    ) -> Result<Self> {
5902        let logical_elements =
5903            ir.logical_len
5904                .checked_mul(ir.batch_count)
5905                .ok_or(VkFftError::ArithmeticOverflow {
5906                    operation: "mixed program logical element count",
5907                })?;
5908        let convolution_elements = rader.convolution_len.checked_mul(rader.batch_count).ok_or(
5909            VkFftError::ArithmeticOverflow {
5910                operation: "mixed program Rader convolution element count",
5911            },
5912        )?;
5913        let external = ExternalBufferLayout {
5914            logical_len: ir.logical_len,
5915            physical_stride: ir.logical_len,
5916            batch_count: ir.batch_count,
5917            element_shape: ProgramElementShape::Complex,
5918        };
5919        let mut builder = RecursiveProgramBuilder {
5920            scalar: ir.scalar,
5921            resources: vec![
5922                ProgramResource {
5923                    id: ProgramResourceId(0),
5924                    name: "input".to_owned(),
5925                    kind: ProgramResourceKind::Input,
5926                    scalar: ir.scalar,
5927                    elements: logical_elements,
5928                    external_layout: Some(external),
5929                    initialization: ProgramResourceInitialization::ExternalInput,
5930                },
5931                ProgramResource {
5932                    id: ProgramResourceId(1),
5933                    name: "prime_input".to_owned(),
5934                    kind: ProgramResourceKind::Scratch,
5935                    scalar: ir.scalar,
5936                    elements: logical_elements,
5937                    external_layout: None,
5938                    initialization: ProgramResourceInitialization::Zeroed,
5939                },
5940                ProgramResource {
5941                    id: ProgramResourceId(2),
5942                    name: "rader_scratch_a".to_owned(),
5943                    kind: ProgramResourceKind::Scratch,
5944                    scalar: ir.scalar,
5945                    elements: convolution_elements,
5946                    external_layout: None,
5947                    initialization: ProgramResourceInitialization::Zeroed,
5948                },
5949                ProgramResource {
5950                    id: ProgramResourceId(3),
5951                    name: "rader_scratch_b".to_owned(),
5952                    kind: ProgramResourceKind::Scratch,
5953                    scalar: ir.scalar,
5954                    elements: convolution_elements,
5955                    external_layout: None,
5956                    initialization: ProgramResourceInitialization::Zeroed,
5957                },
5958                ProgramResource {
5959                    id: ProgramResourceId(4),
5960                    name: "prime_output".to_owned(),
5961                    kind: ProgramResourceKind::Scratch,
5962                    scalar: ir.scalar,
5963                    elements: logical_elements,
5964                    external_layout: None,
5965                    initialization: ProgramResourceInitialization::Zeroed,
5966                },
5967                ProgramResource {
5968                    id: ProgramResourceId(5),
5969                    name: "rader_kernel_spectrum".to_owned(),
5970                    kind: ProgramResourceKind::LookupTable,
5971                    scalar: ir.scalar,
5972                    elements: rader.convolution_len,
5973                    external_layout: None,
5974                    initialization: ProgramResourceInitialization::Complex64(
5975                        rader.kernel_spectrum()?.to_vec(),
5976                    ),
5977                },
5978                ProgramResource {
5979                    id: ProgramResourceId(6),
5980                    name: "stockham_input".to_owned(),
5981                    kind: ProgramResourceKind::Scratch,
5982                    scalar: ir.scalar,
5983                    elements: logical_elements,
5984                    external_layout: None,
5985                    initialization: ProgramResourceInitialization::Zeroed,
5986                },
5987                ProgramResource {
5988                    id: ProgramResourceId(7),
5989                    name: "stockham_output".to_owned(),
5990                    kind: ProgramResourceKind::Scratch,
5991                    scalar: ir.scalar,
5992                    elements: logical_elements,
5993                    external_layout: None,
5994                    initialization: ProgramResourceInitialization::Zeroed,
5995                },
5996                ProgramResource {
5997                    id: ProgramResourceId(8),
5998                    name: "output".to_owned(),
5999                    kind: ProgramResourceKind::Output,
6000                    scalar: ir.scalar,
6001                    elements: logical_elements,
6002                    external_layout: Some(external),
6003                    initialization: ProgramResourceInitialization::Zeroed,
6004                },
6005            ],
6006            passes: vec![two_buffer_pass(
6007                &ir.pack_prime.name,
6008                ir.pack_prime.dispatch,
6009                0,
6010                1,
6011            )],
6012            serial: 0,
6013        };
6014        let forward_input = match rader.input_strategy {
6015            RaderFftInputStrategy::GatherReversePass => {
6016                builder.passes.push(two_buffer_pass(
6017                    &rader.gather.name,
6018                    rader.gather.dispatch,
6019                    1,
6020                    2,
6021                ));
6022                ProgramResourceId(2)
6023            }
6024            RaderFftInputStrategy::GeneratorOrderStockham
6025            | RaderFftInputStrategy::GeneratorOrderRecursive => ProgramResourceId(1),
6026        };
6027        builder.flatten_one_dim(&rader.forward_fft, forward_input, ProgramResourceId(3))?;
6028        if let Some(fused_inverse) = rader.fused_inverse_rader_kernel()? {
6029            builder.push_stockham_pass(
6030                &fused_inverse,
6031                ProgramResourceId(3),
6032                ProgramResourceId(4),
6033                Some(ProgramResourceId(5)),
6034                Some(ProgramResourceId(1)),
6035            )?;
6036        } else if rader.has_fused_recursive_inverse() {
6037            let inverse = rader
6038                .inverse_recursive()
6039                .ok_or(VkFftError::InvalidKernelIr(
6040                    "mixed fused recursive Rader program lost its recursive inverse child",
6041                ))?;
6042            builder.flatten_node_with_boundary_resources(
6043                &inverse.root,
6044                ProgramResourceId(3),
6045                ProgramResourceId(4),
6046                Some(ProgramResourceId(5)),
6047                Some(ProgramResourceId(1)),
6048            )?;
6049        } else {
6050            builder.passes.push(ProgramPass {
6051                name: rader.multiply.name.clone(),
6052                dispatch: rader.multiply.dispatch,
6053                bindings: vec![
6054                    binding(0, 3, BufferRole::Input, BufferAccess::ReadOnly),
6055                    binding(1, 2, BufferRole::Output, BufferAccess::WriteOnly),
6056                    binding(2, 5, BufferRole::LookupTable, BufferAccess::ReadOnly),
6057                ],
6058            });
6059            builder.flatten_one_dim(
6060                &rader.inverse_fft,
6061                ProgramResourceId(2),
6062                ProgramResourceId(3),
6063            )?;
6064            builder.passes.push(ProgramPass {
6065                name: rader.scatter.name.clone(),
6066                dispatch: rader.scatter.dispatch,
6067                bindings: vec![
6068                    binding(0, 3, BufferRole::Input, BufferAccess::ReadOnly),
6069                    binding(1, 4, BufferRole::Output, BufferAccess::WriteOnly),
6070                    binding(2, 1, BufferRole::Auxiliary, BufferAccess::ReadOnly),
6071                ],
6072            });
6073        }
6074        builder.passes.push(two_buffer_pass(
6075            &ir.twiddle_transpose.name,
6076            ir.twiddle_transpose.dispatch,
6077            4,
6078            6,
6079        ));
6080        builder.passes.push(two_buffer_pass(
6081            &ir.stockham_stage.name,
6082            ir.stockham_stage.dispatch,
6083            6,
6084            7,
6085        ));
6086        builder.passes.push(two_buffer_pass(
6087            &ir.scatter_output.name,
6088            ir.scatter_output.dispatch,
6089            7,
6090            8,
6091        ));
6092        let program = Self {
6093            name: format!("vkfft_mixed_rader_stockham_program_{}", ir.logical_len),
6094            scalar: ir.scalar,
6095            resources: builder.resources,
6096            passes: builder.passes,
6097        };
6098        program.validate()?;
6099        Ok(program)
6100    }
6101
6102    fn mixed_rader_stockham_direct(
6103        ir: &MixedRaderStockhamIr,
6104        direct: &RaderDirectIr,
6105    ) -> Result<Self> {
6106        let logical_elements =
6107            ir.logical_len
6108                .checked_mul(ir.batch_count)
6109                .ok_or(VkFftError::ArithmeticOverflow {
6110                    operation: "mixed direct-Rader program logical element count",
6111                })?;
6112        let external = ExternalBufferLayout {
6113            logical_len: ir.logical_len,
6114            physical_stride: ir.logical_len,
6115            batch_count: ir.batch_count,
6116            element_shape: ProgramElementShape::Complex,
6117        };
6118        let program = Self {
6119            name: format!(
6120                "vkfft_mixed_direct_rader_stockham_program_{}",
6121                ir.logical_len
6122            ),
6123            scalar: ir.scalar,
6124            resources: vec![
6125                ProgramResource {
6126                    id: ProgramResourceId(0),
6127                    name: "input".to_owned(),
6128                    kind: ProgramResourceKind::Input,
6129                    scalar: ir.scalar,
6130                    elements: logical_elements,
6131                    external_layout: Some(external),
6132                    initialization: ProgramResourceInitialization::ExternalInput,
6133                },
6134                ProgramResource {
6135                    id: ProgramResourceId(1),
6136                    name: "prime_input".to_owned(),
6137                    kind: ProgramResourceKind::Scratch,
6138                    scalar: ir.scalar,
6139                    elements: logical_elements,
6140                    external_layout: None,
6141                    initialization: ProgramResourceInitialization::Zeroed,
6142                },
6143                ProgramResource {
6144                    id: ProgramResourceId(2),
6145                    name: "prime_output".to_owned(),
6146                    kind: ProgramResourceKind::Scratch,
6147                    scalar: ir.scalar,
6148                    elements: logical_elements,
6149                    external_layout: None,
6150                    initialization: ProgramResourceInitialization::Zeroed,
6151                },
6152                ProgramResource {
6153                    id: ProgramResourceId(3),
6154                    name: "rader_roots".to_owned(),
6155                    kind: ProgramResourceKind::LookupTable,
6156                    scalar: ir.scalar,
6157                    elements: direct.twiddle_lut().len(),
6158                    external_layout: None,
6159                    initialization: ProgramResourceInitialization::Complex64(
6160                        direct.twiddle_lut().to_vec(),
6161                    ),
6162                },
6163                ProgramResource {
6164                    id: ProgramResourceId(4),
6165                    name: "stockham_input".to_owned(),
6166                    kind: ProgramResourceKind::Scratch,
6167                    scalar: ir.scalar,
6168                    elements: logical_elements,
6169                    external_layout: None,
6170                    initialization: ProgramResourceInitialization::Zeroed,
6171                },
6172                ProgramResource {
6173                    id: ProgramResourceId(5),
6174                    name: "stockham_output".to_owned(),
6175                    kind: ProgramResourceKind::Scratch,
6176                    scalar: ir.scalar,
6177                    elements: logical_elements,
6178                    external_layout: None,
6179                    initialization: ProgramResourceInitialization::Zeroed,
6180                },
6181                ProgramResource {
6182                    id: ProgramResourceId(6),
6183                    name: "output".to_owned(),
6184                    kind: ProgramResourceKind::Output,
6185                    scalar: ir.scalar,
6186                    elements: logical_elements,
6187                    external_layout: Some(external),
6188                    initialization: ProgramResourceInitialization::Zeroed,
6189                },
6190            ],
6191            passes: vec![
6192                two_buffer_pass(&ir.pack_prime.name, ir.pack_prime.dispatch, 0, 1),
6193                ProgramPass {
6194                    name: direct.name.clone(),
6195                    dispatch: direct.dispatch,
6196                    bindings: vec![
6197                        binding(0, 1, BufferRole::Input, BufferAccess::ReadOnly),
6198                        binding(1, 2, BufferRole::Output, BufferAccess::WriteOnly),
6199                        binding(2, 3, BufferRole::LookupTable, BufferAccess::ReadOnly),
6200                    ],
6201                },
6202                two_buffer_pass(
6203                    &ir.twiddle_transpose.name,
6204                    ir.twiddle_transpose.dispatch,
6205                    2,
6206                    4,
6207                ),
6208                two_buffer_pass(&ir.stockham_stage.name, ir.stockham_stage.dispatch, 4, 5),
6209                two_buffer_pass(&ir.scatter_output.name, ir.scatter_output.dispatch, 5, 6),
6210            ],
6211        };
6212        program.validate()?;
6213        Ok(program)
6214    }
6215
6216    pub fn recursive_fft(ir: &RecursiveFftIr) -> Result<Self> {
6217        ir.validate()?;
6218        let elements =
6219            ir.logical_len
6220                .checked_mul(ir.batch_count)
6221                .ok_or(VkFftError::ArithmeticOverflow {
6222                    operation: "recursive FFT program element count",
6223                })?;
6224        let external = ExternalBufferLayout {
6225            logical_len: ir.logical_len,
6226            physical_stride: ir.logical_len,
6227            batch_count: ir.batch_count,
6228            element_shape: ProgramElementShape::Complex,
6229        };
6230        let external_scalar = ir.external_storage_scalar();
6231        let mut builder = RecursiveProgramBuilder {
6232            scalar: ir.scalar,
6233            resources: vec![
6234                ProgramResource {
6235                    id: ProgramResourceId(0),
6236                    name: "input".to_owned(),
6237                    kind: ProgramResourceKind::Input,
6238                    scalar: external_scalar,
6239                    elements,
6240                    external_layout: Some(external),
6241                    initialization: ProgramResourceInitialization::ExternalInput,
6242                },
6243                ProgramResource {
6244                    id: ProgramResourceId(1),
6245                    name: "output".to_owned(),
6246                    kind: ProgramResourceKind::Output,
6247                    scalar: external_scalar,
6248                    elements,
6249                    external_layout: Some(external),
6250                    initialization: ProgramResourceInitialization::Zeroed,
6251                },
6252            ],
6253            passes: Vec::new(),
6254            serial: 0,
6255        };
6256        builder.flatten_recursive_ir(ir, ProgramResourceId(0), ProgramResourceId(1))?;
6257        let program = Self {
6258            name: format!("vkfft_recursive_fft_program_{}", ir.logical_len),
6259            scalar: ir.scalar,
6260            resources: builder.resources,
6261            passes: builder.passes,
6262        };
6263        program.validate()?;
6264        Ok(program)
6265    }
6266
6267    pub fn one_dim_fft(ir: &OneDimFftIr) -> Result<Self> {
6268        ir.validate()?;
6269        if let OneDimFftIr::Recursive(recursive) = ir
6270            && ir.zero_pad_pass().is_none()
6271        {
6272            return Self::recursive_fft(recursive);
6273        }
6274        let elements = ir.logical_len().checked_mul(ir.batch_count()).ok_or(
6275            VkFftError::ArithmeticOverflow {
6276                operation: "generic one-dimensional FFT program element count",
6277            },
6278        )?;
6279        let external = ExternalBufferLayout {
6280            logical_len: ir.logical_len(),
6281            physical_stride: ir.logical_len(),
6282            batch_count: ir.batch_count(),
6283            element_shape: ProgramElementShape::Complex,
6284        };
6285        let external_scalar = ir.external_storage_scalar();
6286        let mut builder = RecursiveProgramBuilder {
6287            scalar: ir.scalar(),
6288            resources: vec![
6289                ProgramResource {
6290                    id: ProgramResourceId(0),
6291                    name: "input".to_owned(),
6292                    kind: ProgramResourceKind::Input,
6293                    scalar: external_scalar,
6294                    elements,
6295                    external_layout: Some(external),
6296                    initialization: ProgramResourceInitialization::ExternalInput,
6297                },
6298                ProgramResource {
6299                    id: ProgramResourceId(1),
6300                    name: "output".to_owned(),
6301                    kind: ProgramResourceKind::Output,
6302                    scalar: external_scalar,
6303                    elements,
6304                    external_layout: Some(external),
6305                    initialization: ProgramResourceInitialization::Zeroed,
6306                },
6307            ],
6308            passes: Vec::new(),
6309            serial: 0,
6310        };
6311        builder.flatten_one_dim(ir, ProgramResourceId(0), ProgramResourceId(1))?;
6312        let program = Self {
6313            name: format!("vkfft_1d_fft_program_{}", ir.logical_len()),
6314            scalar: ir.scalar(),
6315            resources: builder.resources,
6316            passes: builder.passes,
6317        };
6318        program.validate()?;
6319        Ok(program)
6320    }
6321
6322    pub fn nd_real_fft(ir: &NdRealFftIr) -> Result<Self> {
6323        ir.validate()?;
6324        let full_elements = ir.full_tensor_len.checked_mul(ir.batch_count).ok_or(
6325            VkFftError::ArithmeticOverflow {
6326                operation: "multidimensional real program full element count",
6327            },
6328        )?;
6329        let compact_elements = ir.compact_tensor_len.checked_mul(ir.batch_count).ok_or(
6330            VkFftError::ArithmeticOverflow {
6331                operation: "multidimensional real program compact element count",
6332            },
6333        )?;
6334        let input_tensor_len = ir.input_tensor_len();
6335        let output_tensor_len = ir.output_tensor_len();
6336        let input_layout = ExternalBufferLayout {
6337            logical_len: if ir.input_formatted_copy.is_some() {
6338                ir.input_external_layout.batch_stride
6339            } else {
6340                input_tensor_len
6341            },
6342            physical_stride: if ir.input_formatted_copy.is_some() {
6343                ir.input_external_layout.batch_stride
6344            } else {
6345                input_tensor_len
6346            },
6347            batch_count: ir.batch_count,
6348            element_shape: ProgramElementShape::Complex,
6349        };
6350        let output_layout = ExternalBufferLayout {
6351            logical_len: if ir.output_formatted_copy.is_some() {
6352                ir.output_external_layout.batch_stride
6353            } else {
6354                output_tensor_len
6355            },
6356            physical_stride: if ir.output_formatted_copy.is_some() {
6357                ir.output_external_layout.batch_stride
6358            } else {
6359                output_tensor_len
6360            },
6361            batch_count: ir.batch_count,
6362            element_shape: ProgramElementShape::Complex,
6363        };
6364        let input_elements = input_layout.physical_elements()?;
6365        let output_elements = output_layout.physical_elements()?;
6366        let mut builder = RecursiveProgramBuilder {
6367            scalar: ir.scalar,
6368            resources: vec![
6369                ProgramResource {
6370                    id: ProgramResourceId(0),
6371                    name: "input".to_owned(),
6372                    kind: ProgramResourceKind::Input,
6373                    scalar: if ir.input_boundary_compute_storage {
6374                        ir.scalar
6375                    } else {
6376                        ir.external_scalar
6377                    },
6378                    elements: input_elements,
6379                    external_layout: Some(input_layout),
6380                    initialization: ProgramResourceInitialization::ExternalInput,
6381                },
6382                ProgramResource {
6383                    id: ProgramResourceId(1),
6384                    name: "output".to_owned(),
6385                    kind: ProgramResourceKind::Output,
6386                    scalar: if ir.output_boundary_compute_storage {
6387                        ir.scalar
6388                    } else {
6389                        ir.external_scalar
6390                    },
6391                    elements: output_elements,
6392                    external_layout: Some(output_layout),
6393                    initialization: ProgramResourceInitialization::Zeroed,
6394                },
6395            ],
6396            passes: Vec::new(),
6397            serial: 0,
6398        };
6399
6400        let dense_input = if let Some(copy) = &ir.input_formatted_copy {
6401            let elements = input_tensor_len.checked_mul(ir.batch_count).ok_or(
6402                VkFftError::ArithmeticOverflow {
6403                    operation: "multidimensional real formatted dense input size",
6404                },
6405            )?;
6406            let dense = builder.add_scratch("nd_real_formatted_input", elements);
6407            builder.passes.push(two_buffer_pass(
6408                &copy.name,
6409                copy.dispatch,
6410                ProgramResourceId(0).0,
6411                dense.0,
6412            ));
6413            dense
6414        } else {
6415            ProgramResourceId(0)
6416        };
6417        let dense_output = if ir.output_formatted_copy.is_some() {
6418            let elements = output_tensor_len.checked_mul(ir.batch_count).ok_or(
6419                VkFftError::ArithmeticOverflow {
6420                    operation: "multidimensional real formatted dense output size",
6421                },
6422            )?;
6423            Some(builder.add_scratch("nd_real_formatted_output", elements))
6424        } else {
6425            None
6426        };
6427
6428        match ir.kind {
6429            RealFftKind::RealToComplex => {
6430                let real_input = if let Some(zero_pad) = &ir.zero_pad_pass {
6431                    let boundary = builder.add_scratch("nd_real_zero_padded_input", full_elements);
6432                    builder.push_nd_zero_pad_pass(zero_pad, dense_input, boundary)?;
6433                    boundary
6434                } else {
6435                    dense_input
6436                };
6437                let final_output = dense_output.unwrap_or(ProgramResourceId(1));
6438                let mut natural = if ir.complex_axes.is_empty() {
6439                    final_output
6440                } else {
6441                    builder.add_scratch("nd_real_compact_natural", compact_elements)
6442                };
6443                builder.flatten_real_fft(&ir.real_axis, real_input, natural)?;
6444                for (axis_index, axis) in ir.complex_axes.iter().enumerate() {
6445                    let packed = builder.add_scratch("nd_real_axis_input", compact_elements);
6446                    let transformed = builder.add_scratch("nd_real_axis_output", compact_elements);
6447                    let next = if axis_index + 1 == ir.complex_axes.len() {
6448                        final_output
6449                    } else {
6450                        builder.add_scratch("nd_real_axis_natural", compact_elements)
6451                    };
6452                    builder.passes.push(two_buffer_pass(
6453                        &axis.pack.name,
6454                        axis.pack.dispatch,
6455                        natural.0,
6456                        packed.0,
6457                    ));
6458                    builder.flatten_one_dim(&axis.transform, packed, transformed)?;
6459                    builder.passes.push(two_buffer_pass(
6460                        &axis.scatter.name,
6461                        axis.scatter.dispatch,
6462                        transformed.0,
6463                        next.0,
6464                    ));
6465                    natural = next;
6466                }
6467                if let Some(copy) = &ir.output_formatted_copy {
6468                    builder.passes.push(two_buffer_pass(
6469                        &copy.name,
6470                        copy.dispatch,
6471                        final_output.0,
6472                        ProgramResourceId(1).0,
6473                    ));
6474                }
6475            }
6476            RealFftKind::ComplexToReal => {
6477                let mut natural = dense_input;
6478                for axis in &ir.complex_axes {
6479                    let packed = builder.add_scratch("nd_real_axis_input", compact_elements);
6480                    let transformed = builder.add_scratch("nd_real_axis_output", compact_elements);
6481                    let next = builder.add_scratch("nd_real_axis_natural", compact_elements);
6482                    builder.passes.push(two_buffer_pass(
6483                        &axis.pack.name,
6484                        axis.pack.dispatch,
6485                        natural.0,
6486                        packed.0,
6487                    ));
6488                    builder.flatten_one_dim(&axis.transform, packed, transformed)?;
6489                    builder.passes.push(two_buffer_pass(
6490                        &axis.scatter.name,
6491                        axis.scatter.dispatch,
6492                        transformed.0,
6493                        next.0,
6494                    ));
6495                    natural = next;
6496                }
6497                let final_output = dense_output.unwrap_or(ProgramResourceId(1));
6498                let real_output = if ir.zero_pad_pass.is_some() {
6499                    builder.add_scratch("nd_real_unpadded_output", full_elements)
6500                } else {
6501                    final_output
6502                };
6503                builder.flatten_real_fft(&ir.real_axis, natural, real_output)?;
6504                let dense_final = if let Some(zero_pad) = &ir.zero_pad_pass {
6505                    builder.push_nd_zero_pad_pass(zero_pad, real_output, final_output)?;
6506                    final_output
6507                } else {
6508                    real_output
6509                };
6510                if let Some(copy) = &ir.output_formatted_copy {
6511                    builder.passes.push(two_buffer_pass(
6512                        &copy.name,
6513                        copy.dispatch,
6514                        dense_final.0,
6515                        ProgramResourceId(1).0,
6516                    ));
6517                }
6518            }
6519        }
6520        let program = Self {
6521            name: format!("vkfft_nd_real_program_{:?}_{:?}", ir.dimensions, ir.kind),
6522            scalar: ir.scalar,
6523            resources: builder.resources,
6524            passes: builder.passes,
6525        };
6526        program.validate()?;
6527        Ok(program)
6528    }
6529
6530    pub fn nd_fft(ir: &NdFftIr) -> Result<Self> {
6531        ir.validate()?;
6532        let elements =
6533            ir.tensor_len
6534                .checked_mul(ir.batch_count)
6535                .ok_or(VkFftError::ArithmeticOverflow {
6536                    operation: "multidimensional FFT program element count",
6537                })?;
6538        let input_external_logical_len = if ir.has_formatted_input_tensor_strides()? {
6539            ir.input_buffer_batch_stride
6540        } else {
6541            ir.tensor_len
6542        };
6543        let output_external_logical_len = if ir.has_formatted_output_tensor_strides()? {
6544            ir.output_buffer_batch_stride
6545        } else {
6546            ir.tensor_len
6547        };
6548        let input_external = ExternalBufferLayout {
6549            logical_len: input_external_logical_len,
6550            physical_stride: ir.input_buffer_batch_stride,
6551            batch_count: ir.batch_count,
6552            element_shape: ProgramElementShape::Complex,
6553        };
6554        let output_external = ExternalBufferLayout {
6555            logical_len: output_external_logical_len,
6556            physical_stride: ir.output_buffer_batch_stride,
6557            batch_count: ir.batch_count,
6558            element_shape: ProgramElementShape::Complex,
6559        };
6560        let input_elements = input_external.physical_elements()?;
6561        let output_elements = output_external.physical_elements()?;
6562        let mut builder = RecursiveProgramBuilder {
6563            scalar: ir.scalar,
6564            resources: vec![
6565                ProgramResource {
6566                    id: ProgramResourceId(0),
6567                    name: "input".to_owned(),
6568                    kind: ProgramResourceKind::Input,
6569                    scalar: ir.external_scalar,
6570                    elements: input_elements,
6571                    external_layout: Some(input_external),
6572                    initialization: ProgramResourceInitialization::ExternalInput,
6573                },
6574                ProgramResource {
6575                    id: ProgramResourceId(1),
6576                    name: "output".to_owned(),
6577                    kind: ProgramResourceKind::Output,
6578                    scalar: ir.external_scalar,
6579                    elements: output_elements,
6580                    external_layout: Some(output_external),
6581                    initialization: ProgramResourceInitialization::Zeroed,
6582                },
6583            ],
6584            passes: Vec::new(),
6585            serial: 0,
6586        };
6587        let mut natural_input = ProgramResourceId(0);
6588        if let Some(copy) = &ir.input_formatted_copy {
6589            let dense = builder.add_scratch("nd_formatted_input", elements);
6590            builder.passes.push(two_buffer_pass(
6591                &copy.name,
6592                copy.dispatch,
6593                natural_input.0,
6594                dense.0,
6595            ));
6596            natural_input = dense;
6597        }
6598        if let Some(pass) = &ir.zero_pad_pass
6599            && pass.operation.is_input_boundary()
6600        {
6601            let boundary = builder.add_scratch("nd_zero_pad_boundary", elements);
6602            builder.push_nd_zero_pad_pass(pass, natural_input, boundary)?;
6603            natural_input = boundary;
6604        }
6605        for (axis_index, axis) in ir.axes.iter().enumerate() {
6606            let packed_input = builder.add_scratch("nd_axis_input", elements);
6607            let packed_output = builder.add_scratch("nd_axis_output", elements);
6608            let natural_output = if axis_index + 1 == ir.axes.len() {
6609                if ir
6610                    .zero_pad_pass
6611                    .as_ref()
6612                    .is_some_and(|pass| pass.operation.is_output_boundary())
6613                    || ir.output_formatted_copy.is_some()
6614                {
6615                    builder.add_scratch("nd_external_boundary", elements)
6616                } else {
6617                    ProgramResourceId(1)
6618                }
6619            } else {
6620                builder.add_scratch("nd_natural_output", elements)
6621            };
6622            builder.passes.push(two_buffer_pass(
6623                &axis.pack.name,
6624                axis.pack.dispatch,
6625                natural_input.0,
6626                packed_input.0,
6627            ));
6628            builder.flatten_one_dim(&axis.transform, packed_input, packed_output)?;
6629            builder.passes.push(two_buffer_pass(
6630                &axis.scatter.name,
6631                axis.scatter.dispatch,
6632                packed_output.0,
6633                natural_output.0,
6634            ));
6635            natural_input = natural_output;
6636        }
6637        if let Some(pass) = &ir.zero_pad_pass
6638            && pass.operation.is_output_boundary()
6639        {
6640            let target = if ir.output_formatted_copy.is_some() {
6641                builder.add_scratch("nd_formatted_output", elements)
6642            } else {
6643                ProgramResourceId(1)
6644            };
6645            builder.push_nd_zero_pad_pass(pass, natural_input, target)?;
6646            natural_input = target;
6647        }
6648        if let Some(copy) = &ir.output_formatted_copy {
6649            builder.passes.push(two_buffer_pass(
6650                &copy.name,
6651                copy.dispatch,
6652                natural_input.0,
6653                ProgramResourceId(1).0,
6654            ));
6655        }
6656        let program = Self {
6657            name: format!("vkfft_nd_fft_program_{:?}", ir.dimensions),
6658            scalar: ir.scalar,
6659            resources: builder.resources,
6660            passes: builder.passes,
6661        };
6662        program.validate()?;
6663        Ok(program)
6664    }
6665
6666    pub fn real_fft(ir: &RealFftIr) -> Result<Self> {
6667        ir.validate()?;
6668        let full_elements =
6669            ir.length
6670                .checked_mul(ir.batch_count)
6671                .ok_or(VkFftError::ArithmeticOverflow {
6672                    operation: "real FFT program full element count",
6673                })?;
6674        let half_elements = ir.half_spectrum_len.checked_mul(ir.batch_count).ok_or(
6675            VkFftError::ArithmeticOverflow {
6676                operation: "real FFT program half-spectrum element count",
6677            },
6678        )?;
6679        let (input_elements, output_elements, input_layout, output_layout) = match ir.kind {
6680            RealFftKind::RealToComplex => (
6681                full_elements,
6682                half_elements,
6683                ExternalBufferLayout {
6684                    logical_len: ir.length,
6685                    physical_stride: ir.length,
6686                    batch_count: ir.batch_count,
6687                    element_shape: ProgramElementShape::Complex,
6688                },
6689                ExternalBufferLayout {
6690                    logical_len: ir.half_spectrum_len,
6691                    physical_stride: ir.half_spectrum_len,
6692                    batch_count: ir.batch_count,
6693                    element_shape: ProgramElementShape::Complex,
6694                },
6695            ),
6696            RealFftKind::ComplexToReal => (
6697                half_elements,
6698                full_elements,
6699                ExternalBufferLayout {
6700                    logical_len: ir.half_spectrum_len,
6701                    physical_stride: ir.half_spectrum_len,
6702                    batch_count: ir.batch_count,
6703                    element_shape: ProgramElementShape::Complex,
6704                },
6705                ExternalBufferLayout {
6706                    logical_len: ir.length,
6707                    physical_stride: ir.length,
6708                    batch_count: ir.batch_count,
6709                    element_shape: ProgramElementShape::Complex,
6710                },
6711            ),
6712        };
6713        let mut builder = RecursiveProgramBuilder {
6714            scalar: ir.scalar,
6715            resources: vec![
6716                ProgramResource {
6717                    id: ProgramResourceId(0),
6718                    name: "input".to_owned(),
6719                    kind: ProgramResourceKind::Input,
6720                    scalar: ir.input_storage_scalar,
6721                    elements: input_elements,
6722                    external_layout: Some(input_layout),
6723                    initialization: ProgramResourceInitialization::ExternalInput,
6724                },
6725                ProgramResource {
6726                    id: ProgramResourceId(1),
6727                    name: "output".to_owned(),
6728                    kind: ProgramResourceKind::Output,
6729                    scalar: ir.output_storage_scalar,
6730                    elements: output_elements,
6731                    external_layout: Some(output_layout),
6732                    initialization: ProgramResourceInitialization::Zeroed,
6733                },
6734            ],
6735            passes: Vec::new(),
6736            serial: 0,
6737        };
6738        let external_input = ProgramResourceId(0);
6739        let external_output = ProgramResourceId(1);
6740        let transform_input = if ir.kind == RealFftKind::RealToComplex {
6741            if let Some(zero_pad) = &ir.zero_pad_pass {
6742                let scratch = builder.add_scratch("real_zero_padded_input", full_elements);
6743                builder.passes.push(two_buffer_pass(
6744                    &zero_pad.name,
6745                    zero_pad.dispatch,
6746                    external_input.0,
6747                    scratch.0,
6748                ));
6749                scratch
6750            } else {
6751                external_input
6752            }
6753        } else {
6754            external_input
6755        };
6756        let transform_output =
6757            if ir.kind == RealFftKind::ComplexToReal && ir.zero_pad_pass.is_some() {
6758                builder.add_scratch("real_unpadded_output", full_elements)
6759            } else {
6760                external_output
6761            };
6762        builder.flatten_real_fft(ir, transform_input, transform_output)?;
6763        if ir.kind == RealFftKind::ComplexToReal
6764            && let Some(zero_pad) = &ir.zero_pad_pass
6765        {
6766            builder.passes.push(two_buffer_pass(
6767                &zero_pad.name,
6768                zero_pad.dispatch,
6769                transform_output.0,
6770                external_output.0,
6771            ));
6772        }
6773        let program = Self {
6774            name: format!(
6775                "vkfft_real_fft_program_{}_{}",
6776                ir.length,
6777                match ir.kind {
6778                    RealFftKind::RealToComplex => "r2c",
6779                    RealFftKind::ComplexToReal => "c2r",
6780                }
6781            ),
6782            scalar: ir.scalar,
6783            resources: builder.resources,
6784            passes: builder.passes,
6785        };
6786        program.validate()?;
6787        Ok(program)
6788    }
6789
6790    /// Build a backend-neutral physical allocation plan for the logical resources.
6791    /// External input/output stay dedicated. Identical immutable LUTs share one
6792    /// allocation, while scratch resources can reuse a slot after their previous
6793    /// lifetime ends when their first access is `WriteOnly` (the full-overwrite
6794    /// contract used by the generated kernels in this crate).
6795    pub fn memory_plan(&self) -> Result<ProgramMemoryPlan> {
6796        self.validate()?;
6797        let mut resource_allocations = vec![ProgramAllocationId(usize::MAX); self.resources.len()];
6798        let mut allocations = Vec::<ProgramAllocation>::new();
6799
6800        let add_dedicated =
6801            |resource: &ProgramResource,
6802             kind: ProgramAllocationKind,
6803             allocations: &mut Vec<ProgramAllocation>,
6804             resource_allocations: &mut Vec<ProgramAllocationId>| {
6805                let id = ProgramAllocationId(allocations.len());
6806                allocations.push(ProgramAllocation {
6807                    id,
6808                    kind,
6809                    scalar: resource.scalar,
6810                    elements: resource.elements,
6811                    resources: vec![resource.id],
6812                });
6813                resource_allocations[resource.id.0] = id;
6814            };
6815
6816        for resource in &self.resources {
6817            match resource.kind {
6818                ProgramResourceKind::Input => add_dedicated(
6819                    resource,
6820                    ProgramAllocationKind::ExternalInput,
6821                    &mut allocations,
6822                    &mut resource_allocations,
6823                ),
6824                ProgramResourceKind::Output => add_dedicated(
6825                    resource,
6826                    ProgramAllocationKind::ExternalOutput,
6827                    &mut allocations,
6828                    &mut resource_allocations,
6829                ),
6830                ProgramResourceKind::Scratch | ProgramResourceKind::LookupTable => {}
6831            }
6832        }
6833
6834        for resource in self
6835            .resources
6836            .iter()
6837            .filter(|resource| resource.kind == ProgramResourceKind::LookupTable)
6838        {
6839            let shared = self.resources[..resource.id.0].iter().find(|candidate| {
6840                candidate.kind == ProgramResourceKind::LookupTable
6841                    && candidate.scalar == resource.scalar
6842                    && candidate.elements == resource.elements
6843                    && candidate.initialization == resource.initialization
6844            });
6845            if let Some(shared) = shared {
6846                let allocation_id = resource_allocations[shared.id.0];
6847                if allocation_id.0 == usize::MAX {
6848                    return Err(VkFftError::InvalidKernelIr(
6849                        "program LUT deduplication encountered an unassigned resource",
6850                    ));
6851                }
6852                resource_allocations[resource.id.0] = allocation_id;
6853                allocations[allocation_id.0].resources.push(resource.id);
6854            } else {
6855                add_dedicated(
6856                    resource,
6857                    ProgramAllocationKind::LookupTable,
6858                    &mut allocations,
6859                    &mut resource_allocations,
6860                );
6861            }
6862        }
6863
6864        #[derive(Clone, Copy)]
6865        struct ScratchLifetime {
6866            resource: ProgramResourceId,
6867            first: usize,
6868            last: usize,
6869            first_access: BufferAccess,
6870        }
6871
6872        let mut scratch_lifetimes = Vec::<ScratchLifetime>::new();
6873        for resource in self
6874            .resources
6875            .iter()
6876            .filter(|resource| resource.kind == ProgramResourceKind::Scratch)
6877        {
6878            let mut first = None::<(usize, BufferAccess)>;
6879            let mut last = None::<usize>;
6880            for (pass_index, pass) in self.passes.iter().enumerate() {
6881                for binding in pass
6882                    .bindings
6883                    .iter()
6884                    .filter(|binding| binding.resource == resource.id)
6885                {
6886                    if first.is_none() {
6887                        first = Some((pass_index, binding.access));
6888                    }
6889                    last = Some(pass_index);
6890                }
6891            }
6892            if let (Some((first, first_access)), Some(last)) = (first, last) {
6893                scratch_lifetimes.push(ScratchLifetime {
6894                    resource: resource.id,
6895                    first,
6896                    last,
6897                    first_access,
6898                });
6899            } else {
6900                add_dedicated(
6901                    resource,
6902                    ProgramAllocationKind::Scratch,
6903                    &mut allocations,
6904                    &mut resource_allocations,
6905                );
6906            }
6907        }
6908        scratch_lifetimes.sort_by_key(|lifetime| (lifetime.first, lifetime.last));
6909
6910        let mut scratch_slots = Vec::<(ProgramAllocationId, usize)>::new();
6911        for lifetime in scratch_lifetimes {
6912            let resource = &self.resources[lifetime.resource.0];
6913            let can_alias = lifetime.first_access == BufferAccess::WriteOnly
6914                && matches!(
6915                    resource.initialization,
6916                    ProgramResourceInitialization::Zeroed
6917                );
6918            let reusable = if can_alias {
6919                scratch_slots
6920                    .iter()
6921                    .enumerate()
6922                    .filter(|(_, (allocation_id, last_pass))| {
6923                        if *last_pass >= lifetime.first {
6924                            return false;
6925                        }
6926                        allocations[allocation_id.0]
6927                            .resources
6928                            .first()
6929                            .map(|resource_id| self.resources[resource_id.0].element_shape())
6930                            == Some(resource.element_shape())
6931                    })
6932                    .min_by_key(|(_, (allocation_id, _))| {
6933                        let allocation = &allocations[allocation_id.0];
6934                        allocation.elements.max(resource.elements)
6935                    })
6936                    .map(|(index, _)| index)
6937            } else {
6938                None
6939            };
6940
6941            if let Some(slot_index) = reusable {
6942                let (allocation_id, last_pass) = &mut scratch_slots[slot_index];
6943                let allocation = &mut allocations[allocation_id.0];
6944                allocation.elements = allocation.elements.max(resource.elements);
6945                allocation.resources.push(resource.id);
6946                resource_allocations[resource.id.0] = *allocation_id;
6947                *last_pass = lifetime.last;
6948            } else {
6949                let id = ProgramAllocationId(allocations.len());
6950                allocations.push(ProgramAllocation {
6951                    id,
6952                    kind: ProgramAllocationKind::Scratch,
6953                    scalar: resource.scalar,
6954                    elements: resource.elements,
6955                    resources: vec![resource.id],
6956                });
6957                resource_allocations[resource.id.0] = id;
6958                scratch_slots.push((id, lifetime.last));
6959            }
6960        }
6961
6962        if resource_allocations.iter().any(|id| id.0 == usize::MAX) {
6963            return Err(VkFftError::InvalidKernelIr(
6964                "program memory planner left a logical resource unassigned",
6965            ));
6966        }
6967        Ok(ProgramMemoryPlan {
6968            allocations,
6969            resource_allocations,
6970        })
6971    }
6972
6973    pub fn validate(&self) -> Result<()> {
6974        if self.resources.is_empty() || self.passes.is_empty() {
6975            return Err(VkFftError::InvalidKernelIr(
6976                "program requires at least one resource and one pass",
6977            ));
6978        }
6979        let mut input_count = 0usize;
6980        let mut output_count = 0usize;
6981        for (index, resource) in self.resources.iter().enumerate() {
6982            let external_mixed_scalar = matches!(
6983                resource.kind,
6984                ProgramResourceKind::Input | ProgramResourceKind::Output
6985            ) && matches!(
6986                (self.scalar, resource.scalar),
6987                (ScalarType::F64, ScalarType::F32)
6988                    | (ScalarType::F32, ScalarType::F16)
6989                    | (ScalarType::DoubleDouble, ScalarType::F64)
6990            );
6991            let boundary_mixed_scratch = resource.kind == ProgramResourceKind::Scratch
6992                && self.scalar == ScalarType::DoubleDouble
6993                && resource.scalar == ScalarType::F64;
6994            if resource.id.0 != index
6995                || resource.elements == 0
6996                || (resource.scalar != self.scalar
6997                    && !external_mixed_scalar
6998                    && !boundary_mixed_scratch)
6999            {
7000                return Err(VkFftError::InvalidKernelIr(
7001                    "program resources must have contiguous IDs, non-zero sizes, and a compute-compatible scalar type",
7002                ));
7003            }
7004            match resource.kind {
7005                ProgramResourceKind::Input => input_count += 1,
7006                ProgramResourceKind::Output => output_count += 1,
7007                ProgramResourceKind::Scratch | ProgramResourceKind::LookupTable => {}
7008            }
7009            if let Some(layout) = resource.external_layout {
7010                layout.validate()?;
7011                if layout.physical_elements()? != resource.elements {
7012                    return Err(VkFftError::InvalidKernelIr(
7013                        "program external resource layout does not cover its physical allocation",
7014                    ));
7015                }
7016            }
7017            match &resource.initialization {
7018                ProgramResourceInitialization::ExternalInput => {
7019                    if resource.kind != ProgramResourceKind::Input
7020                        || resource.external_layout.is_none()
7021                    {
7022                        return Err(VkFftError::InvalidKernelIr(
7023                            "program external input initialization requires an external input resource",
7024                        ));
7025                    }
7026                }
7027                ProgramResourceInitialization::Zeroed => {}
7028                ProgramResourceInitialization::Complex64(values) => {
7029                    if resource.kind != ProgramResourceKind::LookupTable
7030                        || values.len() != resource.elements
7031                        || values
7032                            .iter()
7033                            .any(|value| !value.re.is_finite() || !value.im.is_finite())
7034                    {
7035                        return Err(VkFftError::InvalidKernelIr(
7036                            "program LUT initialization does not match the resource",
7037                        ));
7038                    }
7039                }
7040                ProgramResourceInitialization::ComplexDoubleDouble(values) => {
7041                    if resource.kind != ProgramResourceKind::LookupTable
7042                        || resource.scalar != ScalarType::DoubleDouble
7043                        || values.len() != resource.elements
7044                        || values.iter().any(|value| {
7045                            !value.re.hi.is_finite()
7046                                || !value.re.lo.is_finite()
7047                                || !value.im.hi.is_finite()
7048                                || !value.im.lo.is_finite()
7049                        })
7050                    {
7051                        return Err(VkFftError::InvalidKernelIr(
7052                            "program double-double LUT initialization does not match the resource",
7053                        ));
7054                    }
7055                }
7056                ProgramResourceInitialization::StockhamUnitRoots { len } => {
7057                    if resource.kind != ProgramResourceKind::LookupTable
7058                        || *len == 0
7059                        || *len != resource.elements
7060                    {
7061                        return Err(VkFftError::InvalidKernelIr(
7062                            "program Stockham unit-root initialization does not match the resource",
7063                        ));
7064                    }
7065                }
7066            }
7067        }
7068        if input_count != 1 || output_count != 1 {
7069            return Err(VkFftError::InvalidKernelIr(
7070                "program currently requires exactly one external input and one output resource",
7071            ));
7072        }
7073        let input_scalar = self.input_resource()?.scalar;
7074        let output_scalar = self.output_resource()?.scalar;
7075        let supports_external_scalar = |external_scalar| {
7076            external_scalar == self.scalar
7077                || matches!(
7078                    (self.scalar, external_scalar),
7079                    (ScalarType::F64, ScalarType::F32)
7080                        | (ScalarType::F32, ScalarType::F16)
7081                        | (ScalarType::DoubleDouble, ScalarType::F64)
7082                )
7083        };
7084        if !supports_external_scalar(input_scalar) || !supports_external_scalar(output_scalar) {
7085            return Err(VkFftError::InvalidKernelIr(
7086                "program external storage scalar is inconsistent with compute precision",
7087            ));
7088        }
7089
7090        let input_id = self
7091            .resources
7092            .iter()
7093            .find(|resource| resource.kind == ProgramResourceKind::Input)
7094            .map(|resource| resource.id)
7095            .expect("validated input resource");
7096        let output_id = self
7097            .resources
7098            .iter()
7099            .find(|resource| resource.kind == ProgramResourceKind::Output)
7100            .map(|resource| resource.id)
7101            .expect("validated output resource");
7102        let mut reads_external_input = false;
7103        let mut writes_external_output = false;
7104        for pass in &self.passes {
7105            if pass.name.is_empty()
7106                || pass.dispatch.x == 0
7107                || pass.dispatch.y == 0
7108                || pass.dispatch.z == 0
7109                || pass.bindings.len() < 2
7110            {
7111                return Err(VkFftError::InvalidKernelIr(
7112                    "program pass metadata is incomplete",
7113                ));
7114            }
7115            let mut seen_bindings = Vec::with_capacity(pass.bindings.len());
7116            for pass_binding in &pass.bindings {
7117                if seen_bindings.contains(&pass_binding.binding) {
7118                    return Err(VkFftError::InvalidKernelIr(
7119                        "program pass has duplicate descriptor bindings",
7120                    ));
7121                }
7122                seen_bindings.push(pass_binding.binding);
7123                let resource = self.resource(pass_binding.resource)?;
7124                if matches!(
7125                    pass_binding.role,
7126                    BufferRole::LookupTable | BufferRole::TwiddleLookupTable
7127                ) && pass_binding.access != BufferAccess::ReadOnly
7128                {
7129                    return Err(VkFftError::InvalidKernelIr(
7130                        "program immutable LUT bindings must be read-only",
7131                    ));
7132                }
7133                if pass_binding.role == BufferRole::Auxiliary
7134                    && !(pass_binding.access == BufferAccess::ReadOnly
7135                        || (pass_binding.access == BufferAccess::ReadWrite
7136                            && resource.kind == ProgramResourceKind::Scratch))
7137                {
7138                    return Err(VkFftError::InvalidKernelIr(
7139                        "program auxiliary bindings must be read-only or read-write scratch",
7140                    ));
7141                }
7142                if pass_binding.resource == input_id
7143                    && pass_binding.access == BufferAccess::ReadOnly
7144                {
7145                    reads_external_input = true;
7146                }
7147                if pass_binding.resource == output_id
7148                    && matches!(
7149                        pass_binding.access,
7150                        BufferAccess::WriteOnly | BufferAccess::ReadWrite
7151                    )
7152                {
7153                    writes_external_output = true;
7154                }
7155                if resource.kind == ProgramResourceKind::LookupTable
7156                    && pass_binding.access != BufferAccess::ReadOnly
7157                {
7158                    return Err(VkFftError::InvalidKernelIr(
7159                        "program LUT resources are immutable",
7160                    ));
7161                }
7162            }
7163        }
7164        if !reads_external_input || !writes_external_output {
7165            return Err(VkFftError::InvalidKernelIr(
7166                "program must read its external input and write its external output",
7167            ));
7168        }
7169        Ok(())
7170    }
7171
7172    pub fn resource(&self, id: ProgramResourceId) -> Result<&ProgramResource> {
7173        self.resources
7174            .get(id.0)
7175            .filter(|resource| resource.id == id)
7176            .ok_or(VkFftError::InvalidKernelIr(
7177                "program pass references an unknown resource",
7178            ))
7179    }
7180
7181    pub fn input_resource(&self) -> Result<&ProgramResource> {
7182        self.resources
7183            .iter()
7184            .find(|resource| resource.kind == ProgramResourceKind::Input)
7185            .ok_or(VkFftError::InvalidKernelIr("program has no input resource"))
7186    }
7187
7188    pub fn output_resource(&self) -> Result<&ProgramResource> {
7189        self.resources
7190            .iter()
7191            .find(|resource| resource.kind == ProgramResourceKind::Output)
7192            .ok_or(VkFftError::InvalidKernelIr(
7193                "program has no output resource",
7194            ))
7195    }
7196}
7197
7198fn add_double_double_recursive_scratch(
7199    resources: &mut Vec<ProgramResource>,
7200    serial: &mut usize,
7201    stem: &str,
7202    elements: usize,
7203) -> ProgramResourceId {
7204    *serial += 1;
7205    let id = ProgramResourceId(resources.len());
7206    resources.push(ProgramResource {
7207        id,
7208        name: format!("{stem}_{}", *serial),
7209        kind: ProgramResourceKind::Scratch,
7210        scalar: ScalarType::DoubleDouble,
7211        elements,
7212        external_layout: None,
7213        initialization: ProgramResourceInitialization::Zeroed,
7214    });
7215    id
7216}
7217
7218fn append_double_double_recursive_child(
7219    child: &ProgramIr,
7220    prefix: &str,
7221    input: ProgramResourceId,
7222    output: ProgramResourceId,
7223    resources: &mut Vec<ProgramResource>,
7224    passes: &mut Vec<ProgramPass>,
7225    allow_output_storage_override: bool,
7226) -> Result<()> {
7227    let parent_input_scalar = resources
7228        .get(input.0)
7229        .ok_or(VkFftError::InvalidKernelIr(
7230            "double-double recursive child input resource is missing",
7231        ))?
7232        .scalar;
7233    let parent_output_scalar = resources
7234        .get(output.0)
7235        .ok_or(VkFftError::InvalidKernelIr(
7236            "double-double recursive child output resource is missing",
7237        ))?
7238        .scalar;
7239    let child_output_scalar = child.output_resource()?.scalar;
7240    let output_matches = child_output_scalar == parent_output_scalar
7241        || (allow_output_storage_override
7242            && child_output_scalar == ScalarType::DoubleDouble
7243            && matches!(
7244                parent_output_scalar,
7245                ScalarType::DoubleDouble | ScalarType::F64
7246            ));
7247    if child.scalar != ScalarType::DoubleDouble
7248        || child.input_resource()?.scalar != parent_input_scalar
7249        || !output_matches
7250    {
7251        return Err(VkFftError::InvalidKernelIr(
7252            "double-double recursive leaf ProgramIr boundary storage does not match parent resources",
7253        ));
7254    }
7255    let mut resource_map = vec![None; child.resources.len()];
7256    for resource in &child.resources {
7257        let mapped = match resource.kind {
7258            ProgramResourceKind::Input => input,
7259            ProgramResourceKind::Output => output,
7260            _ => {
7261                let id = ProgramResourceId(resources.len());
7262                let mut cloned = resource.clone();
7263                cloned.id = id;
7264                cloned.name = format!("{prefix}_{}", resource.name);
7265                cloned.external_layout = None;
7266                resources.push(cloned);
7267                id
7268            }
7269        };
7270        resource_map[resource.id.0] = Some(mapped);
7271    }
7272    for child_pass in &child.passes {
7273        let mut pass = child_pass.clone();
7274        pass.name = format!("{prefix}_{}", child_pass.name);
7275        for binding in &mut pass.bindings {
7276            binding.resource = resource_map
7277                .get(binding.resource.0)
7278                .and_then(|mapped| *mapped)
7279                .ok_or(VkFftError::InvalidKernelIr(
7280                    "double-double recursive child resource mapping is incomplete",
7281                ))?;
7282        }
7283        passes.push(pass);
7284    }
7285    Ok(())
7286}
7287
7288fn flatten_double_double_recursive_node(
7289    node: &DoubleDoubleRecursiveFftNodeIr,
7290    input: ProgramResourceId,
7291    output: ProgramResourceId,
7292    resources: &mut Vec<ProgramResource>,
7293    passes: &mut Vec<ProgramPass>,
7294    serial: &mut usize,
7295) -> Result<()> {
7296    node.validate()?;
7297    match node {
7298        DoubleDoubleRecursiveFftNodeIr::Stockham(ir) => {
7299            let child = ProgramIr::double_double_stockham(ir)?;
7300            *serial += 1;
7301            append_double_double_recursive_child(
7302                &child,
7303                &format!("vkfft_dd_recursive_leaf_{}", *serial),
7304                input,
7305                output,
7306                resources,
7307                passes,
7308                false,
7309            )?;
7310        }
7311        DoubleDoubleRecursiveFftNodeIr::DirectRader(ir) => {
7312            let child = ProgramIr::double_double_direct_rader(ir)?;
7313            *serial += 1;
7314            append_double_double_recursive_child(
7315                &child,
7316                &format!("vkfft_dd_recursive_leaf_{}", *serial),
7317                input,
7318                output,
7319                resources,
7320                passes,
7321                false,
7322            )?;
7323        }
7324        DoubleDoubleRecursiveFftNodeIr::FftRader(ir) => {
7325            let child = ProgramIr::double_double_fft_rader(ir)?;
7326            *serial += 1;
7327            append_double_double_recursive_child(
7328                &child,
7329                &format!("vkfft_dd_recursive_leaf_{}", *serial),
7330                input,
7331                output,
7332                resources,
7333                passes,
7334                false,
7335            )?;
7336        }
7337        DoubleDoubleRecursiveFftNodeIr::Bluestein(ir) => {
7338            let child = ProgramIr::double_double_bluestein(ir)?;
7339            *serial += 1;
7340            append_double_double_recursive_child(
7341                &child,
7342                &format!("vkfft_dd_recursive_leaf_{}", *serial),
7343                input,
7344                output,
7345                resources,
7346                passes,
7347                false,
7348            )?;
7349        }
7350        DoubleDoubleRecursiveFftNodeIr::CooleyTukey(ir) => {
7351            if let Some(fused) = ir.fused_small_direct_rader_stockham()? {
7352                let direct_lut_id = ProgramResourceId(resources.len());
7353                resources.push(ProgramResource {
7354                    id: direct_lut_id,
7355                    name: "double_double_composite_direct_rader_roots".to_owned(),
7356                    kind: ProgramResourceKind::LookupTable,
7357                    scalar: ScalarType::DoubleDouble,
7358                    elements: fused.direct.table.twiddles_by_generator_power.len(),
7359                    external_layout: None,
7360                    initialization: ProgramResourceInitialization::ComplexDoubleDouble(
7361                        fused.direct.table.twiddles_by_generator_power.clone(),
7362                    ),
7363                });
7364                let parent_roots_id = ProgramResourceId(resources.len());
7365                resources.push(ProgramResource {
7366                    id: parent_roots_id,
7367                    name: "double_double_composite_parent_roots".to_owned(),
7368                    kind: ProgramResourceKind::LookupTable,
7369                    scalar: ScalarType::DoubleDouble,
7370                    elements: ir.twiddles.len(),
7371                    external_layout: None,
7372                    initialization: ProgramResourceInitialization::ComplexDoubleDouble(
7373                        ir.twiddles.clone(),
7374                    ),
7375                });
7376                passes.push(ProgramPass {
7377                    name: fused.name(),
7378                    dispatch: ir.pack_right.dispatch,
7379                    bindings: vec![
7380                        binding(0, input.0, BufferRole::Input, BufferAccess::ReadOnly),
7381                        binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
7382                        ProgramPassBinding {
7383                            binding: 2,
7384                            resource: direct_lut_id,
7385                            role: BufferRole::LookupTable,
7386                            access: BufferAccess::ReadOnly,
7387                        },
7388                        ProgramPassBinding {
7389                            binding: 3,
7390                            resource: parent_roots_id,
7391                            role: BufferRole::TwiddleLookupTable,
7392                            access: BufferAccess::ReadOnly,
7393                        },
7394                    ],
7395                });
7396                return Ok(());
7397            }
7398            if let Some(fused) = ir.fused_small_fft_rader_stockham()? {
7399                let spectrum_id = ProgramResourceId(resources.len());
7400                resources.push(ProgramResource {
7401                    id: spectrum_id,
7402                    name: "double_double_composite_fft_rader_spectrum".to_owned(),
7403                    kind: ProgramResourceKind::LookupTable,
7404                    scalar: ScalarType::DoubleDouble,
7405                    elements: fused.rader.kernel_spectrum.len(),
7406                    external_layout: None,
7407                    initialization: ProgramResourceInitialization::ComplexDoubleDouble(
7408                        fused.rader.kernel_spectrum.clone(),
7409                    ),
7410                });
7411                let parent_roots_id = ProgramResourceId(resources.len());
7412                resources.push(ProgramResource {
7413                    id: parent_roots_id,
7414                    name: "double_double_composite_fft_rader_parent_roots".to_owned(),
7415                    kind: ProgramResourceKind::LookupTable,
7416                    scalar: ScalarType::DoubleDouble,
7417                    elements: ir.twiddles.len(),
7418                    external_layout: None,
7419                    initialization: ProgramResourceInitialization::ComplexDoubleDouble(
7420                        ir.twiddles.clone(),
7421                    ),
7422                });
7423                passes.push(ProgramPass {
7424                    name: fused.name(),
7425                    dispatch: ir.pack_right.dispatch,
7426                    bindings: vec![
7427                        binding(0, input.0, BufferRole::Input, BufferAccess::ReadOnly),
7428                        binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
7429                        ProgramPassBinding {
7430                            binding: 2,
7431                            resource: spectrum_id,
7432                            role: BufferRole::LookupTable,
7433                            access: BufferAccess::ReadOnly,
7434                        },
7435                        ProgramPassBinding {
7436                            binding: 3,
7437                            resource: parent_roots_id,
7438                            role: BufferRole::TwiddleLookupTable,
7439                            access: BufferAccess::ReadOnly,
7440                        },
7441                    ],
7442                });
7443                return Ok(());
7444            }
7445            if let Some(fused) = ir.fused_dual_fft_rader_supported()? {
7446                let right_spectrum_id = ProgramResourceId(resources.len());
7447                resources.push(ProgramResource {
7448                    id: right_spectrum_id,
7449                    name: "double_double_composite_dual_fft_rader_right_spectrum".to_owned(),
7450                    kind: ProgramResourceKind::LookupTable,
7451                    scalar: ScalarType::DoubleDouble,
7452                    elements: fused.right.kernel_spectrum.len(),
7453                    external_layout: None,
7454                    initialization: ProgramResourceInitialization::ComplexDoubleDouble(
7455                        fused.right.kernel_spectrum.clone(),
7456                    ),
7457                });
7458                let left_spectrum_id = ProgramResourceId(resources.len());
7459                resources.push(ProgramResource {
7460                    id: left_spectrum_id,
7461                    name: "double_double_composite_dual_fft_rader_left_spectrum".to_owned(),
7462                    kind: ProgramResourceKind::LookupTable,
7463                    scalar: ScalarType::DoubleDouble,
7464                    elements: fused.left.kernel_spectrum.len(),
7465                    external_layout: None,
7466                    initialization: ProgramResourceInitialization::ComplexDoubleDouble(
7467                        fused.left.kernel_spectrum.clone(),
7468                    ),
7469                });
7470                let parent_roots_id = ProgramResourceId(resources.len());
7471                resources.push(ProgramResource {
7472                    id: parent_roots_id,
7473                    name: "double_double_composite_dual_fft_rader_parent_roots".to_owned(),
7474                    kind: ProgramResourceKind::LookupTable,
7475                    scalar: ScalarType::DoubleDouble,
7476                    elements: ir.twiddles.len(),
7477                    external_layout: None,
7478                    initialization: ProgramResourceInitialization::ComplexDoubleDouble(
7479                        ir.twiddles.clone(),
7480                    ),
7481                });
7482                passes.push(ProgramPass {
7483                    name: fused.name(),
7484                    dispatch: ir.pack_right.dispatch,
7485                    bindings: vec![
7486                        binding(0, input.0, BufferRole::Input, BufferAccess::ReadOnly),
7487                        binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
7488                        ProgramPassBinding {
7489                            binding: 2,
7490                            resource: right_spectrum_id,
7491                            role: BufferRole::LookupTable,
7492                            access: BufferAccess::ReadOnly,
7493                        },
7494                        ProgramPassBinding {
7495                            binding: 3,
7496                            resource: left_spectrum_id,
7497                            role: BufferRole::LookupTable,
7498                            access: BufferAccess::ReadOnly,
7499                        },
7500                        ProgramPassBinding {
7501                            binding: 4,
7502                            resource: parent_roots_id,
7503                            role: BufferRole::TwiddleLookupTable,
7504                            access: BufferAccess::ReadOnly,
7505                        },
7506                    ],
7507                });
7508                return Ok(());
7509            }
7510            let elements = ir.logical_len.checked_mul(ir.batch_count).ok_or(
7511                VkFftError::ArithmeticOverflow {
7512                    operation: "double-double recursive Cooley-Tukey program element count",
7513                },
7514            )?;
7515            let right_input = add_double_double_recursive_scratch(
7516                resources,
7517                serial,
7518                "vkfft_dd_recursive_right_input",
7519                elements,
7520            );
7521            let right_output = add_double_double_recursive_scratch(
7522                resources,
7523                serial,
7524                "vkfft_dd_recursive_right_output",
7525                elements,
7526            );
7527            let left_input = add_double_double_recursive_scratch(
7528                resources,
7529                serial,
7530                "vkfft_dd_recursive_left_input",
7531                elements,
7532            );
7533            let left_output = add_double_double_recursive_scratch(
7534                resources,
7535                serial,
7536                "vkfft_dd_recursive_left_output",
7537                elements,
7538            );
7539            passes.push(two_buffer_pass(
7540                &ir.pack_right.name,
7541                ir.pack_right.dispatch,
7542                input.0,
7543                right_input.0,
7544            ));
7545            flatten_double_double_recursive_node(
7546                &ir.right,
7547                right_input,
7548                right_output,
7549                resources,
7550                passes,
7551                serial,
7552            )?;
7553            *serial += 1;
7554            let twiddle_id = ProgramResourceId(resources.len());
7555            resources.push(ProgramResource {
7556                id: twiddle_id,
7557                name: format!("vkfft_dd_recursive_twiddles_{}", *serial),
7558                kind: ProgramResourceKind::LookupTable,
7559                scalar: ScalarType::DoubleDouble,
7560                elements: ir.twiddles.len(),
7561                external_layout: None,
7562                initialization: ProgramResourceInitialization::ComplexDoubleDouble(
7563                    ir.twiddles.clone(),
7564                ),
7565            });
7566            passes.push(ProgramPass {
7567                name: ir.twiddle_transpose.name.clone(),
7568                dispatch: ir.twiddle_transpose.dispatch,
7569                bindings: vec![
7570                    binding(0, right_output.0, BufferRole::Input, BufferAccess::ReadOnly),
7571                    binding(1, left_input.0, BufferRole::Output, BufferAccess::WriteOnly),
7572                    ProgramPassBinding {
7573                        binding: 2,
7574                        resource: twiddle_id,
7575                        role: BufferRole::TwiddleLookupTable,
7576                        access: BufferAccess::ReadOnly,
7577                    },
7578                ],
7579            });
7580            flatten_double_double_recursive_node(
7581                &ir.left,
7582                left_input,
7583                left_output,
7584                resources,
7585                passes,
7586                serial,
7587            )?;
7588            let mut scatter_bindings = vec![
7589                binding(0, left_output.0, BufferRole::Input, BufferAccess::ReadOnly),
7590                binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
7591            ];
7592            let mapped_twiddle_period = match ir.scatter_output.output_modifier {
7593                DoubleDoubleCooleyTukeyOutputModifier::FourStepRight(mapping) => {
7594                    Some(mapping.logical_len)
7595                }
7596                DoubleDoubleCooleyTukeyOutputModifier::FourStepThreeUpload2(mapping) => {
7597                    Some(mapping.logical_len)
7598                }
7599                DoubleDoubleCooleyTukeyOutputModifier::FourStepThreeUpload1(mapping) => {
7600                    let [a, b, _] = mapping.axis_split;
7601                    Some(a.checked_mul(b).ok_or(VkFftError::ArithmeticOverflow {
7602                        operation: "double-double recursive three-upload-1 twiddle period",
7603                    })?)
7604                }
7605                _ => None,
7606            };
7607            if let Some(period) = mapped_twiddle_period {
7608                let values = crate::lut::unit_root_table_double_double(period, ir.direction)?;
7609                let twiddle_id = ProgramResourceId(resources.len());
7610                resources.push(ProgramResource {
7611                    id: twiddle_id,
7612                    name: format!("vkfft_dd_recursive_four_step_twiddles_{}", *serial),
7613                    kind: ProgramResourceKind::LookupTable,
7614                    scalar: ScalarType::DoubleDouble,
7615                    elements: values.len(),
7616                    external_layout: None,
7617                    initialization: ProgramResourceInitialization::ComplexDoubleDouble(values),
7618                });
7619                scatter_bindings.push(ProgramPassBinding {
7620                    binding: 2,
7621                    resource: twiddle_id,
7622                    role: BufferRole::TwiddleLookupTable,
7623                    access: BufferAccess::ReadOnly,
7624                });
7625            }
7626            passes.push(ProgramPass {
7627                name: ir.scatter_output.name.clone(),
7628                dispatch: ir.scatter_output.dispatch,
7629                bindings: scatter_bindings,
7630            });
7631        }
7632    }
7633    Ok(())
7634}
7635
7636struct RecursiveProgramBuilder {
7637    scalar: ScalarType,
7638    resources: Vec<ProgramResource>,
7639    passes: Vec<ProgramPass>,
7640    serial: usize,
7641}
7642
7643impl RecursiveProgramBuilder {
7644    fn add_scratch(&mut self, stem: &str, elements: usize) -> ProgramResourceId {
7645        let id = ProgramResourceId(self.resources.len());
7646        self.serial += 1;
7647        self.resources.push(ProgramResource {
7648            id,
7649            name: format!("{stem}_{}", self.serial),
7650            kind: ProgramResourceKind::Scratch,
7651            scalar: self.scalar,
7652            elements,
7653            external_layout: None,
7654            initialization: ProgramResourceInitialization::Zeroed,
7655        });
7656        id
7657    }
7658
7659    fn add_lut(&mut self, stem: &str, values: Vec<Complex64>) -> ProgramResourceId {
7660        if let Some(existing) = self.resources.iter().find(|resource| {
7661            resource.kind == ProgramResourceKind::LookupTable
7662                && resource.scalar == self.scalar
7663                && matches!(
7664                    &resource.initialization,
7665                    ProgramResourceInitialization::Complex64(existing_values)
7666                        if existing_values == &values
7667                )
7668        }) {
7669            return existing.id;
7670        }
7671        let id = ProgramResourceId(self.resources.len());
7672        self.serial += 1;
7673        self.resources.push(ProgramResource {
7674            id,
7675            name: format!("{stem}_{}", self.serial),
7676            kind: ProgramResourceKind::LookupTable,
7677            scalar: self.scalar,
7678            elements: values.len(),
7679            external_layout: None,
7680            initialization: ProgramResourceInitialization::Complex64(values),
7681        });
7682        id
7683    }
7684
7685    fn add_stockham_root_lut(&mut self, len: usize) -> Result<ProgramResourceId> {
7686        if len == 0 {
7687            return Err(VkFftError::InvalidKernelIr(
7688                "Stockham unit-root LUT requires a non-zero period",
7689            ));
7690        }
7691        let initialization = ProgramResourceInitialization::StockhamUnitRoots { len };
7692        if let Some(existing) = self.resources.iter().find(|resource| {
7693            resource.kind == ProgramResourceKind::LookupTable
7694                && resource.scalar == self.scalar
7695                && resource.elements == len
7696                && resource.initialization == initialization
7697        }) {
7698            return Ok(existing.id);
7699        }
7700        let id = ProgramResourceId(self.resources.len());
7701        self.serial += 1;
7702        self.resources.push(ProgramResource {
7703            id,
7704            name: format!("stockham_unit_roots_{}", self.serial),
7705            kind: ProgramResourceKind::LookupTable,
7706            scalar: self.scalar,
7707            elements: len,
7708            external_layout: None,
7709            initialization,
7710        });
7711        Ok(id)
7712    }
7713
7714    fn stockham_pass_bindings(
7715        &mut self,
7716        kernel: &KernelIr,
7717        input: ProgramResourceId,
7718        output: ProgramResourceId,
7719        lookup: Option<ProgramResourceId>,
7720        auxiliary: Option<ProgramResourceId>,
7721    ) -> Result<Vec<ProgramPassBinding>> {
7722        let twiddle = match kernel.twiddle_lut_len() {
7723            Some(len) => Some(self.add_stockham_root_lut(len)?),
7724            None => None,
7725        };
7726        kernel
7727            .bindings
7728            .iter()
7729            .map(|descriptor| {
7730                let resource = match descriptor.role {
7731                    BufferRole::Input => input,
7732                    BufferRole::Output => output,
7733                    BufferRole::LookupTable => lookup.ok_or(VkFftError::InvalidKernelIr(
7734                        "Stockham pass requires a lookup-table resource",
7735                    ))?,
7736                    BufferRole::TwiddleLookupTable => {
7737                        twiddle.ok_or(VkFftError::InvalidKernelIr(
7738                            "Stockham pass requires a twiddle-table resource",
7739                        ))?
7740                    }
7741                    BufferRole::Auxiliary => auxiliary.ok_or(VkFftError::InvalidKernelIr(
7742                        "Stockham pass requires an auxiliary resource",
7743                    ))?,
7744                };
7745                Ok(binding(
7746                    descriptor.binding,
7747                    resource.0,
7748                    descriptor.role,
7749                    descriptor.access,
7750                ))
7751            })
7752            .collect()
7753    }
7754
7755    fn push_stockham_pass(
7756        &mut self,
7757        kernel: &KernelIr,
7758        input: ProgramResourceId,
7759        output: ProgramResourceId,
7760        lookup: Option<ProgramResourceId>,
7761        auxiliary: Option<ProgramResourceId>,
7762    ) -> Result<()> {
7763        let bindings = self.stockham_pass_bindings(kernel, input, output, lookup, auxiliary)?;
7764        self.passes.push(ProgramPass {
7765            name: kernel.name.clone(),
7766            dispatch: kernel.dispatch,
7767            bindings,
7768        });
7769        Ok(())
7770    }
7771
7772    fn flatten_bluestein(
7773        &mut self,
7774        pipeline: &BluesteinPipelineIr,
7775        input: ProgramResourceId,
7776        output: ProgramResourceId,
7777    ) -> Result<()> {
7778        pipeline.validate()?;
7779        let convolution_elements = pipeline
7780            .convolution_len
7781            .checked_mul(pipeline.batch_count)
7782            .ok_or(VkFftError::ArithmeticOverflow {
7783                operation: "generic program Bluestein convolution element count",
7784            })?;
7785        let scratch_a = self.add_scratch("bluestein_scratch_a", convolution_elements);
7786        let scratch_b = self.add_scratch("bluestein_scratch_b", convolution_elements);
7787        let lut = self.add_lut(
7788            "bluestein_kernel_spectrum",
7789            pipeline.kernel_spectrum()?.to_vec(),
7790        );
7791        self.passes.push(two_buffer_pass(
7792            &pipeline.preprocess.name,
7793            pipeline.preprocess.dispatch,
7794            input.0,
7795            scratch_a.0,
7796        ));
7797        self.flatten_recursive_ir(&pipeline.forward_fft, scratch_a, scratch_b)?;
7798        let convolution_output =
7799            if let Some(fused_inverse) = pipeline.fused_inverse_stockham_kernel()? {
7800                self.push_stockham_pass(&fused_inverse, scratch_b, scratch_a, Some(lut), None)?;
7801                scratch_a
7802            } else if pipeline.has_fused_recursive_inverse() {
7803                self.flatten_node_with_boundary_resources(
7804                    &pipeline.inverse_fft.root,
7805                    scratch_b,
7806                    scratch_a,
7807                    Some(lut),
7808                    None,
7809                )?;
7810                scratch_a
7811            } else {
7812                self.passes.push(ProgramPass {
7813                    name: pipeline.multiply.name.clone(),
7814                    dispatch: pipeline.multiply.dispatch,
7815                    bindings: vec![
7816                        binding(0, scratch_b.0, BufferRole::Input, BufferAccess::ReadOnly),
7817                        binding(1, scratch_a.0, BufferRole::Output, BufferAccess::WriteOnly),
7818                        binding(2, lut.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
7819                    ],
7820                });
7821                self.flatten_recursive_ir(&pipeline.inverse_fft, scratch_a, scratch_b)?;
7822                scratch_b
7823            };
7824        self.passes.push(two_buffer_pass(
7825            &pipeline.postprocess.name,
7826            pipeline.postprocess.dispatch,
7827            convolution_output.0,
7828            output.0,
7829        ));
7830        Ok(())
7831    }
7832
7833    fn push_zero_pad_pass(
7834        &mut self,
7835        pass: &crate::ZeroPadPassIr,
7836        input: ProgramResourceId,
7837        output: ProgramResourceId,
7838    ) -> Result<()> {
7839        pass.validate()?;
7840        self.passes.push(ProgramPass {
7841            name: pass.name.clone(),
7842            dispatch: pass.dispatch,
7843            bindings: vec![
7844                binding(0, input.0, BufferRole::Input, BufferAccess::ReadOnly),
7845                binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
7846            ],
7847        });
7848        Ok(())
7849    }
7850
7851    fn push_nd_zero_pad_pass(
7852        &mut self,
7853        pass: &crate::NdZeroPadPassIr,
7854        input: ProgramResourceId,
7855        output: ProgramResourceId,
7856    ) -> Result<()> {
7857        pass.validate()?;
7858        self.passes.push(ProgramPass {
7859            name: pass.name.clone(),
7860            dispatch: pass.dispatch,
7861            bindings: vec![
7862                binding(0, input.0, BufferRole::Input, BufferAccess::ReadOnly),
7863                binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
7864            ],
7865        });
7866        Ok(())
7867    }
7868
7869    fn flatten_recursive_ir(
7870        &mut self,
7871        ir: &RecursiveFftIr,
7872        input: ProgramResourceId,
7873        output: ProgramResourceId,
7874    ) -> Result<()> {
7875        if let Some(uploads) = ir.four_step_stockham_upload_kernels()? {
7876            let elements = ir.logical_len.checked_mul(ir.batch_count).ok_or(
7877                VkFftError::ArithmeticOverflow {
7878                    operation: "recursive Four-step program element count",
7879                },
7880            )?;
7881            let scratch = self.add_scratch("four_step_exchange", elements);
7882            match uploads.as_slice() {
7883                [first, second] => {
7884                    self.push_stockham_pass(first, input, scratch, None, None)?;
7885                    self.push_stockham_pass(second, scratch, output, None, None)?;
7886                }
7887                [first, second, third] => {
7888                    let output_scalar = self
7889                        .resources
7890                        .get(output.0)
7891                        .ok_or(VkFftError::InvalidKernelIr(
7892                            "recursive Four-step output resource is missing",
7893                        ))?
7894                        .scalar;
7895                    if output_scalar == ir.scalar {
7896                        self.push_stockham_pass(first, input, output, None, None)?;
7897                        self.push_stockham_pass(second, output, scratch, None, None)?;
7898                        self.push_stockham_pass(third, scratch, output, None, None)?;
7899                    } else {
7900                        let scratch_b = self.add_scratch("four_step_exchange_b", elements);
7901                        self.push_stockham_pass(first, input, scratch, None, None)?;
7902                        self.push_stockham_pass(second, scratch, scratch_b, None, None)?;
7903                        self.push_stockham_pass(third, scratch_b, output, None, None)?;
7904                    }
7905                }
7906                _ => {
7907                    return Err(VkFftError::InvalidKernelIr(
7908                        "Four-step program currently requires exactly two or three kernels",
7909                    ));
7910                }
7911            }
7912            Ok(())
7913        } else if let Some(uploads) = ir.four_step_rader_upload_nodes()? {
7914            let elements = ir.logical_len.checked_mul(ir.batch_count).ok_or(
7915                VkFftError::ArithmeticOverflow {
7916                    operation: "recursive Rader Four-step program element count",
7917                },
7918            )?;
7919            match uploads.as_slice() {
7920                [first, second] => {
7921                    let scratch = self.add_scratch("rader_four_step_exchange", elements);
7922                    self.flatten_node(first, input, scratch)?;
7923                    self.flatten_node(second, scratch, output)
7924                }
7925                [first, second, third] => {
7926                    let output_scalar = self
7927                        .resources
7928                        .get(output.0)
7929                        .ok_or(VkFftError::InvalidKernelIr(
7930                            "recursive Rader Four-step output resource is missing",
7931                        ))?
7932                        .scalar;
7933                    if output_scalar == ir.scalar {
7934                        let scratch = self.add_scratch("rader_four_step_exchange", elements);
7935                        self.flatten_node(first, input, output)?;
7936                        self.flatten_node(second, output, scratch)?;
7937                        self.flatten_node(third, scratch, output)
7938                    } else {
7939                        let scratch_a = self.add_scratch("rader_four_step_exchange_a", elements);
7940                        let scratch_b = self.add_scratch("rader_four_step_exchange_b", elements);
7941                        self.flatten_node(first, input, scratch_a)?;
7942                        self.flatten_node(second, scratch_a, scratch_b)?;
7943                        self.flatten_node(third, scratch_b, output)
7944                    }
7945                }
7946                _ => Err(VkFftError::InvalidKernelIr(
7947                    "Rader Four-step program currently requires exactly two or three upload nodes",
7948                )),
7949            }
7950        } else {
7951            self.flatten_node(&ir.root, input, output)
7952        }
7953    }
7954
7955    fn flatten_one_dim(
7956        &mut self,
7957        ir: &OneDimFftIr,
7958        input: ProgramResourceId,
7959        output: ProgramResourceId,
7960    ) -> Result<()> {
7961        let Some(zero_pad) = ir.zero_pad_pass() else {
7962            return match ir {
7963                OneDimFftIr::Recursive(recursive) => {
7964                    self.flatten_recursive_ir(recursive, input, output)
7965                }
7966                OneDimFftIr::Bluestein(pipeline) => self.flatten_bluestein(pipeline, input, output),
7967            };
7968        };
7969        let elements = ir.logical_len().checked_mul(ir.batch_count()).ok_or(
7970            VkFftError::ArithmeticOverflow {
7971                operation: "zero-padded one-dimensional program element count",
7972            },
7973        )?;
7974        let boundary = self.add_scratch("zero_pad_boundary", elements);
7975        if zero_pad.operation.is_input_boundary() {
7976            self.push_zero_pad_pass(zero_pad, input, boundary)?;
7977            match ir {
7978                OneDimFftIr::Recursive(recursive) => {
7979                    self.flatten_recursive_ir(recursive, boundary, output)
7980                }
7981                OneDimFftIr::Bluestein(pipeline) => {
7982                    self.flatten_bluestein(pipeline, boundary, output)
7983                }
7984            }
7985        } else {
7986            match ir {
7987                OneDimFftIr::Recursive(recursive) => {
7988                    self.flatten_recursive_ir(recursive, input, boundary)?
7989                }
7990                OneDimFftIr::Bluestein(pipeline) => {
7991                    self.flatten_bluestein(pipeline, input, boundary)?
7992                }
7993            }
7994            self.push_zero_pad_pass(zero_pad, boundary, output)
7995        }
7996    }
7997
7998    fn flatten_r2r(
7999        &mut self,
8000        ir: &R2rIr,
8001        input: ProgramResourceId,
8002        output: ProgramResourceId,
8003    ) -> Result<()> {
8004        ir.validate()?;
8005        if let Some(reduction) = ir.fft_reduction.as_deref() {
8006            let fft_elements = reduction.fft_len.checked_mul(reduction.batch_count).ok_or(
8007                VkFftError::ArithmeticOverflow {
8008                    operation: "R2R FFT reduction program element count",
8009                },
8010            )?;
8011            let fft_input = self.add_scratch("r2r_fft_input", fft_elements);
8012            let fft_output = self.add_scratch("r2r_fft_output", fft_elements);
8013            self.passes.push(two_buffer_pass(
8014                &reduction.preprocess.name,
8015                reduction.preprocess.dispatch,
8016                input.0,
8017                fft_input.0,
8018            ));
8019            self.flatten_one_dim(&reduction.fft, fft_input, fft_output)?;
8020            self.passes.push(two_buffer_pass(
8021                &reduction.postprocess.name,
8022                reduction.postprocess.dispatch,
8023                fft_output.0,
8024                output.0,
8025            ));
8026        } else {
8027            self.passes.push(two_buffer_pass(
8028                &format!("vkfft_r2r_{:?}_{:?}", ir.effective_transform, ir.direction),
8029                ir.dispatch,
8030                input.0,
8031                output.0,
8032            ));
8033        }
8034        Ok(())
8035    }
8036
8037    fn flatten_real_fft(
8038        &mut self,
8039        ir: &RealFftIr,
8040        input: ProgramResourceId,
8041        output: ProgramResourceId,
8042    ) -> Result<()> {
8043        ir.validate()?;
8044        if let Some(fused) = ir.fused_even_input_stockham_kernel()? {
8045            let transform_elements = ir.transform_len().checked_mul(ir.batch_count).ok_or(
8046                VkFftError::ArithmeticOverflow {
8047                    operation: "fused real FFT internal transform element count",
8048                },
8049            )?;
8050            let fuses_postprocess = matches!(
8051                fused.output_modifier,
8052                crate::StockhamOutputModifier::RealEvenPostprocess(_)
8053                    | crate::StockhamOutputModifier::RealEvenUnpack(_)
8054            );
8055            let transform_output = if ir.postprocess.is_some() && !fuses_postprocess {
8056                self.add_scratch("real_transform", transform_elements)
8057            } else {
8058                output
8059            };
8060            let auxiliary = if matches!(
8061                fused.output_modifier,
8062                crate::StockhamOutputModifier::RealEvenPostprocess(_)
8063            ) {
8064                Some(self.add_scratch("real_postprocess_scratch", transform_elements))
8065            } else {
8066                None
8067            };
8068            self.push_stockham_pass(&fused, input, transform_output, None, auxiliary)?;
8069            if !fuses_postprocess && let Some(postprocess) = &ir.postprocess {
8070                self.passes.push(two_buffer_pass(
8071                    &postprocess.name,
8072                    postprocess.dispatch,
8073                    transform_output.0,
8074                    output.0,
8075                ));
8076            }
8077            return Ok(());
8078        }
8079        let transform_elements = ir.transform_len().checked_mul(ir.batch_count).ok_or(
8080            VkFftError::ArithmeticOverflow {
8081                operation: "real FFT program internal transform element count",
8082            },
8083        )?;
8084        if let Some(fused_recursive) = ir.fused_even_recursive_ir()? {
8085            self.flatten_node_with_boundary_resources(
8086                &fused_recursive.root,
8087                input,
8088                output,
8089                None,
8090                None,
8091            )?;
8092            return Ok(());
8093        }
8094        if let Some(fused_bluestein) = ir.fused_even_bluestein_ir()? {
8095            self.flatten_bluestein(&fused_bluestein, input, output)?;
8096            return Ok(());
8097        }
8098
8099        let transform_input = if let Some(preprocess) = &ir.preprocess {
8100            let scratch = self.add_scratch("real_preprocess", transform_elements);
8101            self.passes.push(two_buffer_pass(
8102                &preprocess.name,
8103                preprocess.dispatch,
8104                input.0,
8105                scratch.0,
8106            ));
8107            scratch
8108        } else {
8109            input
8110        };
8111        let transform_output = if ir.postprocess.is_some() {
8112            self.add_scratch("real_transform", transform_elements)
8113        } else {
8114            output
8115        };
8116        self.flatten_one_dim(&ir.transform, transform_input, transform_output)?;
8117        if let Some(postprocess) = &ir.postprocess {
8118            self.passes.push(two_buffer_pass(
8119                &postprocess.name,
8120                postprocess.dispatch,
8121                transform_output.0,
8122                output.0,
8123            ));
8124        }
8125        Ok(())
8126    }
8127
8128    fn flatten_node_with_boundary_resources(
8129        &mut self,
8130        node: &RecursiveFftNodeIr,
8131        input: ProgramResourceId,
8132        output: ProgramResourceId,
8133        lookup: Option<ProgramResourceId>,
8134        auxiliary: Option<ProgramResourceId>,
8135    ) -> Result<()> {
8136        node.validate()?;
8137        let RecursiveFftNodeIr::CooleyTukey(ir) = node else {
8138            return Err(VkFftError::UnsupportedKernelPath(
8139                "recursive boundary resources require a Cooley-Tukey root",
8140            ));
8141        };
8142        let elements =
8143            ir.logical_len
8144                .checked_mul(ir.batch_count)
8145                .ok_or(VkFftError::ArithmeticOverflow {
8146                    operation: "recursive program fused Cooley-Tukey element count",
8147                })?;
8148        let right_input = self.add_scratch("recursive_right_input", elements);
8149        let right_output = self.add_scratch("recursive_right_output", elements);
8150        let left_input = self.add_scratch("recursive_left_input", elements);
8151        let left_output = self.add_scratch("recursive_left_output", elements);
8152
8153        let mut pack_bindings = vec![
8154            binding(0, input.0, BufferRole::Input, BufferAccess::ReadOnly),
8155            binding(
8156                1,
8157                right_input.0,
8158                BufferRole::Output,
8159                BufferAccess::WriteOnly,
8160            ),
8161        ];
8162        if ir.pack_right.input_modifier == CooleyTukeyInputModifier::MultiplyLookupTable {
8163            let lookup = lookup.ok_or(VkFftError::InvalidKernelIr(
8164                "recursive fused Cooley-Tukey pack requires a lookup resource",
8165            ))?;
8166            pack_bindings.push(binding(
8167                2,
8168                lookup.0,
8169                BufferRole::LookupTable,
8170                BufferAccess::ReadOnly,
8171            ));
8172        }
8173        self.passes.push(ProgramPass {
8174            name: ir.pack_right.name.clone(),
8175            dispatch: ir.pack_right.dispatch,
8176            bindings: pack_bindings,
8177        });
8178        self.flatten_node(&ir.right, right_input, right_output)?;
8179        self.passes.push(two_buffer_pass(
8180            &ir.twiddle_transpose.name,
8181            ir.twiddle_transpose.dispatch,
8182            right_output.0,
8183            left_input.0,
8184        ));
8185        self.flatten_node(&ir.left, left_input, left_output)?;
8186
8187        let mut scatter_bindings = vec![
8188            binding(0, left_output.0, BufferRole::Input, BufferAccess::ReadOnly),
8189            binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
8190        ];
8191        if matches!(
8192            ir.scatter_output.output_modifier,
8193            CooleyTukeyOutputModifier::RaderScatter(_)
8194        ) {
8195            let auxiliary = auxiliary.ok_or(VkFftError::InvalidKernelIr(
8196                "recursive fused Cooley-Tukey scatter requires an auxiliary resource",
8197            ))?;
8198            scatter_bindings.push(binding(
8199                2,
8200                auxiliary.0,
8201                BufferRole::Auxiliary,
8202                BufferAccess::ReadOnly,
8203            ));
8204        }
8205        self.passes.push(ProgramPass {
8206            name: ir.scatter_output.name.clone(),
8207            dispatch: ir.scatter_output.dispatch,
8208            bindings: scatter_bindings,
8209        });
8210        Ok(())
8211    }
8212
8213    fn flatten_node(
8214        &mut self,
8215        node: &RecursiveFftNodeIr,
8216        input: ProgramResourceId,
8217        output: ProgramResourceId,
8218    ) -> Result<()> {
8219        node.validate()?;
8220        match node {
8221            RecursiveFftNodeIr::Stockham(kernel) => {
8222                self.push_stockham_pass(kernel, input, output, None, None)?;
8223            }
8224            RecursiveFftNodeIr::DirectRader(direct) => {
8225                let lut = self.add_lut("rader_roots", direct.twiddle_lut().to_vec());
8226                self.passes.push(ProgramPass {
8227                    name: direct.name.clone(),
8228                    dispatch: direct.dispatch,
8229                    bindings: vec![
8230                        binding(0, input.0, BufferRole::Input, BufferAccess::ReadOnly),
8231                        binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
8232                        binding(2, lut.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
8233                    ],
8234                });
8235            }
8236            RecursiveFftNodeIr::FftRader(rader) => {
8237                let convolution_elements = rader
8238                    .convolution_len
8239                    .checked_mul(rader.batch_count)
8240                    .ok_or(VkFftError::ArithmeticOverflow {
8241                        operation: "recursive program Rader convolution element count",
8242                    })?;
8243                let scratch_a = self.add_scratch("rader_scratch_a", convolution_elements);
8244                let scratch_b = self.add_scratch("rader_scratch_b", convolution_elements);
8245                let lut = self.add_lut("rader_kernel_spectrum", rader.kernel_spectrum()?.to_vec());
8246                let forward_input = match rader.input_strategy {
8247                    RaderFftInputStrategy::GatherReversePass => {
8248                        self.passes.push(two_buffer_pass(
8249                            &rader.gather.name,
8250                            rader.gather.dispatch,
8251                            input.0,
8252                            scratch_a.0,
8253                        ));
8254                        scratch_a
8255                    }
8256                    RaderFftInputStrategy::GeneratorOrderStockham
8257                    | RaderFftInputStrategy::GeneratorOrderRecursive => input,
8258                };
8259                self.flatten_one_dim(&rader.forward_fft, forward_input, scratch_b)?;
8260                if let Some(fused_inverse) = rader.fused_inverse_rader_kernel()? {
8261                    self.push_stockham_pass(
8262                        &fused_inverse,
8263                        scratch_b,
8264                        output,
8265                        Some(lut),
8266                        Some(input),
8267                    )?;
8268                } else if rader.has_fused_recursive_inverse() {
8269                    let inverse = rader
8270                        .inverse_recursive()
8271                        .ok_or(VkFftError::InvalidKernelIr(
8272                            "recursive fused Rader program lost its recursive inverse child",
8273                        ))?;
8274                    self.flatten_node_with_boundary_resources(
8275                        &inverse.root,
8276                        scratch_b,
8277                        output,
8278                        Some(lut),
8279                        Some(input),
8280                    )?;
8281                } else {
8282                    self.passes.push(ProgramPass {
8283                        name: rader.multiply.name.clone(),
8284                        dispatch: rader.multiply.dispatch,
8285                        bindings: vec![
8286                            binding(0, scratch_b.0, BufferRole::Input, BufferAccess::ReadOnly),
8287                            binding(1, scratch_a.0, BufferRole::Output, BufferAccess::WriteOnly),
8288                            binding(2, lut.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
8289                        ],
8290                    });
8291                    self.flatten_one_dim(&rader.inverse_fft, scratch_a, scratch_b)?;
8292                    self.passes.push(ProgramPass {
8293                        name: rader.scatter.name.clone(),
8294                        dispatch: rader.scatter.dispatch,
8295                        bindings: vec![
8296                            binding(0, scratch_b.0, BufferRole::Input, BufferAccess::ReadOnly),
8297                            binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
8298                            binding(2, input.0, BufferRole::Auxiliary, BufferAccess::ReadOnly),
8299                        ],
8300                    });
8301                }
8302            }
8303            RecursiveFftNodeIr::CooleyTukey(ir) => {
8304                if let Some(fused) = ir.fused_small_direct_rader_stockham()? {
8305                    let lut = self.add_lut(
8306                        "composite_direct_rader_roots",
8307                        fused.direct.twiddle_lut().to_vec(),
8308                    );
8309                    self.passes.push(ProgramPass {
8310                        name: fused.name(),
8311                        dispatch: ir.pack_right.dispatch,
8312                        bindings: vec![
8313                            binding(0, input.0, BufferRole::Input, BufferAccess::ReadOnly),
8314                            binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
8315                            binding(2, lut.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
8316                        ],
8317                    });
8318                    return Ok(());
8319                }
8320                if let Some(fused) = ir.fused_small_fft_rader_stockham()? {
8321                    let spectrum = self.add_lut(
8322                        "composite_fft_rader_spectrum",
8323                        fused.rader.kernel_spectrum()?.to_vec(),
8324                    );
8325                    let mut bindings = vec![
8326                        binding(0, input.0, BufferRole::Input, BufferAccess::ReadOnly),
8327                        binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
8328                        binding(
8329                            2,
8330                            spectrum.0,
8331                            BufferRole::LookupTable,
8332                            BufferAccess::ReadOnly,
8333                        ),
8334                    ];
8335                    if let Some(len) = fused.forward.twiddle_lut_len() {
8336                        let twiddle = self.add_stockham_root_lut(len)?;
8337                        bindings.push(binding(
8338                            3,
8339                            twiddle.0,
8340                            BufferRole::TwiddleLookupTable,
8341                            BufferAccess::ReadOnly,
8342                        ));
8343                    }
8344                    self.passes.push(ProgramPass {
8345                        name: fused.name(),
8346                        dispatch: ir.pack_right.dispatch,
8347                        bindings,
8348                    });
8349                    return Ok(());
8350                }
8351                if let Some((mapped_right, mapped_left)) = ir.fused_fft_rader_cooley_boundaries()? {
8352                    let elements = ir.logical_len.checked_mul(ir.batch_count).ok_or(
8353                        VkFftError::ArithmeticOverflow {
8354                            operation: "recursive program fused Cooley-right element count",
8355                        },
8356                    )?;
8357                    let right_output = self.add_scratch("recursive_right_output", elements);
8358                    let mapped_right = RecursiveFftNodeIr::FftRader(Box::new(mapped_right));
8359                    self.flatten_node(&mapped_right, input, right_output)?;
8360                    self.push_stockham_pass(&mapped_left, right_output, output, None, None)?;
8361                    return Ok(());
8362                }
8363                if let Some(mapped_left) = ir.fused_fft_rader_left_stockham()? {
8364                    let elements = ir.logical_len.checked_mul(ir.batch_count).ok_or(
8365                        VkFftError::ArithmeticOverflow {
8366                            operation: "recursive program fused Cooley-left element count",
8367                        },
8368                    )?;
8369                    let right_input = self.add_scratch("recursive_right_input", elements);
8370                    let right_output = self.add_scratch("recursive_right_output", elements);
8371                    self.passes.push(two_buffer_pass(
8372                        &ir.pack_right.name,
8373                        ir.pack_right.dispatch,
8374                        input.0,
8375                        right_input.0,
8376                    ));
8377                    self.flatten_node(&ir.right, right_input, right_output)?;
8378                    self.push_stockham_pass(&mapped_left, right_output, output, None, None)?;
8379                    return Ok(());
8380                }
8381                if ir.pack_right.input_modifier == CooleyTukeyInputModifier::MultiplyLookupTable
8382                    || !matches!(
8383                        ir.scatter_output.output_modifier,
8384                        CooleyTukeyOutputModifier::None
8385                            | CooleyTukeyOutputModifier::FourStepRight(_)
8386                            | CooleyTukeyOutputModifier::FourStepLeft(_)
8387                            | CooleyTukeyOutputModifier::FourStepThreeUpload2(_)
8388                            | CooleyTukeyOutputModifier::FourStepThreeUpload1(_)
8389                            | CooleyTukeyOutputModifier::FourStepThreeUpload0(_)
8390                    )
8391                {
8392                    return Err(VkFftError::InvalidKernelIr(
8393                        "modified recursive Cooley-Tukey boundaries require explicit resources",
8394                    ));
8395                }
8396                let elements = ir.logical_len.checked_mul(ir.batch_count).ok_or(
8397                    VkFftError::ArithmeticOverflow {
8398                        operation: "recursive program Cooley-Tukey element count",
8399                    },
8400                )?;
8401                let right_input = self.add_scratch("recursive_right_input", elements);
8402                let right_output = self.add_scratch("recursive_right_output", elements);
8403                let left_input = self.add_scratch("recursive_left_input", elements);
8404                let left_output = self.add_scratch("recursive_left_output", elements);
8405                self.passes.push(two_buffer_pass(
8406                    &ir.pack_right.name,
8407                    ir.pack_right.dispatch,
8408                    input.0,
8409                    right_input.0,
8410                ));
8411                self.flatten_node(&ir.right, right_input, right_output)?;
8412                self.passes.push(two_buffer_pass(
8413                    &ir.twiddle_transpose.name,
8414                    ir.twiddle_transpose.dispatch,
8415                    right_output.0,
8416                    left_input.0,
8417                ));
8418                self.flatten_node(&ir.left, left_input, left_output)?;
8419                self.passes.push(two_buffer_pass(
8420                    &ir.scatter_output.name,
8421                    ir.scatter_output.dispatch,
8422                    left_output.0,
8423                    output.0,
8424                ));
8425            }
8426        }
8427        Ok(())
8428    }
8429}
8430
8431fn double_double_stockham_input_lookup_eligible(ir: &DoubleDoubleStockhamIr) -> bool {
8432    ir.sequence_len > 64
8433        && ir.external_storage == crate::PrecisionStorage::DoubleDouble
8434        && ir.zero_pad_pass.is_none()
8435        && !ir.stages.is_empty()
8436}
8437
8438fn attach_double_double_stockham_input_lookup(
8439    pass: &mut ProgramPass,
8440    lookup: ProgramResourceId,
8441) -> Result<()> {
8442    if pass.bindings.iter().any(|binding| binding.binding == 3) {
8443        return Err(VkFftError::InvalidKernelIr(
8444            "fused DD Stockham input lookup binding 3 is already occupied",
8445        ));
8446    }
8447    pass.bindings.push(ProgramPassBinding {
8448        binding: 3,
8449        resource: lookup,
8450        role: BufferRole::LookupTable,
8451        access: BufferAccess::ReadOnly,
8452    });
8453    pass.name.push_str("_input_mul_lut");
8454    Ok(())
8455}
8456
8457fn binding(
8458    binding: u32,
8459    resource: usize,
8460    role: BufferRole,
8461    access: BufferAccess,
8462) -> ProgramPassBinding {
8463    ProgramPassBinding {
8464        binding,
8465        resource: ProgramResourceId(resource),
8466        role,
8467        access,
8468    }
8469}
8470
8471fn two_buffer_pass(
8472    name: &str,
8473    dispatch: DispatchGeometry,
8474    input: usize,
8475    output: usize,
8476) -> ProgramPass {
8477    ProgramPass {
8478        name: name.to_owned(),
8479        dispatch,
8480        bindings: vec![
8481            binding(0, input, BufferRole::Input, BufferAccess::ReadOnly),
8482            binding(1, output, BufferRole::Output, BufferAccess::WriteOnly),
8483        ],
8484    }
8485}
8486
8487#[cfg(test)]
8488mod tests {
8489    use super::*;
8490    use crate::config::{
8491        Backend, DeviceProfile, Direction, GpuVendor, Precision, TransformKind, ZeroPaddingDomain,
8492    };
8493    use crate::{FftConfig, FftPlan};
8494
8495    fn device() -> DeviceProfile {
8496        DeviceProfile {
8497            shared_memory_bytes: 128 * 1024,
8498            shared_memory_pow2_bytes: 128 * 1024,
8499            ..DeviceProfile::generic(Backend::Vulkan, GpuVendor::Nvidia)
8500        }
8501    }
8502
8503    #[test]
8504    fn double_double_even_real_program_uses_half_size_child_and_root_lut() {
8505        for precision in [Precision::DoubleDouble, Precision::DoubleDoubleF64Storage] {
8506            for transform in [TransformKind::RealToComplex, TransformKind::ComplexToReal] {
8507                let plan = FftPlan::build(
8508                    FftConfig::new(vec![16])
8509                        .with_batch_count(2)
8510                        .with_transform(transform)
8511                        .with_precision(precision)
8512                        .with_inverse_normalization(transform == TransformKind::ComplexToReal),
8513                )
8514                .unwrap();
8515                let ir = crate::DoubleDoubleRealFftIr::build(&plan).unwrap();
8516                assert!(ir.even_half_size);
8517                assert_eq!(ir.transform.sequence_len(), 8);
8518                assert_eq!(ir.even_roots.len(), 9);
8519                let program = ProgramIr::double_double_real(&ir).unwrap();
8520                assert_eq!(program.resources[2].elements, 16);
8521                assert_eq!(program.resources[3].elements, 16);
8522                let roots = program
8523                    .resources
8524                    .iter()
8525                    .find(|resource| resource.name == "double_double_real_even_roots")
8526                    .expect("even DD real ProgramIr must own a root LUT");
8527                assert_eq!(roots.scalar, ScalarType::DoubleDouble);
8528                assert_eq!(roots.elements, 9);
8529                let ProgramResourceInitialization::ComplexDoubleDouble(values) =
8530                    &roots.initialization
8531                else {
8532                    panic!("even DD real roots must preserve DD words");
8533                };
8534                assert_eq!(values, &ir.even_roots);
8535                let first = program.passes.first().unwrap();
8536                let last = program.passes.last().unwrap();
8537                match transform {
8538                    TransformKind::RealToComplex => {
8539                        assert_eq!(first.bindings.len(), 2);
8540                        assert_eq!(last.bindings.len(), 3);
8541                        assert!(last.bindings.iter().any(|binding| {
8542                            binding.binding == 2
8543                                && binding.resource == roots.id
8544                                && binding.role == BufferRole::LookupTable
8545                        }));
8546                    }
8547                    TransformKind::ComplexToReal => {
8548                        assert_eq!(first.bindings.len(), 3);
8549                        assert_eq!(last.bindings.len(), 2);
8550                        assert!(first.bindings.iter().any(|binding| {
8551                            binding.binding == 2
8552                                && binding.resource == roots.id
8553                                && binding.role == BufferRole::LookupTable
8554                        }));
8555                    }
8556                    _ => unreachable!(),
8557                }
8558            }
8559        }
8560    }
8561
8562    #[test]
8563    fn double_double_real_program_separates_real_compact_and_full_dd_resources() {
8564        for precision in [Precision::DoubleDouble, Precision::DoubleDoubleF64Storage] {
8565            let external_scalar = if precision == Precision::DoubleDouble {
8566                ScalarType::DoubleDouble
8567            } else {
8568                ScalarType::F64
8569            };
8570            for (transform, kind, input_len, output_len) in [
8571                (
8572                    TransformKind::RealToComplex,
8573                    crate::RealFftKind::RealToComplex,
8574                    15usize,
8575                    8usize,
8576                ),
8577                (
8578                    TransformKind::ComplexToReal,
8579                    crate::RealFftKind::ComplexToReal,
8580                    8usize,
8581                    15usize,
8582                ),
8583            ] {
8584                let plan = FftPlan::build(
8585                    FftConfig::new(vec![15])
8586                        .with_batch_count(2)
8587                        .with_transform(transform)
8588                        .with_precision(precision)
8589                        .with_inverse_normalization(transform == TransformKind::ComplexToReal),
8590                )
8591                .unwrap();
8592                let ir = crate::DoubleDoubleRealFftIr::build(&plan).unwrap();
8593                assert_eq!(ir.kind, kind);
8594                let child = ProgramIr::double_double_one_dim(&ir.transform).unwrap();
8595                let program = ProgramIr::double_double_real(&ir).unwrap();
8596                let input = program.input_resource().unwrap();
8597                let output = program.output_resource().unwrap();
8598                assert_eq!(input.scalar, external_scalar);
8599                assert_eq!(output.scalar, external_scalar);
8600                assert_eq!(input.external_layout.unwrap().logical_len, input_len);
8601                assert_eq!(output.external_layout.unwrap().logical_len, output_len);
8602                assert_eq!(program.resources[2].scalar, ScalarType::DoubleDouble);
8603                assert_eq!(program.resources[2].elements, 30);
8604                assert_eq!(program.resources[3].scalar, ScalarType::DoubleDouble);
8605                assert_eq!(program.resources[3].elements, 30);
8606                assert_eq!(program.passes.len(), child.passes.len() + 2);
8607                assert_eq!(
8608                    program.passes.first().unwrap().bindings[0].resource,
8609                    ProgramResourceId(0)
8610                );
8611                assert_eq!(
8612                    program.passes.first().unwrap().bindings[1].resource,
8613                    ProgramResourceId(2)
8614                );
8615                assert_eq!(
8616                    program.passes.last().unwrap().bindings[0].resource,
8617                    ProgramResourceId(3)
8618                );
8619                assert_eq!(
8620                    program.passes.last().unwrap().bindings[1].resource,
8621                    ProgramResourceId(1)
8622                );
8623                for child_resource in child.resources.iter().filter(|resource| {
8624                    !matches!(
8625                        resource.kind,
8626                        ProgramResourceKind::Input | ProgramResourceKind::Output
8627                    )
8628                }) {
8629                    let name = format!("double_double_real_{}", child_resource.name);
8630                    let mapped = program
8631                        .resources
8632                        .iter()
8633                        .find(|resource| resource.name == name)
8634                        .expect("DD real child resource must be cloned into parent");
8635                    assert_eq!(mapped.scalar, ScalarType::DoubleDouble);
8636                    assert_eq!(mapped.initialization, child_resource.initialization);
8637                    assert!(mapped.external_layout.is_none());
8638                }
8639            }
8640        }
8641    }
8642
8643    #[test]
8644    fn double_double_real_padding_stays_fused_in_true_real_boundaries() {
8645        for length in [15usize, 16usize] {
8646            for precision in [Precision::DoubleDouble, Precision::DoubleDoubleF64Storage] {
8647                let external_scalar = if precision == Precision::DoubleDouble {
8648                    ScalarType::DoubleDouble
8649                } else {
8650                    ScalarType::F64
8651                };
8652                for transform in [TransformKind::RealToComplex, TransformKind::ComplexToReal] {
8653                    let config = FftConfig::new(vec![length])
8654                        .with_batch_count(7)
8655                        .with_transform(transform)
8656                        .with_precision(precision)
8657                        .with_inverse_normalization(transform == TransformKind::ComplexToReal)
8658                        .with_grouped_batch(0, 3)
8659                        .unwrap()
8660                        .with_zero_padding(0, 3, 7)
8661                        .unwrap();
8662                    let ir = crate::DoubleDoubleRealFftIr::build(&FftPlan::build(config).unwrap())
8663                        .unwrap();
8664                    assert!(ir.has_spatial_zero_padding());
8665                    assert_eq!(ir.grouped_batch, 3);
8666                    assert_eq!(ir.transform.grouped_batch(), 3);
8667                    assert_eq!(ir.batch_group_count(), 3);
8668                    let child = ProgramIr::double_double_one_dim(&ir.transform).unwrap();
8669                    let program = ProgramIr::double_double_real(&ir).unwrap();
8670                    assert!(program.passes.iter().all(|pass| pass.dispatch.x == 3));
8671                    assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
8672                    assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
8673                    assert_eq!(program.passes.len(), child.passes.len() + 2);
8674                    assert!(
8675                        !program
8676                            .passes
8677                            .iter()
8678                            .any(|pass| pass.name.contains("zero_pad"))
8679                    );
8680                    assert_eq!(
8681                        program.passes.first().unwrap().bindings[0].resource,
8682                        ProgramResourceId(0)
8683                    );
8684                    assert_eq!(
8685                        program.passes.last().unwrap().bindings[1].resource,
8686                        ProgramResourceId(1)
8687                    );
8688                    assert!(
8689                        program
8690                            .resources
8691                            .iter()
8692                            .skip(2)
8693                            .all(|resource| { resource.scalar == ScalarType::DoubleDouble })
8694                    );
8695                    assert_eq!(ir.even_half_size, length.is_multiple_of(2));
8696                    if ir.even_half_size {
8697                        assert_eq!(ir.transform.sequence_len(), length / 2);
8698                    } else {
8699                        assert_eq!(ir.transform.sequence_len(), length);
8700                    }
8701                }
8702            }
8703        }
8704    }
8705
8706    #[test]
8707    fn double_double_nd_real_program_keeps_scalar_full_and_complex_compact_storage() {
8708        for last_len in [7usize, 8usize] {
8709            for precision in [Precision::DoubleDouble, Precision::DoubleDoubleF64Storage] {
8710                let external_scalar = if precision == Precision::DoubleDouble {
8711                    ScalarType::DoubleDouble
8712                } else {
8713                    ScalarType::F64
8714                };
8715                for transform in [TransformKind::RealToComplex, TransformKind::ComplexToReal] {
8716                    let plan = FftPlan::build(
8717                        FftConfig::new(vec![3, last_len])
8718                            .with_batch_count(7)
8719                            .with_precision(precision)
8720                            .with_transform(transform)
8721                            .with_inverse_normalization(transform == TransformKind::ComplexToReal)
8722                            .with_grouped_batch(0, 3)
8723                            .unwrap()
8724                            .with_grouped_batch(1, 3)
8725                            .unwrap(),
8726                    )
8727                    .unwrap();
8728                    let ir = crate::DoubleDoubleNdRealFftIr::build(&plan).unwrap();
8729                    let program = ProgramIr::double_double_nd_real(&ir).unwrap();
8730                    let compact_last = last_len / 2 + 1;
8731                    assert_eq!(ir.real_grouped_batch, 3);
8732                    assert_eq!(ir.real_axis.grouped_batch, 3);
8733                    assert_eq!(ir.batch_group_count(), 3);
8734                    assert_eq!(ir.complex_axes.len(), 1);
8735                    assert_eq!(ir.complex_axes[0].grouped_batch, 3);
8736                    assert_eq!(ir.complex_axes[0].transform.grouped_batch(), 3);
8737                    assert!(program.passes.iter().any(|pass| pass.dispatch.x == 3));
8738                    assert!(program.passes.iter().any(|pass| pass.dispatch.x == 7));
8739                    let outer_groups = (7 * compact_last).div_ceil(3);
8740                    assert!(
8741                        program
8742                            .passes
8743                            .iter()
8744                            .any(|pass| pass.dispatch.x == outer_groups as u32)
8745                    );
8746                    let input = program.input_resource().unwrap();
8747                    let output = program.output_resource().unwrap();
8748                    assert_eq!(input.scalar, external_scalar);
8749                    assert_eq!(output.scalar, external_scalar);
8750                    assert_eq!(
8751                        input.element_shape(),
8752                        if transform == TransformKind::RealToComplex {
8753                            ProgramElementShape::Scalar
8754                        } else {
8755                            ProgramElementShape::Complex
8756                        }
8757                    );
8758                    assert_eq!(
8759                        output.element_shape(),
8760                        if transform == TransformKind::RealToComplex {
8761                            ProgramElementShape::Complex
8762                        } else {
8763                            ProgramElementShape::Scalar
8764                        }
8765                    );
8766                    assert_eq!(program.resources[2].scalar, ScalarType::DoubleDouble);
8767                    assert_eq!(
8768                        program.resources[2].element_shape(),
8769                        ProgramElementShape::Scalar
8770                    );
8771                    assert_eq!(program.resources[2].element_bytes(), 16);
8772                    for resource in program.resources.iter().skip(3) {
8773                        assert_eq!(resource.scalar, ScalarType::DoubleDouble);
8774                    }
8775                    assert!(
8776                        program
8777                            .passes
8778                            .first()
8779                            .unwrap()
8780                            .name
8781                            .contains("external_input_promote")
8782                    );
8783                    assert!(
8784                        program
8785                            .passes
8786                            .last()
8787                            .unwrap()
8788                            .name
8789                            .contains("external_output_finalize")
8790                    );
8791                    assert!(
8792                        program
8793                            .passes
8794                            .iter()
8795                            .any(|pass| pass.name.contains("last_axis_vkfft_dd_real_"))
8796                    );
8797                    assert!(
8798                        program
8799                            .passes
8800                            .iter()
8801                            .any(|pass| pass.name.contains("axis_0_pack"))
8802                    );
8803                    let memory = program.memory_plan().unwrap();
8804                    for allocation in &memory.allocations {
8805                        let Some(first) = allocation.resources.first().copied() else {
8806                            continue;
8807                        };
8808                        let shape = program.resources[first.0].element_shape();
8809                        assert!(
8810                            allocation
8811                                .resources
8812                                .iter()
8813                                .all(|resource| program.resources[resource.0].element_shape()
8814                                    == shape)
8815                        );
8816                    }
8817                }
8818            }
8819        }
8820    }
8821
8822    #[test]
8823    fn double_double_nd_real_padding_stays_fused_in_full_real_boundary() {
8824        for (precision, external_scalar) in [
8825            (Precision::DoubleDouble, ScalarType::DoubleDouble),
8826            (Precision::DoubleDoubleF64Storage, ScalarType::F64),
8827        ] {
8828            for transform in [TransformKind::RealToComplex, TransformKind::ComplexToReal] {
8829                let config = FftConfig::new(vec![3, 8])
8830                    .with_batch_count(7)
8831                    .with_precision(precision)
8832                    .with_transform(transform)
8833                    .with_inverse_normalization(transform == TransformKind::ComplexToReal)
8834                    .with_grouped_batch(0, 3)
8835                    .unwrap()
8836                    .with_grouped_batch(1, 3)
8837                    .unwrap()
8838                    .with_zero_padding(0, 1, 2)
8839                    .unwrap()
8840                    .with_zero_padding(1, 2, 4)
8841                    .unwrap();
8842                let ir = crate::DoubleDoubleNdRealFftIr::build(&FftPlan::build(config).unwrap())
8843                    .unwrap();
8844                assert!(ir.has_spatial_zero_padding());
8845                let program = ProgramIr::double_double_nd_real(&ir).unwrap();
8846                assert_eq!(ir.real_grouped_batch, 3);
8847                assert_eq!(ir.real_axis.grouped_batch, 3);
8848                assert_eq!(ir.complex_axes[0].transform.grouped_batch(), 3);
8849                assert_eq!(ir.batch_group_count(), 3);
8850                assert!(program.passes.iter().any(|pass| pass.dispatch.x == 3));
8851                assert!(program.passes.iter().any(|pass| pass.dispatch.x == 7));
8852                assert!(program.passes.iter().any(|pass| pass.dispatch.x == 12));
8853                assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
8854                assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
8855                assert!(
8856                    program
8857                        .passes
8858                        .first()
8859                        .unwrap()
8860                        .name
8861                        .contains("external_input_promote")
8862                );
8863                assert!(
8864                    program
8865                        .passes
8866                        .last()
8867                        .unwrap()
8868                        .name
8869                        .contains("external_output_finalize")
8870                );
8871                assert!(
8872                    !program
8873                        .passes
8874                        .iter()
8875                        .any(|pass| pass.name.contains("zero_pad"))
8876                );
8877                assert_eq!(
8878                    program.resources[2].element_shape(),
8879                    ProgramElementShape::Scalar
8880                );
8881                assert_eq!(program.resources[2].element_bytes(), 16);
8882                assert!(program.resources.iter().skip(2).all(|resource| {
8883                    resource.scalar == ScalarType::DoubleDouble
8884                        && resource.kind != ProgramResourceKind::Input
8885                        && resource.kind != ProgramResourceKind::Output
8886                }));
8887            }
8888        }
8889    }
8890
8891    #[test]
8892    fn double_double_nd_padding_stays_fused_in_external_pack_scatter() {
8893        for (precision, external_scalar) in [
8894            (Precision::DoubleDouble, ScalarType::DoubleDouble),
8895            (Precision::DoubleDoubleF64Storage, ScalarType::F64),
8896        ] {
8897            for direction in [Direction::Forward, Direction::Inverse] {
8898                let config = FftConfig::new(vec![3, 4])
8899                    .with_batch_count(2)
8900                    .with_precision(precision)
8901                    .with_inverse_normalization(direction == Direction::Inverse)
8902                    .with_zero_padding(1, 1, 3)
8903                    .unwrap();
8904                let ir =
8905                    crate::DoubleDoubleNdFftIr::build(&FftPlan::build(config).unwrap(), direction)
8906                        .unwrap();
8907                assert!(ir.has_spatial_zero_padding());
8908                let program = ProgramIr::double_double_nd(&ir).unwrap();
8909                assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
8910                assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
8911                assert!(
8912                    !program
8913                        .passes
8914                        .iter()
8915                        .any(|pass| pass.name.contains("zero_pad"))
8916                );
8917                assert_eq!(
8918                    program.passes.first().unwrap().bindings[0].resource,
8919                    ProgramResourceId(0)
8920                );
8921                assert_eq!(
8922                    program.passes.last().unwrap().bindings[1].resource,
8923                    ProgramResourceId(1)
8924                );
8925                assert!(
8926                    program
8927                        .resources
8928                        .iter()
8929                        .skip(2)
8930                        .all(|resource| { resource.scalar == ScalarType::DoubleDouble })
8931                );
8932            }
8933        }
8934    }
8935
8936    #[test]
8937    fn double_double_nd_program_maps_fft_rader_child_resources_and_boundaries() {
8938        for (precision, external_scalar) in [
8939            (Precision::DoubleDouble, ScalarType::DoubleDouble),
8940            (Precision::DoubleDoubleF64Storage, ScalarType::F64),
8941        ] {
8942            let plan =
8943                FftPlan::build(FftConfig::new(vec![3, 17]).with_precision(precision)).unwrap();
8944            let ir = crate::DoubleDoubleNdFftIr::build(&plan, Direction::Forward).unwrap();
8945            let crate::DoubleDoubleOneDimIr::FftRader(rader) = &ir.axes[0].transform else {
8946                panic!("DD ND [3,17] fastest axis must use FFT Rader");
8947            };
8948            assert_eq!(rader.prime, 17);
8949            let child = ProgramIr::double_double_one_dim(&ir.axes[0].transform).unwrap();
8950            let program = ProgramIr::double_double_nd(&ir).unwrap();
8951            assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
8952            assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
8953            assert!(
8954                program
8955                    .resources
8956                    .iter()
8957                    .skip(2)
8958                    .all(|resource| { resource.scalar == ScalarType::DoubleDouble })
8959            );
8960            for child_resource in child.resources.iter().filter(|resource| {
8961                !matches!(
8962                    resource.kind,
8963                    ProgramResourceKind::Input | ProgramResourceKind::Output
8964                )
8965            }) {
8966                let name = format!("double_double_nd_axis_1_{}", child_resource.name);
8967                let mapped = program
8968                    .resources
8969                    .iter()
8970                    .find(|resource| resource.name == name)
8971                    .expect("FFT-Rader child resource must be cloned into ND parent");
8972                assert_eq!(mapped.scalar, ScalarType::DoubleDouble);
8973                assert_eq!(mapped.elements, child_resource.elements);
8974                assert_eq!(mapped.initialization, child_resource.initialization);
8975                assert!(mapped.external_layout.is_none());
8976            }
8977            let gather = program
8978                .passes
8979                .iter()
8980                .find(|pass| pass.name.contains("axis_1_") && pass.name.contains("_gather"))
8981                .expect("ND parent must include FFT-Rader gather pass");
8982            assert_eq!(gather.bindings[0].resource, ProgramResourceId(4));
8983            let rader_scatter = program
8984                .passes
8985                .iter()
8986                .find(|pass| pass.name.contains("axis_1_") && pass.name.contains("_scatter"))
8987                .expect("ND parent must include FFT-Rader scatter pass");
8988            assert_eq!(rader_scatter.bindings[0].resource, ProgramResourceId(4));
8989            assert_eq!(rader_scatter.bindings[1].resource, ProgramResourceId(5));
8990            assert!(rader_scatter.bindings.iter().any(|binding| {
8991                binding.role == BufferRole::Auxiliary
8992                    && binding.resource != ProgramResourceId(4)
8993                    && binding.resource != ProgramResourceId(5)
8994            }));
8995            assert_eq!(
8996                program.passes.first().unwrap().bindings[0].resource,
8997                ProgramResourceId(0)
8998            );
8999            assert_eq!(
9000                program.passes.last().unwrap().bindings[1].resource,
9001                ProgramResourceId(1)
9002            );
9003        }
9004    }
9005
9006    #[test]
9007    fn double_double_nd_program_keeps_only_tensor_boundaries_external() {
9008        for (precision, external_scalar) in [
9009            (Precision::DoubleDouble, ScalarType::DoubleDouble),
9010            (Precision::DoubleDoubleF64Storage, ScalarType::F64),
9011        ] {
9012            let plan = FftPlan::build(
9013                FftConfig::new(vec![3, 4])
9014                    .with_batch_count(2)
9015                    .with_precision(precision),
9016            )
9017            .unwrap();
9018            let ir = crate::DoubleDoubleNdFftIr::build(&plan, Direction::Forward).unwrap();
9019            let program = ProgramIr::double_double_nd(&ir).unwrap();
9020            assert_eq!(program.scalar, ScalarType::DoubleDouble);
9021            assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
9022            assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
9023            let child_programs = ir
9024                .axes
9025                .iter()
9026                .map(|axis| ProgramIr::double_double_one_dim(&axis.transform).unwrap())
9027                .collect::<Vec<_>>();
9028            let expected_internal_resources = child_programs
9029                .iter()
9030                .map(|child| {
9031                    child
9032                        .resources
9033                        .iter()
9034                        .filter(|resource| {
9035                            !matches!(
9036                                resource.kind,
9037                                ProgramResourceKind::Input | ProgramResourceKind::Output
9038                            )
9039                        })
9040                        .count()
9041                })
9042                .sum::<usize>();
9043            assert_eq!(program.resources.len(), 6 + expected_internal_resources);
9044            for resource in &program.resources[2..] {
9045                assert_eq!(resource.scalar, ScalarType::DoubleDouble);
9046            }
9047            let expected_passes = child_programs
9048                .iter()
9049                .map(|child| child.passes.len() + 2)
9050                .sum::<usize>();
9051            assert_eq!(program.passes.len(), expected_passes);
9052            assert_eq!(
9053                program.passes.first().unwrap().bindings[0].resource,
9054                ProgramResourceId(0)
9055            );
9056            assert_eq!(
9057                program.passes.last().unwrap().bindings[1].resource,
9058                ProgramResourceId(1)
9059            );
9060            for (axis, child) in ir.axes.iter().zip(&child_programs) {
9061                for child_resource in child.resources.iter().filter(|resource| {
9062                    !matches!(
9063                        resource.kind,
9064                        ProgramResourceKind::Input | ProgramResourceKind::Output
9065                    )
9066                }) {
9067                    let expected_name = format!(
9068                        "double_double_nd_axis_{}_{}",
9069                        axis.axis, child_resource.name
9070                    );
9071                    let cloned = program
9072                        .resources
9073                        .iter()
9074                        .find(|resource| resource.name == expected_name)
9075                        .expect("DD ND child internal resource must be cloned into parent");
9076                    assert_eq!(cloned.kind, child_resource.kind);
9077                    assert_eq!(cloned.scalar, child_resource.scalar);
9078                    assert_eq!(cloned.elements, child_resource.elements);
9079                    assert_eq!(cloned.initialization, child_resource.initialization);
9080                    assert!(cloned.external_layout.is_none());
9081                }
9082            }
9083        }
9084    }
9085
9086    #[test]
9087    fn double_double_stockham_large_program_uses_dd_ping_pong_stages() {
9088        for (precision, external_scalar) in [
9089            (Precision::DoubleDouble, ScalarType::DoubleDouble),
9090            (Precision::DoubleDoubleF64Storage, ScalarType::F64),
9091        ] {
9092            let plan = FftPlan::build(
9093                FftConfig::new(vec![1024])
9094                    .with_batch_count(2)
9095                    .with_precision(precision),
9096            )
9097            .unwrap();
9098            let ir = crate::DoubleDoubleStockhamIr::build(&plan, Direction::Forward).unwrap();
9099            assert!(ir.stages.len() >= 2);
9100            let program = ProgramIr::double_double_stockham(&ir).unwrap();
9101            assert_eq!(program.resources.len(), 5);
9102            assert_eq!(program.passes.len(), ir.stages.len() + 2);
9103            assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
9104            assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
9105            assert_eq!(program.resources[2].scalar, ScalarType::DoubleDouble);
9106            assert_eq!(program.resources[3].scalar, ScalarType::DoubleDouble);
9107            assert_eq!(program.resources[4].scalar, ScalarType::DoubleDouble);
9108            assert_eq!(program.passes.first().unwrap().bindings.len(), 2);
9109            assert_eq!(program.passes.last().unwrap().bindings.len(), 2);
9110            for pass in &program.passes[1..program.passes.len() - 1] {
9111                assert_eq!(pass.bindings.len(), 3);
9112                assert_eq!(pass.bindings[2].role, BufferRole::TwiddleLookupTable);
9113            }
9114            let ProgramResourceInitialization::ComplexDoubleDouble(values) =
9115                &program.resources[4].initialization
9116            else {
9117                panic!("large DD Stockham twiddle resource must preserve DD words");
9118            };
9119            assert_eq!(values, &ir.twiddles.packed_values());
9120        }
9121    }
9122
9123    #[test]
9124    fn double_double_stockham_program_separates_compute_and_external_storage() {
9125        for (precision, external_scalar) in [
9126            (Precision::DoubleDouble, ScalarType::DoubleDouble),
9127            (Precision::DoubleDoubleF64Storage, ScalarType::F64),
9128        ] {
9129            let plan = FftPlan::build(
9130                FftConfig::new(vec![15])
9131                    .with_batch_count(2)
9132                    .with_precision(precision),
9133            )
9134            .unwrap();
9135            let ir = crate::DoubleDoubleStockhamIr::build(&plan, Direction::Forward).unwrap();
9136            let program = ProgramIr::double_double_stockham(&ir).unwrap();
9137            assert_eq!(program.scalar, ScalarType::DoubleDouble);
9138            assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
9139            assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
9140            assert_eq!(program.passes.len(), 1);
9141            assert_eq!(program.passes[0].bindings.len(), 3);
9142            assert_eq!(
9143                program.passes[0].bindings[2].role,
9144                BufferRole::TwiddleLookupTable
9145            );
9146
9147            let lut = program
9148                .resources
9149                .iter()
9150                .find(|resource| resource.kind == ProgramResourceKind::LookupTable)
9151                .unwrap();
9152            assert_eq!(lut.scalar, ScalarType::DoubleDouble);
9153            let ProgramResourceInitialization::ComplexDoubleDouble(values) = &lut.initialization
9154            else {
9155                panic!("double-double twiddle resource must retain DD words");
9156            };
9157            assert_eq!(values.len(), lut.elements);
9158            assert_eq!(values, &ir.twiddles.packed_values());
9159
9160            let memory = program.memory_plan().unwrap();
9161            let input_allocation = memory.allocation_for(ProgramResourceId(0)).unwrap();
9162            let output_allocation = memory.allocation_for(ProgramResourceId(1)).unwrap();
9163            let lut_allocation = memory.allocation_for(lut.id).unwrap();
9164            assert_eq!(
9165                memory.allocations[input_allocation.0].scalar,
9166                external_scalar
9167            );
9168            assert_eq!(
9169                memory.allocations[output_allocation.0].scalar,
9170                external_scalar
9171            );
9172            assert_eq!(
9173                memory.allocations[lut_allocation.0].scalar,
9174                ScalarType::DoubleDouble
9175            );
9176            assert_eq!(ScalarType::DoubleDouble.complex_bytes(), 32);
9177            assert_eq!(
9178                memory.allocations[lut_allocation.0].elements
9179                    * ScalarType::DoubleDouble.complex_bytes(),
9180                lut.elements * 32
9181            );
9182            let expected_external_bytes = if external_scalar == ScalarType::DoubleDouble {
9183                15 * 2 * 32
9184            } else {
9185                15 * 2 * 16
9186            };
9187            assert_eq!(
9188                memory.allocations[input_allocation.0].elements * external_scalar.complex_bytes(),
9189                expected_external_bytes
9190            );
9191        }
9192    }
9193
9194    #[test]
9195    fn double_double_direct_rader_program_keeps_lut_at_compute_precision() {
9196        for (precision, external_scalar) in [
9197            (Precision::DoubleDouble, ScalarType::DoubleDouble),
9198            (Precision::DoubleDoubleF64Storage, ScalarType::F64),
9199        ] {
9200            let plan = FftPlan::build(
9201                FftConfig::new(vec![47])
9202                    .with_batch_count(2)
9203                    .with_precision(precision),
9204            )
9205            .unwrap();
9206            let ir = crate::DoubleDoubleDirectRaderIr::build(&plan, Direction::Forward).unwrap();
9207            let program = ProgramIr::double_double_direct_rader(&ir).unwrap();
9208            assert_eq!(program.scalar, ScalarType::DoubleDouble);
9209            assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
9210            assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
9211            assert_eq!(program.resources.len(), 3);
9212            assert_eq!(program.passes.len(), 1);
9213            assert_eq!(program.passes[0].bindings.len(), 3);
9214            assert_eq!(program.passes[0].bindings[2].role, BufferRole::LookupTable);
9215            let lut = &program.resources[2];
9216            assert_eq!(lut.kind, ProgramResourceKind::LookupTable);
9217            assert_eq!(lut.scalar, ScalarType::DoubleDouble);
9218            assert_eq!(lut.elements, 46);
9219            let ProgramResourceInitialization::ComplexDoubleDouble(values) = &lut.initialization
9220            else {
9221                panic!("DD direct-Rader roots must keep four-word initialization");
9222            };
9223            assert_eq!(values, &ir.table.twiddles_by_generator_power);
9224            let memory = program.memory_plan().unwrap();
9225            assert_eq!(
9226                memory.allocations[memory.allocation_for(lut.id).unwrap().0].scalar,
9227                ScalarType::DoubleDouble
9228            );
9229        }
9230    }
9231
9232    #[test]
9233    fn double_double_bluestein_program_keeps_chirp_and_convolution_at_dd_precision() {
9234        for (precision, external_scalar) in [
9235            (Precision::DoubleDouble, ScalarType::DoubleDouble),
9236            (Precision::DoubleDoubleF64Storage, ScalarType::F64),
9237        ] {
9238            let mut tuning = crate::PlannerTuning::portable();
9239            tuning.max_rader_fft_prime = 100;
9240            let plan = FftPlan::build(
9241                FftConfig::new(vec![103])
9242                    .with_batch_count(7)
9243                    .with_precision(precision)
9244                    .with_tuning(tuning)
9245                    .with_grouped_batch(0, 3)
9246                    .unwrap(),
9247            )
9248            .unwrap();
9249            let ir = crate::DoubleDoubleBluesteinIr::build(&plan, Direction::Forward).unwrap();
9250            assert_eq!(ir.convolution_len, 210);
9251            assert_eq!(ir.batch_count, 7);
9252            assert_eq!(ir.grouped_batch, 3);
9253            assert_eq!(ir.batch_group_count(), 3);
9254            assert_eq!(ir.forward_fft.grouped_batch(), 3);
9255            assert_eq!(ir.inverse_fft.grouped_batch(), 3);
9256            let program = ProgramIr::double_double_bluestein(&ir).unwrap();
9257            assert_eq!(program.scalar, ScalarType::DoubleDouble);
9258            assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
9259            assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
9260            let parent_scratch = [
9261                "double_double_bluestein_forward_input",
9262                "double_double_bluestein_forward_output",
9263                "double_double_bluestein_inverse_output",
9264            ]
9265            .map(|name| {
9266                program
9267                    .resources
9268                    .iter()
9269                    .find(|resource| resource.name == name)
9270                    .unwrap_or_else(|| panic!("missing fused DD Bluestein scratch {name}"))
9271            });
9272            for resource in parent_scratch {
9273                assert_eq!(resource.kind, ProgramResourceKind::Scratch);
9274                assert_eq!(resource.scalar, ScalarType::DoubleDouble);
9275                assert_eq!(resource.elements, 210 * 7);
9276            }
9277            assert!(
9278                !program
9279                    .resources
9280                    .iter()
9281                    .any(|resource| { resource.name == "double_double_bluestein_inverse_input" })
9282            );
9283            let forward_input = program
9284                .resources
9285                .iter()
9286                .find(|resource| resource.name == "double_double_bluestein_forward_input")
9287                .unwrap();
9288            let forward_output = program
9289                .resources
9290                .iter()
9291                .find(|resource| resource.name == "double_double_bluestein_forward_output")
9292                .unwrap();
9293            let inverse_output = program
9294                .resources
9295                .iter()
9296                .find(|resource| resource.name == "double_double_bluestein_inverse_output")
9297                .unwrap();
9298            let chirp = program
9299                .resources
9300                .iter()
9301                .find(|resource| resource.name == "double_double_bluestein_chirp")
9302                .unwrap();
9303            let kernel = program
9304                .resources
9305                .iter()
9306                .find(|resource| resource.name == "double_double_bluestein_kernel_spectrum")
9307                .unwrap();
9308            assert!(program.resources.len() > 8);
9309            assert!(program.passes.len() > 3);
9310            assert!(program.passes.iter().all(|pass| pass.dispatch.x == 3));
9311            assert!(
9312                program
9313                    .passes
9314                    .iter()
9315                    .all(|pass| !pass.name.ends_with("_multiply"))
9316            );
9317            let inverse_stage0 = program
9318                .passes
9319                .iter()
9320                .find(|pass| {
9321                    pass.name.contains("vkfft_dd_bluestein_inverse")
9322                        && pass.name.contains("stage_0")
9323                })
9324                .expect("DD Bluestein inverse Stockham stage0");
9325            assert!(inverse_stage0.name.ends_with("_input_mul_lut"));
9326            assert!(inverse_stage0.bindings.iter().any(|binding| {
9327                binding.binding == 3
9328                    && binding.role == BufferRole::LookupTable
9329                    && binding.access == BufferAccess::ReadOnly
9330            }));
9331            let preprocess = &program.passes[0];
9332            assert_eq!(preprocess.bindings.len(), 3);
9333            assert_eq!(preprocess.bindings[0].resource, ProgramResourceId(0));
9334            assert_eq!(preprocess.bindings[1].resource, forward_input.id);
9335            assert_eq!(preprocess.bindings[2].resource, chirp.id);
9336            assert_eq!(preprocess.bindings[2].role, BufferRole::LookupTable);
9337            let postprocess = program.passes.last().unwrap();
9338            assert_eq!(postprocess.bindings[0].role, BufferRole::Input);
9339            assert_eq!(postprocess.bindings[0].resource, inverse_output.id);
9340            assert_eq!(postprocess.bindings[1].resource, ProgramResourceId(1));
9341            assert_eq!(postprocess.bindings[2].resource, chirp.id);
9342            for resource in program
9343                .resources
9344                .iter()
9345                .filter(|resource| resource.kind == ProgramResourceKind::LookupTable)
9346            {
9347                assert_eq!(resource.scalar, ScalarType::DoubleDouble);
9348                let ProgramResourceInitialization::ComplexDoubleDouble(values) =
9349                    &resource.initialization
9350                else {
9351                    panic!("DD Bluestein immutable tables must retain all four words");
9352                };
9353                assert_eq!(values.len(), resource.elements);
9354                assert!(
9355                    values
9356                        .iter()
9357                        .any(|value| value.re.lo != 0.0 || value.im.lo != 0.0)
9358                );
9359            }
9360            assert_eq!(chirp.elements, 103);
9361            assert_eq!(kernel.elements, 210);
9362            let memory = program.memory_plan().unwrap();
9363            let parent_scratch_allocations =
9364                [forward_input.id, forward_output.id, inverse_output.id]
9365                    .map(|resource| memory.allocation_for(resource).unwrap());
9366            assert!(
9367                parent_scratch_allocations
9368                    .iter()
9369                    .all(|allocation| *allocation == parent_scratch_allocations[0])
9370            );
9371            let scratch_allocations = memory
9372                .allocations
9373                .iter()
9374                .filter(|allocation| allocation.kind == ProgramAllocationKind::Scratch)
9375                .collect::<Vec<_>>();
9376            assert_eq!(scratch_allocations.len(), 4);
9377            assert!(scratch_allocations.iter().all(|allocation| {
9378                allocation.scalar == ScalarType::DoubleDouble && allocation.elements == 210 * 7
9379            }));
9380            for resource in &program.resources[2..] {
9381                assert_eq!(
9382                    memory.allocations[memory.allocation_for(resource.id).unwrap().0].scalar,
9383                    ScalarType::DoubleDouble
9384                );
9385            }
9386        }
9387    }
9388
9389    #[test]
9390    fn double_double_fft_rader_program_keeps_all_convolution_tables_at_dd_precision() {
9391        for (precision, external_scalar) in [
9392            (Precision::DoubleDouble, ScalarType::DoubleDouble),
9393            (Precision::DoubleDoubleF64Storage, ScalarType::F64),
9394        ] {
9395            let plan = FftPlan::build(
9396                FftConfig::new(vec![257])
9397                    .with_batch_count(2)
9398                    .with_precision(precision),
9399            )
9400            .unwrap();
9401            let ir = crate::DoubleDoubleFftRaderIr::build(&plan, Direction::Forward).unwrap();
9402            let program = ProgramIr::double_double_fft_rader(&ir).unwrap();
9403            assert_eq!(program.scalar, ScalarType::DoubleDouble);
9404            assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
9405            assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
9406            let child_resources = |child: &crate::DoubleDoubleBluesteinConvolutionIr| match child {
9407                crate::DoubleDoubleBluesteinConvolutionIr::Stockham(child) => {
9408                    ProgramIr::double_double_stockham(child)
9409                        .unwrap()
9410                        .resources
9411                        .len()
9412                        - 2
9413                }
9414                crate::DoubleDoubleBluesteinConvolutionIr::Recursive(child) => {
9415                    ProgramIr::double_double_recursive(child)
9416                        .unwrap()
9417                        .resources
9418                        .len()
9419                        - 2
9420                }
9421                crate::DoubleDoubleBluesteinConvolutionIr::Bluestein(child) => {
9422                    ProgramIr::double_double_bluestein(child)
9423                        .unwrap()
9424                        .resources
9425                        .len()
9426                        - 2
9427                }
9428            };
9429            assert_eq!(
9430                program.resources.len(),
9431                5 + child_resources(&ir.forward_fft) + child_resources(&ir.inverse_fft)
9432            );
9433            assert_eq!(program.resources[2].kind, ProgramResourceKind::Scratch);
9434            assert_eq!(program.resources[3].kind, ProgramResourceKind::Scratch);
9435            assert_eq!(program.resources[2].elements, 256 * 2);
9436            assert_eq!(program.resources[3].elements, 256 * 2);
9437            let child_passes = |child: &crate::DoubleDoubleBluesteinConvolutionIr| match child {
9438                crate::DoubleDoubleBluesteinConvolutionIr::Stockham(child) => {
9439                    ProgramIr::double_double_stockham_internal(child)
9440                        .unwrap()
9441                        .passes
9442                        .len()
9443                }
9444                crate::DoubleDoubleBluesteinConvolutionIr::Recursive(child) => {
9445                    ProgramIr::double_double_recursive(child)
9446                        .unwrap()
9447                        .passes
9448                        .len()
9449                }
9450                crate::DoubleDoubleBluesteinConvolutionIr::Bluestein(child) => {
9451                    ProgramIr::double_double_bluestein(child)
9452                        .unwrap()
9453                        .passes
9454                        .len()
9455                }
9456            };
9457            let expected_passes = 2 + child_passes(&ir.forward_fft) + child_passes(&ir.inverse_fft);
9458            assert_eq!(program.passes.len(), expected_passes);
9459            assert!(
9460                program
9461                    .passes
9462                    .iter()
9463                    .all(|pass| !pass.name.ends_with("_multiply"))
9464            );
9465            assert_eq!(program.passes[0].bindings.len(), 2);
9466            assert_eq!(program.passes[0].bindings[0].resource, ProgramResourceId(0));
9467            assert_eq!(program.passes[0].bindings[1].resource, ProgramResourceId(2));
9468            let kernel_resource = program
9469                .resources
9470                .iter()
9471                .find(|resource| resource.name == "double_double_rader_kernel_spectrum")
9472                .expect("DD FFT-Rader kernel spectrum resource");
9473            assert_eq!(kernel_resource.elements, 256);
9474            let inverse_stage0 = program
9475                .passes
9476                .iter()
9477                .find(|pass| {
9478                    pass.name.contains("vkfft_dd_fft_rader_inverse")
9479                        && pass.name.contains("stage_0")
9480                })
9481                .expect("DD FFT-Rader inverse Stockham stage0");
9482            assert!(inverse_stage0.name.ends_with("_input_mul_lut"));
9483            assert!(inverse_stage0.bindings.iter().any(|binding| {
9484                binding.binding == 3
9485                    && binding.resource == kernel_resource.id
9486                    && binding.role == BufferRole::LookupTable
9487                    && binding.access == BufferAccess::ReadOnly
9488            }));
9489            assert!(program.passes.iter().any(|pass| {
9490                pass.bindings.iter().any(|binding| {
9491                    binding.resource == kernel_resource.id
9492                        && binding.role == BufferRole::LookupTable
9493                })
9494            }));
9495            assert!(
9496                program
9497                    .passes
9498                    .iter()
9499                    .filter(|pass| {
9500                        pass.bindings
9501                            .iter()
9502                            .any(|binding| binding.role == BufferRole::TwiddleLookupTable)
9503                    })
9504                    .count()
9505                    >= 2
9506            );
9507            let scatter = program.passes.last().unwrap();
9508            assert_eq!(scatter.bindings.len(), 3);
9509            assert_eq!(scatter.bindings[0].resource, ProgramResourceId(0));
9510            assert_eq!(scatter.bindings[1].resource, ProgramResourceId(1));
9511            assert_eq!(scatter.bindings[2].role, BufferRole::Auxiliary);
9512            assert!(matches!(scatter.bindings[2].resource.0, 2 | 3));
9513            for resource in program
9514                .resources
9515                .iter()
9516                .filter(|resource| resource.kind == ProgramResourceKind::LookupTable)
9517            {
9518                assert_eq!(resource.scalar, ScalarType::DoubleDouble);
9519                let ProgramResourceInitialization::ComplexDoubleDouble(values) =
9520                    &resource.initialization
9521                else {
9522                    panic!("DD FFT-Rader immutable tables must keep four-word initialization");
9523                };
9524                assert_eq!(values.len(), resource.elements);
9525                assert!(
9526                    values
9527                        .iter()
9528                        .any(|value| value.re.lo != 0.0 || value.im.lo != 0.0)
9529                );
9530            }
9531            let memory = program.memory_plan().unwrap();
9532            for resource in &program.resources[2..] {
9533                assert_eq!(
9534                    memory.allocations[memory.allocation_for(resource.id).unwrap().0].scalar,
9535                    ScalarType::DoubleDouble
9536                );
9537            }
9538        }
9539    }
9540
9541    #[test]
9542    fn bluestein_program_describes_ping_pong_and_lut_resources() {
9543        let mut bluestein_tuning = crate::PlannerTuning::portable();
9544        bluestein_tuning.max_rader_fft_prime = 100;
9545        let plan = FftPlan::build(
9546            FftConfig::new(vec![103])
9547                .with_batch_count(2)
9548                .with_tuning(bluestein_tuning),
9549        )
9550        .unwrap();
9551        let pipeline = BluesteinPipelineIr::build(&plan, Direction::Forward, device()).unwrap();
9552        let program = ProgramIr::bluestein(&pipeline).unwrap();
9553        assert_eq!(program.resources.len(), 5);
9554        assert_eq!(program.passes.len(), 4);
9555        assert_eq!(
9556            program.input_resource().unwrap().elements,
9557            pipeline.logical_len * 2
9558        );
9559        assert_eq!(
9560            program.output_resource().unwrap().elements,
9561            pipeline.logical_len * 2
9562        );
9563        assert_eq!(program.resources[2].kind, ProgramResourceKind::LookupTable);
9564        assert_eq!(program.resources[3].elements, pipeline.convolution_len * 2);
9565        assert_eq!(program.resources[4].elements, pipeline.convolution_len * 2);
9566        assert_eq!(program.passes[2].bindings[2].role, BufferRole::LookupTable);
9567        assert_eq!(program.passes[2].bindings[2].resource, ProgramResourceId(2));
9568        assert!(
9569            !program
9570                .passes
9571                .iter()
9572                .any(|pass| pass.name == pipeline.multiply.name)
9573        );
9574        assert_eq!(program.passes[0].bindings[0].resource, ProgramResourceId(0));
9575        assert_eq!(program.passes[0].bindings[1].resource, ProgramResourceId(3));
9576        assert_eq!(program.passes[3].bindings[0].resource, ProgramResourceId(3));
9577        assert_eq!(program.passes[3].bindings[1].resource, ProgramResourceId(1));
9578    }
9579
9580    #[test]
9581    fn grouped_bluestein_program_keeps_parent_tail_and_single_kernel_lut() {
9582        let mut tuning = crate::PlannerTuning::portable();
9583        tuning.max_rader_fft_prime = 100;
9584        let config = FftConfig::new(vec![103])
9585            .with_batch_count(7)
9586            .with_grouped_batch(0, 3)
9587            .unwrap()
9588            .with_precision(Precision::F16StorageF32Compute)
9589            .with_tuning(tuning)
9590            .with_zero_padding(0, 19, 31)
9591            .unwrap();
9592        let plan = FftPlan::build(config).unwrap();
9593        let ir = crate::OneDimFftIr::build(&plan, Direction::Forward, device()).unwrap();
9594        let crate::OneDimFftIr::Bluestein(pipeline) = &ir else {
9595            panic!("forced p103 grouped plan should use Bluestein");
9596        };
9597        assert_eq!(pipeline.grouped_batch, 3);
9598        let program = ProgramIr::one_dim_fft(&ir).unwrap();
9599        assert!(
9600            program.passes.iter().all(|pass| pass.dispatch.x == 3),
9601            "grouped Bluestein dispatches: {:?}",
9602            program
9603                .passes
9604                .iter()
9605                .map(|pass| (pass.name.as_str(), pass.dispatch.x))
9606                .collect::<Vec<_>>()
9607        );
9608        let kernel_luts = program
9609            .resources
9610            .iter()
9611            .filter(|resource| {
9612                resource.kind == ProgramResourceKind::LookupTable
9613                    && resource.name.contains("bluestein_kernel_spectrum")
9614            })
9615            .collect::<Vec<_>>();
9616        assert_eq!(kernel_luts.len(), 1);
9617        assert_eq!(kernel_luts[0].elements, pipeline.convolution_len);
9618        assert_eq!(program.input_resource().unwrap().elements, 103 * 7);
9619        assert_eq!(program.output_resource().unwrap().elements, 103 * 7);
9620        assert!(
9621            program
9622                .passes
9623                .first()
9624                .unwrap()
9625                .name
9626                .contains("zero_pad_forward_input")
9627        );
9628        program.validate().unwrap();
9629    }
9630
9631    #[test]
9632    fn multidimensional_real_and_r2r_programs_use_axis_composition() {
9633        let r2c_plan = FftPlan::build(
9634            FftConfig::new(vec![3, 8]).with_transform(crate::TransformKind::RealToComplex),
9635        )
9636        .unwrap();
9637        let r2c = crate::NdRealFftIr::build(&r2c_plan, device()).unwrap();
9638        let r2c_program = ProgramIr::nd_real_fft(&r2c).unwrap();
9639        assert_eq!(r2c_program.input_resource().unwrap().elements, 24);
9640        assert_eq!(r2c_program.output_resource().unwrap().elements, 15);
9641        assert_eq!(r2c_program.passes.len(), 4);
9642
9643        let r2r_plan = FftPlan::build(
9644            FftConfig::new(vec![3, 4])
9645                .with_transform(crate::TransformKind::Dct(crate::DctType::II)),
9646        )
9647        .unwrap();
9648        let r2r = crate::NdR2rIr::build(&r2r_plan, Direction::Forward, device()).unwrap();
9649        let r2r_program = ProgramIr::nd_r2r(&r2r).unwrap();
9650        assert_eq!(r2r_program.input_resource().unwrap().elements, 12);
9651        assert_eq!(r2r_program.output_resource().unwrap().elements, 12);
9652        assert_eq!(r2r_program.passes.len(), 10);
9653        assert!(r2r_program.passes[0].name.contains("nd_r2r_pack_axis_1"));
9654    }
9655
9656    #[test]
9657    fn r2r_fft_reduction_programs_cover_all_i_through_iv_shapes() {
9658        for (transform, label, length, fft_elements) in [
9659            (
9660                crate::TransformKind::Dct(crate::DctType::I),
9661                "dct1",
9662                9usize,
9663                32usize,
9664            ),
9665            (
9666                crate::TransformKind::Dst(crate::DstType::I),
9667                "dst1",
9668                9usize,
9669                40usize,
9670            ),
9671            (
9672                crate::TransformKind::Dct(crate::DctType::II),
9673                "dct2",
9674                9usize,
9675                18usize,
9676            ),
9677            (
9678                crate::TransformKind::Dct(crate::DctType::III),
9679                "dct3",
9680                9usize,
9681                18usize,
9682            ),
9683            (
9684                crate::TransformKind::Dst(crate::DstType::II),
9685                "dst2",
9686                9usize,
9687                18usize,
9688            ),
9689            (
9690                crate::TransformKind::Dst(crate::DstType::III),
9691                "dst3",
9692                9usize,
9693                18usize,
9694            ),
9695            (
9696                crate::TransformKind::Dct(crate::DctType::IV),
9697                "dct4_even",
9698                8usize,
9699                8usize,
9700            ),
9701            (
9702                crate::TransformKind::Dst(crate::DstType::IV),
9703                "dst4_even",
9704                8usize,
9705                8usize,
9706            ),
9707            (
9708                crate::TransformKind::Dct(crate::DctType::IV),
9709                "dct4_odd",
9710                9usize,
9711                36usize,
9712            ),
9713            (
9714                crate::TransformKind::Dst(crate::DstType::IV),
9715                "dst4_odd",
9716                9usize,
9717                36usize,
9718            ),
9719        ] {
9720            let plan = FftPlan::build(
9721                FftConfig::new(vec![length])
9722                    .with_batch_count(2)
9723                    .with_transform(transform),
9724            )
9725            .unwrap();
9726            let ir = crate::R2rIr::build(&plan, Direction::Forward, device()).unwrap();
9727            assert!(ir.fft_reduction.is_some());
9728            let program = ProgramIr::r2r(&ir).unwrap();
9729            assert_eq!(program.resources.len(), 4);
9730            assert_eq!(program.passes.len(), 3);
9731            assert_eq!(program.input_resource().unwrap().elements, 2 * length);
9732            assert_eq!(program.output_resource().unwrap().elements, 2 * length);
9733            assert_eq!(program.resources[2].elements, fft_elements);
9734            assert_eq!(program.resources[3].elements, fft_elements);
9735            assert!(
9736                program
9737                    .passes
9738                    .first()
9739                    .unwrap()
9740                    .name
9741                    .contains(&format!("r2r_fft_{label}_pre"))
9742            );
9743            assert!(
9744                program
9745                    .passes
9746                    .last()
9747                    .unwrap()
9748                    .name
9749                    .contains(&format!("r2r_fft_{label}_post"))
9750            );
9751        }
9752    }
9753
9754    #[test]
9755    fn one_dim_zero_padding_wraps_forward_and_inverse_at_external_boundaries() {
9756        let config = FftConfig::new(vec![64])
9757            .with_zero_padding(0, 32, 64)
9758            .unwrap();
9759        let plan = FftPlan::build(config).unwrap();
9760        let forward = OneDimFftIr::build(&plan, Direction::Forward, device()).unwrap();
9761        let forward_program = ProgramIr::one_dim_fft(&forward).unwrap();
9762        assert!(
9763            forward_program
9764                .passes
9765                .first()
9766                .unwrap()
9767                .name
9768                .contains("zero_pad_forward_input")
9769        );
9770        assert_eq!(
9771            forward_program.passes.first().unwrap().bindings[0].resource,
9772            ProgramResourceId(0)
9773        );
9774        assert_eq!(
9775            forward_program
9776                .resource(forward_program.passes.first().unwrap().bindings[1].resource)
9777                .unwrap()
9778                .kind,
9779            ProgramResourceKind::Scratch
9780        );
9781
9782        let config = FftConfig::new(vec![64])
9783            .with_inverse_normalization(true)
9784            .with_zero_padding(0, 32, 64)
9785            .unwrap();
9786        let plan = FftPlan::build(config).unwrap();
9787        let inverse = OneDimFftIr::build(&plan, Direction::Inverse, device()).unwrap();
9788        let inverse_program = ProgramIr::one_dim_fft(&inverse).unwrap();
9789        assert!(
9790            inverse_program
9791                .passes
9792                .last()
9793                .unwrap()
9794                .name
9795                .contains("zero_pad_inverse_output")
9796        );
9797        assert_eq!(
9798            inverse_program.passes.last().unwrap().bindings[1].resource,
9799            ProgramResourceId(1)
9800        );
9801        assert_eq!(
9802            inverse_program
9803                .resource(inverse_program.passes.last().unwrap().bindings[0].resource)
9804                .unwrap()
9805                .kind,
9806            ProgramResourceKind::Scratch
9807        );
9808    }
9809
9810    #[test]
9811    fn frequency_zero_padding_reverses_mixed_storage_program_boundary_order() {
9812        let build = |direction| {
9813            let config = FftConfig::new(vec![64])
9814                .with_batch_count(2)
9815                .with_precision(Precision::F16StorageF32Compute)
9816                .with_inverse_normalization(direction == Direction::Inverse)
9817                .with_zero_padding(0, 16, 32)
9818                .unwrap()
9819                .with_zero_padding_domain(ZeroPaddingDomain::Frequency);
9820            let plan = FftPlan::build(config).unwrap();
9821            let ir = OneDimFftIr::build(&plan, direction, device()).unwrap();
9822            ProgramIr::one_dim_fft(&ir).unwrap()
9823        };
9824
9825        let forward = build(Direction::Forward);
9826        let forward_pad = forward.passes.last().unwrap();
9827        assert!(forward_pad.name.contains("zero_pad_forward_output"));
9828        assert_eq!(forward_pad.bindings[1].resource, ProgramResourceId(1));
9829        assert_eq!(
9830            forward
9831                .resource(forward_pad.bindings[0].resource)
9832                .unwrap()
9833                .kind,
9834            ProgramResourceKind::Scratch
9835        );
9836        assert_eq!(
9837            forward
9838                .resource(forward_pad.bindings[0].resource)
9839                .unwrap()
9840                .scalar,
9841            ScalarType::F32
9842        );
9843        assert_eq!(forward.output_resource().unwrap().scalar, ScalarType::F16);
9844
9845        let inverse = build(Direction::Inverse);
9846        let inverse_pad = inverse.passes.first().unwrap();
9847        assert!(inverse_pad.name.contains("zero_pad_inverse_input"));
9848        assert_eq!(inverse_pad.bindings[0].resource, ProgramResourceId(0));
9849        assert_eq!(inverse.input_resource().unwrap().scalar, ScalarType::F16);
9850        assert_eq!(
9851            inverse
9852                .resource(inverse_pad.bindings[1].resource)
9853                .unwrap()
9854                .kind,
9855            ProgramResourceKind::Scratch
9856        );
9857        assert_eq!(
9858            inverse
9859                .resource(inverse_pad.bindings[1].resource)
9860                .unwrap()
9861                .scalar,
9862            ScalarType::F32
9863        );
9864    }
9865
9866    #[test]
9867    fn generic_one_dim_program_covers_bluestein_with_compact_external_buffers() {
9868        let mut bluestein_tuning = crate::PlannerTuning::portable();
9869        bluestein_tuning.max_rader_fft_prime = 100;
9870        let plan = FftPlan::build(
9871            FftConfig::new(vec![103])
9872                .with_batch_count(2)
9873                .with_tuning(bluestein_tuning),
9874        )
9875        .unwrap();
9876        let ir = crate::OneDimFftIr::build(&plan, Direction::Forward, device()).unwrap();
9877        assert!(matches!(ir, crate::OneDimFftIr::Bluestein(_)));
9878        let program = ProgramIr::one_dim_fft(&ir).unwrap();
9879        assert_eq!(program.input_resource().unwrap().elements, 206);
9880        assert_eq!(program.output_resource().unwrap().elements, 206);
9881        assert_eq!(program.passes.len(), 4);
9882        assert_eq!(program.passes[2].bindings.len(), 3);
9883        assert_eq!(program.passes[2].bindings[2].role, BufferRole::LookupTable);
9884        assert!(
9885            program
9886                .resources
9887                .iter()
9888                .any(|resource| resource.kind == ProgramResourceKind::LookupTable)
9889        );
9890        let memory = program.memory_plan().unwrap();
9891        assert!(memory.allocations.len() <= program.resources.len());
9892    }
9893
9894    #[test]
9895    fn mixed_program_flattens_outer_and_nested_fft_rader_passes() {
9896        for length in [34usize, 514] {
9897            let plan = FftPlan::build(FftConfig::new(vec![length]).with_batch_count(2)).unwrap();
9898            let ir =
9899                crate::MixedRaderStockhamIr::build(&plan, Direction::Forward, device()).unwrap();
9900            let program = ProgramIr::mixed_rader_stockham(&ir).unwrap();
9901            assert_eq!(program.resources.len(), 9);
9902            assert_eq!(program.passes.len(), 6);
9903            assert_eq!(program.input_resource().unwrap().elements, 2 * length);
9904            assert_eq!(program.output_resource().unwrap().elements, 2 * length);
9905            assert_eq!(program.resources[5].kind, ProgramResourceKind::LookupTable);
9906            assert_eq!(program.passes[2].bindings[2].role, BufferRole::LookupTable);
9907            assert_eq!(program.passes[2].bindings[2].resource, ProgramResourceId(5));
9908            assert_eq!(program.passes[2].bindings[3].role, BufferRole::Auxiliary);
9909            assert_eq!(program.passes[2].bindings[3].resource, ProgramResourceId(1));
9910            if length == 514 {
9911                // The fused generator-order forward FFT sees four logical 256-point
9912                // containers; both internal FFTs physically launch two grouped workgroups.
9913                assert_eq!(program.passes[1].dispatch.x, 2);
9914                assert_eq!(program.passes[2].dispatch.x, 2);
9915            }
9916        }
9917    }
9918
9919    #[test]
9920    fn p257_rader_transpose_fuses_generator_gather_into_forward_stockham() {
9921        let plan = FftPlan::build(FftConfig::new(vec![2056])).unwrap();
9922        let ir = crate::MixedRaderStockhamIr::build(&plan, Direction::Forward, device()).unwrap();
9923        let MixedPrimeRaderIr::FftConvolution(rader) = &ir.prime_stage else {
9924            panic!("2056 should use FFT-convolution Rader");
9925        };
9926        assert_eq!(
9927            rader.input_strategy,
9928            RaderFftInputStrategy::GeneratorOrderStockham
9929        );
9930        let program = ProgramIr::mixed_rader_stockham(&ir).unwrap();
9931        assert_eq!(program.resources.len(), 9);
9932        assert_eq!(program.passes.len(), 6);
9933        assert!(
9934            !program
9935                .passes
9936                .iter()
9937                .any(|pass| pass.name == rader.gather.name)
9938        );
9939        assert_eq!(program.passes[1].bindings[0].resource, ProgramResourceId(1));
9940        assert_eq!(program.passes[1].bindings[0].role, BufferRole::Input);
9941        assert_eq!(program.passes[2].bindings[2].role, BufferRole::LookupTable);
9942        assert_eq!(program.passes[2].bindings[2].resource, ProgramResourceId(5));
9943        assert_eq!(program.passes[2].bindings[3].role, BufferRole::Auxiliary);
9944        assert_eq!(program.passes[2].bindings[3].resource, ProgramResourceId(1));
9945        assert_eq!(program.passes[2].bindings[1].resource, ProgramResourceId(4));
9946        assert!(
9947            !program
9948                .passes
9949                .iter()
9950                .any(|pass| pass.name == rader.scatter.name)
9951        );
9952    }
9953
9954    #[test]
9955    fn mixed_program_flattens_direct_rader_stage() {
9956        let plan = FftPlan::build(FftConfig::new(vec![94]).with_batch_count(2)).unwrap();
9957        let ir = crate::MixedRaderStockhamIr::build(&plan, Direction::Forward, device()).unwrap();
9958        let program = ProgramIr::mixed_rader_stockham(&ir).unwrap();
9959        assert_eq!(program.resources.len(), 7);
9960        assert_eq!(program.passes.len(), 5);
9961        assert_eq!(program.resources[3].kind, ProgramResourceKind::LookupTable);
9962        assert_eq!(program.passes[1].bindings[2].resource, ProgramResourceId(3));
9963    }
9964
9965    #[test]
9966    fn fft_rader_program_preserves_prime_input_for_scatter() {
9967        let plan = FftPlan::build(FftConfig::new(vec![17]).with_batch_count(2)).unwrap();
9968        let pipeline = RaderFftPipelineIr::build(&plan, Direction::Forward, device()).unwrap();
9969        let program = ProgramIr::rader_fft(&pipeline).unwrap();
9970        assert_eq!(program.resources.len(), 5);
9971        assert_eq!(program.passes.len(), 2);
9972        assert_eq!(
9973            pipeline.input_strategy,
9974            RaderFftInputStrategy::GeneratorOrderStockham
9975        );
9976        let fused_inverse = &program.passes[1];
9977        assert_eq!(fused_inverse.bindings[2].role, BufferRole::LookupTable);
9978        assert_eq!(fused_inverse.bindings[2].resource, ProgramResourceId(4));
9979        assert_eq!(fused_inverse.bindings[3].role, BufferRole::Auxiliary);
9980        assert_eq!(fused_inverse.bindings[3].resource, ProgramResourceId(0));
9981        assert_eq!(fused_inverse.bindings[1].resource, ProgramResourceId(3));
9982        assert!(
9983            !program
9984                .passes
9985                .iter()
9986                .any(|pass| pass.name == pipeline.scatter.name)
9987        );
9988        assert_eq!(program.resources[4].kind, ProgramResourceKind::LookupTable);
9989    }
9990
9991    #[test]
9992    fn recursive_program_flattens_repeated_and_multi_prime_rader_trees() {
9993        for (length, expected_passes, expected_luts) in
9994            [(289usize, 7usize, 1usize), (323, 7, 2), (578, 8, 1)]
9995        {
9996            let plan = FftPlan::build(FftConfig::new(vec![length])).unwrap();
9997            let ir = crate::RecursiveFftIr::build(&plan, Direction::Forward, device()).unwrap();
9998            let program = ProgramIr::recursive_fft(&ir).unwrap();
9999            assert_eq!(program.passes.len(), expected_passes);
10000            assert_eq!(program.input_resource().unwrap().elements, length);
10001            assert_eq!(program.output_resource().unwrap().elements, length);
10002            assert_eq!(
10003                program
10004                    .resources
10005                    .iter()
10006                    .filter(|resource| resource.kind == ProgramResourceKind::LookupTable)
10007                    .count(),
10008                expected_luts
10009            );
10010            assert!(program.passes.iter().any(|pass| {
10011                pass.bindings
10012                    .iter()
10013                    .any(|binding| binding.role == BufferRole::Auxiliary)
10014            }));
10015            if length == 578 {
10016                assert!(
10017                    !program
10018                        .passes
10019                        .iter()
10020                        .any(|pass| pass.name == "vkfft_recursive_twiddle_34_forward")
10021                );
10022                assert!(
10023                    !program
10024                        .passes
10025                        .iter()
10026                        .any(|pass| pass.name.contains("recursive_scatter_34"))
10027                );
10028                assert!(
10029                    !program
10030                        .passes
10031                        .iter()
10032                        .any(|pass| pass.name.contains("recursive_pack_34"))
10033                );
10034            }
10035        }
10036    }
10037
10038    #[test]
10039    fn real_fft_program_uses_compact_external_layouts() {
10040        let length = 17usize;
10041        let batch_count = 2usize;
10042        let r2c_plan = FftPlan::build(
10043            FftConfig::new(vec![length])
10044                .with_batch_count(batch_count)
10045                .with_transform(crate::TransformKind::RealToComplex),
10046        )
10047        .unwrap();
10048        let r2c = crate::RealFftIr::build(&r2c_plan, device()).unwrap();
10049        let r2c_program = ProgramIr::real_fft(&r2c).unwrap();
10050        assert_eq!(
10051            r2c_program.input_resource().unwrap().elements,
10052            length * batch_count
10053        );
10054        assert_eq!(
10055            r2c_program.output_resource().unwrap().elements,
10056            (length / 2 + 1) * batch_count
10057        );
10058        assert!(r2c_program.passes.last().unwrap().name.contains("real_r2c"));
10059
10060        let c2r_plan = FftPlan::build(
10061            FftConfig::new(vec![length])
10062                .with_batch_count(batch_count)
10063                .with_transform(crate::TransformKind::ComplexToReal)
10064                .with_inverse_normalization(true),
10065        )
10066        .unwrap();
10067        let c2r = crate::RealFftIr::build(&c2r_plan, device()).unwrap();
10068        let c2r_program = ProgramIr::real_fft(&c2r).unwrap();
10069        assert_eq!(
10070            c2r_program.input_resource().unwrap().elements,
10071            (length / 2 + 1) * batch_count
10072        );
10073        assert_eq!(
10074            c2r_program.output_resource().unwrap().elements,
10075            length * batch_count
10076        );
10077        assert!(
10078            c2r_program
10079                .passes
10080                .first()
10081                .unwrap()
10082                .name
10083                .contains("real_c2r")
10084        );
10085    }
10086
10087    #[test]
10088    fn real_zero_padding_wraps_only_external_spatial_boundaries() {
10089        let r2c_plan = FftPlan::build(
10090            FftConfig::new(vec![34])
10091                .with_transform(crate::TransformKind::RealToComplex)
10092                .with_zero_padding(0, 17, 34)
10093                .unwrap(),
10094        )
10095        .unwrap();
10096        let r2c = crate::RealFftIr::build(&r2c_plan, device()).unwrap();
10097        let r2c_program = ProgramIr::real_fft(&r2c).unwrap();
10098        assert!(
10099            r2c_program
10100                .passes
10101                .first()
10102                .unwrap()
10103                .name
10104                .contains("zero_pad_forward_input")
10105        );
10106        assert_eq!(
10107            r2c_program.passes.first().unwrap().bindings[0].resource,
10108            ProgramResourceId(0)
10109        );
10110        assert_eq!(
10111            r2c_program
10112                .resource(r2c_program.passes.first().unwrap().bindings[1].resource)
10113                .unwrap()
10114                .kind,
10115            ProgramResourceKind::Scratch
10116        );
10117
10118        let c2r_plan = FftPlan::build(
10119            FftConfig::new(vec![34])
10120                .with_transform(crate::TransformKind::ComplexToReal)
10121                .with_inverse_normalization(true)
10122                .with_zero_padding(0, 17, 34)
10123                .unwrap(),
10124        )
10125        .unwrap();
10126        let c2r = crate::RealFftIr::build(&c2r_plan, device()).unwrap();
10127        let c2r_program = ProgramIr::real_fft(&c2r).unwrap();
10128        assert!(
10129            c2r_program
10130                .passes
10131                .last()
10132                .unwrap()
10133                .name
10134                .contains("zero_pad_inverse_output")
10135        );
10136        assert_eq!(
10137            c2r_program.passes.last().unwrap().bindings[1].resource,
10138            ProgramResourceId(1)
10139        );
10140
10141        let nd_plan = FftPlan::build(
10142            FftConfig::new(vec![3, 8])
10143                .with_transform(crate::TransformKind::RealToComplex)
10144                .with_zero_padding(0, 2, 3)
10145                .unwrap()
10146                .with_zero_padding(1, 6, 8)
10147                .unwrap(),
10148        )
10149        .unwrap();
10150        let nd = crate::NdRealFftIr::build(&nd_plan, device()).unwrap();
10151        assert!(nd.real_axis.zero_pad_pass.is_none());
10152        let nd_program = ProgramIr::nd_real_fft(&nd).unwrap();
10153        assert!(
10154            nd_program
10155                .passes
10156                .first()
10157                .unwrap()
10158                .name
10159                .contains("nd_zero_pad_forward_input")
10160        );
10161        assert_eq!(
10162            nd_program.passes.first().unwrap().bindings[0].resource,
10163            ProgramResourceId(0)
10164        );
10165    }
10166
10167    #[test]
10168    fn even_real_stockham_program_fuses_io_passes() {
10169        let length = 16usize;
10170        let batch_count = 2usize;
10171        let r2c_plan = FftPlan::build(
10172            FftConfig::new(vec![length])
10173                .with_batch_count(batch_count)
10174                .with_transform(crate::TransformKind::RealToComplex),
10175        )
10176        .unwrap();
10177        let r2c = crate::RealFftIr::build(&r2c_plan, device()).unwrap();
10178        let fused_r2c = r2c
10179            .fused_even_input_stockham_kernel()
10180            .unwrap()
10181            .expect("16-point R2C should have a single half-size Stockham root");
10182        assert!(matches!(
10183            fused_r2c.io_mapping,
10184            crate::StockhamIoMapping::RealEvenPack(_)
10185        ));
10186        assert!(matches!(
10187            fused_r2c.output_modifier,
10188            crate::StockhamOutputModifier::RealEvenPostprocess(_)
10189        ));
10190        let r2c_program = ProgramIr::real_fft(&r2c).unwrap();
10191        assert_eq!(r2c_program.passes.len(), 1);
10192        assert!(r2c_program.passes[0].name.contains("stockham"));
10193        assert_eq!(r2c_program.passes[0].bindings.len(), 3);
10194        assert_eq!(
10195            r2c_program.passes[0].bindings[2].role,
10196            BufferRole::Auxiliary
10197        );
10198        assert_eq!(
10199            r2c_program.passes[0].bindings[2].access,
10200            BufferAccess::ReadWrite
10201        );
10202        assert_eq!(
10203            r2c_program
10204                .resource(r2c_program.passes[0].bindings[2].resource)
10205                .unwrap()
10206                .kind,
10207            ProgramResourceKind::Scratch
10208        );
10209
10210        let c2r_plan = FftPlan::build(
10211            FftConfig::new(vec![length])
10212                .with_batch_count(batch_count)
10213                .with_transform(crate::TransformKind::ComplexToReal)
10214                .with_inverse_normalization(true),
10215        )
10216        .unwrap();
10217        let c2r = crate::RealFftIr::build(&c2r_plan, device()).unwrap();
10218        let fused_c2r = c2r
10219            .fused_even_input_stockham_kernel()
10220            .unwrap()
10221            .expect("16-point C2R should have a single half-size Stockham root");
10222        assert!(matches!(
10223            fused_c2r.io_mapping,
10224            crate::StockhamIoMapping::RealEvenInversePreprocess(_)
10225        ));
10226        assert!(matches!(
10227            fused_c2r.output_modifier,
10228            crate::StockhamOutputModifier::RealEvenUnpack(_)
10229        ));
10230        let c2r_program = ProgramIr::real_fft(&c2r).unwrap();
10231        assert_eq!(c2r_program.passes.len(), 1);
10232        assert_eq!(
10233            c2r_program.passes[0].bindings[0].resource,
10234            ProgramResourceId(0)
10235        );
10236        assert_eq!(
10237            c2r_program.passes[0].bindings[1].resource,
10238            ProgramResourceId(1)
10239        );
10240    }
10241
10242    #[test]
10243    fn recursive_even_real_program_fuses_root_boundaries() {
10244        let length = 578usize;
10245        for transform in [
10246            crate::TransformKind::RealToComplex,
10247            crate::TransformKind::ComplexToReal,
10248        ] {
10249            let plan = FftPlan::build(
10250                FftConfig::new(vec![length])
10251                    .with_transform(transform)
10252                    .with_inverse_normalization(transform == crate::TransformKind::ComplexToReal),
10253            )
10254            .unwrap();
10255            let ir = crate::RealFftIr::build(&plan, device()).unwrap();
10256            assert!(ir.fused_even_input_stockham_kernel().unwrap().is_none());
10257            assert!(ir.fused_even_recursive_ir().unwrap().is_some());
10258            let program = ProgramIr::real_fft(&ir).unwrap();
10259            assert!(
10260                program
10261                    .resources
10262                    .iter()
10263                    .all(|resource| !resource.name.contains("real_recursive_transform"))
10264            );
10265            assert!(program.passes.iter().all(|pass| {
10266                !pass.name.contains("pack_even_odd")
10267                    && !pass.name.contains("preprocess_even_half")
10268                    && !pass.name.contains("postprocess_even_half")
10269                    && !pass.name.contains("unpack_even_odd")
10270            }));
10271            match transform {
10272                crate::TransformKind::RealToComplex => {
10273                    assert!(
10274                        program
10275                            .passes
10276                            .first()
10277                            .unwrap()
10278                            .name
10279                            .contains("real_even_pack")
10280                    );
10281                    assert!(
10282                        program
10283                            .passes
10284                            .last()
10285                            .unwrap()
10286                            .name
10287                            .contains("real_even_postprocess")
10288                    );
10289                }
10290                crate::TransformKind::ComplexToReal => {
10291                    assert!(
10292                        program
10293                            .passes
10294                            .first()
10295                            .unwrap()
10296                            .name
10297                            .contains("real_even_inverse_preprocess")
10298                    );
10299                    assert!(
10300                        program
10301                            .passes
10302                            .last()
10303                            .unwrap()
10304                            .name
10305                            .contains("real_even_unpack")
10306                    );
10307                }
10308                _ => unreachable!(),
10309            }
10310        }
10311    }
10312
10313    #[test]
10314    fn bluestein_even_real_program_fuses_outer_boundaries() {
10315        let length = 206usize;
10316        let mut tuning = crate::PlannerTuning::portable();
10317        tuning.max_rader_fft_prime = 100;
10318        for transform in [
10319            crate::TransformKind::RealToComplex,
10320            crate::TransformKind::ComplexToReal,
10321        ] {
10322            let plan = FftPlan::build(
10323                FftConfig::new(vec![length])
10324                    .with_transform(transform)
10325                    .with_inverse_normalization(transform == crate::TransformKind::ComplexToReal)
10326                    .with_tuning(tuning),
10327            )
10328            .unwrap();
10329            let ir = crate::RealFftIr::build(&plan, device()).unwrap();
10330            assert!(ir.fused_even_input_stockham_kernel().unwrap().is_none());
10331            assert!(ir.fused_even_recursive_ir().unwrap().is_none());
10332            assert!(ir.fused_even_bluestein_ir().unwrap().is_some());
10333            let program = ProgramIr::real_fft(&ir).unwrap();
10334            assert!(program.passes.iter().all(|pass| {
10335                !pass.name.contains("pack_even_odd")
10336                    && !pass.name.contains("preprocess_even_half")
10337                    && !pass.name.contains("postprocess_even_half")
10338                    && !pass.name.contains("unpack_even_odd")
10339            }));
10340            assert!(program.resources.iter().all(|resource| {
10341                !resource.name.contains("real_preprocess")
10342                    && !resource.name.contains("real_transform")
10343            }));
10344            match transform {
10345                crate::TransformKind::RealToComplex => {
10346                    assert!(
10347                        program
10348                            .passes
10349                            .first()
10350                            .unwrap()
10351                            .name
10352                            .contains("real_even_pack")
10353                    );
10354                    assert!(
10355                        program
10356                            .passes
10357                            .last()
10358                            .unwrap()
10359                            .name
10360                            .contains("real_even_postprocess")
10361                    );
10362                }
10363                crate::TransformKind::ComplexToReal => {
10364                    assert!(
10365                        program
10366                            .passes
10367                            .first()
10368                            .unwrap()
10369                            .name
10370                            .contains("real_even_inverse_preprocess")
10371                    );
10372                    assert!(
10373                        program
10374                            .passes
10375                            .last()
10376                            .unwrap()
10377                            .name
10378                            .contains("real_even_unpack")
10379                    );
10380                }
10381                _ => unreachable!(),
10382            }
10383            assert!(
10384                program
10385                    .resources
10386                    .iter()
10387                    .any(|resource| resource.name.contains("bluestein_scratch"))
10388            );
10389        }
10390    }
10391
10392    #[test]
10393    fn multidimensional_program_flattens_axis_pack_transform_scatter() {
10394        let plan = FftPlan::build(FftConfig::new(vec![3, 4]).with_batch_count(2)).unwrap();
10395        let ir = crate::NdFftIr::build(&plan, Direction::Forward, device()).unwrap();
10396        let program = ProgramIr::nd_fft(&ir).unwrap();
10397        assert_eq!(program.passes.len(), 6);
10398        assert_eq!(program.input_resource().unwrap().elements, 24);
10399        assert_eq!(program.output_resource().unwrap().elements, 24);
10400        assert!(program.passes[0].name.contains("nd_pack_axis_1"));
10401        assert!(program.passes[2].name.contains("nd_scatter_axis_1"));
10402        assert!(program.passes[3].name.contains("nd_pack_axis_0"));
10403        assert_eq!(program.passes[4].dispatch.x, 2);
10404        let outer = ir
10405            .axes
10406            .iter()
10407            .find(|axis| axis.axis == 0)
10408            .expect("outer ND axis");
10409        let crate::OneDimFftIr::Recursive(outer_recursive) = &outer.transform else {
10410            panic!("outer ND Stockham axis unexpectedly selected Bluestein");
10411        };
10412        let crate::RecursiveFftNodeIr::Stockham(outer_kernel) = &outer_recursive.root else {
10413            panic!("outer ND Stockham axis unexpectedly selected Rader");
10414        };
10415        assert_eq!(
10416            [outer_kernel.workgroup_size.x, outer_kernel.workgroup_size.y],
10417            [4, 1]
10418        );
10419        assert_eq!(outer_kernel.dispatch.x, 2);
10420        let memory = program.memory_plan().unwrap();
10421        assert!(memory.allocations.len() < program.resources.len());
10422    }
10423
10424    #[test]
10425    fn omit_dimension_removes_only_selected_axis_from_program_graph() {
10426        let plan = FftPlan::build(
10427            FftConfig::new(vec![3, 4])
10428                .with_batch_count(2)
10429                .with_omit_dimension(1, true)
10430                .unwrap(),
10431        )
10432        .unwrap();
10433        let ir = crate::NdFftIr::build(&plan, Direction::Forward, device()).unwrap();
10434        let program = ProgramIr::nd_fft(&ir).unwrap();
10435        assert_eq!(
10436            ir.axes.iter().map(|axis| axis.axis).collect::<Vec<_>>(),
10437            vec![0]
10438        );
10439        assert_eq!(program.input_resource().unwrap().elements, 24);
10440        assert_eq!(program.output_resource().unwrap().elements, 24);
10441        assert!(
10442            program
10443                .passes
10444                .iter()
10445                .any(|pass| pass.name.contains("nd_pack_axis_0"))
10446        );
10447        assert!(
10448            program
10449                .passes
10450                .iter()
10451                .any(|pass| pass.name.contains("nd_scatter_axis_0"))
10452        );
10453        assert!(
10454            !program
10455                .passes
10456                .iter()
10457                .any(|pass| pass.name.contains("axis_1"))
10458        );
10459
10460        let dd_plan = FftPlan::build(
10461            FftConfig::new(vec![3, 4])
10462                .with_batch_count(2)
10463                .with_precision(Precision::DoubleDouble)
10464                .with_omit_dimension(1, true)
10465                .unwrap(),
10466        )
10467        .unwrap();
10468        let dd_ir = crate::DoubleDoubleNdFftIr::build(&dd_plan, Direction::Forward).unwrap();
10469        let dd_program = ProgramIr::double_double_nd(&dd_ir).unwrap();
10470        assert_eq!(
10471            dd_ir.axes.iter().map(|axis| axis.axis).collect::<Vec<_>>(),
10472            vec![0]
10473        );
10474        assert_eq!(dd_program.input_resource().unwrap().elements, 24);
10475        assert_eq!(dd_program.output_resource().unwrap().elements, 24);
10476        assert!(
10477            !dd_program
10478                .passes
10479                .iter()
10480                .any(|pass| pass.name.contains("axis_1"))
10481        );
10482    }
10483
10484    #[test]
10485    fn formatted_batch_stride_nd_expands_only_external_allocations() {
10486        let plan = FftPlan::build(
10487            FftConfig::new(vec![3, 4])
10488                .with_batch_count(2)
10489                .with_input_buffer_batch_stride(17)
10490                .with_output_buffer_batch_stride(19),
10491        )
10492        .unwrap();
10493        let ir = crate::NdFftIr::build(&plan, Direction::Forward, device()).unwrap();
10494        let program = ProgramIr::nd_fft(&ir).unwrap();
10495        let input = program.input_resource().unwrap();
10496        let output = program.output_resource().unwrap();
10497        assert_eq!(input.elements, 34);
10498        assert_eq!(output.elements, 38);
10499        assert_eq!(input.external_layout.unwrap().logical_len, 12);
10500        assert_eq!(input.external_layout.unwrap().physical_stride, 17);
10501        assert_eq!(output.external_layout.unwrap().logical_len, 12);
10502        assert_eq!(output.external_layout.unwrap().physical_stride, 19);
10503        assert!(
10504            program
10505                .resources
10506                .iter()
10507                .filter(|resource| resource.kind == ProgramResourceKind::Scratch)
10508                .all(|resource| resource.elements == 24
10509                    || resource.kind == ProgramResourceKind::LookupTable)
10510        );
10511        program.validate().unwrap();
10512    }
10513
10514    #[test]
10515    fn formatted_tensor_stride_nd_exposes_physical_external_batches_and_dense_scratch() {
10516        let plan = FftPlan::build(
10517            FftConfig::new(vec![2, 3, 4])
10518                .with_batch_count(2)
10519                .with_input_buffer_axis_stride(1, 6)
10520                .unwrap()
10521                .with_input_buffer_axis_stride(0, 20)
10522                .unwrap()
10523                .with_output_buffer_axis_stride(1, 7)
10524                .unwrap()
10525                .with_output_buffer_axis_stride(0, 24)
10526                .unwrap(),
10527        )
10528        .unwrap();
10529        let ir = crate::NdFftIr::build(&plan, Direction::Forward, device()).unwrap();
10530        let program = ProgramIr::nd_fft(&ir).unwrap();
10531        let input = program.input_resource().unwrap();
10532        let output = program.output_resource().unwrap();
10533        assert_eq!(input.elements, 80);
10534        assert_eq!(output.elements, 96);
10535        assert_eq!(input.external_layout.unwrap().logical_len, 40);
10536        assert_eq!(input.external_layout.unwrap().physical_stride, 40);
10537        assert_eq!(output.external_layout.unwrap().logical_len, 48);
10538        assert_eq!(output.external_layout.unwrap().physical_stride, 48);
10539        assert!(
10540            program
10541                .resources
10542                .iter()
10543                .filter(|resource| resource.kind == ProgramResourceKind::Scratch)
10544                .all(|resource| resource.elements == 48)
10545        );
10546        program.validate().unwrap();
10547    }
10548
10549    #[test]
10550    fn formatted_zero_padding_nd_program_keeps_padding_on_dense_compute_boundaries() {
10551        let mut device = device();
10552        device.supports_f64 = true;
10553        for (domain, input_boundary) in [
10554            (ZeroPaddingDomain::Spatial, true),
10555            (ZeroPaddingDomain::Frequency, false),
10556        ] {
10557            let plan = FftPlan::build(
10558                FftConfig::new(vec![3, 4])
10559                    .with_batch_count(2)
10560                    .with_precision(Precision::F64ComputeF32Storage)
10561                    .with_input_buffer_axis_stride(0, 7)
10562                    .unwrap()
10563                    .with_output_buffer_axis_stride(0, 9)
10564                    .unwrap()
10565                    .with_zero_padding(1, 1, 3)
10566                    .unwrap()
10567                    .with_zero_padding_domain(domain),
10568            )
10569            .unwrap();
10570            let ir = crate::NdFftIr::build(&plan, Direction::Forward, device).unwrap();
10571            let program = ProgramIr::nd_fft(&ir).unwrap();
10572            assert_eq!(program.input_resource().unwrap().scalar, ScalarType::F32);
10573            assert_eq!(program.output_resource().unwrap().scalar, ScalarType::F32);
10574            assert_eq!(program.input_resource().unwrap().elements, 42);
10575            assert_eq!(program.output_resource().unwrap().elements, 54);
10576            let names = program
10577                .passes
10578                .iter()
10579                .map(|pass| pass.name.as_str())
10580                .collect::<Vec<_>>();
10581            if input_boundary {
10582                assert_eq!(names[0], "vkfft_nd_gather_formatted_input_before_zero_pad");
10583                assert!(names[1].contains("nd_zero_pad_forward_input"));
10584                assert!(ir.input_formatted_copy.is_some());
10585                assert!(ir.output_formatted_copy.is_none());
10586            } else {
10587                assert!(names[names.len() - 2].contains("nd_zero_pad_forward_output"));
10588                assert_eq!(
10589                    names[names.len() - 1],
10590                    "vkfft_nd_scatter_formatted_output_after_zero_pad"
10591                );
10592                assert!(ir.input_formatted_copy.is_none());
10593                assert!(ir.output_formatted_copy.is_some());
10594            }
10595            let zero_pad = ir.zero_pad_pass.as_ref().unwrap();
10596            assert_eq!(zero_pad.input_storage_scalar, ScalarType::F64);
10597            assert_eq!(zero_pad.output_storage_scalar, ScalarType::F64);
10598            assert!(program.resources.iter().any(|resource| {
10599                resource.kind == ProgramResourceKind::Scratch
10600                    && resource.scalar == ScalarType::F64
10601                    && resource.elements == 24
10602                    && resource.name.contains("formatted")
10603            }));
10604            program.validate().unwrap();
10605        }
10606    }
10607
10608    #[test]
10609    fn formatted_real_tensor_strides_insert_physical_gather_and_scatter_around_dense_program() {
10610        for (transform, input_stride, output_stride, input_elements, output_elements) in [
10611            (
10612                TransformKind::RealToComplex,
10613                11usize,
10614                7usize,
10615                66usize,
10616                42usize,
10617            ),
10618            (
10619                TransformKind::ComplexToReal,
10620                7usize,
10621                11usize,
10622                42usize,
10623                66usize,
10624            ),
10625        ] {
10626            let plan = FftPlan::build(
10627                FftConfig::new(vec![3usize, 8])
10628                    .with_batch_count(2)
10629                    .with_transform(transform)
10630                    .with_inverse_normalization(transform == TransformKind::ComplexToReal)
10631                    .with_input_buffer_axis_stride(0, input_stride)
10632                    .unwrap()
10633                    .with_output_buffer_axis_stride(0, output_stride)
10634                    .unwrap(),
10635            )
10636            .unwrap();
10637            let ir = crate::NdRealFftIr::build(&plan, device()).unwrap();
10638            let program = ProgramIr::nd_real_fft(&ir).unwrap();
10639            let input = program.input_resource().unwrap();
10640            let output = program.output_resource().unwrap();
10641            assert_eq!(input.elements, input_elements);
10642            assert_eq!(output.elements, output_elements);
10643            assert_eq!(
10644                input.external_layout.unwrap().logical_len,
10645                ir.input_external_layout.batch_stride
10646            );
10647            assert_eq!(
10648                output.external_layout.unwrap().logical_len,
10649                ir.output_external_layout.batch_stride
10650            );
10651            assert!(
10652                program
10653                    .passes
10654                    .first()
10655                    .unwrap()
10656                    .name
10657                    .contains("nd_real_gather_formatted_input")
10658            );
10659            assert!(
10660                program
10661                    .passes
10662                    .last()
10663                    .unwrap()
10664                    .name
10665                    .contains("nd_real_scatter_formatted_output")
10666            );
10667            assert!(program.resources.iter().any(|resource| {
10668                resource.name.contains("nd_real_formatted_input")
10669                    && resource.elements == ir.input_tensor_len() * ir.batch_count
10670            }));
10671            assert!(program.resources.iter().any(|resource| {
10672                resource.name.contains("nd_real_formatted_output")
10673                    && resource.elements == ir.output_tensor_len() * ir.batch_count
10674            }));
10675            program.validate().unwrap();
10676        }
10677    }
10678
10679    #[test]
10680    fn formatted_real_zero_padding_program_keeps_pad_between_dense_copies() {
10681        let mut device = device();
10682        device.supports_f64 = true;
10683        for (transform, input_elements, output_elements, pad_at_start) in [
10684            (TransformKind::RealToComplex, 66usize, 42usize, true),
10685            (TransformKind::ComplexToReal, 42usize, 66usize, false),
10686        ] {
10687            let (input_stride, output_stride) = match transform {
10688                TransformKind::RealToComplex => (11usize, 7usize),
10689                TransformKind::ComplexToReal => (7usize, 11usize),
10690                _ => unreachable!(),
10691            };
10692            let plan = FftPlan::build(
10693                FftConfig::new(vec![3usize, 8])
10694                    .with_batch_count(2)
10695                    .with_transform(transform)
10696                    .with_precision(Precision::F64ComputeF32Storage)
10697                    .with_inverse_normalization(transform == TransformKind::ComplexToReal)
10698                    .with_input_buffer_axis_stride(0, input_stride)
10699                    .unwrap()
10700                    .with_output_buffer_axis_stride(0, output_stride)
10701                    .unwrap()
10702                    .with_zero_padding(0, 1, 2)
10703                    .unwrap()
10704                    .with_zero_padding(1, 2, 4)
10705                    .unwrap(),
10706            )
10707            .unwrap();
10708            let ir = crate::NdRealFftIr::build(&plan, device).unwrap();
10709            let program = ProgramIr::nd_real_fft(&ir).unwrap();
10710            assert_eq!(program.input_resource().unwrap().elements, input_elements);
10711            assert_eq!(program.output_resource().unwrap().elements, output_elements);
10712            assert_eq!(program.input_resource().unwrap().scalar, ScalarType::F32);
10713            assert_eq!(program.output_resource().unwrap().scalar, ScalarType::F32);
10714            let zero_pad = ir.zero_pad_pass.as_ref().unwrap();
10715            assert_eq!(zero_pad.input_storage_scalar, ScalarType::F64);
10716            assert_eq!(zero_pad.output_storage_scalar, ScalarType::F64);
10717            let names = program
10718                .passes
10719                .iter()
10720                .map(|pass| pass.name.as_str())
10721                .collect::<Vec<_>>();
10722            assert_eq!(
10723                names.first().copied(),
10724                Some("vkfft_nd_real_gather_formatted_input")
10725            );
10726            assert_eq!(
10727                names.last().copied(),
10728                Some("vkfft_nd_real_scatter_formatted_output")
10729            );
10730            if pad_at_start {
10731                assert!(names[1].contains("nd_zero_pad_forward_input"));
10732            } else {
10733                assert!(names[names.len() - 2].contains("nd_zero_pad_inverse_output"));
10734            }
10735            assert!(program.resources.iter().any(|resource| {
10736                resource.kind == ProgramResourceKind::Scratch
10737                    && resource.scalar == ScalarType::F64
10738                    && resource.name.contains("formatted")
10739            }));
10740            program.validate().unwrap();
10741        }
10742    }
10743
10744    #[test]
10745    fn formatted_nd_r2r_program_wraps_dense_pipeline_with_physical_copies() {
10746        let plan = FftPlan::build(
10747            FftConfig::new(vec![2usize, 3, 4])
10748                .with_batch_count(2)
10749                .with_transform(TransformKind::Dct(crate::DctType::II))
10750                .with_input_buffer_axis_stride(1, 6)
10751                .unwrap()
10752                .with_input_buffer_axis_stride(0, 20)
10753                .unwrap()
10754                .with_output_buffer_axis_stride(1, 7)
10755                .unwrap()
10756                .with_output_buffer_axis_stride(0, 24)
10757                .unwrap(),
10758        )
10759        .unwrap();
10760        let ir = crate::NdR2rIr::build(&plan, Direction::Forward, device()).unwrap();
10761        let program = ProgramIr::nd_r2r(&ir).unwrap();
10762        let input = program.input_resource().unwrap();
10763        let output = program.output_resource().unwrap();
10764        assert_eq!(input.elements, 80);
10765        assert_eq!(output.elements, 96);
10766        assert_eq!(input.external_layout.unwrap().physical_stride, 40);
10767        assert_eq!(output.external_layout.unwrap().physical_stride, 48);
10768        assert_eq!(
10769            program.passes.first().unwrap().name,
10770            "vkfft_nd_r2r_gather_formatted_input"
10771        );
10772        assert_eq!(
10773            program.passes.last().unwrap().name,
10774            "vkfft_nd_r2r_scatter_formatted_output"
10775        );
10776        let formatted_input = program
10777            .resources
10778            .iter()
10779            .find(|resource| resource.name.starts_with("nd_r2r_formatted_input"))
10780            .unwrap();
10781        let formatted_output = program
10782            .resources
10783            .iter()
10784            .find(|resource| resource.name.starts_with("nd_r2r_formatted_output"))
10785            .unwrap();
10786        assert_eq!(formatted_input.kind, ProgramResourceKind::Scratch);
10787        assert_eq!(formatted_output.kind, ProgramResourceKind::Scratch);
10788        assert_eq!(formatted_input.elements, 48);
10789        assert_eq!(formatted_output.elements, 48);
10790        assert!(formatted_input.external_layout.is_none());
10791        assert!(formatted_output.external_layout.is_none());
10792        program.validate().unwrap();
10793    }
10794
10795    #[test]
10796    fn formatted_double_double_nd_c2c_program_keeps_dense_dd_scratch() {
10797        for (precision, external_scalar) in [
10798            (Precision::DoubleDouble, ScalarType::DoubleDouble),
10799            (Precision::DoubleDoubleF64Storage, ScalarType::F64),
10800        ] {
10801            let plan = FftPlan::build(
10802                FftConfig::new(vec![2usize, 3, 4])
10803                    .with_batch_count(2)
10804                    .with_precision(precision)
10805                    .with_input_buffer_axis_stride(1, 6)
10806                    .unwrap()
10807                    .with_input_buffer_axis_stride(0, 20)
10808                    .unwrap()
10809                    .with_output_buffer_axis_stride(1, 7)
10810                    .unwrap()
10811                    .with_output_buffer_axis_stride(0, 24)
10812                    .unwrap(),
10813            )
10814            .unwrap();
10815            let ir = crate::DoubleDoubleNdFftIr::build(&plan, Direction::Forward).unwrap();
10816            let program = ProgramIr::double_double_nd(&ir).unwrap();
10817            let input = program.input_resource().unwrap();
10818            let output = program.output_resource().unwrap();
10819            assert_eq!(input.scalar, external_scalar);
10820            assert_eq!(output.scalar, external_scalar);
10821            assert_eq!(input.elements, 80);
10822            assert_eq!(output.elements, 96);
10823            assert_eq!(input.external_layout.unwrap().physical_stride, 40);
10824            assert_eq!(output.external_layout.unwrap().physical_stride, 48);
10825            assert_eq!(
10826                program.passes.first().unwrap().name,
10827                "vkfft_dd_nd_gather_formatted_input"
10828            );
10829            assert_eq!(
10830                program.passes.last().unwrap().name,
10831                "vkfft_dd_nd_scatter_formatted_output"
10832            );
10833            for name in ["double_double_nd_tensor_a", "double_double_nd_tensor_b"] {
10834                let scratch = program
10835                    .resources
10836                    .iter()
10837                    .find(|resource| resource.name == name)
10838                    .unwrap();
10839                assert_eq!(scratch.kind, ProgramResourceKind::Scratch);
10840                assert_eq!(scratch.scalar, ScalarType::DoubleDouble);
10841                assert_eq!(scratch.elements, 48);
10842                assert!(scratch.external_layout.is_none());
10843            }
10844            program.validate().unwrap();
10845        }
10846    }
10847
10848    #[test]
10849    fn formatted_double_double_real_and_r2r_programs_keep_dense_dd_scratch() {
10850        for (precision, external_scalar) in [
10851            (Precision::DoubleDouble, ScalarType::DoubleDouble),
10852            (Precision::DoubleDoubleF64Storage, ScalarType::F64),
10853        ] {
10854            let real_plan = FftPlan::build(
10855                FftConfig::new(vec![2usize, 3, 8])
10856                    .with_batch_count(2)
10857                    .with_transform(TransformKind::RealToComplex)
10858                    .with_precision(precision)
10859                    .with_input_buffer_axis_stride(1, 11)
10860                    .unwrap()
10861                    .with_input_buffer_axis_stride(0, 40)
10862                    .unwrap()
10863                    .with_output_buffer_axis_stride(1, 7)
10864                    .unwrap()
10865                    .with_output_buffer_axis_stride(0, 24)
10866                    .unwrap(),
10867            )
10868            .unwrap();
10869            let real_ir = crate::DoubleDoubleNdRealFftIr::build(&real_plan).unwrap();
10870            let real_program = ProgramIr::double_double_nd_real(&real_ir).unwrap();
10871            let real_input = real_program.input_resource().unwrap();
10872            let real_output = real_program.output_resource().unwrap();
10873            assert_eq!(real_input.scalar, external_scalar);
10874            assert_eq!(real_output.scalar, external_scalar);
10875            assert_eq!(real_input.elements, 160);
10876            assert_eq!(real_output.elements, 96);
10877            assert_eq!(real_input.external_layout.unwrap().physical_stride, 80);
10878            assert_eq!(real_output.external_layout.unwrap().physical_stride, 48);
10879            assert_eq!(
10880                real_program.passes.first().unwrap().name,
10881                "vkfft_dd_nd_real_gather_formatted_input"
10882            );
10883            assert_eq!(
10884                real_program.passes.last().unwrap().name,
10885                "vkfft_dd_nd_real_scatter_formatted_output"
10886            );
10887            let full_scratch = real_program
10888                .resources
10889                .iter()
10890                .find(|resource| resource.name == "double_double_nd_real_full_scalar")
10891                .unwrap();
10892            assert_eq!(full_scratch.scalar, ScalarType::DoubleDouble);
10893            assert_eq!(full_scratch.elements, 96);
10894            let compact_scratch = real_program
10895                .resources
10896                .iter()
10897                .find(|resource| resource.name == "double_double_nd_real_compact_a")
10898                .unwrap();
10899            assert_eq!(compact_scratch.scalar, ScalarType::DoubleDouble);
10900            assert_eq!(compact_scratch.elements, 60);
10901            real_program.validate().unwrap();
10902
10903            let r2r_plan = FftPlan::build(
10904                FftConfig::new(vec![2usize, 3, 4])
10905                    .with_batch_count(2)
10906                    .with_transform(TransformKind::Dct(crate::DctType::II))
10907                    .with_precision(precision)
10908                    .with_input_buffer_axis_stride(1, 6)
10909                    .unwrap()
10910                    .with_input_buffer_axis_stride(0, 20)
10911                    .unwrap()
10912                    .with_output_buffer_axis_stride(1, 7)
10913                    .unwrap()
10914                    .with_output_buffer_axis_stride(0, 24)
10915                    .unwrap(),
10916            )
10917            .unwrap();
10918            let r2r_ir = crate::DoubleDoubleNdR2rIr::build(&r2r_plan, Direction::Forward).unwrap();
10919            let r2r_program = ProgramIr::double_double_nd_r2r(&r2r_ir).unwrap();
10920            let r2r_input = r2r_program.input_resource().unwrap();
10921            let r2r_output = r2r_program.output_resource().unwrap();
10922            assert_eq!(r2r_input.scalar, external_scalar);
10923            assert_eq!(r2r_output.scalar, external_scalar);
10924            assert_eq!(r2r_input.elements, 80);
10925            assert_eq!(r2r_output.elements, 96);
10926            assert_eq!(r2r_input.external_layout.unwrap().physical_stride, 40);
10927            assert_eq!(r2r_output.external_layout.unwrap().physical_stride, 48);
10928            assert_eq!(
10929                r2r_program.passes.first().unwrap().name,
10930                "vkfft_dd_nd_r2r_gather_formatted_input"
10931            );
10932            assert_eq!(
10933                r2r_program.passes.last().unwrap().name,
10934                "vkfft_dd_nd_r2r_scatter_formatted_output"
10935            );
10936            for name in [
10937                "double_double_nd_r2r_tensor_a",
10938                "double_double_nd_r2r_tensor_b",
10939            ] {
10940                let scratch = r2r_program
10941                    .resources
10942                    .iter()
10943                    .find(|resource| resource.name == name)
10944                    .unwrap();
10945                assert_eq!(scratch.scalar, ScalarType::DoubleDouble);
10946                assert_eq!(scratch.elements, 48);
10947            }
10948            r2r_program.validate().unwrap();
10949        }
10950    }
10951
10952    #[test]
10953    fn double_double_omit_dimension_programs_drop_omitted_axis_passes() {
10954        let real_plan = FftPlan::build(
10955            FftConfig::new(vec![3usize, 8])
10956                .with_transform(TransformKind::RealToComplex)
10957                .with_precision(Precision::DoubleDouble)
10958                .with_omit_dimension(0, true)
10959                .unwrap(),
10960        )
10961        .unwrap();
10962        let real_ir = crate::DoubleDoubleNdRealFftIr::build(&real_plan).unwrap();
10963        assert!(real_ir.complex_axes.is_empty());
10964        let real_program = ProgramIr::double_double_nd_real(&real_ir).unwrap();
10965        assert!(
10966            !real_program
10967                .passes
10968                .iter()
10969                .any(|pass| pass.name.contains("nd_real_axis_0"))
10970        );
10971        assert!(
10972            real_program
10973                .passes
10974                .iter()
10975                .any(|pass| pass.name.contains("nd_real_last_axis"))
10976        );
10977        real_program.validate().unwrap();
10978
10979        let r2r_plan = FftPlan::build(
10980            FftConfig::new(vec![3usize, 4])
10981                .with_transform(TransformKind::Dct(crate::DctType::II))
10982                .with_precision(Precision::DoubleDouble)
10983                .with_omit_dimension(1, true)
10984                .unwrap(),
10985        )
10986        .unwrap();
10987        let r2r_ir = crate::DoubleDoubleNdR2rIr::build(&r2r_plan, Direction::Forward).unwrap();
10988        assert_eq!(
10989            r2r_ir.axes.iter().map(|axis| axis.axis).collect::<Vec<_>>(),
10990            vec![0]
10991        );
10992        let r2r_program = ProgramIr::double_double_nd_r2r(&r2r_ir).unwrap();
10993        assert!(
10994            !r2r_program
10995                .passes
10996                .iter()
10997                .any(|pass| pass.name.contains("nd_r2r_axis_1"))
10998        );
10999        assert!(
11000            r2r_program
11001                .passes
11002                .iter()
11003                .any(|pass| pass.name.contains("nd_r2r_axis_0"))
11004        );
11005        r2r_program.validate().unwrap();
11006    }
11007
11008    #[test]
11009    fn f64_four_step_uses_one_lazy_full_period_twiddle_lut() {
11010        let length = 1_048_576usize;
11011        let mut profile = DeviceProfile::generic(Backend::Vulkan, GpuVendor::Nvidia);
11012        profile.shared_memory_bytes = 48 * 1024;
11013        profile.shared_memory_pow2_bytes = 48 * 1024;
11014        profile.max_threads_per_block = 1024;
11015        profile.supports_f64 = true;
11016        let plan =
11017            FftPlan::build(FftConfig::new(vec![length]).with_precision(Precision::F64)).unwrap();
11018        let ir = crate::RecursiveFftIr::build(&plan, Direction::Forward, profile).unwrap();
11019        assert!(ir.four_step_plan.is_some());
11020        let kernels = ir.four_step_stockham_upload_kernels().unwrap().unwrap();
11021        assert!(
11022            kernels
11023                .iter()
11024                .all(|kernel| kernel.twiddle_lut_len() == Some(length))
11025        );
11026
11027        let program = ProgramIr::recursive_fft(&ir).unwrap();
11028        let root_luts = program
11029            .resources
11030            .iter()
11031            .filter(|resource| {
11032                matches!(
11033                    resource.initialization,
11034                    ProgramResourceInitialization::StockhamUnitRoots { len } if len == length
11035                )
11036            })
11037            .collect::<Vec<_>>();
11038        assert_eq!(root_luts.len(), 1);
11039        assert_eq!(root_luts[0].elements, length);
11040        assert!(program.passes.iter().all(|pass| {
11041            pass.bindings
11042                .iter()
11043                .any(|binding| binding.role == BufferRole::TwiddleLookupTable)
11044        }));
11045        assert!(!program.resources.iter().any(|resource| {
11046            matches!(
11047                &resource.initialization,
11048                ProgramResourceInitialization::Complex64(values) if values.len() == length
11049            )
11050        }));
11051    }
11052
11053    #[test]
11054    fn two_upload_four_step_program_collapses_root_to_two_fft_passes() {
11055        let length = 1_048_576usize;
11056        let mut profile = DeviceProfile::generic(Backend::Vulkan, GpuVendor::Nvidia);
11057        profile.shared_memory_bytes = 48 * 1024;
11058        profile.shared_memory_pow2_bytes = 48 * 1024;
11059        profile.max_threads_per_block = 1024;
11060        let plan = FftPlan::build(FftConfig::new(vec![length])).unwrap();
11061        let ir = crate::RecursiveFftIr::build(&plan, Direction::Forward, profile).unwrap();
11062        assert!(ir.four_step_plan.is_some());
11063
11064        let program = ProgramIr::recursive_fft(&ir).unwrap();
11065        assert_eq!(program.passes.len(), 2);
11066        assert_eq!(program.resources.len(), 3);
11067        assert_eq!(
11068            program
11069                .resources
11070                .iter()
11071                .filter(|resource| resource.kind == ProgramResourceKind::Scratch)
11072                .count(),
11073            1
11074        );
11075        assert!(program.passes[0].name.ends_with("four_step_upload_1"));
11076        assert!(program.passes[1].name.ends_with("four_step_upload_0"));
11077        assert_eq!(program.passes[0].dispatch.x, 256);
11078        assert_eq!(program.passes[1].dispatch.x, 256);
11079        program.validate().unwrap();
11080    }
11081
11082    #[test]
11083    fn three_upload_four_step_program_collapses_root_to_three_fft_passes() {
11084        let length = 8_388_608usize;
11085        let mut profile = DeviceProfile::generic(Backend::Vulkan, GpuVendor::Nvidia);
11086        profile.shared_memory_bytes = 48 * 1024;
11087        profile.shared_memory_pow2_bytes = 48 * 1024;
11088        profile.max_threads_per_block = 1024;
11089        let plan = FftPlan::build(FftConfig::new(vec![length])).unwrap();
11090        let ir = crate::RecursiveFftIr::build(&plan, Direction::Forward, profile).unwrap();
11091        assert!(ir.four_step_plan.is_some());
11092
11093        let program = ProgramIr::recursive_fft(&ir).unwrap();
11094        assert_eq!(program.passes.len(), 3);
11095        assert_eq!(program.resources.len(), 3);
11096        let scratch = program
11097            .resources
11098            .iter()
11099            .find(|resource| resource.kind == ProgramResourceKind::Scratch)
11100            .unwrap()
11101            .id;
11102        assert_eq!(scratch, ProgramResourceId(2));
11103        assert!(program.passes[0].name.ends_with("four_step_upload_2"));
11104        assert!(program.passes[1].name.ends_with("four_step_upload_1"));
11105        assert!(program.passes[2].name.ends_with("four_step_upload_0"));
11106        assert_eq!(
11107            program
11108                .passes
11109                .iter()
11110                .map(|pass| pass.dispatch.x)
11111                .collect::<Vec<_>>(),
11112            vec![2048, 4096, 2048]
11113        );
11114        assert_eq!(program.passes[0].bindings[0].resource, ProgramResourceId(0));
11115        assert_eq!(program.passes[0].bindings[1].resource, ProgramResourceId(1));
11116        assert_eq!(program.passes[1].bindings[0].resource, ProgramResourceId(1));
11117        assert_eq!(program.passes[1].bindings[1].resource, scratch);
11118        assert_eq!(program.passes[2].bindings[0].resource, scratch);
11119        assert_eq!(program.passes[2].bindings[1].resource, ProgramResourceId(1));
11120        program.validate().unwrap();
11121    }
11122
11123    #[test]
11124    fn rader_three_upload_reuses_output_only_for_same_scalar_storage() {
11125        let length = 1_922usize;
11126        let base_profile = DeviceProfile {
11127            shared_memory_bytes: 1024,
11128            shared_memory_pow2_bytes: 1024,
11129            max_threads_per_block: 32,
11130            max_workgroup_size: [32, 32, 64],
11131            ..DeviceProfile::generic(Backend::Vulkan, GpuVendor::Nvidia)
11132        };
11133
11134        for (precision, expected_exchange_scratch, expected_output_scalar) in [
11135            (Precision::F32, 1usize, ScalarType::F32),
11136            (Precision::F16StorageF32Compute, 2usize, ScalarType::F16),
11137        ] {
11138            let mut profile = base_profile;
11139            if precision == Precision::F16StorageF32Compute {
11140                profile.shared_memory_bytes = 2 * 1024;
11141                profile.shared_memory_pow2_bytes = 2 * 1024;
11142            }
11143            let plan =
11144                FftPlan::build(FftConfig::new(vec![length]).with_precision(precision)).unwrap();
11145            let ir = crate::RecursiveFftIr::build(&plan, Direction::Forward, profile).unwrap();
11146            assert_eq!(ir.scalar, ScalarType::F32);
11147            assert_eq!(ir.external_scalar, expected_output_scalar);
11148            assert_eq!(
11149                ir.rader_forced_upload_schedule
11150                    .as_ref()
11151                    .expect("N1922 must retain the forced-Rader schedule")
11152                    .axis_split,
11153                vec![2, 31, 31]
11154            );
11155            assert_eq!(
11156                ir.four_step_rader_upload_nodes()
11157                    .unwrap()
11158                    .expect("N1922 must materialize three Rader Four-step uploads")
11159                    .len(),
11160                3
11161            );
11162
11163            let program = ProgramIr::recursive_fft(&ir).unwrap();
11164            let output = program.output_resource().unwrap();
11165            assert_eq!(output.scalar, expected_output_scalar);
11166            let exchange = program
11167                .resources
11168                .iter()
11169                .filter(|resource| {
11170                    resource.kind == ProgramResourceKind::Scratch
11171                        && resource.name.starts_with("rader_four_step_exchange")
11172                })
11173                .collect::<Vec<_>>();
11174            assert_eq!(exchange.len(), expected_exchange_scratch, "{precision:?}");
11175            if precision == Precision::F32 {
11176                assert!(program.passes.iter().any(|pass| {
11177                    pass.bindings
11178                        .iter()
11179                        .any(|binding| binding.resource == output.id)
11180                }));
11181            } else {
11182                assert_ne!(exchange[0].id, exchange[1].id);
11183                assert!(
11184                    exchange
11185                        .iter()
11186                        .all(|resource| resource.scalar == ScalarType::F32)
11187                );
11188            }
11189            program.validate().unwrap();
11190        }
11191    }
11192
11193    #[test]
11194    fn double_double_three_upload_reuses_output_only_for_dd_storage() {
11195        let stockham_device = DeviceProfile {
11196            shared_memory_bytes: 1024,
11197            shared_memory_pow2_bytes: 1024,
11198            max_threads_per_block: 1024,
11199            max_workgroup_size: [1024, 1024, 64],
11200            supports_f64: true,
11201            ..DeviceProfile::generic(Backend::Vulkan, GpuVendor::Nvidia)
11202        };
11203        let forced_rader_device = DeviceProfile {
11204            shared_memory_bytes: 8 * 1024,
11205            shared_memory_pow2_bytes: 8 * 1024,
11206            max_threads_per_block: 1024,
11207            max_workgroup_size: [1024, 1024, 64],
11208            supports_f64: true,
11209            ..DeviceProfile::generic(Backend::Vulkan, GpuVendor::Nvidia)
11210        };
11211
11212        for (precision, expected_scratch, expected_output_scalar) in [
11213            (Precision::DoubleDouble, 1usize, ScalarType::DoubleDouble),
11214            (Precision::DoubleDoubleF64Storage, 2usize, ScalarType::F64),
11215        ] {
11216            let stockham_plan = FftPlan::build_for_device(
11217                FftConfig::new(vec![8_192]).with_precision(precision),
11218                stockham_device,
11219            )
11220            .unwrap();
11221            let stockham_ir = crate::DoubleDoubleRecursiveFftIr::build_for_device(
11222                &stockham_plan,
11223                Direction::Forward,
11224                stockham_device,
11225            )
11226            .unwrap();
11227            assert_eq!(
11228                stockham_ir
11229                    .three_upload_four_step_plan
11230                    .expect("constrained DD N8192 must keep three-upload Four-step metadata")
11231                    .axis_split,
11232                [32, 16, 16]
11233            );
11234            let stockham_program = ProgramIr::double_double_recursive(&stockham_ir).unwrap();
11235            assert_eq!(
11236                stockham_program.output_resource().unwrap().scalar,
11237                expected_output_scalar
11238            );
11239            let stockham_exchange = stockham_program
11240                .resources
11241                .iter()
11242                .filter(|resource| {
11243                    resource.kind == ProgramResourceKind::Scratch
11244                        && resource
11245                            .name
11246                            .starts_with("double_double_three_upload_four_step_scratch")
11247                })
11248                .collect::<Vec<_>>();
11249            assert_eq!(stockham_exchange.len(), expected_scratch, "{precision:?}");
11250            assert!(
11251                stockham_exchange
11252                    .iter()
11253                    .all(|resource| resource.scalar == ScalarType::DoubleDouble)
11254            );
11255            stockham_program.validate().unwrap();
11256
11257            let forced_length = 17usize * 47 * 128;
11258            let forced_plan =
11259                FftPlan::build(FftConfig::new(vec![forced_length]).with_precision(precision))
11260                    .unwrap();
11261            let forced_ir = crate::DoubleDoubleRecursiveFftIr::build_for_device(
11262                &forced_plan,
11263                Direction::Forward,
11264                forced_rader_device,
11265            )
11266            .unwrap();
11267            let forced_schedule = forced_ir
11268                .rader_forced_upload_schedule
11269                .as_ref()
11270                .expect("DD p47 composite must retain forced-Rader upload metadata");
11271            assert_eq!(forced_schedule.axis_split, vec![64, 47, 34]);
11272            assert_eq!(forced_schedule.upload_count, 3);
11273            let forced_program = ProgramIr::double_double_recursive(&forced_ir).unwrap();
11274            assert_eq!(
11275                forced_program.output_resource().unwrap().scalar,
11276                expected_output_scalar
11277            );
11278            let forced_exchange = forced_program
11279                .resources
11280                .iter()
11281                .filter(|resource| {
11282                    resource.kind == ProgramResourceKind::Scratch
11283                        && resource
11284                            .name
11285                            .starts_with("double_double_forced_rader_three_upload_scratch")
11286                })
11287                .collect::<Vec<_>>();
11288            assert_eq!(forced_exchange.len(), expected_scratch, "{precision:?}");
11289            assert!(
11290                forced_exchange
11291                    .iter()
11292                    .all(|resource| resource.scalar == ScalarType::DoubleDouble)
11293            );
11294            forced_program.validate().unwrap();
11295        }
11296    }
11297
11298    #[test]
11299    fn memory_plan_reuses_dead_scratch_and_preserves_external_buffers() {
11300        let plan = FftPlan::build(FftConfig::new(vec![578])).unwrap();
11301        let ir = crate::RecursiveFftIr::build(&plan, Direction::Forward, device()).unwrap();
11302        let program = ProgramIr::recursive_fft(&ir).unwrap();
11303        let memory = program.memory_plan().unwrap();
11304        let input_allocation = memory
11305            .allocation_for(program.input_resource().unwrap().id)
11306            .unwrap();
11307        let output_allocation = memory
11308            .allocation_for(program.output_resource().unwrap().id)
11309            .unwrap();
11310        assert_ne!(input_allocation, output_allocation);
11311        assert_eq!(
11312            memory
11313                .allocations
11314                .iter()
11315                .filter(|allocation| allocation.kind == ProgramAllocationKind::LookupTable)
11316                .count(),
11317            1
11318        );
11319        assert!(memory.allocations.len() < program.resources.len());
11320        let logical_elements = program
11321            .resources
11322            .iter()
11323            .map(|resource| resource.elements)
11324            .sum::<usize>();
11325        assert!(memory.allocated_elements().unwrap() < logical_elements);
11326        assert!(memory.allocations.iter().any(|allocation| allocation.kind
11327            == ProgramAllocationKind::Scratch
11328            && allocation.resources.len() > 1));
11329    }
11330
11331    #[test]
11332    fn double_double_r2r_program_uses_scalar_boundaries_and_dd_internal_resources() {
11333        for precision in [Precision::DoubleDouble, Precision::DoubleDoubleF64Storage] {
11334            let external_scalar = if precision == Precision::DoubleDouble {
11335                ScalarType::DoubleDouble
11336            } else {
11337                ScalarType::F64
11338            };
11339            for transform in [
11340                TransformKind::Dct(crate::DctType::I),
11341                TransformKind::Dst(crate::DstType::I),
11342                TransformKind::Dst(crate::DstType::II),
11343                TransformKind::Dst(crate::DstType::III),
11344                TransformKind::Dct(crate::DctType::II),
11345                TransformKind::Dct(crate::DctType::III),
11346                TransformKind::Dct(crate::DctType::IV),
11347                TransformKind::Dst(crate::DstType::IV),
11348            ] {
11349                let plan = FftPlan::build(
11350                    FftConfig::new(vec![9])
11351                        .with_batch_count(7)
11352                        .with_precision(precision)
11353                        .with_transform(transform)
11354                        .with_grouped_batch(0, 3)
11355                        .unwrap(),
11356                )
11357                .unwrap();
11358                let ir = crate::DoubleDoubleR2rIr::build(&plan, Direction::Forward).unwrap();
11359                let crate::DoubleDoubleR2rAlgorithm::FftReduction { fft, fft_len, .. } =
11360                    &ir.algorithm
11361                else {
11362                    panic!("all DD DCT/DST I-IV families must use FFT reduction");
11363                };
11364                assert_eq!(
11365                    *fft_len,
11366                    match transform {
11367                        TransformKind::Dct(crate::DctType::I) => 16,
11368                        TransformKind::Dst(crate::DstType::I) => 20,
11369                        TransformKind::Dct(crate::DctType::II | crate::DctType::III)
11370                        | TransformKind::Dst(crate::DstType::II | crate::DstType::III) => 9,
11371                        TransformKind::Dct(crate::DctType::IV)
11372                        | TransformKind::Dst(crate::DstType::IV) => 18,
11373                        _ => unreachable!(),
11374                    }
11375                );
11376                assert_eq!(ir.grouped_batch, 3);
11377                assert_eq!(fft.grouped_batch(), 3);
11378                assert_eq!(ir.batch_group_count(), 3);
11379                let child = ProgramIr::double_double_one_dim(fft).unwrap();
11380                let program = ProgramIr::double_double_r2r(&ir).unwrap();
11381                assert_eq!(program.passes.len(), child.passes.len() + 2);
11382                assert!(program.passes.iter().all(|pass| pass.dispatch.x == 3));
11383                for resource in [
11384                    program.input_resource().unwrap(),
11385                    program.output_resource().unwrap(),
11386                ] {
11387                    assert_eq!(resource.scalar, external_scalar);
11388                    assert_eq!(
11389                        resource.external_layout.unwrap().element_shape,
11390                        ProgramElementShape::Scalar
11391                    );
11392                }
11393                let phases = program
11394                    .resources
11395                    .iter()
11396                    .find(|resource| resource.name == "double_double_r2r_phases");
11397                if matches!(
11398                    transform,
11399                    TransformKind::Dct(
11400                        crate::DctType::II | crate::DctType::III | crate::DctType::IV
11401                    ) | TransformKind::Dst(
11402                        crate::DstType::II | crate::DstType::III | crate::DstType::IV
11403                    )
11404                ) {
11405                    let phases = phases.unwrap();
11406                    assert_eq!(phases.scalar, ScalarType::DoubleDouble);
11407                    assert_eq!(phases.elements, 9);
11408                    assert!(matches!(
11409                        phases.initialization,
11410                        ProgramResourceInitialization::ComplexDoubleDouble(_)
11411                    ));
11412                } else {
11413                    assert!(phases.is_none());
11414                }
11415                assert!(
11416                    program
11417                        .resources
11418                        .iter()
11419                        .skip(2)
11420                        .all(|resource| { resource.scalar == ScalarType::DoubleDouble })
11421                );
11422            }
11423        }
11424    }
11425
11426    #[test]
11427    fn double_double_nd_r2r_program_preserves_scalar_tensor_and_complex_child_shapes() {
11428        for (precision, external_scalar) in [
11429            (Precision::DoubleDouble, ScalarType::DoubleDouble),
11430            (Precision::DoubleDoubleF64Storage, ScalarType::F64),
11431        ] {
11432            let plan = FftPlan::build(
11433                FftConfig::new(vec![3, 4])
11434                    .with_batch_count(7)
11435                    .with_precision(precision)
11436                    .with_transform(TransformKind::Dct(crate::DctType::II))
11437                    .with_grouped_batch(0, 3)
11438                    .unwrap()
11439                    .with_grouped_batch(1, 3)
11440                    .unwrap(),
11441            )
11442            .unwrap();
11443            let ir = crate::double_double_ir::DoubleDoubleNdR2rIr::build(&plan, Direction::Forward)
11444                .unwrap();
11445            let program = ProgramIr::double_double_nd_r2r(&ir).unwrap();
11446            assert!(ir.axes.iter().all(|axis| axis.grouped_batch == 3));
11447            assert_eq!(ir.axes[0].transform.grouped_batch, 3);
11448            assert_eq!(ir.axes[1].transform.grouped_batch, 3);
11449            assert!(program.passes.iter().any(|pass| pass.dispatch.x == 3));
11450            assert!(program.passes.iter().any(|pass| pass.dispatch.x == 7));
11451            assert!(program.passes.iter().any(|pass| pass.dispatch.x == 10));
11452            assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
11453            assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
11454            assert_eq!(
11455                program.input_resource().unwrap().element_shape(),
11456                ProgramElementShape::Scalar
11457            );
11458            assert_eq!(
11459                program.output_resource().unwrap().element_shape(),
11460                ProgramElementShape::Scalar
11461            );
11462            for resource_id in 2..=5 {
11463                let resource = &program.resources[resource_id];
11464                assert_eq!(resource.scalar, ScalarType::DoubleDouble);
11465                assert_eq!(resource.element_shape(), ProgramElementShape::Scalar);
11466                assert_eq!(resource.element_bytes(), 16);
11467            }
11468            assert!(program.resources.iter().any(
11469                |resource| resource.name.contains("r2r_fft_input")
11470                    && resource.element_shape() == ProgramElementShape::Complex
11471                    && resource.element_bytes() == 32
11472            ));
11473            assert!(program.passes.first().unwrap().name.contains("axis_1_pack"));
11474            assert!(
11475                program
11476                    .passes
11477                    .last()
11478                    .unwrap()
11479                    .name
11480                    .contains("axis_0_scatter")
11481            );
11482            assert!(
11483                program
11484                    .passes
11485                    .iter()
11486                    .any(|pass| pass.name.contains("axis_1_vkfft_dd_r2r_fft_dct2_pre"))
11487            );
11488            let memory = program.memory_plan().unwrap();
11489            for resource_id in 2..=5 {
11490                let allocation_id = memory
11491                    .allocation_for(ProgramResourceId(resource_id))
11492                    .unwrap();
11493                let allocation = &memory.allocations[allocation_id.0];
11494                let first = allocation.resources.first().copied().unwrap();
11495                assert_eq!(
11496                    program.resources[first.0].element_shape(),
11497                    ProgramElementShape::Scalar
11498                );
11499            }
11500            for allocation in &memory.allocations {
11501                let first_shape = allocation
11502                    .resources
11503                    .first()
11504                    .map(|resource| program.resources[resource.0].element_shape())
11505                    .unwrap();
11506                assert!(
11507                    allocation
11508                        .resources
11509                        .iter()
11510                        .all(
11511                            |resource| program.resources[resource.0].element_shape() == first_shape
11512                        ),
11513                    "scratch alias mixed scalar and complex shapes"
11514                );
11515            }
11516        }
11517    }
11518
11519    #[test]
11520    fn double_double_nd_r2r_padding_stays_fused_in_scalar_tensor_boundaries() {
11521        for (precision, external_scalar) in [
11522            (Precision::DoubleDouble, ScalarType::DoubleDouble),
11523            (Precision::DoubleDoubleF64Storage, ScalarType::F64),
11524        ] {
11525            for direction in [Direction::Forward, Direction::Inverse] {
11526                let config = FftConfig::new(vec![3, 4])
11527                    .with_batch_count(7)
11528                    .with_precision(precision)
11529                    .with_transform(TransformKind::Dct(crate::DctType::II))
11530                    .with_inverse_normalization(direction == Direction::Inverse)
11531                    .with_grouped_batch(0, 3)
11532                    .unwrap()
11533                    .with_grouped_batch(1, 3)
11534                    .unwrap()
11535                    .with_zero_padding(0, 1, 2)
11536                    .unwrap()
11537                    .with_zero_padding(1, 1, 3)
11538                    .unwrap();
11539                let ir =
11540                    crate::DoubleDoubleNdR2rIr::build(&FftPlan::build(config).unwrap(), direction)
11541                        .unwrap();
11542                assert!(ir.has_spatial_zero_padding());
11543                let program = ProgramIr::double_double_nd_r2r(&ir).unwrap();
11544                assert!(ir.axes.iter().all(|axis| axis.grouped_batch == 3));
11545                assert!(ir.axes.iter().all(|axis| axis.transform.grouped_batch == 3));
11546                assert!(program.passes.iter().any(|pass| pass.dispatch.x == 3));
11547                assert!(program.passes.iter().any(|pass| pass.dispatch.x == 7));
11548                assert!(program.passes.iter().any(|pass| pass.dispatch.x == 10));
11549                assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
11550                assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
11551                assert!(
11552                    !program
11553                        .passes
11554                        .iter()
11555                        .any(|pass| pass.name.contains("zero_pad"))
11556                );
11557                for resource_id in 2..=5 {
11558                    let resource = &program.resources[resource_id];
11559                    assert_eq!(resource.scalar, ScalarType::DoubleDouble);
11560                    assert_eq!(resource.element_shape(), ProgramElementShape::Scalar);
11561                    assert_eq!(resource.element_bytes(), 16);
11562                }
11563                assert!(program.resources.iter().any(|resource| {
11564                    resource.name.contains("r2r_fft_input")
11565                        && resource.scalar == ScalarType::DoubleDouble
11566                        && resource.element_shape() == ProgramElementShape::Complex
11567                        && resource.element_bytes() == 32
11568                }));
11569                assert_eq!(
11570                    program.passes.first().unwrap().bindings[0].resource,
11571                    ProgramResourceId(0)
11572                );
11573                assert_eq!(
11574                    program.passes.last().unwrap().bindings[1].resource,
11575                    ProgramResourceId(1)
11576                );
11577            }
11578        }
11579    }
11580}