1use crate::bluestein_ir::BluesteinPipelineIr;
9use crate::complex::Complex64;
10use crate::convolution_ir::{ConvolutionIr, NdConvolutionIr, NdRealConvolutionIr};
11use crate::double_double::ComplexDoubleDouble;
12use crate::double_double_ir::{
13 DoubleDoubleBluesteinIr, DoubleDoubleDirectRaderIr, DoubleDoubleFftRaderIr,
14 DoubleDoubleNdFftIr, DoubleDoubleNdR2rIr, DoubleDoubleNdRealFftIr, DoubleDoubleOneDimIr,
15 DoubleDoubleR2rAlgorithm, DoubleDoubleR2rIr, DoubleDoubleRealFftIr, DoubleDoubleStockhamIr,
16};
17use crate::double_double_recursive_ir::{
18 DoubleDoubleCooleyTukeyOutputModifier, DoubleDoubleForcedRaderThreeUploadComponentIr,
19 DoubleDoubleRecursiveFftIr, DoubleDoubleRecursiveFftNodeIr,
20};
21use crate::error::{Result, VkFftError};
22use crate::kernel_ir::{BufferAccess, BufferRole, DispatchGeometry, KernelIr, ScalarType};
23use crate::mixed_ir::{MixedPrimeRaderIr, MixedRaderStockhamIr};
24use crate::nd_ir::NdFftIr;
25use crate::nd_real_ir::NdRealFftIr;
26use crate::one_dim_ir::OneDimFftIr;
27use crate::r2r_ir::{NdR2rIr, R2rIr};
28use crate::rader_ir::{RaderDirectIr, RaderFftInputStrategy, RaderFftPipelineIr};
29use crate::real_ir::{RealFftIr, RealFftKind};
30use crate::recursive_ir::{
31 CooleyTukeyInputModifier, CooleyTukeyOutputModifier, RecursiveFftIr, RecursiveFftNodeIr,
32};
33use crate::zero_pad_ir::ZeroPadPassIr;
34
35#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
36pub struct ProgramResourceId(pub usize);
37
38#[derive(Debug, Clone, Copy, PartialEq, Eq)]
39pub enum ProgramResourceKind {
40 Input,
41 Output,
42 Scratch,
43 LookupTable,
44}
45
46#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
47pub enum ProgramElementShape {
48 Complex,
49 Scalar,
50}
51
52impl ProgramElementShape {
53 pub const fn element_bytes(self, scalar: ScalarType) -> usize {
54 match self {
55 Self::Complex => scalar.complex_bytes(),
56 Self::Scalar => scalar.bytes(),
57 }
58 }
59}
60
61#[derive(Debug, Clone, Copy, PartialEq, Eq)]
62pub struct ExternalBufferLayout {
63 pub logical_len: usize,
64 pub physical_stride: usize,
65 pub batch_count: usize,
66 pub element_shape: ProgramElementShape,
67}
68
69impl ExternalBufferLayout {
70 pub fn logical_elements(self) -> Result<usize> {
71 self.logical_len
72 .checked_mul(self.batch_count)
73 .ok_or(VkFftError::ArithmeticOverflow {
74 operation: "program external logical element count",
75 })
76 }
77
78 pub fn physical_elements(self) -> Result<usize> {
79 self.physical_stride
80 .checked_mul(self.batch_count)
81 .ok_or(VkFftError::ArithmeticOverflow {
82 operation: "program external physical element count",
83 })
84 }
85
86 fn validate(self) -> Result<()> {
87 if self.logical_len == 0 || self.physical_stride == 0 || self.batch_count == 0 {
88 return Err(VkFftError::InvalidKernelIr(
89 "program external buffer layout must be non-zero",
90 ));
91 }
92 if self.logical_len > self.physical_stride {
93 return Err(VkFftError::InvalidKernelIr(
94 "program external logical length must fit inside physical stride",
95 ));
96 }
97 let _ = self.logical_elements()?;
98 let _ = self.physical_elements()?;
99 Ok(())
100 }
101}
102
103impl ProgramResource {
104 pub const fn element_shape(&self) -> ProgramElementShape {
105 match self.external_layout {
106 Some(layout) => layout.element_shape,
107 None => ProgramElementShape::Complex,
108 }
109 }
110
111 pub const fn element_bytes(&self) -> usize {
112 self.element_shape().element_bytes(self.scalar)
113 }
114}
115
116#[derive(Debug, Clone, PartialEq)]
117pub enum ProgramResourceInitialization {
118 ExternalInput,
119 Zeroed,
120 Complex64(Vec<Complex64>),
121 ComplexDoubleDouble(Vec<ComplexDoubleDouble>),
124 StockhamUnitRoots {
128 len: usize,
129 },
130}
131
132#[derive(Debug, Clone, PartialEq)]
133pub struct ProgramResource {
134 pub id: ProgramResourceId,
135 pub name: String,
136 pub kind: ProgramResourceKind,
137 pub scalar: ScalarType,
138 pub elements: usize,
139 pub external_layout: Option<ExternalBufferLayout>,
140 pub initialization: ProgramResourceInitialization,
141}
142
143#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
144pub struct ProgramAllocationId(pub usize);
145
146#[derive(Debug, Clone, Copy, PartialEq, Eq)]
147pub enum ProgramAllocationKind {
148 ExternalInput,
149 ExternalOutput,
150 Scratch,
151 LookupTable,
152}
153
154#[derive(Debug, Clone, PartialEq, Eq)]
155pub struct ProgramAllocation {
156 pub id: ProgramAllocationId,
157 pub kind: ProgramAllocationKind,
158 pub scalar: ScalarType,
159 pub elements: usize,
160 pub resources: Vec<ProgramResourceId>,
161}
162
163#[derive(Debug, Clone, PartialEq, Eq)]
164pub struct ProgramMemoryPlan {
165 pub allocations: Vec<ProgramAllocation>,
166 pub resource_allocations: Vec<ProgramAllocationId>,
167}
168
169impl ProgramMemoryPlan {
170 pub fn allocation_for(&self, resource: ProgramResourceId) -> Result<ProgramAllocationId> {
171 self.resource_allocations
172 .get(resource.0)
173 .copied()
174 .ok_or(VkFftError::InvalidKernelIr(
175 "program memory plan references an unknown logical resource",
176 ))
177 }
178
179 pub fn allocated_elements(&self) -> Result<usize> {
180 self.allocations
181 .iter()
182 .try_fold(0usize, |total, allocation| {
183 total
184 .checked_add(allocation.elements)
185 .ok_or(VkFftError::ArithmeticOverflow {
186 operation: "program physical allocation element count",
187 })
188 })
189 }
190}
191
192#[derive(Debug, Clone, Copy, PartialEq, Eq)]
193pub struct ProgramPassBinding {
194 pub binding: u32,
195 pub resource: ProgramResourceId,
196 pub role: BufferRole,
197 pub access: BufferAccess,
198}
199
200#[derive(Debug, Clone, PartialEq, Eq)]
201pub struct ProgramPass {
202 pub name: String,
203 pub dispatch: DispatchGeometry,
204 pub bindings: Vec<ProgramPassBinding>,
205}
206
207#[derive(Debug, Clone, PartialEq)]
208pub struct ProgramIr {
209 pub name: String,
210 pub scalar: ScalarType,
211 pub resources: Vec<ProgramResource>,
212 pub passes: Vec<ProgramPass>,
213}
214
215impl ProgramIr {
216 pub fn double_double_stockham(ir: &DoubleDoubleStockhamIr) -> Result<Self> {
217 ir.validate()?;
218 if ir.sequence_len > 64 {
219 let core = Self::double_double_stockham_multi_pass(ir)?;
220 return Self::wrap_double_double_zero_pad(
221 ir.zero_pad_pass.as_ref(),
222 ir.sequence_len,
223 ir.batch_count,
224 "double_double_stockham",
225 core,
226 );
227 }
228 let elements =
229 ir.sequence_len
230 .checked_mul(ir.batch_count)
231 .ok_or(VkFftError::ArithmeticOverflow {
232 operation: "double-double Stockham program element count",
233 })?;
234 let external_scalar = match ir.external_storage {
235 crate::PrecisionStorage::DoubleDouble => ScalarType::DoubleDouble,
236 crate::PrecisionStorage::F64 => ScalarType::F64,
237 _ => {
238 return Err(VkFftError::InvalidKernelIr(
239 "double-double Stockham program requires DD or F64 external storage",
240 ));
241 }
242 };
243 let layout = ExternalBufferLayout {
244 logical_len: ir.sequence_len,
245 physical_stride: ir.sequence_len,
246 batch_count: ir.batch_count,
247 element_shape: ProgramElementShape::Complex,
248 };
249 let twiddles = ir.twiddles.packed_values();
250 let mut resources = vec![
251 ProgramResource {
252 id: ProgramResourceId(0),
253 name: "input".to_owned(),
254 kind: ProgramResourceKind::Input,
255 scalar: external_scalar,
256 elements,
257 external_layout: Some(layout),
258 initialization: ProgramResourceInitialization::ExternalInput,
259 },
260 ProgramResource {
261 id: ProgramResourceId(1),
262 name: "output".to_owned(),
263 kind: ProgramResourceKind::Output,
264 scalar: external_scalar,
265 elements,
266 external_layout: Some(layout),
267 initialization: ProgramResourceInitialization::Zeroed,
268 },
269 ];
270 let mut bindings = vec![
271 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
272 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
273 ];
274 if !twiddles.is_empty() {
275 let lut_id = ProgramResourceId(resources.len());
276 resources.push(ProgramResource {
277 id: lut_id,
278 name: "double_double_stockham_twiddles".to_owned(),
279 kind: ProgramResourceKind::LookupTable,
280 scalar: ScalarType::DoubleDouble,
281 elements: twiddles.len(),
282 external_layout: None,
283 initialization: ProgramResourceInitialization::ComplexDoubleDouble(twiddles),
284 });
285 bindings.push(ProgramPassBinding {
286 binding: 2,
287 resource: lut_id,
288 role: BufferRole::TwiddleLookupTable,
289 access: BufferAccess::ReadOnly,
290 });
291 }
292 let dispatch_x =
293 u32::try_from(ir.batch_group_count()).map_err(|_| VkFftError::ValueOutOfRange {
294 field: "double-double Stockham program dispatch count",
295 })?;
296 let program = Self {
297 name: format!("{}_program", ir.name),
298 scalar: ScalarType::DoubleDouble,
299 resources,
300 passes: vec![ProgramPass {
301 name: ir.name.clone(),
302 dispatch: DispatchGeometry {
303 x: dispatch_x,
304 y: 1,
305 z: 1,
306 },
307 bindings,
308 }],
309 };
310 program.validate()?;
311 Self::wrap_double_double_zero_pad(
312 ir.zero_pad_pass.as_ref(),
313 ir.sequence_len,
314 ir.batch_count,
315 "double_double_stockham",
316 program,
317 )
318 }
319
320 fn double_double_stockham_internal(ir: &DoubleDoubleStockhamIr) -> Result<Self> {
321 if ir.external_storage != crate::PrecisionStorage::DoubleDouble
322 || ir.zero_pad_pass.is_some()
323 {
324 return Err(VkFftError::InvalidKernelIr(
325 "internal double-double Stockham child requires unmodified DD boundaries",
326 ));
327 }
328 let mut program = Self::double_double_stockham(ir)?;
329 if ir.sequence_len <= 64 {
330 return Ok(program);
331 }
332 if program.passes.len() != ir.stages.len() + 2 || ir.stages.is_empty() {
333 return Err(VkFftError::InvalidKernelIr(
334 "internal double-double Stockham child has an unexpected pass graph",
335 ));
336 }
337 program.passes.remove(0);
338 program.passes.pop();
339 let first_input = program
340 .passes
341 .first_mut()
342 .and_then(|pass| {
343 pass.bindings
344 .iter_mut()
345 .find(|binding| binding.binding == 0)
346 })
347 .ok_or(VkFftError::InvalidKernelIr(
348 "internal double-double Stockham first stage is missing input binding",
349 ))?;
350 first_input.resource = ProgramResourceId(0);
351 let final_output = program
352 .passes
353 .last_mut()
354 .and_then(|pass| {
355 pass.bindings
356 .iter_mut()
357 .find(|binding| binding.binding == 1)
358 })
359 .ok_or(VkFftError::InvalidKernelIr(
360 "internal double-double Stockham final stage is missing output binding",
361 ))?;
362 final_output.resource = ProgramResourceId(1);
363 program.name = format!("{}_internal_program", ir.name);
364 program.validate()?;
365 Ok(program)
366 }
367
368 fn double_double_stockham_multi_pass(ir: &DoubleDoubleStockhamIr) -> Result<Self> {
369 let elements =
370 ir.sequence_len
371 .checked_mul(ir.batch_count)
372 .ok_or(VkFftError::ArithmeticOverflow {
373 operation: "double-double Stockham multi-pass element count",
374 })?;
375 let external_scalar = match ir.external_storage {
376 crate::PrecisionStorage::DoubleDouble => ScalarType::DoubleDouble,
377 crate::PrecisionStorage::F64 => ScalarType::F64,
378 _ => {
379 return Err(VkFftError::InvalidKernelIr(
380 "double-double Stockham multi-pass requires DD or F64 external storage",
381 ));
382 }
383 };
384 let layout = ExternalBufferLayout {
385 logical_len: ir.sequence_len,
386 physical_stride: ir.sequence_len,
387 batch_count: ir.batch_count,
388 element_shape: ProgramElementShape::Complex,
389 };
390 let twiddles = ir.twiddles.packed_values();
391 let resources = vec![
392 ProgramResource {
393 id: ProgramResourceId(0),
394 name: "input".to_owned(),
395 kind: ProgramResourceKind::Input,
396 scalar: external_scalar,
397 elements,
398 external_layout: Some(layout),
399 initialization: ProgramResourceInitialization::ExternalInput,
400 },
401 ProgramResource {
402 id: ProgramResourceId(1),
403 name: "output".to_owned(),
404 kind: ProgramResourceKind::Output,
405 scalar: external_scalar,
406 elements,
407 external_layout: Some(layout),
408 initialization: ProgramResourceInitialization::Zeroed,
409 },
410 ProgramResource {
411 id: ProgramResourceId(2),
412 name: "double_double_stockham_scratch_a".to_owned(),
413 kind: ProgramResourceKind::Scratch,
414 scalar: ScalarType::DoubleDouble,
415 elements,
416 external_layout: None,
417 initialization: ProgramResourceInitialization::Zeroed,
418 },
419 ProgramResource {
420 id: ProgramResourceId(3),
421 name: "double_double_stockham_scratch_b".to_owned(),
422 kind: ProgramResourceKind::Scratch,
423 scalar: ScalarType::DoubleDouble,
424 elements,
425 external_layout: None,
426 initialization: ProgramResourceInitialization::Zeroed,
427 },
428 ProgramResource {
429 id: ProgramResourceId(4),
430 name: "double_double_stockham_twiddles".to_owned(),
431 kind: ProgramResourceKind::LookupTable,
432 scalar: ScalarType::DoubleDouble,
433 elements: twiddles.len(),
434 external_layout: None,
435 initialization: ProgramResourceInitialization::ComplexDoubleDouble(twiddles),
436 },
437 ];
438 let dispatch = DispatchGeometry {
439 x: u32::try_from(ir.batch_group_count()).map_err(|_| VkFftError::ValueOutOfRange {
440 field: "double-double Stockham multi-pass dispatch count",
441 })?,
442 y: 1,
443 z: 1,
444 };
445 let mut passes = Vec::with_capacity(ir.stages.len() + 2);
446 passes.push(ProgramPass {
447 name: format!("{}_promote", ir.name),
448 dispatch,
449 bindings: vec![
450 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
451 binding(1, 2, BufferRole::Output, BufferAccess::WriteOnly),
452 ],
453 });
454 let mut source = 2usize;
455 let mut target = 3usize;
456 for stage in &ir.stages {
457 passes.push(ProgramPass {
458 name: format!("{}_stage_{}", ir.name, stage.index),
459 dispatch,
460 bindings: vec![
461 binding(0, source, BufferRole::Input, BufferAccess::ReadOnly),
462 binding(1, target, BufferRole::Output, BufferAccess::WriteOnly),
463 binding(2, 4, BufferRole::TwiddleLookupTable, BufferAccess::ReadOnly),
464 ],
465 });
466 core::mem::swap(&mut source, &mut target);
467 }
468 passes.push(ProgramPass {
469 name: format!("{}_finalize", ir.name),
470 dispatch,
471 bindings: vec![
472 binding(0, source, BufferRole::Input, BufferAccess::ReadOnly),
473 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
474 ],
475 });
476 let program = Self {
477 name: format!("{}_program", ir.name),
478 scalar: ScalarType::DoubleDouble,
479 resources,
480 passes,
481 };
482 program.validate()?;
483 Ok(program)
484 }
485
486 pub fn double_double_direct_rader(ir: &DoubleDoubleDirectRaderIr) -> Result<Self> {
487 ir.validate()?;
488 let elements =
489 ir.prime
490 .checked_mul(ir.batch_count)
491 .ok_or(VkFftError::ArithmeticOverflow {
492 operation: "double-double direct Rader program element count",
493 })?;
494 let storage_scalar = |storage| match storage {
495 crate::PrecisionStorage::DoubleDouble => Ok(ScalarType::DoubleDouble),
496 crate::PrecisionStorage::F64 => Ok(ScalarType::F64),
497 _ => Err(VkFftError::InvalidKernelIr(
498 "double-double direct Rader program requires DD or F64 caller storage",
499 )),
500 };
501 let input_scalar = storage_scalar(ir.input_storage)?;
502 let output_scalar = storage_scalar(ir.output_storage)?;
503 let layout = ExternalBufferLayout {
504 logical_len: ir.prime,
505 physical_stride: ir.prime,
506 batch_count: ir.batch_count,
507 element_shape: ProgramElementShape::Complex,
508 };
509 let lut = ir.table.twiddles_by_generator_power.clone();
510 let mut resources = vec![
511 ProgramResource {
512 id: ProgramResourceId(0),
513 name: "input".to_owned(),
514 kind: ProgramResourceKind::Input,
515 scalar: input_scalar,
516 elements,
517 external_layout: Some(layout),
518 initialization: ProgramResourceInitialization::ExternalInput,
519 },
520 ProgramResource {
521 id: ProgramResourceId(1),
522 name: "output".to_owned(),
523 kind: ProgramResourceKind::Output,
524 scalar: output_scalar,
525 elements,
526 external_layout: Some(layout),
527 initialization: ProgramResourceInitialization::Zeroed,
528 },
529 ProgramResource {
530 id: ProgramResourceId(2),
531 name: "double_double_rader_twiddles".to_owned(),
532 kind: ProgramResourceKind::LookupTable,
533 scalar: ScalarType::DoubleDouble,
534 elements: lut.len(),
535 external_layout: None,
536 initialization: ProgramResourceInitialization::ComplexDoubleDouble(lut),
537 },
538 ];
539 let mut bindings = vec![
540 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
541 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
542 binding(2, 2, BufferRole::LookupTable, BufferAccess::ReadOnly),
543 ];
544 let parent_period = match ir.io_mapping {
545 crate::StockhamIoMapping::FourStepRight(mapping) => Some(mapping.logical_len),
546 crate::StockhamIoMapping::FourStepThreeUpload2(mapping) => Some(mapping.logical_len),
547 crate::StockhamIoMapping::FourStepThreeUpload1(mapping) => {
548 let [a, b, _] = mapping.axis_split;
549 Some(a.checked_mul(b).ok_or(VkFftError::ArithmeticOverflow {
550 operation: "double-double direct Rader three-upload parent root period",
551 })?)
552 }
553 _ => None,
554 };
555 if let Some(period) = parent_period {
556 let roots = crate::lut::unit_root_table_double_double(period, ir.direction)?;
557 let root_id = ProgramResourceId(resources.len());
558 resources.push(ProgramResource {
559 id: root_id,
560 name: "double_double_rader_four_step_roots".to_owned(),
561 kind: ProgramResourceKind::LookupTable,
562 scalar: ScalarType::DoubleDouble,
563 elements: roots.len(),
564 external_layout: None,
565 initialization: ProgramResourceInitialization::ComplexDoubleDouble(roots),
566 });
567 bindings.push(ProgramPassBinding {
568 binding: 3,
569 resource: root_id,
570 role: BufferRole::TwiddleLookupTable,
571 access: BufferAccess::ReadOnly,
572 });
573 }
574 let program = Self {
575 name: format!("{}_program", ir.name),
576 scalar: ScalarType::DoubleDouble,
577 resources,
578 passes: vec![ProgramPass {
579 name: ir.name.clone(),
580 dispatch: DispatchGeometry {
581 x: u32::try_from(ir.batch_group_count()).map_err(|_| {
582 VkFftError::ValueOutOfRange {
583 field: "double-double direct Rader program dispatch count",
584 }
585 })?,
586 y: 1,
587 z: 1,
588 },
589 bindings,
590 }],
591 };
592 program.validate()?;
593 Self::wrap_double_double_zero_pad(
594 ir.zero_pad_pass.as_ref(),
595 ir.prime,
596 ir.batch_count,
597 "double_double_direct_rader",
598 program,
599 )
600 }
601
602 pub fn double_double_one_dim(ir: &DoubleDoubleOneDimIr) -> Result<Self> {
603 match ir {
604 DoubleDoubleOneDimIr::Stockham(ir) => Self::double_double_stockham(ir),
605 DoubleDoubleOneDimIr::DirectRader(ir) => Self::double_double_direct_rader(ir),
606 DoubleDoubleOneDimIr::FftRader(ir) => Self::double_double_fft_rader(ir),
607 DoubleDoubleOneDimIr::Bluestein(ir) => Self::double_double_bluestein(ir),
608 DoubleDoubleOneDimIr::Recursive(ir) => Self::double_double_recursive(ir),
609 }
610 }
611
612 pub fn double_double_recursive(ir: &DoubleDoubleRecursiveFftIr) -> Result<Self> {
613 ir.validate()?;
614 let elements =
615 ir.logical_len
616 .checked_mul(ir.batch_count)
617 .ok_or(VkFftError::ArithmeticOverflow {
618 operation: "double-double recursive program element count",
619 })?;
620 let external_scalar = match ir.external_storage {
621 crate::PrecisionStorage::DoubleDouble => ScalarType::DoubleDouble,
622 crate::PrecisionStorage::F64 => ScalarType::F64,
623 _ => {
624 return Err(VkFftError::InvalidKernelIr(
625 "double-double recursive ProgramIr requires DD or F64 external storage",
626 ));
627 }
628 };
629 let layout = ExternalBufferLayout {
630 logical_len: ir.logical_len,
631 physical_stride: ir.logical_len,
632 batch_count: ir.batch_count,
633 element_shape: ProgramElementShape::Complex,
634 };
635 let mut resources = vec![
636 ProgramResource {
637 id: ProgramResourceId(0),
638 name: "input".to_owned(),
639 kind: ProgramResourceKind::Input,
640 scalar: external_scalar,
641 elements,
642 external_layout: Some(layout),
643 initialization: ProgramResourceInitialization::ExternalInput,
644 },
645 ProgramResource {
646 id: ProgramResourceId(1),
647 name: "output".to_owned(),
648 kind: ProgramResourceKind::Output,
649 scalar: external_scalar,
650 elements,
651 external_layout: Some(layout),
652 initialization: ProgramResourceInitialization::Zeroed,
653 },
654 ];
655 let core = if ir.two_upload_four_step_plan.is_some() {
656 Self::double_double_recursive_two_upload_four_step(ir, resources)?
657 } else if ir.three_upload_four_step_plan.is_some() {
658 Self::double_double_recursive_three_upload_four_step(ir, resources)?
659 } else if let Some(components) = ir.forced_rader_three_upload_mapped_components()? {
660 Self::double_double_recursive_forced_rader_three_upload(ir, resources, components)?
661 } else if let Some((high, _mapping)) = ir.forced_rader_two_upload_mapped_high_stockham()? {
662 Self::double_double_recursive_forced_rader_stockham_high_boundary(ir, resources, high)?
663 } else if let Some(mapped_high) = ir.forced_rader_two_upload_mapped_high_component()? {
664 Self::double_double_recursive_forced_rader_high_boundary(ir, resources, mapped_high)?
665 } else {
666 let mut passes = Vec::new();
667 let mut serial = 0usize;
668 flatten_double_double_recursive_node(
669 &ir.root,
670 ProgramResourceId(0),
671 ProgramResourceId(1),
672 &mut resources,
673 &mut passes,
674 &mut serial,
675 )?;
676 let program = Self {
677 name: format!("{}_program", ir.name),
678 scalar: ScalarType::DoubleDouble,
679 resources,
680 passes,
681 };
682 program.validate()?;
683 program
684 };
685 Self::wrap_double_double_zero_pad(
686 ir.zero_pad_pass.as_ref(),
687 ir.logical_len,
688 ir.batch_count,
689 "double_double_recursive",
690 core,
691 )
692 }
693
694 fn wrap_double_double_zero_pad(
695 pass: Option<&ZeroPadPassIr>,
696 logical_len: usize,
697 batch_count: usize,
698 stem: &str,
699 mut program: Self,
700 ) -> Result<Self> {
701 let Some(pass) = pass else {
702 return Ok(program);
703 };
704 pass.validate()?;
705 let input = program.input_resource()?.id;
706 let output = program.output_resource()?.id;
707 let elements =
708 logical_len
709 .checked_mul(batch_count)
710 .ok_or(VkFftError::ArithmeticOverflow {
711 operation: "double-double zero-pad scratch element count",
712 })?;
713 let boundary = ProgramResourceId(program.resources.len());
714 let input_boundary = pass.operation.is_input_boundary();
715 program.resources.push(ProgramResource {
716 id: boundary,
717 name: format!(
718 "{stem}_zero_padded_{}",
719 if input_boundary { "input" } else { "output" }
720 ),
721 kind: ProgramResourceKind::Scratch,
722 scalar: if input_boundary {
723 pass.output_storage_scalar
724 } else {
725 pass.input_storage_scalar
726 },
727 elements,
728 external_layout: None,
729 initialization: ProgramResourceInitialization::Zeroed,
730 });
731 if input_boundary {
732 for program_pass in &mut program.passes {
733 for pass_binding in &mut program_pass.bindings {
734 if pass_binding.resource == input {
735 pass_binding.resource = boundary;
736 }
737 }
738 }
739 program.passes.insert(
740 0,
741 ProgramPass {
742 name: pass.name.clone(),
743 dispatch: pass.dispatch,
744 bindings: vec![
745 ProgramPassBinding {
746 binding: 0,
747 resource: input,
748 role: BufferRole::Input,
749 access: BufferAccess::ReadOnly,
750 },
751 ProgramPassBinding {
752 binding: 1,
753 resource: boundary,
754 role: BufferRole::Output,
755 access: BufferAccess::WriteOnly,
756 },
757 ],
758 },
759 );
760 } else {
761 for program_pass in &mut program.passes {
762 for pass_binding in &mut program_pass.bindings {
763 if pass_binding.resource == output {
764 pass_binding.resource = boundary;
765 }
766 }
767 }
768 program.passes.push(ProgramPass {
769 name: pass.name.clone(),
770 dispatch: pass.dispatch,
771 bindings: vec![
772 ProgramPassBinding {
773 binding: 0,
774 resource: boundary,
775 role: BufferRole::Input,
776 access: BufferAccess::ReadOnly,
777 },
778 ProgramPassBinding {
779 binding: 1,
780 resource: output,
781 role: BufferRole::Output,
782 access: BufferAccess::WriteOnly,
783 },
784 ],
785 });
786 }
787 program.validate()?;
788 Ok(program)
789 }
790
791 fn double_double_recursive_forced_rader_stockham_high_boundary(
792 ir: &DoubleDoubleRecursiveFftIr,
793 mut resources: Vec<ProgramResource>,
794 high: &DoubleDoubleStockhamIr,
795 ) -> Result<Self> {
796 let DoubleDoubleRecursiveFftNodeIr::CooleyTukey(root) = &ir.root else {
797 return Err(VkFftError::InvalidKernelIr(
798 "double-double Stockham-high forced-Rader boundary requires a Cooley-Tukey root",
799 ));
800 };
801 let elements =
802 ir.logical_len
803 .checked_mul(ir.batch_count)
804 .ok_or(VkFftError::ArithmeticOverflow {
805 operation: "double-double Stockham-high forced-Rader scratch element count",
806 })?;
807 let exchange = ProgramResourceId(resources.len());
808 resources.push(ProgramResource {
809 id: exchange,
810 name: "double_double_rader_four_step_exchange".to_owned(),
811 kind: ProgramResourceKind::Scratch,
812 scalar: ScalarType::DoubleDouble,
813 elements,
814 external_layout: None,
815 initialization: ProgramResourceInitialization::Zeroed,
816 });
817 let high_twiddle_values = high.twiddles.packed_values();
818 let high_twiddles = ProgramResourceId(resources.len());
819 resources.push(ProgramResource {
820 id: high_twiddles,
821 name: "double_double_forced_rader_two_upload_1_stockham_twiddles".to_owned(),
822 kind: ProgramResourceKind::LookupTable,
823 scalar: ScalarType::DoubleDouble,
824 elements: high_twiddle_values.len(),
825 external_layout: None,
826 initialization: ProgramResourceInitialization::ComplexDoubleDouble(high_twiddle_values),
827 });
828 let parent_roots = ProgramResourceId(resources.len());
829 resources.push(ProgramResource {
830 id: parent_roots,
831 name: "double_double_forced_rader_two_upload_1_parent_roots".to_owned(),
832 kind: ProgramResourceKind::LookupTable,
833 scalar: ScalarType::DoubleDouble,
834 elements: root.twiddles.len(),
835 external_layout: None,
836 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
837 root.twiddles.clone(),
838 ),
839 });
840 let passes = vec![ProgramPass {
841 name: format!("{}_forced_rader_two_upload_1", ir.name),
842 dispatch: DispatchGeometry {
843 x: u32::try_from(high.batch_group_count()).map_err(|_| {
844 VkFftError::ValueOutOfRange {
845 field: "double-double forced-Rader high Stockham dispatch count",
846 }
847 })?,
848 y: 1,
849 z: 1,
850 },
851 bindings: vec![
852 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
853 ProgramPassBinding {
854 binding: 1,
855 resource: exchange,
856 role: BufferRole::Output,
857 access: BufferAccess::WriteOnly,
858 },
859 ProgramPassBinding {
860 binding: 2,
861 resource: high_twiddles,
862 role: BufferRole::TwiddleLookupTable,
863 access: BufferAccess::ReadOnly,
864 },
865 ProgramPassBinding {
866 binding: 3,
867 resource: parent_roots,
868 role: BufferRole::Auxiliary,
869 access: BufferAccess::ReadOnly,
870 },
871 ],
872 }];
873 Self::finish_double_double_recursive_forced_rader_two_upload(
874 ir, resources, passes, 0, exchange, elements,
875 )
876 }
877
878 fn double_double_recursive_forced_rader_high_boundary(
879 ir: &DoubleDoubleRecursiveFftIr,
880 mut resources: Vec<ProgramResource>,
881 mapped_high: DoubleDoubleRecursiveFftNodeIr,
882 ) -> Result<Self> {
883 let DoubleDoubleRecursiveFftNodeIr::CooleyTukey(_) = &ir.root else {
884 return Err(VkFftError::InvalidKernelIr(
885 "double-double mapped forced-Rader boundary requires a Cooley-Tukey root",
886 ));
887 };
888 let elements =
889 ir.logical_len
890 .checked_mul(ir.batch_count)
891 .ok_or(VkFftError::ArithmeticOverflow {
892 operation: "double-double mapped forced-Rader scratch element count",
893 })?;
894 let exchange = ProgramResourceId(resources.len());
895 resources.push(ProgramResource {
896 id: exchange,
897 name: "double_double_rader_four_step_exchange".to_owned(),
898 kind: ProgramResourceKind::Scratch,
899 scalar: ScalarType::DoubleDouble,
900 elements,
901 external_layout: None,
902 initialization: ProgramResourceInitialization::Zeroed,
903 });
904 let mut passes = Vec::new();
905 let mut serial = 0usize;
906 flatten_double_double_recursive_node(
907 &mapped_high,
908 ProgramResourceId(0),
909 exchange,
910 &mut resources,
911 &mut passes,
912 &mut serial,
913 )?;
914 Self::finish_double_double_recursive_forced_rader_two_upload(
915 ir, resources, passes, serial, exchange, elements,
916 )
917 }
918
919 fn finish_double_double_recursive_forced_rader_two_upload(
920 ir: &DoubleDoubleRecursiveFftIr,
921 mut resources: Vec<ProgramResource>,
922 mut passes: Vec<ProgramPass>,
923 mut serial: usize,
924 exchange: ProgramResourceId,
925 elements: usize,
926 ) -> Result<Self> {
927 let DoubleDoubleRecursiveFftNodeIr::CooleyTukey(root) = &ir.root else {
928 return Err(VkFftError::InvalidKernelIr(
929 "double-double mapped forced-Rader finalize requires a Cooley-Tukey root",
930 ));
931 };
932 if let Some(mapped_low) = ir.forced_rader_two_upload_mapped_low_component()? {
933 flatten_double_double_recursive_node(
934 &mapped_low,
935 exchange,
936 ProgramResourceId(1),
937 &mut resources,
938 &mut passes,
939 &mut serial,
940 )?;
941 } else if let Some((low, _mapping)) = ir.forced_rader_two_upload_mapped_low_stockham()? {
942 let child = ProgramIr::double_double_stockham(low)?;
943 serial += 1;
944 append_double_double_recursive_child(
945 &child,
946 &format!("vkfft_dd_recursive_leaf_{}", serial),
947 exchange,
948 ProgramResourceId(1),
949 &mut resources,
950 &mut passes,
951 true,
952 )?;
953 let final_pass = passes.last_mut().ok_or(VkFftError::InvalidKernelIr(
954 "double-double mapped forced-Rader low upload emitted no final pass",
955 ))?;
956 if low.sequence_len <= 64 {
957 if !final_pass.name.ends_with(&low.name) {
958 return Err(VkFftError::InvalidKernelIr(
959 "double-double mapped forced-Rader monolithic low upload does not end in its Stockham pass",
960 ));
961 }
962 final_pass.name = format!("{}_four_step_left", low.name);
963 } else {
964 let expected_finalize = format!("{}_finalize", low.name);
965 if !final_pass.name.ends_with(&expected_finalize) {
966 return Err(VkFftError::InvalidKernelIr(
967 "double-double mapped forced-Rader low upload does not end in a Stockham finalize pass",
968 ));
969 }
970 final_pass.name = format!("{}_four_step_left_finalize", low.name);
971 }
972 } else {
973 let left_output = ProgramResourceId(resources.len());
974 resources.push(ProgramResource {
975 id: left_output,
976 name: "double_double_rader_four_step_left_output".to_owned(),
977 kind: ProgramResourceKind::Scratch,
978 scalar: ScalarType::DoubleDouble,
979 elements,
980 external_layout: None,
981 initialization: ProgramResourceInitialization::Zeroed,
982 });
983 flatten_double_double_recursive_node(
984 &root.left,
985 exchange,
986 left_output,
987 &mut resources,
988 &mut passes,
989 &mut serial,
990 )?;
991 passes.push(two_buffer_pass(
992 &root.scatter_output.name,
993 root.scatter_output.dispatch,
994 left_output.0,
995 1,
996 ));
997 }
998 let program = Self {
999 name: format!("{}_mapped_rader_four_step_program", ir.name),
1000 scalar: ScalarType::DoubleDouble,
1001 resources,
1002 passes,
1003 };
1004 program.validate()?;
1005 Ok(program)
1006 }
1007
1008 fn double_double_recursive_forced_rader_three_upload(
1009 ir: &DoubleDoubleRecursiveFftIr,
1010 mut resources: Vec<ProgramResource>,
1011 components: Vec<DoubleDoubleForcedRaderThreeUploadComponentIr>,
1012 ) -> Result<Self> {
1013 if components.len() != 3
1014 || components
1015 .iter()
1016 .map(DoubleDoubleForcedRaderThreeUploadComponentIr::upload_id)
1017 .collect::<Vec<_>>()
1018 != vec![2, 1, 0]
1019 {
1020 return Err(VkFftError::InvalidKernelIr(
1021 "double-double forced-Rader three-upload components are out of execution order",
1022 ));
1023 }
1024 let schedule =
1025 ir.rader_forced_upload_schedule
1026 .as_ref()
1027 .ok_or(VkFftError::InvalidKernelIr(
1028 "double-double forced-Rader three-upload ProgramIr requires schedule metadata",
1029 ))?;
1030 let [a, b, c]: [usize; 3] = schedule.axis_split.as_slice().try_into().map_err(|_| {
1031 VkFftError::InvalidKernelIr(
1032 "double-double forced-Rader three-upload ProgramIr requires three axis splits",
1033 )
1034 })?;
1035 let component_lengths = components
1036 .iter()
1037 .map(DoubleDoubleForcedRaderThreeUploadComponentIr::logical_len)
1038 .collect::<Vec<_>>();
1039 if component_lengths != vec![c, b, a] {
1040 return Err(VkFftError::InvalidKernelIr(
1041 "double-double forced-Rader three-upload component lengths do not match axisSplit",
1042 ));
1043 }
1044 let elements =
1045 ir.logical_len
1046 .checked_mul(ir.batch_count)
1047 .ok_or(VkFftError::ArithmeticOverflow {
1048 operation: "double-double forced-Rader three-upload scratch element count",
1049 })?;
1050 let output_is_compute = resources
1051 .get(1)
1052 .ok_or(VkFftError::InvalidKernelIr(
1053 "double-double forced-Rader three-upload output resource is missing",
1054 ))?
1055 .scalar
1056 == ScalarType::DoubleDouble;
1057 let scratch0 = ProgramResourceId(resources.len());
1058 resources.push(ProgramResource {
1059 id: scratch0,
1060 name: "double_double_forced_rader_three_upload_scratch_0".to_owned(),
1061 kind: ProgramResourceKind::Scratch,
1062 scalar: ScalarType::DoubleDouble,
1063 elements,
1064 external_layout: None,
1065 initialization: ProgramResourceInitialization::Zeroed,
1066 });
1067 let scratch1 = if output_is_compute {
1068 None
1069 } else {
1070 let scratch = ProgramResourceId(resources.len());
1071 resources.push(ProgramResource {
1072 id: scratch,
1073 name: "double_double_forced_rader_three_upload_scratch_1".to_owned(),
1074 kind: ProgramResourceKind::Scratch,
1075 scalar: ScalarType::DoubleDouble,
1076 elements,
1077 external_layout: None,
1078 initialization: ProgramResourceInitialization::Zeroed,
1079 });
1080 Some(scratch)
1081 };
1082 let (exchange0, exchange1) = if output_is_compute {
1083 (ProgramResourceId(1), scratch0)
1084 } else {
1085 (
1086 scratch0,
1087 scratch1.expect("mixed-storage path allocates scratch1"),
1088 )
1089 };
1090 let mut passes = Vec::new();
1091 let mut serial = 0usize;
1092 for component in components {
1093 let upload_id = component.upload_id();
1094 let (input, output) = match upload_id {
1095 2 => (ProgramResourceId(0), exchange0),
1096 1 => (exchange0, exchange1),
1097 0 => (exchange1, ProgramResourceId(1)),
1098 _ => unreachable!(),
1099 };
1100 match component {
1101 DoubleDoubleForcedRaderThreeUploadComponentIr::Recursive { ir: node, .. } => {
1102 flatten_double_double_recursive_node(
1103 &node,
1104 input,
1105 output,
1106 &mut resources,
1107 &mut passes,
1108 &mut serial,
1109 )?;
1110 }
1111 DoubleDoubleForcedRaderThreeUploadComponentIr::Stockham {
1112 ir: stockham, ..
1113 } => {
1114 let twiddle_values = stockham.twiddles.packed_values();
1115 let twiddle_id = ProgramResourceId(resources.len());
1116 resources.push(ProgramResource {
1117 id: twiddle_id,
1118 name: format!(
1119 "double_double_forced_rader_three_upload_{upload_id}_stockham_twiddles"
1120 ),
1121 kind: ProgramResourceKind::LookupTable,
1122 scalar: ScalarType::DoubleDouble,
1123 elements: twiddle_values.len(),
1124 external_layout: None,
1125 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
1126 twiddle_values,
1127 ),
1128 });
1129 let mut bindings = vec![
1130 ProgramPassBinding {
1131 binding: 0,
1132 resource: input,
1133 role: BufferRole::Input,
1134 access: BufferAccess::ReadOnly,
1135 },
1136 ProgramPassBinding {
1137 binding: 1,
1138 resource: output,
1139 role: BufferRole::Output,
1140 access: BufferAccess::WriteOnly,
1141 },
1142 ProgramPassBinding {
1143 binding: 2,
1144 resource: twiddle_id,
1145 role: BufferRole::TwiddleLookupTable,
1146 access: BufferAccess::ReadOnly,
1147 },
1148 ];
1149 let parent_period = match upload_id {
1150 2 => Some(ir.logical_len),
1151 1 => Some(a.checked_mul(b).ok_or(VkFftError::ArithmeticOverflow {
1152 operation: "double-double forced-Rader three-upload AB period",
1153 })?),
1154 0 => None,
1155 _ => unreachable!(),
1156 };
1157 if let Some(period) = parent_period {
1158 let roots =
1159 crate::lut::unit_root_table_double_double(period, ir.direction)?;
1160 let root_id = ProgramResourceId(resources.len());
1161 resources.push(ProgramResource {
1162 id: root_id,
1163 name: format!(
1164 "double_double_forced_rader_three_upload_{upload_id}_parent_roots"
1165 ),
1166 kind: ProgramResourceKind::LookupTable,
1167 scalar: ScalarType::DoubleDouble,
1168 elements: roots.len(),
1169 external_layout: None,
1170 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
1171 roots,
1172 ),
1173 });
1174 bindings.push(ProgramPassBinding {
1175 binding: 3,
1176 resource: root_id,
1177 role: BufferRole::Auxiliary,
1178 access: BufferAccess::ReadOnly,
1179 });
1180 }
1181 passes.push(ProgramPass {
1182 name: format!("{}_forced_rader_three_upload_{upload_id}", ir.name),
1183 dispatch: DispatchGeometry {
1184 x: u32::try_from(stockham.batch_group_count()).map_err(|_| {
1185 VkFftError::ValueOutOfRange {
1186 field: "double-double forced-Rader three-upload Stockham dispatch count",
1187 }
1188 })?,
1189 y: 1,
1190 z: 1,
1191 },
1192 bindings,
1193 });
1194 }
1195 }
1196 }
1197 let program = Self {
1198 name: format!("{}_forced_rader_three_upload_program", ir.name),
1199 scalar: ScalarType::DoubleDouble,
1200 resources,
1201 passes,
1202 };
1203 program.validate()?;
1204 Ok(program)
1205 }
1206
1207 fn double_double_recursive_two_upload_four_step(
1208 ir: &DoubleDoubleRecursiveFftIr,
1209 mut resources: Vec<ProgramResource>,
1210 ) -> Result<Self> {
1211 let plan = ir
1212 .two_upload_four_step_plan
1213 .ok_or(VkFftError::InvalidKernelIr(
1214 "double-double Four-step ProgramIr requires plan metadata",
1215 ))?;
1216 plan.validate()?;
1217 let DoubleDoubleRecursiveFftNodeIr::CooleyTukey(root) = &ir.root else {
1218 return Err(VkFftError::InvalidKernelIr(
1219 "double-double Four-step ProgramIr requires a Cooley-Tukey root",
1220 ));
1221 };
1222 let (
1223 DoubleDoubleRecursiveFftNodeIr::Stockham(left),
1224 DoubleDoubleRecursiveFftNodeIr::Stockham(right),
1225 ) = (&root.left, &root.right)
1226 else {
1227 return Err(VkFftError::InvalidKernelIr(
1228 "double-double Four-step ProgramIr requires Stockham leaves",
1229 ));
1230 };
1231 let elements =
1232 ir.logical_len
1233 .checked_mul(ir.batch_count)
1234 .ok_or(VkFftError::ArithmeticOverflow {
1235 operation: "double-double Four-step scratch element count",
1236 })?;
1237 let scratch = ProgramResourceId(resources.len());
1238 resources.push(ProgramResource {
1239 id: scratch,
1240 name: "double_double_four_step_scratch".to_owned(),
1241 kind: ProgramResourceKind::Scratch,
1242 scalar: ScalarType::DoubleDouble,
1243 elements,
1244 external_layout: None,
1245 initialization: ProgramResourceInitialization::Zeroed,
1246 });
1247 let right_twiddle_values = right.twiddles.packed_values();
1248 let right_twiddles = ProgramResourceId(resources.len());
1249 resources.push(ProgramResource {
1250 id: right_twiddles,
1251 name: "double_double_four_step_upload_1_twiddles".to_owned(),
1252 kind: ProgramResourceKind::LookupTable,
1253 scalar: ScalarType::DoubleDouble,
1254 elements: right_twiddle_values.len(),
1255 external_layout: None,
1256 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
1257 right_twiddle_values,
1258 ),
1259 });
1260 let root_twiddles = ProgramResourceId(resources.len());
1261 resources.push(ProgramResource {
1262 id: root_twiddles,
1263 name: "double_double_four_step_root_twiddles".to_owned(),
1264 kind: ProgramResourceKind::LookupTable,
1265 scalar: ScalarType::DoubleDouble,
1266 elements: root.twiddles.len(),
1267 external_layout: None,
1268 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
1269 root.twiddles.clone(),
1270 ),
1271 });
1272 let left_twiddle_values = left.twiddles.packed_values();
1273 let left_twiddles = ProgramResourceId(resources.len());
1274 resources.push(ProgramResource {
1275 id: left_twiddles,
1276 name: "double_double_four_step_upload_0_twiddles".to_owned(),
1277 kind: ProgramResourceKind::LookupTable,
1278 scalar: ScalarType::DoubleDouble,
1279 elements: left_twiddle_values.len(),
1280 external_layout: None,
1281 initialization: ProgramResourceInitialization::ComplexDoubleDouble(left_twiddle_values),
1282 });
1283 let right_dispatch = DispatchGeometry {
1284 x: u32::try_from(right.batch_group_count()).map_err(|_| {
1285 VkFftError::ValueOutOfRange {
1286 field: "double-double Four-step upload 1 dispatch count",
1287 }
1288 })?,
1289 y: 1,
1290 z: 1,
1291 };
1292 let left_dispatch = DispatchGeometry {
1293 x: u32::try_from(left.batch_group_count()).map_err(|_| {
1294 VkFftError::ValueOutOfRange {
1295 field: "double-double Four-step upload 0 dispatch count",
1296 }
1297 })?,
1298 y: 1,
1299 z: 1,
1300 };
1301 let passes = vec![
1302 ProgramPass {
1303 name: format!("{}_four_step_upload_1", ir.name),
1304 dispatch: right_dispatch,
1305 bindings: vec![
1306 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
1307 ProgramPassBinding {
1308 binding: 1,
1309 resource: scratch,
1310 role: BufferRole::Output,
1311 access: BufferAccess::WriteOnly,
1312 },
1313 ProgramPassBinding {
1314 binding: 2,
1315 resource: right_twiddles,
1316 role: BufferRole::TwiddleLookupTable,
1317 access: BufferAccess::ReadOnly,
1318 },
1319 ProgramPassBinding {
1320 binding: 3,
1321 resource: root_twiddles,
1322 role: BufferRole::Auxiliary,
1323 access: BufferAccess::ReadOnly,
1324 },
1325 ],
1326 },
1327 ProgramPass {
1328 name: format!("{}_four_step_upload_0", ir.name),
1329 dispatch: left_dispatch,
1330 bindings: vec![
1331 ProgramPassBinding {
1332 binding: 0,
1333 resource: scratch,
1334 role: BufferRole::Input,
1335 access: BufferAccess::ReadOnly,
1336 },
1337 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
1338 ProgramPassBinding {
1339 binding: 2,
1340 resource: left_twiddles,
1341 role: BufferRole::TwiddleLookupTable,
1342 access: BufferAccess::ReadOnly,
1343 },
1344 ],
1345 },
1346 ];
1347 let program = Self {
1348 name: format!("{}_four_step_program", ir.name),
1349 scalar: ScalarType::DoubleDouble,
1350 resources,
1351 passes,
1352 };
1353 program.validate()?;
1354 Ok(program)
1355 }
1356
1357 fn double_double_recursive_three_upload_four_step(
1358 ir: &DoubleDoubleRecursiveFftIr,
1359 mut resources: Vec<ProgramResource>,
1360 ) -> Result<Self> {
1361 let plan = ir
1362 .three_upload_four_step_plan
1363 .ok_or(VkFftError::InvalidKernelIr(
1364 "double-double three-upload Four-step ProgramIr requires plan metadata",
1365 ))?;
1366 plan.validate()?;
1367 let DoubleDoubleRecursiveFftNodeIr::CooleyTukey(root) = &ir.root else {
1368 return Err(VkFftError::InvalidKernelIr(
1369 "double-double three-upload Four-step ProgramIr requires a Cooley-Tukey root",
1370 ));
1371 };
1372 let DoubleDoubleRecursiveFftNodeIr::Stockham(low) = &root.left else {
1373 return Err(VkFftError::InvalidKernelIr(
1374 "double-double three-upload Four-step ProgramIr requires a low Stockham leaf",
1375 ));
1376 };
1377 let DoubleDoubleRecursiveFftNodeIr::CooleyTukey(upper) = &root.right else {
1378 return Err(VkFftError::InvalidKernelIr(
1379 "double-double three-upload Four-step ProgramIr requires an upper Cooley-Tukey node",
1380 ));
1381 };
1382 let (
1383 DoubleDoubleRecursiveFftNodeIr::Stockham(middle),
1384 DoubleDoubleRecursiveFftNodeIr::Stockham(high),
1385 ) = (&upper.left, &upper.right)
1386 else {
1387 return Err(VkFftError::InvalidKernelIr(
1388 "double-double three-upload Four-step ProgramIr requires middle/high Stockham leaves",
1389 ));
1390 };
1391 let [a, b, c] = plan.axis_split;
1392 if low.sequence_len != a || middle.sequence_len != b || high.sequence_len != c {
1393 return Err(VkFftError::InvalidKernelIr(
1394 "double-double three-upload Four-step ProgramIr leaf lengths do not match axisSplit",
1395 ));
1396 }
1397 let elements =
1398 ir.logical_len
1399 .checked_mul(ir.batch_count)
1400 .ok_or(VkFftError::ArithmeticOverflow {
1401 operation: "double-double three-upload Four-step scratch element count",
1402 })?;
1403 let output_is_compute = resources
1404 .get(1)
1405 .ok_or(VkFftError::InvalidKernelIr(
1406 "double-double three-upload Four-step output resource is missing",
1407 ))?
1408 .scalar
1409 == ScalarType::DoubleDouble;
1410 let scratch0 = ProgramResourceId(resources.len());
1411 resources.push(ProgramResource {
1412 id: scratch0,
1413 name: "double_double_three_upload_four_step_scratch_0".to_owned(),
1414 kind: ProgramResourceKind::Scratch,
1415 scalar: ScalarType::DoubleDouble,
1416 elements,
1417 external_layout: None,
1418 initialization: ProgramResourceInitialization::Zeroed,
1419 });
1420 let scratch1 = if output_is_compute {
1421 None
1422 } else {
1423 let scratch = ProgramResourceId(resources.len());
1424 resources.push(ProgramResource {
1425 id: scratch,
1426 name: "double_double_three_upload_four_step_scratch_1".to_owned(),
1427 kind: ProgramResourceKind::Scratch,
1428 scalar: ScalarType::DoubleDouble,
1429 elements,
1430 external_layout: None,
1431 initialization: ProgramResourceInitialization::Zeroed,
1432 });
1433 Some(scratch)
1434 };
1435 let (exchange0, exchange1) = if output_is_compute {
1436 (ProgramResourceId(1), scratch0)
1437 } else {
1438 (
1439 scratch0,
1440 scratch1.expect("mixed-storage path allocates scratch1"),
1441 )
1442 };
1443
1444 let high_twiddle_values = high.twiddles.packed_values();
1445 let high_twiddles = ProgramResourceId(resources.len());
1446 resources.push(ProgramResource {
1447 id: high_twiddles,
1448 name: "double_double_three_upload_four_step_upload_2_twiddles".to_owned(),
1449 kind: ProgramResourceKind::LookupTable,
1450 scalar: ScalarType::DoubleDouble,
1451 elements: high_twiddle_values.len(),
1452 external_layout: None,
1453 initialization: ProgramResourceInitialization::ComplexDoubleDouble(high_twiddle_values),
1454 });
1455 let root_twiddles = ProgramResourceId(resources.len());
1456 resources.push(ProgramResource {
1457 id: root_twiddles,
1458 name: "double_double_three_upload_four_step_root_twiddles".to_owned(),
1459 kind: ProgramResourceKind::LookupTable,
1460 scalar: ScalarType::DoubleDouble,
1461 elements: root.twiddles.len(),
1462 external_layout: None,
1463 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
1464 root.twiddles.clone(),
1465 ),
1466 });
1467
1468 let middle_twiddle_values = middle.twiddles.packed_values();
1469 let middle_twiddles = ProgramResourceId(resources.len());
1470 resources.push(ProgramResource {
1471 id: middle_twiddles,
1472 name: "double_double_three_upload_four_step_upload_1_twiddles".to_owned(),
1473 kind: ProgramResourceKind::LookupTable,
1474 scalar: ScalarType::DoubleDouble,
1475 elements: middle_twiddle_values.len(),
1476 external_layout: None,
1477 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
1478 middle_twiddle_values,
1479 ),
1480 });
1481 let ab = a.checked_mul(b).ok_or(VkFftError::ArithmeticOverflow {
1482 operation: "double-double three-upload Four-step AB twiddle period",
1483 })?;
1484 let ab_twiddle_values = crate::lut::unit_root_table_double_double(ab, ir.direction)?;
1485 let ab_twiddles = ProgramResourceId(resources.len());
1486 resources.push(ProgramResource {
1487 id: ab_twiddles,
1488 name: "double_double_three_upload_four_step_ab_twiddles".to_owned(),
1489 kind: ProgramResourceKind::LookupTable,
1490 scalar: ScalarType::DoubleDouble,
1491 elements: ab_twiddle_values.len(),
1492 external_layout: None,
1493 initialization: ProgramResourceInitialization::ComplexDoubleDouble(ab_twiddle_values),
1494 });
1495
1496 let low_twiddle_values = low.twiddles.packed_values();
1497 let low_twiddles = ProgramResourceId(resources.len());
1498 resources.push(ProgramResource {
1499 id: low_twiddles,
1500 name: "double_double_three_upload_four_step_upload_0_twiddles".to_owned(),
1501 kind: ProgramResourceKind::LookupTable,
1502 scalar: ScalarType::DoubleDouble,
1503 elements: low_twiddle_values.len(),
1504 external_layout: None,
1505 initialization: ProgramResourceInitialization::ComplexDoubleDouble(low_twiddle_values),
1506 });
1507
1508 let dispatch = |leaf: &DoubleDoubleStockhamIr, field| {
1509 Ok(DispatchGeometry {
1510 x: u32::try_from(leaf.batch_group_count())
1511 .map_err(|_| VkFftError::ValueOutOfRange { field })?,
1512 y: 1,
1513 z: 1,
1514 })
1515 };
1516 let passes = vec![
1517 ProgramPass {
1518 name: format!("{}_four_step_upload_2", ir.name),
1519 dispatch: dispatch(high, "double-double Four-step upload 2 dispatch count")?,
1520 bindings: vec![
1521 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
1522 ProgramPassBinding {
1523 binding: 1,
1524 resource: exchange0,
1525 role: BufferRole::Output,
1526 access: BufferAccess::WriteOnly,
1527 },
1528 ProgramPassBinding {
1529 binding: 2,
1530 resource: high_twiddles,
1531 role: BufferRole::TwiddleLookupTable,
1532 access: BufferAccess::ReadOnly,
1533 },
1534 ProgramPassBinding {
1535 binding: 3,
1536 resource: root_twiddles,
1537 role: BufferRole::Auxiliary,
1538 access: BufferAccess::ReadOnly,
1539 },
1540 ],
1541 },
1542 ProgramPass {
1543 name: format!("{}_four_step_upload_1", ir.name),
1544 dispatch: dispatch(middle, "double-double Four-step upload 1 dispatch count")?,
1545 bindings: vec![
1546 ProgramPassBinding {
1547 binding: 0,
1548 resource: exchange0,
1549 role: BufferRole::Input,
1550 access: BufferAccess::ReadOnly,
1551 },
1552 ProgramPassBinding {
1553 binding: 1,
1554 resource: exchange1,
1555 role: BufferRole::Output,
1556 access: BufferAccess::WriteOnly,
1557 },
1558 ProgramPassBinding {
1559 binding: 2,
1560 resource: middle_twiddles,
1561 role: BufferRole::TwiddleLookupTable,
1562 access: BufferAccess::ReadOnly,
1563 },
1564 ProgramPassBinding {
1565 binding: 3,
1566 resource: ab_twiddles,
1567 role: BufferRole::Auxiliary,
1568 access: BufferAccess::ReadOnly,
1569 },
1570 ],
1571 },
1572 ProgramPass {
1573 name: format!("{}_four_step_upload_0", ir.name),
1574 dispatch: dispatch(low, "double-double Four-step upload 0 dispatch count")?,
1575 bindings: vec![
1576 ProgramPassBinding {
1577 binding: 0,
1578 resource: exchange1,
1579 role: BufferRole::Input,
1580 access: BufferAccess::ReadOnly,
1581 },
1582 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
1583 ProgramPassBinding {
1584 binding: 2,
1585 resource: low_twiddles,
1586 role: BufferRole::TwiddleLookupTable,
1587 access: BufferAccess::ReadOnly,
1588 },
1589 ],
1590 },
1591 ];
1592 let program = Self {
1593 name: format!("{}_three_upload_four_step_program", ir.name),
1594 scalar: ScalarType::DoubleDouble,
1595 resources,
1596 passes,
1597 };
1598 program.validate()?;
1599 Ok(program)
1600 }
1601
1602 pub fn double_double_real(ir: &DoubleDoubleRealFftIr) -> Result<Self> {
1603 ir.validate()?;
1604 let full_elements =
1605 ir.length
1606 .checked_mul(ir.batch_count)
1607 .ok_or(VkFftError::ArithmeticOverflow {
1608 operation: "double-double real full element count",
1609 })?;
1610 let compact_elements = ir.half_spectrum_len.checked_mul(ir.batch_count).ok_or(
1611 VkFftError::ArithmeticOverflow {
1612 operation: "double-double real compact element count",
1613 },
1614 )?;
1615 let child_elements = ir
1616 .transform
1617 .sequence_len()
1618 .checked_mul(ir.batch_count)
1619 .ok_or(VkFftError::ArithmeticOverflow {
1620 operation: "double-double real child element count",
1621 })?;
1622 let external_scalar = match ir.external_storage {
1623 crate::PrecisionStorage::DoubleDouble => ScalarType::DoubleDouble,
1624 crate::PrecisionStorage::F64 => ScalarType::F64,
1625 _ => {
1626 return Err(VkFftError::InvalidKernelIr(
1627 "double-double real ProgramIr requires DD or F64 external storage",
1628 ));
1629 }
1630 };
1631 let (input_elements, output_elements, input_len, output_len) = match ir.kind {
1632 crate::RealFftKind::RealToComplex => (
1633 full_elements,
1634 compact_elements,
1635 ir.length,
1636 ir.half_spectrum_len,
1637 ),
1638 crate::RealFftKind::ComplexToReal => (
1639 compact_elements,
1640 full_elements,
1641 ir.half_spectrum_len,
1642 ir.length,
1643 ),
1644 };
1645 let mut resources = vec![
1646 ProgramResource {
1647 id: ProgramResourceId(0),
1648 name: "input".to_owned(),
1649 kind: ProgramResourceKind::Input,
1650 scalar: external_scalar,
1651 elements: input_elements,
1652 external_layout: Some(ExternalBufferLayout {
1653 logical_len: input_len,
1654 physical_stride: input_len,
1655 batch_count: ir.batch_count,
1656 element_shape: match ir.kind {
1657 crate::RealFftKind::RealToComplex => ProgramElementShape::Scalar,
1658 crate::RealFftKind::ComplexToReal => ProgramElementShape::Complex,
1659 },
1660 }),
1661 initialization: ProgramResourceInitialization::ExternalInput,
1662 },
1663 ProgramResource {
1664 id: ProgramResourceId(1),
1665 name: "output".to_owned(),
1666 kind: ProgramResourceKind::Output,
1667 scalar: external_scalar,
1668 elements: output_elements,
1669 external_layout: Some(ExternalBufferLayout {
1670 logical_len: output_len,
1671 physical_stride: output_len,
1672 batch_count: ir.batch_count,
1673 element_shape: match ir.kind {
1674 crate::RealFftKind::RealToComplex => ProgramElementShape::Complex,
1675 crate::RealFftKind::ComplexToReal => ProgramElementShape::Scalar,
1676 },
1677 }),
1678 initialization: ProgramResourceInitialization::Zeroed,
1679 },
1680 ProgramResource {
1681 id: ProgramResourceId(2),
1682 name: "double_double_real_child_input".to_owned(),
1683 kind: ProgramResourceKind::Scratch,
1684 scalar: ScalarType::DoubleDouble,
1685 elements: child_elements,
1686 external_layout: None,
1687 initialization: ProgramResourceInitialization::Zeroed,
1688 },
1689 ProgramResource {
1690 id: ProgramResourceId(3),
1691 name: "double_double_real_child_output".to_owned(),
1692 kind: ProgramResourceKind::Scratch,
1693 scalar: ScalarType::DoubleDouble,
1694 elements: child_elements,
1695 external_layout: None,
1696 initialization: ProgramResourceInitialization::Zeroed,
1697 },
1698 ];
1699 let child = Self::double_double_one_dim(&ir.transform)?;
1700 if child.scalar != ScalarType::DoubleDouble
1701 || child.input_resource()?.scalar != ScalarType::DoubleDouble
1702 || child.output_resource()?.scalar != ScalarType::DoubleDouble
1703 || child.input_resource()?.elements != child_elements
1704 || child.output_resource()?.elements != child_elements
1705 {
1706 return Err(VkFftError::InvalidKernelIr(
1707 "double-double real child ProgramIr must expose child-sized DD boundaries",
1708 ));
1709 }
1710 let mut resource_map = vec![None; child.resources.len()];
1711 for resource in &child.resources {
1712 let mapped = match resource.kind {
1713 ProgramResourceKind::Input => ProgramResourceId(2),
1714 ProgramResourceKind::Output => ProgramResourceId(3),
1715 _ => {
1716 let new_id = ProgramResourceId(resources.len());
1717 let mut cloned = resource.clone();
1718 cloned.id = new_id;
1719 cloned.name = format!("double_double_real_{}", resource.name);
1720 cloned.external_layout = None;
1721 resources.push(cloned);
1722 new_id
1723 }
1724 };
1725 resource_map[resource.id.0] = Some(mapped);
1726 }
1727 let even_roots_resource = if ir.even_half_size {
1728 let id = ProgramResourceId(resources.len());
1729 resources.push(ProgramResource {
1730 id,
1731 name: "double_double_real_even_roots".to_owned(),
1732 kind: ProgramResourceKind::LookupTable,
1733 scalar: ScalarType::DoubleDouble,
1734 elements: ir.even_roots.len(),
1735 external_layout: None,
1736 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
1737 ir.even_roots.clone(),
1738 ),
1739 });
1740 Some(id)
1741 } else {
1742 None
1743 };
1744 let boundary_dispatch = DispatchGeometry {
1745 x: u32::try_from(ir.batch_group_count()).map_err(|_| VkFftError::ValueOutOfRange {
1746 field: "double-double real boundary dispatch count",
1747 })?,
1748 y: 1,
1749 z: 1,
1750 };
1751 let mut first_bindings = vec![
1752 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
1753 binding(1, 2, BufferRole::Output, BufferAccess::WriteOnly),
1754 ];
1755 if ir.kind == crate::RealFftKind::ComplexToReal
1756 && let Some(roots) = even_roots_resource
1757 {
1758 first_bindings.push(ProgramPassBinding {
1759 binding: 2,
1760 resource: roots,
1761 role: BufferRole::LookupTable,
1762 access: BufferAccess::ReadOnly,
1763 });
1764 }
1765 let mut passes = Vec::with_capacity(child.passes.len() + 2);
1766 passes.push(ProgramPass {
1767 name: match (ir.kind, ir.even_half_size) {
1768 (crate::RealFftKind::RealToComplex, true) => {
1769 "vkfft_dd_real_r2c_half_size_pack".to_owned()
1770 }
1771 (crate::RealFftKind::ComplexToReal, true) => {
1772 "vkfft_dd_real_c2r_half_size_preprocess".to_owned()
1773 }
1774 (crate::RealFftKind::RealToComplex, false) => {
1775 "vkfft_dd_real_r2c_promote".to_owned()
1776 }
1777 (crate::RealFftKind::ComplexToReal, false) => "vkfft_dd_real_c2r_expand".to_owned(),
1778 },
1779 dispatch: boundary_dispatch,
1780 bindings: first_bindings,
1781 });
1782 for child_pass in &child.passes {
1783 let mut pass = child_pass.clone();
1784 pass.name = format!("vkfft_dd_real_{}", child_pass.name);
1785 for binding in &mut pass.bindings {
1786 binding.resource = resource_map
1787 .get(binding.resource.0)
1788 .and_then(|mapped| *mapped)
1789 .ok_or(VkFftError::InvalidKernelIr(
1790 "double-double real child resource mapping is incomplete",
1791 ))?;
1792 }
1793 passes.push(pass);
1794 }
1795 let mut last_bindings = vec![
1796 binding(0, 3, BufferRole::Input, BufferAccess::ReadOnly),
1797 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
1798 ];
1799 if ir.kind == crate::RealFftKind::RealToComplex
1800 && let Some(roots) = even_roots_resource
1801 {
1802 last_bindings.push(ProgramPassBinding {
1803 binding: 2,
1804 resource: roots,
1805 role: BufferRole::LookupTable,
1806 access: BufferAccess::ReadOnly,
1807 });
1808 }
1809 passes.push(ProgramPass {
1810 name: match (ir.kind, ir.even_half_size) {
1811 (crate::RealFftKind::RealToComplex, true) => {
1812 "vkfft_dd_real_r2c_half_size_postprocess".to_owned()
1813 }
1814 (crate::RealFftKind::ComplexToReal, true) => {
1815 "vkfft_dd_real_c2r_half_size_unpack".to_owned()
1816 }
1817 (crate::RealFftKind::RealToComplex, false) => {
1818 "vkfft_dd_real_r2c_compact".to_owned()
1819 }
1820 (crate::RealFftKind::ComplexToReal, false) => {
1821 "vkfft_dd_real_c2r_finalize".to_owned()
1822 }
1823 },
1824 dispatch: boundary_dispatch,
1825 bindings: last_bindings,
1826 });
1827 let program = Self {
1828 name: format!(
1829 "vkfft_dd_real_{}_{}",
1830 match ir.kind {
1831 crate::RealFftKind::RealToComplex => "r2c",
1832 crate::RealFftKind::ComplexToReal => "c2r",
1833 },
1834 ir.length
1835 ),
1836 scalar: ScalarType::DoubleDouble,
1837 resources,
1838 passes,
1839 };
1840 program.validate()?;
1841 Ok(program)
1842 }
1843
1844 pub fn double_double_nd_real(ir: &DoubleDoubleNdRealFftIr) -> Result<Self> {
1845 ir.validate()?;
1846 let full_elements = ir.full_tensor_len.checked_mul(ir.batch_count).ok_or(
1847 VkFftError::ArithmeticOverflow {
1848 operation: "double-double ND real full element count",
1849 },
1850 )?;
1851 let compact_elements = ir.compact_tensor_len.checked_mul(ir.batch_count).ok_or(
1852 VkFftError::ArithmeticOverflow {
1853 operation: "double-double ND real compact element count",
1854 },
1855 )?;
1856 let external_scalar = match ir.external_storage {
1857 crate::PrecisionStorage::DoubleDouble => ScalarType::DoubleDouble,
1858 crate::PrecisionStorage::F64 => ScalarType::F64,
1859 _ => {
1860 return Err(VkFftError::InvalidKernelIr(
1861 "double-double ND real ProgramIr requires DD or F64 external storage",
1862 ));
1863 }
1864 };
1865 let formatted_io = ir.formatted_io.as_ref();
1866 let input_elements = formatted_io
1867 .input_external_layout
1868 .batch_stride
1869 .checked_mul(ir.batch_count)
1870 .ok_or(VkFftError::ArithmeticOverflow {
1871 operation: "double-double ND real physical input element count",
1872 })?;
1873 let output_elements = formatted_io
1874 .output_external_layout
1875 .batch_stride
1876 .checked_mul(ir.batch_count)
1877 .ok_or(VkFftError::ArithmeticOverflow {
1878 operation: "double-double ND real physical output element count",
1879 })?;
1880 let input_layout = ExternalBufferLayout {
1881 logical_len: if formatted_io.input_formatted_copy.is_some() {
1882 formatted_io.input_external_layout.batch_stride
1883 } else {
1884 ir.input_tensor_len()
1885 },
1886 physical_stride: formatted_io.input_external_layout.batch_stride,
1887 batch_count: ir.batch_count,
1888 element_shape: match ir.kind {
1889 crate::RealFftKind::RealToComplex => ProgramElementShape::Scalar,
1890 crate::RealFftKind::ComplexToReal => ProgramElementShape::Complex,
1891 },
1892 };
1893 let output_layout = ExternalBufferLayout {
1894 logical_len: if formatted_io.output_formatted_copy.is_some() {
1895 formatted_io.output_external_layout.batch_stride
1896 } else {
1897 ir.output_tensor_len()
1898 },
1899 physical_stride: formatted_io.output_external_layout.batch_stride,
1900 batch_count: ir.batch_count,
1901 element_shape: match ir.kind {
1902 crate::RealFftKind::RealToComplex => ProgramElementShape::Complex,
1903 crate::RealFftKind::ComplexToReal => ProgramElementShape::Scalar,
1904 },
1905 };
1906 let full_scalar_layout = ExternalBufferLayout {
1907 logical_len: full_elements,
1908 physical_stride: full_elements,
1909 batch_count: 1,
1910 element_shape: ProgramElementShape::Scalar,
1911 };
1912 let mut resources = vec![
1913 ProgramResource {
1914 id: ProgramResourceId(0),
1915 name: "input".to_owned(),
1916 kind: ProgramResourceKind::Input,
1917 scalar: external_scalar,
1918 elements: input_elements,
1919 external_layout: Some(input_layout),
1920 initialization: ProgramResourceInitialization::ExternalInput,
1921 },
1922 ProgramResource {
1923 id: ProgramResourceId(1),
1924 name: "output".to_owned(),
1925 kind: ProgramResourceKind::Output,
1926 scalar: external_scalar,
1927 elements: output_elements,
1928 external_layout: Some(output_layout),
1929 initialization: ProgramResourceInitialization::Zeroed,
1930 },
1931 ProgramResource {
1932 id: ProgramResourceId(2),
1933 name: "double_double_nd_real_full_scalar".to_owned(),
1934 kind: ProgramResourceKind::Scratch,
1935 scalar: ScalarType::DoubleDouble,
1936 elements: full_elements,
1937 external_layout: Some(full_scalar_layout),
1938 initialization: ProgramResourceInitialization::Zeroed,
1939 },
1940 ProgramResource {
1941 id: ProgramResourceId(3),
1942 name: "double_double_nd_real_compact_a".to_owned(),
1943 kind: ProgramResourceKind::Scratch,
1944 scalar: ScalarType::DoubleDouble,
1945 elements: compact_elements,
1946 external_layout: None,
1947 initialization: ProgramResourceInitialization::Zeroed,
1948 },
1949 ProgramResource {
1950 id: ProgramResourceId(4),
1951 name: "double_double_nd_real_compact_b".to_owned(),
1952 kind: ProgramResourceKind::Scratch,
1953 scalar: ScalarType::DoubleDouble,
1954 elements: compact_elements,
1955 external_layout: None,
1956 initialization: ProgramResourceInitialization::Zeroed,
1957 },
1958 ProgramResource {
1959 id: ProgramResourceId(5),
1960 name: "double_double_nd_real_lines_a".to_owned(),
1961 kind: ProgramResourceKind::Scratch,
1962 scalar: ScalarType::DoubleDouble,
1963 elements: compact_elements,
1964 external_layout: None,
1965 initialization: ProgramResourceInitialization::Zeroed,
1966 },
1967 ProgramResource {
1968 id: ProgramResourceId(6),
1969 name: "double_double_nd_real_lines_b".to_owned(),
1970 kind: ProgramResourceKind::Scratch,
1971 scalar: ScalarType::DoubleDouble,
1972 elements: compact_elements,
1973 external_layout: None,
1974 initialization: ProgramResourceInitialization::Zeroed,
1975 },
1976 ];
1977 let boundary_dispatch = DispatchGeometry {
1978 x: u32::try_from(ir.batch_group_count()).map_err(|_| VkFftError::ValueOutOfRange {
1979 field: "double-double ND real boundary dispatch count",
1980 })?,
1981 y: 1,
1982 z: 1,
1983 };
1984 let mut passes = Vec::new();
1985 let input_internal = match ir.kind {
1986 crate::RealFftKind::RealToComplex => ProgramResourceId(2),
1987 crate::RealFftKind::ComplexToReal => ProgramResourceId(3),
1988 };
1989 if let Some(copy) = &formatted_io.input_formatted_copy {
1990 passes.push(ProgramPass {
1991 name: copy.name.clone(),
1992 dispatch: copy.dispatch,
1993 bindings: vec![
1994 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
1995 ProgramPassBinding {
1996 binding: 1,
1997 resource: input_internal,
1998 role: BufferRole::Output,
1999 access: BufferAccess::WriteOnly,
2000 },
2001 ],
2002 });
2003 } else {
2004 passes.push(ProgramPass {
2005 name: "vkfft_dd_nd_real_external_input_promote".to_owned(),
2006 dispatch: boundary_dispatch,
2007 bindings: vec![
2008 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
2009 ProgramPassBinding {
2010 binding: 1,
2011 resource: input_internal,
2012 role: BufferRole::Output,
2013 access: BufferAccess::WriteOnly,
2014 },
2015 ],
2016 });
2017 }
2018
2019 let append_child = |child: &ProgramIr,
2020 prefix: &str,
2021 input: ProgramResourceId,
2022 output: ProgramResourceId,
2023 resources: &mut Vec<ProgramResource>,
2024 passes: &mut Vec<ProgramPass>|
2025 -> Result<()> {
2026 let mut resource_map = vec![None; child.resources.len()];
2027 for resource in &child.resources {
2028 let mapped = match resource.kind {
2029 ProgramResourceKind::Input => input,
2030 ProgramResourceKind::Output => output,
2031 _ => {
2032 let new_id = ProgramResourceId(resources.len());
2033 let mut cloned = resource.clone();
2034 cloned.id = new_id;
2035 cloned.name = format!("{prefix}_{}", resource.name);
2036 cloned.external_layout = None;
2037 resources.push(cloned);
2038 new_id
2039 }
2040 };
2041 resource_map[resource.id.0] = Some(mapped);
2042 }
2043 for child_pass in &child.passes {
2044 let mut pass = child_pass.clone();
2045 pass.name = format!("{prefix}_{}", child_pass.name);
2046 for binding in &mut pass.bindings {
2047 binding.resource = resource_map
2048 .get(binding.resource.0)
2049 .and_then(|mapped| *mapped)
2050 .ok_or(VkFftError::InvalidKernelIr(
2051 "double-double ND real child resource mapping is incomplete",
2052 ))?;
2053 }
2054 passes.push(pass);
2055 }
2056 Ok(())
2057 };
2058
2059 let real_child = Self::double_double_real(&ir.real_axis)?;
2060 let expected_real_input = match ir.kind {
2061 crate::RealFftKind::RealToComplex => full_elements,
2062 crate::RealFftKind::ComplexToReal => compact_elements,
2063 };
2064 let expected_real_output = match ir.kind {
2065 crate::RealFftKind::RealToComplex => compact_elements,
2066 crate::RealFftKind::ComplexToReal => full_elements,
2067 };
2068 if real_child.input_resource()?.scalar != ScalarType::DoubleDouble
2069 || real_child.output_resource()?.scalar != ScalarType::DoubleDouble
2070 || real_child.input_resource()?.elements != expected_real_input
2071 || real_child.output_resource()?.elements != expected_real_output
2072 {
2073 return Err(VkFftError::InvalidKernelIr(
2074 "double-double ND real last-axis child has incompatible boundaries",
2075 ));
2076 }
2077
2078 let mut compact_source = ProgramResourceId(3);
2079 let mut compact_target = ProgramResourceId(4);
2080 if ir.kind == crate::RealFftKind::RealToComplex {
2081 append_child(
2082 &real_child,
2083 "vkfft_dd_nd_real_last_axis",
2084 ProgramResourceId(2),
2085 compact_source,
2086 &mut resources,
2087 &mut passes,
2088 )?;
2089 }
2090
2091 for axis in &ir.complex_axes {
2092 let dispatch = DispatchGeometry {
2093 x: u32::try_from(ir.batch_count.div_ceil(axis.grouped_batch)).map_err(|_| {
2094 VkFftError::ValueOutOfRange {
2095 field: "double-double ND real complex-axis dispatch count",
2096 }
2097 })?,
2098 y: 1,
2099 z: 1,
2100 };
2101 passes.push(ProgramPass {
2102 name: format!("vkfft_dd_nd_real_axis_{}_pack", axis.axis),
2103 dispatch,
2104 bindings: vec![
2105 ProgramPassBinding {
2106 binding: 0,
2107 resource: compact_source,
2108 role: BufferRole::Input,
2109 access: BufferAccess::ReadOnly,
2110 },
2111 binding(1, 5, BufferRole::Output, BufferAccess::WriteOnly),
2112 ],
2113 });
2114 let child = Self::double_double_one_dim(&axis.transform)?;
2115 if child.input_resource()?.scalar != ScalarType::DoubleDouble
2116 || child.output_resource()?.scalar != ScalarType::DoubleDouble
2117 || child.input_resource()?.elements != compact_elements
2118 || child.output_resource()?.elements != compact_elements
2119 {
2120 return Err(VkFftError::InvalidKernelIr(
2121 "double-double ND real complex-axis child has incompatible boundaries",
2122 ));
2123 }
2124 append_child(
2125 &child,
2126 &format!("vkfft_dd_nd_real_axis_{}", axis.axis),
2127 ProgramResourceId(5),
2128 ProgramResourceId(6),
2129 &mut resources,
2130 &mut passes,
2131 )?;
2132 passes.push(ProgramPass {
2133 name: format!("vkfft_dd_nd_real_axis_{}_scatter", axis.axis),
2134 dispatch,
2135 bindings: vec![
2136 binding(0, 6, BufferRole::Input, BufferAccess::ReadOnly),
2137 ProgramPassBinding {
2138 binding: 1,
2139 resource: compact_target,
2140 role: BufferRole::Output,
2141 access: BufferAccess::WriteOnly,
2142 },
2143 ],
2144 });
2145 compact_source = compact_target;
2146 compact_target = if compact_target == ProgramResourceId(3) {
2147 ProgramResourceId(4)
2148 } else {
2149 ProgramResourceId(3)
2150 };
2151 }
2152
2153 if ir.kind == crate::RealFftKind::ComplexToReal {
2154 append_child(
2155 &real_child,
2156 "vkfft_dd_nd_real_last_axis",
2157 compact_source,
2158 ProgramResourceId(2),
2159 &mut resources,
2160 &mut passes,
2161 )?;
2162 }
2163 let output_internal = match ir.kind {
2164 crate::RealFftKind::RealToComplex => compact_source,
2165 crate::RealFftKind::ComplexToReal => ProgramResourceId(2),
2166 };
2167 if let Some(copy) = &formatted_io.output_formatted_copy {
2168 passes.push(ProgramPass {
2169 name: copy.name.clone(),
2170 dispatch: copy.dispatch,
2171 bindings: vec![
2172 ProgramPassBinding {
2173 binding: 0,
2174 resource: output_internal,
2175 role: BufferRole::Input,
2176 access: BufferAccess::ReadOnly,
2177 },
2178 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
2179 ],
2180 });
2181 } else {
2182 passes.push(ProgramPass {
2183 name: "vkfft_dd_nd_real_external_output_finalize".to_owned(),
2184 dispatch: boundary_dispatch,
2185 bindings: vec![
2186 ProgramPassBinding {
2187 binding: 0,
2188 resource: output_internal,
2189 role: BufferRole::Input,
2190 access: BufferAccess::ReadOnly,
2191 },
2192 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
2193 ],
2194 });
2195 }
2196 let program = Self {
2197 name: format!("vkfft_dd_nd_real_{:?}_{:?}", ir.dimensions, ir.kind),
2198 scalar: ScalarType::DoubleDouble,
2199 resources,
2200 passes,
2201 };
2202 program.validate()?;
2203 Ok(program)
2204 }
2205
2206 pub fn double_double_nd(ir: &DoubleDoubleNdFftIr) -> Result<Self> {
2207 ir.validate()?;
2208 let elements =
2209 ir.tensor_len
2210 .checked_mul(ir.batch_count)
2211 .ok_or(VkFftError::ArithmeticOverflow {
2212 operation: "double-double ND program element count",
2213 })?;
2214 let external_scalar = match ir.external_storage {
2215 crate::PrecisionStorage::DoubleDouble => ScalarType::DoubleDouble,
2216 crate::PrecisionStorage::F64 => ScalarType::F64,
2217 _ => {
2218 return Err(VkFftError::InvalidKernelIr(
2219 "double-double ND program requires DD or F64 external storage",
2220 ));
2221 }
2222 };
2223 let input_layout = ExternalBufferLayout {
2224 logical_len: if ir.input_formatted_copy.is_some() {
2225 ir.input_external_layout.batch_stride
2226 } else {
2227 ir.tensor_len
2228 },
2229 physical_stride: ir.input_external_layout.batch_stride,
2230 batch_count: ir.batch_count,
2231 element_shape: ProgramElementShape::Complex,
2232 };
2233 let output_layout = ExternalBufferLayout {
2234 logical_len: if ir.output_formatted_copy.is_some() {
2235 ir.output_external_layout.batch_stride
2236 } else {
2237 ir.tensor_len
2238 },
2239 physical_stride: ir.output_external_layout.batch_stride,
2240 batch_count: ir.batch_count,
2241 element_shape: ProgramElementShape::Complex,
2242 };
2243 let input_elements = input_layout.physical_elements()?;
2244 let output_elements = output_layout.physical_elements()?;
2245 let mut resources = vec![
2246 ProgramResource {
2247 id: ProgramResourceId(0),
2248 name: "input".to_owned(),
2249 kind: ProgramResourceKind::Input,
2250 scalar: external_scalar,
2251 elements: input_elements,
2252 external_layout: Some(input_layout),
2253 initialization: ProgramResourceInitialization::ExternalInput,
2254 },
2255 ProgramResource {
2256 id: ProgramResourceId(1),
2257 name: "output".to_owned(),
2258 kind: ProgramResourceKind::Output,
2259 scalar: external_scalar,
2260 elements: output_elements,
2261 external_layout: Some(output_layout),
2262 initialization: ProgramResourceInitialization::Zeroed,
2263 },
2264 ProgramResource {
2265 id: ProgramResourceId(2),
2266 name: "double_double_nd_tensor_a".to_owned(),
2267 kind: ProgramResourceKind::Scratch,
2268 scalar: ScalarType::DoubleDouble,
2269 elements,
2270 external_layout: None,
2271 initialization: ProgramResourceInitialization::Zeroed,
2272 },
2273 ProgramResource {
2274 id: ProgramResourceId(3),
2275 name: "double_double_nd_tensor_b".to_owned(),
2276 kind: ProgramResourceKind::Scratch,
2277 scalar: ScalarType::DoubleDouble,
2278 elements,
2279 external_layout: None,
2280 initialization: ProgramResourceInitialization::Zeroed,
2281 },
2282 ProgramResource {
2283 id: ProgramResourceId(4),
2284 name: "double_double_nd_lines_a".to_owned(),
2285 kind: ProgramResourceKind::Scratch,
2286 scalar: ScalarType::DoubleDouble,
2287 elements,
2288 external_layout: None,
2289 initialization: ProgramResourceInitialization::Zeroed,
2290 },
2291 ProgramResource {
2292 id: ProgramResourceId(5),
2293 name: "double_double_nd_lines_b".to_owned(),
2294 kind: ProgramResourceKind::Scratch,
2295 scalar: ScalarType::DoubleDouble,
2296 elements,
2297 external_layout: None,
2298 initialization: ProgramResourceInitialization::Zeroed,
2299 },
2300 ];
2301 let mut passes = Vec::new();
2302 let mut tensor_source = if let Some(copy) = &ir.input_formatted_copy {
2303 passes.push(two_buffer_pass(©.name, copy.dispatch, 0, 2));
2304 ProgramResourceId(2)
2305 } else {
2306 ProgramResourceId(0)
2307 };
2308 let mut tensor_target = if tensor_source == ProgramResourceId(2) {
2309 ProgramResourceId(3)
2310 } else {
2311 ProgramResourceId(2)
2312 };
2313 let mut formatted_output_source = None;
2314 for (axis_index, axis) in ir.axes.iter().enumerate() {
2315 let dispatch = DispatchGeometry {
2316 x: u32::try_from(ir.batch_count.div_ceil(axis.grouped_batch)).map_err(|_| {
2317 VkFftError::ValueOutOfRange {
2318 field: "double-double ND grouped dispatch count",
2319 }
2320 })?,
2321 y: 1,
2322 z: 1,
2323 };
2324 passes.push(ProgramPass {
2325 name: format!("vkfft_dd_nd_axis_{}_pack", axis.axis),
2326 dispatch,
2327 bindings: vec![
2328 ProgramPassBinding {
2329 binding: 0,
2330 resource: tensor_source,
2331 role: BufferRole::Input,
2332 access: BufferAccess::ReadOnly,
2333 },
2334 binding(1, 4, BufferRole::Output, BufferAccess::WriteOnly),
2335 ],
2336 });
2337
2338 let child = Self::double_double_one_dim(&axis.transform)?;
2339 if child.scalar != ScalarType::DoubleDouble
2340 || child.input_resource()?.scalar != ScalarType::DoubleDouble
2341 || child.output_resource()?.scalar != ScalarType::DoubleDouble
2342 || child.input_resource()?.elements != elements
2343 || child.output_resource()?.elements != elements
2344 {
2345 return Err(VkFftError::InvalidKernelIr(
2346 "double-double ND child ProgramIr must expose DD tensor-sized boundaries",
2347 ));
2348 }
2349 let mut resource_map = vec![None; child.resources.len()];
2350 for resource in &child.resources {
2351 let mapped = match resource.kind {
2352 ProgramResourceKind::Input => ProgramResourceId(4),
2353 ProgramResourceKind::Output => ProgramResourceId(5),
2354 _ => {
2355 let new_id = ProgramResourceId(resources.len());
2356 let mut cloned = resource.clone();
2357 cloned.id = new_id;
2358 cloned.name =
2359 format!("double_double_nd_axis_{}_{}", axis.axis, resource.name);
2360 cloned.external_layout = None;
2361 resources.push(cloned);
2362 new_id
2363 }
2364 };
2365 resource_map[resource.id.0] = Some(mapped);
2366 }
2367 for child_pass in &child.passes {
2368 let mut pass = child_pass.clone();
2369 pass.name = format!("vkfft_dd_nd_axis_{}_{}", axis.axis, child_pass.name);
2370 for binding in &mut pass.bindings {
2371 binding.resource = resource_map
2372 .get(binding.resource.0)
2373 .and_then(|mapped| *mapped)
2374 .ok_or(VkFftError::InvalidKernelIr(
2375 "double-double ND child resource mapping is incomplete",
2376 ))?;
2377 }
2378 passes.push(pass);
2379 }
2380
2381 let last_axis = axis_index + 1 == ir.axes.len();
2382 let scatter_target = if last_axis && ir.output_formatted_copy.is_none() {
2383 ProgramResourceId(1)
2384 } else {
2385 tensor_target
2386 };
2387 passes.push(ProgramPass {
2388 name: format!("vkfft_dd_nd_axis_{}_scatter", axis.axis),
2389 dispatch,
2390 bindings: vec![
2391 binding(0, 5, BufferRole::Input, BufferAccess::ReadOnly),
2392 ProgramPassBinding {
2393 binding: 1,
2394 resource: scatter_target,
2395 role: BufferRole::Output,
2396 access: BufferAccess::WriteOnly,
2397 },
2398 ],
2399 });
2400 if last_axis {
2401 formatted_output_source = ir.output_formatted_copy.as_ref().map(|_| scatter_target);
2402 } else {
2403 tensor_source = tensor_target;
2404 tensor_target = if tensor_target == ProgramResourceId(2) {
2405 ProgramResourceId(3)
2406 } else {
2407 ProgramResourceId(2)
2408 };
2409 }
2410 }
2411 if let (Some(copy), Some(source)) = (&ir.output_formatted_copy, formatted_output_source) {
2412 passes.push(ProgramPass {
2413 name: copy.name.clone(),
2414 dispatch: copy.dispatch,
2415 bindings: vec![
2416 ProgramPassBinding {
2417 binding: 0,
2418 resource: source,
2419 role: BufferRole::Input,
2420 access: BufferAccess::ReadOnly,
2421 },
2422 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
2423 ],
2424 });
2425 }
2426 let program = Self {
2427 name: format!("vkfft_dd_nd_program_{}d", ir.dimensions.len()),
2428 scalar: ScalarType::DoubleDouble,
2429 resources,
2430 passes,
2431 };
2432 program.validate()?;
2433 Ok(program)
2434 }
2435
2436 pub fn double_double_nd_r2r(ir: &DoubleDoubleNdR2rIr) -> Result<Self> {
2437 ir.validate()?;
2438 let elements =
2439 ir.tensor_len
2440 .checked_mul(ir.batch_count)
2441 .ok_or(VkFftError::ArithmeticOverflow {
2442 operation: "double-double ND R2R program element count",
2443 })?;
2444 let external_scalar = match ir.external_storage {
2445 crate::PrecisionStorage::DoubleDouble => ScalarType::DoubleDouble,
2446 crate::PrecisionStorage::F64 => ScalarType::F64,
2447 _ => {
2448 return Err(VkFftError::InvalidKernelIr(
2449 "double-double ND R2R program requires DD or F64 external storage",
2450 ));
2451 }
2452 };
2453 let input_external_stride = if ir.input_formatted_copy.is_some() {
2454 ir.input_external_layout.batch_stride
2455 } else {
2456 ir.tensor_len
2457 };
2458 let output_external_stride = if ir.output_formatted_copy.is_some() {
2459 ir.output_external_layout.batch_stride
2460 } else {
2461 ir.tensor_len
2462 };
2463 let input_external_layout = ExternalBufferLayout {
2464 logical_len: input_external_stride,
2465 physical_stride: input_external_stride,
2466 batch_count: ir.batch_count,
2467 element_shape: ProgramElementShape::Scalar,
2468 };
2469 let output_external_layout = ExternalBufferLayout {
2470 logical_len: output_external_stride,
2471 physical_stride: output_external_stride,
2472 batch_count: ir.batch_count,
2473 element_shape: ProgramElementShape::Scalar,
2474 };
2475 let input_external_elements = input_external_layout.physical_elements()?;
2476 let output_external_elements = output_external_layout.physical_elements()?;
2477 let scalar_scratch_layout = ExternalBufferLayout {
2478 logical_len: elements,
2479 physical_stride: elements,
2480 batch_count: 1,
2481 element_shape: ProgramElementShape::Scalar,
2482 };
2483 let mut resources = vec![
2484 ProgramResource {
2485 id: ProgramResourceId(0),
2486 name: "input".to_owned(),
2487 kind: ProgramResourceKind::Input,
2488 scalar: external_scalar,
2489 elements: input_external_elements,
2490 external_layout: Some(input_external_layout),
2491 initialization: ProgramResourceInitialization::ExternalInput,
2492 },
2493 ProgramResource {
2494 id: ProgramResourceId(1),
2495 name: "output".to_owned(),
2496 kind: ProgramResourceKind::Output,
2497 scalar: external_scalar,
2498 elements: output_external_elements,
2499 external_layout: Some(output_external_layout),
2500 initialization: ProgramResourceInitialization::Zeroed,
2501 },
2502 ProgramResource {
2503 id: ProgramResourceId(2),
2504 name: "double_double_nd_r2r_tensor_a".to_owned(),
2505 kind: ProgramResourceKind::Scratch,
2506 scalar: ScalarType::DoubleDouble,
2507 elements,
2508 external_layout: Some(scalar_scratch_layout),
2509 initialization: ProgramResourceInitialization::Zeroed,
2510 },
2511 ProgramResource {
2512 id: ProgramResourceId(3),
2513 name: "double_double_nd_r2r_tensor_b".to_owned(),
2514 kind: ProgramResourceKind::Scratch,
2515 scalar: ScalarType::DoubleDouble,
2516 elements,
2517 external_layout: Some(scalar_scratch_layout),
2518 initialization: ProgramResourceInitialization::Zeroed,
2519 },
2520 ProgramResource {
2521 id: ProgramResourceId(4),
2522 name: "double_double_nd_r2r_lines_a".to_owned(),
2523 kind: ProgramResourceKind::Scratch,
2524 scalar: ScalarType::DoubleDouble,
2525 elements,
2526 external_layout: Some(scalar_scratch_layout),
2527 initialization: ProgramResourceInitialization::Zeroed,
2528 },
2529 ProgramResource {
2530 id: ProgramResourceId(5),
2531 name: "double_double_nd_r2r_lines_b".to_owned(),
2532 kind: ProgramResourceKind::Scratch,
2533 scalar: ScalarType::DoubleDouble,
2534 elements,
2535 external_layout: Some(scalar_scratch_layout),
2536 initialization: ProgramResourceInitialization::Zeroed,
2537 },
2538 ];
2539 let mut passes = Vec::new();
2540 let mut tensor_source = if let Some(copy) = &ir.input_formatted_copy {
2541 let dense = ProgramResourceId(resources.len());
2542 resources.push(ProgramResource {
2543 id: dense,
2544 name: "double_double_nd_r2r_formatted_input".to_owned(),
2545 kind: ProgramResourceKind::Scratch,
2546 scalar: ScalarType::DoubleDouble,
2547 elements,
2548 external_layout: Some(scalar_scratch_layout),
2549 initialization: ProgramResourceInitialization::Zeroed,
2550 });
2551 passes.push(ProgramPass {
2552 name: copy.name.clone(),
2553 dispatch: copy.dispatch,
2554 bindings: vec![
2555 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
2556 ProgramPassBinding {
2557 binding: 1,
2558 resource: dense,
2559 role: BufferRole::Output,
2560 access: BufferAccess::WriteOnly,
2561 },
2562 ],
2563 });
2564 dense
2565 } else {
2566 ProgramResourceId(0)
2567 };
2568 let formatted_output = if ir.output_formatted_copy.is_some() {
2569 let dense = ProgramResourceId(resources.len());
2570 resources.push(ProgramResource {
2571 id: dense,
2572 name: "double_double_nd_r2r_formatted_output".to_owned(),
2573 kind: ProgramResourceKind::Scratch,
2574 scalar: ScalarType::DoubleDouble,
2575 elements,
2576 external_layout: Some(scalar_scratch_layout),
2577 initialization: ProgramResourceInitialization::Zeroed,
2578 });
2579 Some(dense)
2580 } else {
2581 None
2582 };
2583 let mut tensor_target = ProgramResourceId(2);
2584 for (axis_index, axis) in ir.axes.iter().enumerate() {
2585 let dispatch = DispatchGeometry {
2586 x: u32::try_from(ir.batch_count.div_ceil(axis.grouped_batch)).map_err(|_| {
2587 VkFftError::ValueOutOfRange {
2588 field: "double-double ND R2R grouped dispatch count",
2589 }
2590 })?,
2591 y: 1,
2592 z: 1,
2593 };
2594 passes.push(ProgramPass {
2595 name: format!("vkfft_dd_nd_r2r_axis_{}_pack", axis.axis),
2596 dispatch,
2597 bindings: vec![
2598 ProgramPassBinding {
2599 binding: 0,
2600 resource: tensor_source,
2601 role: BufferRole::Input,
2602 access: BufferAccess::ReadOnly,
2603 },
2604 binding(1, 4, BufferRole::Output, BufferAccess::WriteOnly),
2605 ],
2606 });
2607 let child = Self::double_double_r2r(&axis.transform)?;
2608 if child.scalar != ScalarType::DoubleDouble
2609 || child.input_resource()?.scalar != ScalarType::DoubleDouble
2610 || child.output_resource()?.scalar != ScalarType::DoubleDouble
2611 || child.input_resource()?.element_shape() != ProgramElementShape::Scalar
2612 || child.output_resource()?.element_shape() != ProgramElementShape::Scalar
2613 || child.input_resource()?.elements != elements
2614 || child.output_resource()?.elements != elements
2615 {
2616 return Err(VkFftError::InvalidKernelIr(
2617 "double-double ND R2R child ProgramIr must expose tensor-sized scalar DD boundaries",
2618 ));
2619 }
2620 let mut resource_map = vec![None; child.resources.len()];
2621 for resource in &child.resources {
2622 let mapped = match resource.kind {
2623 ProgramResourceKind::Input => ProgramResourceId(4),
2624 ProgramResourceKind::Output => ProgramResourceId(5),
2625 _ => {
2626 let new_id = ProgramResourceId(resources.len());
2627 let mut cloned = resource.clone();
2628 cloned.id = new_id;
2629 cloned.name =
2630 format!("double_double_nd_r2r_axis_{}_{}", axis.axis, resource.name);
2631 cloned.external_layout = None;
2632 resources.push(cloned);
2633 new_id
2634 }
2635 };
2636 resource_map[resource.id.0] = Some(mapped);
2637 }
2638 for child_pass in &child.passes {
2639 let mut pass = child_pass.clone();
2640 pass.name = format!("vkfft_dd_nd_r2r_axis_{}_{}", axis.axis, child_pass.name);
2641 for binding in &mut pass.bindings {
2642 binding.resource = resource_map
2643 .get(binding.resource.0)
2644 .and_then(|mapped| *mapped)
2645 .ok_or(VkFftError::InvalidKernelIr(
2646 "double-double ND R2R child resource mapping is incomplete",
2647 ))?;
2648 }
2649 passes.push(pass);
2650 }
2651 let last_axis = axis_index + 1 == ir.axes.len();
2652 let scatter_target = if last_axis {
2653 formatted_output.unwrap_or(ProgramResourceId(1))
2654 } else {
2655 tensor_target
2656 };
2657 passes.push(ProgramPass {
2658 name: format!("vkfft_dd_nd_r2r_axis_{}_scatter", axis.axis),
2659 dispatch,
2660 bindings: vec![
2661 binding(0, 5, BufferRole::Input, BufferAccess::ReadOnly),
2662 ProgramPassBinding {
2663 binding: 1,
2664 resource: scatter_target,
2665 role: BufferRole::Output,
2666 access: BufferAccess::WriteOnly,
2667 },
2668 ],
2669 });
2670 if !last_axis {
2671 tensor_source = tensor_target;
2672 tensor_target = if tensor_target == ProgramResourceId(2) {
2673 ProgramResourceId(3)
2674 } else {
2675 ProgramResourceId(2)
2676 };
2677 }
2678 }
2679 if let Some(copy) = &ir.output_formatted_copy {
2680 passes.push(ProgramPass {
2681 name: copy.name.clone(),
2682 dispatch: copy.dispatch,
2683 bindings: vec![
2684 ProgramPassBinding {
2685 binding: 0,
2686 resource: formatted_output.expect("formatted DD R2R output scratch"),
2687 role: BufferRole::Input,
2688 access: BufferAccess::ReadOnly,
2689 },
2690 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
2691 ],
2692 });
2693 }
2694 let program = Self {
2695 name: format!("vkfft_dd_nd_r2r_program_{}d", ir.dimensions.len()),
2696 scalar: ScalarType::DoubleDouble,
2697 resources,
2698 passes,
2699 };
2700 program.validate()?;
2701 Ok(program)
2702 }
2703
2704 pub fn double_double_bluestein(ir: &DoubleDoubleBluesteinIr) -> Result<Self> {
2705 ir.validate()?;
2706 let logical_elements =
2707 ir.logical_len
2708 .checked_mul(ir.batch_count)
2709 .ok_or(VkFftError::ArithmeticOverflow {
2710 operation: "double-double Bluestein program logical element count",
2711 })?;
2712 let convolution_elements = ir.convolution_len.checked_mul(ir.batch_count).ok_or(
2713 VkFftError::ArithmeticOverflow {
2714 operation: "double-double Bluestein program convolution element count",
2715 },
2716 )?;
2717 let external_scalar = match ir.external_storage {
2718 crate::PrecisionStorage::DoubleDouble => ScalarType::DoubleDouble,
2719 crate::PrecisionStorage::F64 => ScalarType::F64,
2720 _ => {
2721 return Err(VkFftError::InvalidKernelIr(
2722 "double-double Bluestein program requires DD or F64 external storage",
2723 ));
2724 }
2725 };
2726 let fused_inverse_stockham = matches!(
2727 &ir.inverse_fft,
2728 crate::DoubleDoubleBluesteinConvolutionIr::Stockham(child)
2729 if double_double_stockham_input_lookup_eligible(child)
2730 );
2731 let layout = ExternalBufferLayout {
2732 logical_len: ir.logical_len,
2733 physical_stride: ir.logical_len,
2734 batch_count: ir.batch_count,
2735 element_shape: ProgramElementShape::Complex,
2736 };
2737 let mut resources = vec![
2738 ProgramResource {
2739 id: ProgramResourceId(0),
2740 name: "input".to_owned(),
2741 kind: ProgramResourceKind::Input,
2742 scalar: external_scalar,
2743 elements: logical_elements,
2744 external_layout: Some(layout),
2745 initialization: ProgramResourceInitialization::ExternalInput,
2746 },
2747 ProgramResource {
2748 id: ProgramResourceId(1),
2749 name: "output".to_owned(),
2750 kind: ProgramResourceKind::Output,
2751 scalar: external_scalar,
2752 elements: logical_elements,
2753 external_layout: Some(layout),
2754 initialization: ProgramResourceInitialization::Zeroed,
2755 },
2756 ];
2757 let push_convolution_scratch =
2758 |resources: &mut Vec<ProgramResource>, name: &str| -> ProgramResourceId {
2759 let id = ProgramResourceId(resources.len());
2760 resources.push(ProgramResource {
2761 id,
2762 name: name.to_owned(),
2763 kind: ProgramResourceKind::Scratch,
2764 scalar: ScalarType::DoubleDouble,
2765 elements: convolution_elements,
2766 external_layout: None,
2767 initialization: ProgramResourceInitialization::Zeroed,
2768 });
2769 id
2770 };
2771 let forward_input_id =
2772 push_convolution_scratch(&mut resources, "double_double_bluestein_forward_input");
2773 let forward_output_id =
2774 push_convolution_scratch(&mut resources, "double_double_bluestein_forward_output");
2775 let inverse_input_id = if fused_inverse_stockham {
2776 None
2777 } else {
2778 Some(push_convolution_scratch(
2779 &mut resources,
2780 "double_double_bluestein_inverse_input",
2781 ))
2782 };
2783 let inverse_output_id =
2784 push_convolution_scratch(&mut resources, "double_double_bluestein_inverse_output");
2785 let chirp_id = ProgramResourceId(resources.len());
2786 resources.push(ProgramResource {
2787 id: chirp_id,
2788 name: "double_double_bluestein_chirp".to_owned(),
2789 kind: ProgramResourceKind::LookupTable,
2790 scalar: ScalarType::DoubleDouble,
2791 elements: ir.table.chirp.len(),
2792 external_layout: None,
2793 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
2794 ir.table.chirp.clone(),
2795 ),
2796 });
2797 let kernel_id = ProgramResourceId(resources.len());
2798 resources.push(ProgramResource {
2799 id: kernel_id,
2800 name: "double_double_bluestein_kernel_spectrum".to_owned(),
2801 kind: ProgramResourceKind::LookupTable,
2802 scalar: ScalarType::DoubleDouble,
2803 elements: ir.kernel_spectrum.len(),
2804 external_layout: None,
2805 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
2806 ir.kernel_spectrum.clone(),
2807 ),
2808 });
2809 let dispatch = DispatchGeometry {
2810 x: u32::try_from(ir.batch_group_count()).map_err(|_| VkFftError::ValueOutOfRange {
2811 field: "double-double Bluestein program dispatch count",
2812 })?,
2813 y: 1,
2814 z: 1,
2815 };
2816 let mut passes = Vec::new();
2817 passes.push(ProgramPass {
2818 name: format!("{}_preprocess", ir.name),
2819 dispatch,
2820 bindings: vec![
2821 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
2822 binding(
2823 1,
2824 forward_input_id.0,
2825 BufferRole::Output,
2826 BufferAccess::WriteOnly,
2827 ),
2828 ProgramPassBinding {
2829 binding: 2,
2830 resource: chirp_id,
2831 role: BufferRole::LookupTable,
2832 access: BufferAccess::ReadOnly,
2833 },
2834 ],
2835 });
2836
2837 let forward_child = match &ir.forward_fft {
2838 crate::DoubleDoubleBluesteinConvolutionIr::Stockham(child) => {
2839 ProgramIr::double_double_stockham_internal(child)?
2840 }
2841 crate::DoubleDoubleBluesteinConvolutionIr::Recursive(child) => {
2842 ProgramIr::double_double_recursive(child)?
2843 }
2844 crate::DoubleDoubleBluesteinConvolutionIr::Bluestein(child) => {
2845 ProgramIr::double_double_bluestein(child)?
2846 }
2847 };
2848 append_double_double_recursive_child(
2849 &forward_child,
2850 "vkfft_dd_bluestein_forward",
2851 forward_input_id,
2852 forward_output_id,
2853 &mut resources,
2854 &mut passes,
2855 false,
2856 )?;
2857
2858 if !fused_inverse_stockham {
2859 passes.push(ProgramPass {
2860 name: format!("{}_multiply", ir.name),
2861 dispatch,
2862 bindings: vec![
2863 binding(
2864 0,
2865 forward_output_id.0,
2866 BufferRole::Input,
2867 BufferAccess::ReadOnly,
2868 ),
2869 binding(
2870 1,
2871 inverse_input_id
2872 .expect("explicit DD Bluestein multiply requires inverse input scratch")
2873 .0,
2874 BufferRole::Output,
2875 BufferAccess::WriteOnly,
2876 ),
2877 ProgramPassBinding {
2878 binding: 2,
2879 resource: kernel_id,
2880 role: BufferRole::LookupTable,
2881 access: BufferAccess::ReadOnly,
2882 },
2883 ],
2884 });
2885 }
2886
2887 let inverse_child = match &ir.inverse_fft {
2888 crate::DoubleDoubleBluesteinConvolutionIr::Stockham(child) => {
2889 ProgramIr::double_double_stockham_internal(child)?
2890 }
2891 crate::DoubleDoubleBluesteinConvolutionIr::Recursive(child) => {
2892 ProgramIr::double_double_recursive(child)?
2893 }
2894 crate::DoubleDoubleBluesteinConvolutionIr::Bluestein(child) => {
2895 ProgramIr::double_double_bluestein(child)?
2896 }
2897 };
2898 let inverse_pass_start = passes.len();
2899 append_double_double_recursive_child(
2900 &inverse_child,
2901 "vkfft_dd_bluestein_inverse",
2902 if fused_inverse_stockham {
2903 forward_output_id
2904 } else {
2905 inverse_input_id.expect("explicit DD Bluestein inverse requires input scratch")
2906 },
2907 inverse_output_id,
2908 &mut resources,
2909 &mut passes,
2910 false,
2911 )?;
2912 if fused_inverse_stockham {
2913 attach_double_double_stockham_input_lookup(
2914 passes
2915 .get_mut(inverse_pass_start)
2916 .ok_or(VkFftError::InvalidKernelIr(
2917 "fused DD Bluestein inverse Stockham child emitted no pass",
2918 ))?,
2919 kernel_id,
2920 )?;
2921 }
2922
2923 passes.push(ProgramPass {
2924 name: format!("{}_postprocess", ir.name),
2925 dispatch,
2926 bindings: vec![
2927 binding(
2928 0,
2929 inverse_output_id.0,
2930 BufferRole::Input,
2931 BufferAccess::ReadOnly,
2932 ),
2933 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
2934 ProgramPassBinding {
2935 binding: 2,
2936 resource: chirp_id,
2937 role: BufferRole::LookupTable,
2938 access: BufferAccess::ReadOnly,
2939 },
2940 ],
2941 });
2942 let program = Self {
2943 name: format!("{}_program", ir.name),
2944 scalar: ScalarType::DoubleDouble,
2945 resources,
2946 passes,
2947 };
2948 program.validate()?;
2949 Ok(program)
2950 }
2951
2952 pub fn double_double_fft_rader(ir: &DoubleDoubleFftRaderIr) -> Result<Self> {
2953 ir.validate()?;
2954 let external_elements =
2955 ir.prime
2956 .checked_mul(ir.batch_count)
2957 .ok_or(VkFftError::ArithmeticOverflow {
2958 operation: "double-double FFT Rader program external element count",
2959 })?;
2960 let scratch_elements = ir.convolution_len.checked_mul(ir.batch_count).ok_or(
2961 VkFftError::ArithmeticOverflow {
2962 operation: "double-double FFT Rader program scratch element count",
2963 },
2964 )?;
2965 let storage_scalar = |storage| match storage {
2966 crate::PrecisionStorage::DoubleDouble => Ok(ScalarType::DoubleDouble),
2967 crate::PrecisionStorage::F64 => Ok(ScalarType::F64),
2968 _ => Err(VkFftError::InvalidKernelIr(
2969 "double-double FFT Rader program requires DD or F64 caller storage",
2970 )),
2971 };
2972 let input_scalar = storage_scalar(ir.input_storage)?;
2973 let output_scalar = storage_scalar(ir.output_storage)?;
2974 let layout = ExternalBufferLayout {
2975 logical_len: ir.prime,
2976 physical_stride: ir.prime,
2977 batch_count: ir.batch_count,
2978 element_shape: ProgramElementShape::Complex,
2979 };
2980 let mut resources = vec![
2981 ProgramResource {
2982 id: ProgramResourceId(0),
2983 name: "input".to_owned(),
2984 kind: ProgramResourceKind::Input,
2985 scalar: input_scalar,
2986 elements: external_elements,
2987 external_layout: Some(layout),
2988 initialization: ProgramResourceInitialization::ExternalInput,
2989 },
2990 ProgramResource {
2991 id: ProgramResourceId(1),
2992 name: "output".to_owned(),
2993 kind: ProgramResourceKind::Output,
2994 scalar: output_scalar,
2995 elements: external_elements,
2996 external_layout: Some(layout),
2997 initialization: ProgramResourceInitialization::Zeroed,
2998 },
2999 ProgramResource {
3000 id: ProgramResourceId(2),
3001 name: "double_double_rader_scratch_a".to_owned(),
3002 kind: ProgramResourceKind::Scratch,
3003 scalar: ScalarType::DoubleDouble,
3004 elements: scratch_elements,
3005 external_layout: None,
3006 initialization: ProgramResourceInitialization::Zeroed,
3007 },
3008 ProgramResource {
3009 id: ProgramResourceId(3),
3010 name: "double_double_rader_scratch_b".to_owned(),
3011 kind: ProgramResourceKind::Scratch,
3012 scalar: ScalarType::DoubleDouble,
3013 elements: scratch_elements,
3014 external_layout: None,
3015 initialization: ProgramResourceInitialization::Zeroed,
3016 },
3017 ];
3018 let kernel_id = ProgramResourceId(resources.len());
3019 resources.push(ProgramResource {
3020 id: kernel_id,
3021 name: "double_double_rader_kernel_spectrum".to_owned(),
3022 kind: ProgramResourceKind::LookupTable,
3023 scalar: ScalarType::DoubleDouble,
3024 elements: ir.kernel_spectrum.len(),
3025 external_layout: None,
3026 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
3027 ir.kernel_spectrum.clone(),
3028 ),
3029 });
3030 let dispatch = DispatchGeometry {
3031 x: u32::try_from(ir.batch_group_count()).map_err(|_| VkFftError::ValueOutOfRange {
3032 field: "double-double FFT Rader program dispatch count",
3033 })?,
3034 y: 1,
3035 z: 1,
3036 };
3037 let convolution_grouped_batch = ir
3038 .forward_fft
3039 .stockham_axis_batch_block()
3040 .map_or(ir.grouped_batch, |block| block.grouped_batch);
3041 let convolution_dispatch = DispatchGeometry {
3042 x: u32::try_from(ir.batch_count.div_ceil(convolution_grouped_batch)).map_err(|_| {
3043 VkFftError::ValueOutOfRange {
3044 field: "double-double FFT Rader convolution dispatch count",
3045 }
3046 })?,
3047 y: 1,
3048 z: 1,
3049 };
3050 let mut passes = Vec::new();
3051 passes.push(ProgramPass {
3052 name: format!("{}_gather", ir.name),
3053 dispatch,
3054 bindings: vec![
3055 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
3056 binding(1, 2, BufferRole::Output, BufferAccess::WriteOnly),
3057 ],
3058 });
3059
3060 let forward_child = match &ir.forward_fft {
3061 crate::DoubleDoubleBluesteinConvolutionIr::Stockham(child) => {
3062 ProgramIr::double_double_stockham_internal(child)?
3063 }
3064 crate::DoubleDoubleBluesteinConvolutionIr::Recursive(child) => {
3065 ProgramIr::double_double_recursive(child)?
3066 }
3067 crate::DoubleDoubleBluesteinConvolutionIr::Bluestein(child) => {
3068 ProgramIr::double_double_bluestein(child)?
3069 }
3070 };
3071 append_double_double_recursive_child(
3072 &forward_child,
3073 "vkfft_dd_fft_rader_forward",
3074 ProgramResourceId(2),
3075 ProgramResourceId(3),
3076 &mut resources,
3077 &mut passes,
3078 false,
3079 )?;
3080 let fused_inverse_stockham = matches!(
3081 &ir.inverse_fft,
3082 crate::DoubleDoubleBluesteinConvolutionIr::Stockham(child)
3083 if double_double_stockham_input_lookup_eligible(child)
3084 );
3085 if !fused_inverse_stockham {
3086 passes.push(ProgramPass {
3087 name: format!("{}_multiply", ir.name),
3088 dispatch: convolution_dispatch,
3089 bindings: vec![
3090 binding(0, 3, BufferRole::Input, BufferAccess::ReadOnly),
3091 binding(1, 2, BufferRole::Output, BufferAccess::WriteOnly),
3092 ProgramPassBinding {
3093 binding: 2,
3094 resource: kernel_id,
3095 role: BufferRole::LookupTable,
3096 access: BufferAccess::ReadOnly,
3097 },
3098 ],
3099 });
3100 }
3101 let inverse_child = match &ir.inverse_fft {
3102 crate::DoubleDoubleBluesteinConvolutionIr::Stockham(child) => {
3103 ProgramIr::double_double_stockham_internal(child)?
3104 }
3105 crate::DoubleDoubleBluesteinConvolutionIr::Recursive(child) => {
3106 ProgramIr::double_double_recursive(child)?
3107 }
3108 crate::DoubleDoubleBluesteinConvolutionIr::Bluestein(child) => {
3109 ProgramIr::double_double_bluestein(child)?
3110 }
3111 };
3112 let inverse_pass_start = passes.len();
3113 append_double_double_recursive_child(
3114 &inverse_child,
3115 "vkfft_dd_fft_rader_inverse",
3116 if fused_inverse_stockham {
3117 ProgramResourceId(3)
3118 } else {
3119 ProgramResourceId(2)
3120 },
3121 if fused_inverse_stockham {
3122 ProgramResourceId(2)
3123 } else {
3124 ProgramResourceId(3)
3125 },
3126 &mut resources,
3127 &mut passes,
3128 false,
3129 )?;
3130 if fused_inverse_stockham {
3131 attach_double_double_stockham_input_lookup(
3132 passes
3133 .get_mut(inverse_pass_start)
3134 .ok_or(VkFftError::InvalidKernelIr(
3135 "fused DD FFT-Rader inverse Stockham child emitted no pass",
3136 ))?,
3137 kernel_id,
3138 )?;
3139 }
3140 let inverse_output = if fused_inverse_stockham { 2 } else { 3 };
3141 let mut scatter_bindings = vec![
3142 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
3143 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
3144 binding(
3145 2,
3146 inverse_output,
3147 BufferRole::Auxiliary,
3148 BufferAccess::ReadOnly,
3149 ),
3150 ];
3151 let parent_period = match ir.io_mapping {
3152 crate::StockhamIoMapping::FourStepRight(mapping) => Some(mapping.logical_len),
3153 crate::StockhamIoMapping::FourStepThreeUpload2(mapping) => Some(mapping.logical_len),
3154 crate::StockhamIoMapping::FourStepThreeUpload1(mapping) => {
3155 let [a, b, _] = mapping.axis_split;
3156 Some(a.checked_mul(b).ok_or(VkFftError::ArithmeticOverflow {
3157 operation: "double-double FFT Rader three-upload parent root period",
3158 })?)
3159 }
3160 _ => None,
3161 };
3162 if let Some(period) = parent_period {
3163 let roots = crate::lut::unit_root_table_double_double(period, ir.direction)?;
3164 let root_id = ProgramResourceId(resources.len());
3165 resources.push(ProgramResource {
3166 id: root_id,
3167 name: "double_double_fft_rader_four_step_roots".to_owned(),
3168 kind: ProgramResourceKind::LookupTable,
3169 scalar: ScalarType::DoubleDouble,
3170 elements: roots.len(),
3171 external_layout: None,
3172 initialization: ProgramResourceInitialization::ComplexDoubleDouble(roots),
3173 });
3174 scatter_bindings.push(ProgramPassBinding {
3175 binding: 3,
3176 resource: root_id,
3177 role: BufferRole::TwiddleLookupTable,
3178 access: BufferAccess::ReadOnly,
3179 });
3180 }
3181 passes.push(ProgramPass {
3182 name: format!("{}_scatter", ir.name),
3183 dispatch,
3184 bindings: scatter_bindings,
3185 });
3186 let program = Self {
3187 name: format!("{}_program", ir.name),
3188 scalar: ScalarType::DoubleDouble,
3189 resources,
3190 passes,
3191 };
3192 program.validate()?;
3193 Self::wrap_double_double_zero_pad(
3194 ir.zero_pad_pass.as_ref(),
3195 ir.prime,
3196 ir.batch_count,
3197 "double_double_fft_rader",
3198 program,
3199 )
3200 }
3201
3202 pub fn stockham(kernel: &KernelIr) -> Result<Self> {
3203 kernel.validate()?;
3204 let elements = kernel.sequence_len.checked_mul(kernel.batch_count).ok_or(
3205 VkFftError::ArithmeticOverflow {
3206 operation: "Stockham program element count",
3207 },
3208 )?;
3209 let layout = ExternalBufferLayout {
3210 logical_len: kernel.sequence_len,
3211 physical_stride: kernel.sequence_len,
3212 batch_count: kernel.batch_count,
3213 element_shape: ProgramElementShape::Complex,
3214 };
3215 let mut builder = RecursiveProgramBuilder {
3216 scalar: kernel.scalar,
3217 resources: vec![
3218 ProgramResource {
3219 id: ProgramResourceId(0),
3220 name: "input".to_owned(),
3221 kind: ProgramResourceKind::Input,
3222 scalar: kernel.bindings[0].scalar,
3223 elements,
3224 external_layout: Some(layout),
3225 initialization: ProgramResourceInitialization::ExternalInput,
3226 },
3227 ProgramResource {
3228 id: ProgramResourceId(1),
3229 name: "output".to_owned(),
3230 kind: ProgramResourceKind::Output,
3231 scalar: kernel.bindings[1].scalar,
3232 elements,
3233 external_layout: Some(layout),
3234 initialization: ProgramResourceInitialization::Zeroed,
3235 },
3236 ],
3237 passes: Vec::new(),
3238 serial: 0,
3239 };
3240 builder.push_stockham_pass(
3241 kernel,
3242 ProgramResourceId(0),
3243 ProgramResourceId(1),
3244 None,
3245 None,
3246 )?;
3247 let program = Self {
3248 name: format!("{}_program", kernel.name),
3249 scalar: kernel.scalar,
3250 resources: builder.resources,
3251 passes: builder.passes,
3252 };
3253 program.validate()?;
3254 Ok(program)
3255 }
3256
3257 fn double_double_r2r_fused_physical_stockham(
3258 ir: &DoubleDoubleR2rIr,
3259 elements: usize,
3260 external_scalar: ScalarType,
3261 external: ExternalBufferLayout,
3262 ) -> Result<Option<Self>> {
3263 let DoubleDoubleR2rAlgorithm::FftReduction { fft, phases, .. } = &ir.algorithm else {
3264 return Ok(None);
3265 };
3266 if !matches!(
3267 ir.effective_transform,
3268 crate::R2rTransform::Dct(crate::DctType::II | crate::DctType::III)
3269 | crate::R2rTransform::Dst(crate::DstType::II | crate::DstType::III)
3270 ) {
3271 return Ok(None);
3272 }
3273 let DoubleDoubleOneDimIr::Stockham(stockham) = fft.as_ref() else {
3274 return Ok(None);
3275 };
3276 if stockham.axis_batch_block.is_none()
3277 || stockham.external_storage != crate::PrecisionStorage::DoubleDouble
3278 || stockham.zero_pad_pass.is_some()
3279 || stockham.normalize
3280 || stockham.sequence_len != ir.length
3281 || phases.len() != ir.length
3282 {
3283 return Ok(None);
3284 }
3285
3286 let mut resources = vec![
3287 ProgramResource {
3288 id: ProgramResourceId(0),
3289 name: "input".to_owned(),
3290 kind: ProgramResourceKind::Input,
3291 scalar: external_scalar,
3292 elements,
3293 external_layout: Some(external),
3294 initialization: ProgramResourceInitialization::ExternalInput,
3295 },
3296 ProgramResource {
3297 id: ProgramResourceId(1),
3298 name: "output".to_owned(),
3299 kind: ProgramResourceKind::Output,
3300 scalar: external_scalar,
3301 elements,
3302 external_layout: Some(external),
3303 initialization: ProgramResourceInitialization::Zeroed,
3304 },
3305 ];
3306 let mut bindings = vec![
3307 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
3308 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
3309 ];
3310 let twiddles = stockham.twiddles.packed_values();
3311 if !twiddles.is_empty() {
3312 let twiddle_id = ProgramResourceId(resources.len());
3313 resources.push(ProgramResource {
3314 id: twiddle_id,
3315 name: "double_double_r2r_fused_stockham_twiddles".to_owned(),
3316 kind: ProgramResourceKind::LookupTable,
3317 scalar: ScalarType::DoubleDouble,
3318 elements: twiddles.len(),
3319 external_layout: None,
3320 initialization: ProgramResourceInitialization::ComplexDoubleDouble(twiddles),
3321 });
3322 bindings.push(ProgramPassBinding {
3323 binding: 2,
3324 resource: twiddle_id,
3325 role: BufferRole::TwiddleLookupTable,
3326 access: BufferAccess::ReadOnly,
3327 });
3328 }
3329 let phases_id = ProgramResourceId(resources.len());
3330 resources.push(ProgramResource {
3331 id: phases_id,
3332 name: "double_double_r2r_phases".to_owned(),
3333 kind: ProgramResourceKind::LookupTable,
3334 scalar: ScalarType::DoubleDouble,
3335 elements: phases.len(),
3336 external_layout: None,
3337 initialization: ProgramResourceInitialization::ComplexDoubleDouble(phases.clone()),
3338 });
3339 bindings.push(ProgramPassBinding {
3340 binding: 3,
3341 resource: phases_id,
3342 role: BufferRole::LookupTable,
3343 access: BufferAccess::ReadOnly,
3344 });
3345 let label = match ir.effective_transform {
3346 crate::R2rTransform::Dct(crate::DctType::II) => "dct2",
3347 crate::R2rTransform::Dct(crate::DctType::III) => "dct3",
3348 crate::R2rTransform::Dst(crate::DstType::II) => "dst2",
3349 crate::R2rTransform::Dst(crate::DstType::III) => "dst3",
3350 _ => unreachable!("validated fused DD Type-II/III transform"),
3351 };
3352 let program = Self {
3353 name: format!("vkfft_dd_r2r_{label}_stockham_fused_{}_program", ir.length),
3354 scalar: ScalarType::DoubleDouble,
3355 resources,
3356 passes: vec![ProgramPass {
3357 name: format!("vkfft_dd_r2r_{label}_stockham_fused_{}", ir.length),
3358 dispatch: DispatchGeometry {
3359 x: u32::try_from(stockham.batch_group_count()).map_err(|_| {
3360 VkFftError::ValueOutOfRange {
3361 field: "fused double-double R2R Stockham dispatch count",
3362 }
3363 })?,
3364 y: 1,
3365 z: 1,
3366 },
3367 bindings,
3368 }],
3369 };
3370 program.validate()?;
3371 Ok(Some(program))
3372 }
3373
3374 fn double_double_r2r_fused_two_upload_four_step(
3375 ir: &DoubleDoubleR2rIr,
3376 elements: usize,
3377 external_scalar: ScalarType,
3378 external: ExternalBufferLayout,
3379 ) -> Result<Option<Self>> {
3380 let DoubleDoubleR2rAlgorithm::FftReduction { fft, phases, .. } = &ir.algorithm else {
3381 return Ok(None);
3382 };
3383 if !matches!(
3384 ir.effective_transform,
3385 crate::R2rTransform::Dct(crate::DctType::II | crate::DctType::III)
3386 | crate::R2rTransform::Dst(crate::DstType::II | crate::DstType::III)
3387 ) {
3388 return Ok(None);
3389 }
3390 let DoubleDoubleOneDimIr::Recursive(recursive) = fft.as_ref() else {
3391 return Ok(None);
3392 };
3393 if recursive.two_upload_four_step_plan.is_none()
3394 || recursive.three_upload_four_step_plan.is_some()
3395 || recursive.rader_forced_upload_schedule.is_some()
3396 || recursive.external_storage != crate::PrecisionStorage::DoubleDouble
3397 || recursive.zero_pad_pass.is_some()
3398 || recursive.normalize
3399 || recursive.logical_len != ir.length
3400 || recursive.batch_count != ir.batch_count
3401 || recursive.grouped_batch != ir.grouped_batch
3402 || phases.len() != ir.length
3403 {
3404 return Ok(None);
3405 }
3406
3407 let mut program = Self::double_double_recursive(recursive)?;
3408 if program.passes.len() != 2
3409 || program.input_resource()?.scalar != ScalarType::DoubleDouble
3410 || program.output_resource()?.scalar != ScalarType::DoubleDouble
3411 {
3412 return Ok(None);
3413 }
3414 for resource in &mut program.resources {
3415 match resource.kind {
3416 ProgramResourceKind::Input => {
3417 resource.scalar = external_scalar;
3418 resource.elements = elements;
3419 resource.external_layout = Some(external);
3420 }
3421 ProgramResourceKind::Output => {
3422 resource.scalar = external_scalar;
3423 resource.elements = elements;
3424 resource.external_layout = Some(external);
3425 }
3426 _ => {}
3427 }
3428 }
3429 let phases_id = ProgramResourceId(program.resources.len());
3430 program.resources.push(ProgramResource {
3431 id: phases_id,
3432 name: "double_double_r2r_phases".to_owned(),
3433 kind: ProgramResourceKind::LookupTable,
3434 scalar: ScalarType::DoubleDouble,
3435 elements: phases.len(),
3436 external_layout: None,
3437 initialization: ProgramResourceInitialization::ComplexDoubleDouble(phases.clone()),
3438 });
3439 let label = match ir.effective_transform {
3440 crate::R2rTransform::Dct(crate::DctType::II) => "dct2",
3441 crate::R2rTransform::Dct(crate::DctType::III) => "dct3",
3442 crate::R2rTransform::Dst(crate::DstType::II) => "dst2",
3443 crate::R2rTransform::Dst(crate::DstType::III) => "dst3",
3444 _ => unreachable!("validated two-upload DD Type-II/III fusion"),
3445 };
3446 let phase_binding = ProgramPassBinding {
3447 binding: if matches!(
3448 ir.effective_transform,
3449 crate::R2rTransform::Dct(crate::DctType::III)
3450 | crate::R2rTransform::Dst(crate::DstType::III)
3451 ) {
3452 4
3453 } else {
3454 3
3455 },
3456 resource: phases_id,
3457 role: BufferRole::LookupTable,
3458 access: BufferAccess::ReadOnly,
3459 };
3460 if matches!(
3461 ir.effective_transform,
3462 crate::R2rTransform::Dct(crate::DctType::III)
3463 | crate::R2rTransform::Dst(crate::DstType::III)
3464 ) {
3465 program.passes[0].bindings.push(phase_binding);
3466 } else {
3467 program.passes[1].bindings.push(phase_binding);
3468 }
3469 program.passes[0].name = format!("vkfft_dd_r2r_{label}_four_step_upload_1");
3470 program.passes[1].name = format!("vkfft_dd_r2r_{label}_four_step_upload_0");
3471 program.name = format!("vkfft_dd_r2r_{label}_four_step_fused_{}_program", ir.length);
3472 program.validate()?;
3473 Ok(Some(program))
3474 }
3475
3476 fn double_double_r2r_fused_forced_rader_two_upload(
3477 ir: &DoubleDoubleR2rIr,
3478 elements: usize,
3479 external_scalar: ScalarType,
3480 external: ExternalBufferLayout,
3481 ) -> Result<Option<Self>> {
3482 let DoubleDoubleR2rAlgorithm::FftReduction { fft, phases, .. } = &ir.algorithm else {
3483 return Ok(None);
3484 };
3485 if !matches!(
3486 ir.effective_transform,
3487 crate::R2rTransform::Dct(crate::DctType::II | crate::DctType::III)
3488 | crate::R2rTransform::Dst(crate::DstType::II | crate::DstType::III)
3489 ) {
3490 return Ok(None);
3491 }
3492 let DoubleDoubleOneDimIr::Recursive(recursive) = fft.as_ref() else {
3493 return Ok(None);
3494 };
3495 if recursive.stockham_upload_schedule.is_some()
3496 || recursive.two_upload_four_step_plan.is_some()
3497 || recursive.three_upload_four_step_plan.is_some()
3498 || recursive.external_storage != crate::PrecisionStorage::DoubleDouble
3499 || recursive.zero_pad_pass.is_some()
3500 || recursive.normalize
3501 || recursive.logical_len != ir.length
3502 || recursive.batch_count != ir.batch_count
3503 || recursive.grouped_batch != ir.grouped_batch
3504 || phases.len() != ir.length
3505 {
3506 return Ok(None);
3507 }
3508 let Some(mapped_high) = recursive.forced_rader_two_upload_mapped_high_component()? else {
3509 return Ok(None);
3510 };
3511 let crate::DoubleDoubleRecursiveFftNodeIr::CooleyTukey(high) = mapped_high else {
3512 return Ok(None);
3513 };
3514 if !matches!(
3515 high.pack_right.input_modifier,
3516 crate::double_double_recursive_ir::DoubleDoubleCooleyTukeyInputModifier::FourStepRight(
3517 _
3518 )
3519 ) || high.pack_right.input_storage != crate::PrecisionStorage::DoubleDouble
3520 || high.pack_right.output_storage != crate::PrecisionStorage::DoubleDouble
3521 {
3522 return Ok(None);
3523 }
3524 let Some((low, mapping)) = recursive.forced_rader_two_upload_mapped_low_stockham()? else {
3525 return Ok(None);
3526 };
3527 if low.sequence_len > 64 || low.axis_batch_block.is_none() {
3528 return Ok(None);
3529 }
3530 let crate::double_double_recursive_ir::DoubleDoubleCooleyTukeyInputModifier::FourStepRight(
3531 high_mapping,
3532 ) = high.pack_right.input_modifier
3533 else {
3534 unreachable!("validated forced-two recursive high mapping");
3535 };
3536 if high_mapping != mapping {
3537 return Ok(None);
3538 }
3539
3540 let mut program = Self::double_double_recursive(recursive)?;
3541 let expected_last = format!("{}_four_step_left", low.name);
3542 if program.passes.len() < 2
3543 || program.input_resource()?.scalar != ScalarType::DoubleDouble
3544 || program.output_resource()?.scalar != ScalarType::DoubleDouble
3545 || program
3546 .passes
3547 .first()
3548 .is_none_or(|pass| pass.name != high.pack_right.name)
3549 || program
3550 .passes
3551 .last()
3552 .is_none_or(|pass| !pass.name.ends_with(&expected_last))
3553 {
3554 return Ok(None);
3555 }
3556 for resource in &mut program.resources {
3557 match resource.kind {
3558 ProgramResourceKind::Input => {
3559 resource.scalar = external_scalar;
3560 resource.elements = elements;
3561 resource.external_layout = Some(external);
3562 }
3563 ProgramResourceKind::Output => {
3564 resource.scalar = external_scalar;
3565 resource.elements = elements;
3566 resource.external_layout = Some(external);
3567 }
3568 _ => {}
3569 }
3570 }
3571 let phases_id = ProgramResourceId(program.resources.len());
3572 program.resources.push(ProgramResource {
3573 id: phases_id,
3574 name: "double_double_r2r_phases".to_owned(),
3575 kind: ProgramResourceKind::LookupTable,
3576 scalar: ScalarType::DoubleDouble,
3577 elements: phases.len(),
3578 external_layout: None,
3579 initialization: ProgramResourceInitialization::ComplexDoubleDouble(phases.clone()),
3580 });
3581 let label = match ir.effective_transform {
3582 crate::R2rTransform::Dct(crate::DctType::II) => "dct2",
3583 crate::R2rTransform::Dct(crate::DctType::III) => "dct3",
3584 crate::R2rTransform::Dst(crate::DstType::II) => "dst2",
3585 crate::R2rTransform::Dst(crate::DstType::III) => "dst3",
3586 _ => unreachable!("validated forced-two-upload DD Type-II/III fusion"),
3587 };
3588 let phase_binding = ProgramPassBinding {
3589 binding: if matches!(
3590 ir.effective_transform,
3591 crate::R2rTransform::Dct(crate::DctType::III)
3592 | crate::R2rTransform::Dst(crate::DstType::III)
3593 ) {
3594 2
3595 } else {
3596 3
3597 },
3598 resource: phases_id,
3599 role: BufferRole::LookupTable,
3600 access: BufferAccess::ReadOnly,
3601 };
3602 let last_pass = program.passes.len() - 1;
3603 if matches!(
3604 ir.effective_transform,
3605 crate::R2rTransform::Dct(crate::DctType::III)
3606 | crate::R2rTransform::Dst(crate::DstType::III)
3607 ) {
3608 program.passes[0].bindings.push(phase_binding);
3609 } else {
3610 program.passes[last_pass].bindings.push(phase_binding);
3611 }
3612 program.passes[0].name = format!("vkfft_dd_r2r_{label}_forced_rader_two_upload_1");
3613 program.passes[last_pass].name = format!("vkfft_dd_r2r_{label}_forced_rader_two_upload_0");
3614 program.name = format!(
3615 "vkfft_dd_r2r_{label}_forced_rader_two_upload_fused_{}_program",
3616 ir.length
3617 );
3618 program.validate()?;
3619 Ok(Some(program))
3620 }
3621
3622 fn double_double_r2r_fused_forced_rader_two_upload_recursive_output(
3623 ir: &DoubleDoubleR2rIr,
3624 elements: usize,
3625 external_scalar: ScalarType,
3626 external: ExternalBufferLayout,
3627 ) -> Result<Option<Self>> {
3628 let DoubleDoubleR2rAlgorithm::FftReduction { fft, phases, .. } = &ir.algorithm else {
3629 return Ok(None);
3630 };
3631 if !matches!(
3632 ir.effective_transform,
3633 crate::R2rTransform::Dct(crate::DctType::II | crate::DctType::III)
3634 | crate::R2rTransform::Dst(crate::DstType::II | crate::DstType::III)
3635 ) {
3636 return Ok(None);
3637 }
3638 let DoubleDoubleOneDimIr::Recursive(recursive) = fft.as_ref() else {
3639 return Ok(None);
3640 };
3641 if recursive.stockham_upload_schedule.is_some()
3642 || recursive.two_upload_four_step_plan.is_some()
3643 || recursive.three_upload_four_step_plan.is_some()
3644 || recursive.external_storage != crate::PrecisionStorage::DoubleDouble
3645 || recursive.zero_pad_pass.is_some()
3646 || recursive.normalize
3647 || recursive.logical_len != ir.length
3648 || recursive.batch_count != ir.batch_count
3649 || recursive.grouped_batch != ir.grouped_batch
3650 || phases.len() != ir.length
3651 {
3652 return Ok(None);
3653 }
3654 let (mapping, first_pass_suffix, fft_rader_scatter_suffix) = if let Some((high, mapping)) =
3655 recursive.forced_rader_two_upload_mapped_high_stockham()?
3656 {
3657 if high.axis_batch_block.is_none() {
3658 return Ok(None);
3659 }
3660 (mapping, "forced_rader_two_upload_1".to_owned(), None)
3661 } else {
3662 let Some(mapped_high) = recursive.forced_rader_two_upload_mapped_high_component()?
3663 else {
3664 return Ok(None);
3665 };
3666 match mapped_high {
3667 crate::DoubleDoubleRecursiveFftNodeIr::DirectRader(high) => {
3668 let crate::StockhamIoMapping::FourStepRight(mapping) = high.io_mapping else {
3669 return Ok(None);
3670 };
3671 if high.prime != mapping.right_len
3672 || high.axis_batch_block.is_none()
3673 || high.input_storage != crate::PrecisionStorage::DoubleDouble
3674 || high.output_storage != crate::PrecisionStorage::DoubleDouble
3675 || high.zero_pad_pass.is_some()
3676 {
3677 return Ok(None);
3678 }
3679 (mapping, high.name.clone(), None)
3680 }
3681 crate::DoubleDoubleRecursiveFftNodeIr::FftRader(high) => {
3682 let crate::StockhamIoMapping::FourStepRight(mapping) = high.io_mapping else {
3683 return Ok(None);
3684 };
3685 if high.prime != mapping.right_len
3686 || high.caller_axis_batch_block.is_none()
3687 || high.external_storage != crate::PrecisionStorage::DoubleDouble
3688 || high.input_storage != crate::PrecisionStorage::DoubleDouble
3689 || high.output_storage != crate::PrecisionStorage::DoubleDouble
3690 || high.zero_pad_pass.is_some()
3691 {
3692 return Ok(None);
3693 }
3694 (
3695 mapping,
3696 format!("{}_gather", high.name),
3697 Some(format!("{}_scatter", high.name)),
3698 )
3699 }
3700 _ => return Ok(None),
3701 }
3702 };
3703 let Some(mapped_low) = recursive.forced_rader_two_upload_mapped_low_component()? else {
3704 return Ok(None);
3705 };
3706 let (low_pass_name, low_pass_exact, low_phase_binding) = match mapped_low {
3707 crate::DoubleDoubleRecursiveFftNodeIr::CooleyTukey(low) => {
3708 if !matches!(
3709 low.scatter_output.output_modifier,
3710 crate::double_double_recursive_ir::DoubleDoubleCooleyTukeyOutputModifier::FourStepLeft(low_mapping)
3711 if low_mapping == mapping
3712 ) || low.scatter_output.input_storage != crate::PrecisionStorage::DoubleDouble
3713 || low.scatter_output.output_storage != crate::PrecisionStorage::DoubleDouble
3714 {
3715 return Ok(None);
3716 }
3717 (low.scatter_output.name.clone(), true, 2u32)
3718 }
3719 crate::DoubleDoubleRecursiveFftNodeIr::DirectRader(low) => {
3720 let crate::StockhamIoMapping::FourStepLeft(low_mapping) = low.io_mapping else {
3721 return Ok(None);
3722 };
3723 if low_mapping != mapping
3724 || low.prime != mapping.left_len
3725 || low.axis_batch_block.is_none()
3726 || low.input_storage != crate::PrecisionStorage::DoubleDouble
3727 || low.output_storage != crate::PrecisionStorage::DoubleDouble
3728 || low.zero_pad_pass.is_some()
3729 {
3730 return Ok(None);
3731 }
3732 (low.name.clone(), false, 3u32)
3733 }
3734 crate::DoubleDoubleRecursiveFftNodeIr::FftRader(low) => {
3735 let crate::StockhamIoMapping::FourStepLeft(low_mapping) = low.io_mapping else {
3736 return Ok(None);
3737 };
3738 if low_mapping != mapping
3739 || low.prime != mapping.left_len
3740 || low.caller_axis_batch_block.is_none()
3741 || low.external_storage != crate::PrecisionStorage::DoubleDouble
3742 || low.input_storage != crate::PrecisionStorage::DoubleDouble
3743 || low.output_storage != crate::PrecisionStorage::DoubleDouble
3744 || low.zero_pad_pass.is_some()
3745 {
3746 return Ok(None);
3747 }
3748 (format!("{}_scatter", low.name), false, 3u32)
3749 }
3750 _ => return Ok(None),
3751 };
3752
3753 let mut program = Self::double_double_recursive(recursive)?;
3754 if program.passes.len() < 2
3755 || program.input_resource()?.scalar != ScalarType::DoubleDouble
3756 || program.output_resource()?.scalar != ScalarType::DoubleDouble
3757 || !program
3758 .passes
3759 .first()
3760 .is_some_and(|pass| pass.name.ends_with(&first_pass_suffix))
3761 || !program.passes.last().is_some_and(|pass| {
3762 if low_pass_exact {
3763 pass.name == low_pass_name
3764 } else {
3765 pass.name.ends_with(&low_pass_name)
3766 }
3767 })
3768 {
3769 return Ok(None);
3770 }
3771 for resource in &mut program.resources {
3772 match resource.kind {
3773 ProgramResourceKind::Input => {
3774 resource.scalar = external_scalar;
3775 resource.elements = elements;
3776 resource.external_layout = Some(external);
3777 }
3778 ProgramResourceKind::Output => {
3779 resource.scalar = external_scalar;
3780 resource.elements = elements;
3781 resource.external_layout = Some(external);
3782 }
3783 _ => {}
3784 }
3785 }
3786 let phases_id = ProgramResourceId(program.resources.len());
3787 program.resources.push(ProgramResource {
3788 id: phases_id,
3789 name: "double_double_r2r_phases".to_owned(),
3790 kind: ProgramResourceKind::LookupTable,
3791 scalar: ScalarType::DoubleDouble,
3792 elements: phases.len(),
3793 external_layout: None,
3794 initialization: ProgramResourceInitialization::ComplexDoubleDouble(phases.clone()),
3795 });
3796 let label = match ir.effective_transform {
3797 crate::R2rTransform::Dct(crate::DctType::II) => "dct2",
3798 crate::R2rTransform::Dct(crate::DctType::III) => "dct3",
3799 crate::R2rTransform::Dst(crate::DstType::II) => "dst2",
3800 crate::R2rTransform::Dst(crate::DstType::III) => "dst3",
3801 _ => unreachable!("validated forced-two recursive-output DD Type-II/III fusion"),
3802 };
3803 let last_pass = program.passes.len() - 1;
3804 if matches!(
3805 ir.effective_transform,
3806 crate::R2rTransform::Dct(crate::DctType::III)
3807 | crate::R2rTransform::Dst(crate::DstType::III)
3808 ) {
3809 if let Some(scatter_suffix) = fft_rader_scatter_suffix.as_deref() {
3810 program.passes[0].bindings.push(ProgramPassBinding {
3811 binding: 2,
3812 resource: phases_id,
3813 role: BufferRole::LookupTable,
3814 access: BufferAccess::ReadOnly,
3815 });
3816 let Some(scatter) = program
3817 .passes
3818 .iter_mut()
3819 .take(last_pass)
3820 .find(|pass| pass.name.ends_with(scatter_suffix))
3821 else {
3822 return Ok(None);
3823 };
3824 scatter.bindings.push(ProgramPassBinding {
3825 binding: 4,
3826 resource: phases_id,
3827 role: BufferRole::LookupTable,
3828 access: BufferAccess::ReadOnly,
3829 });
3830 } else {
3831 program.passes[0].bindings.push(ProgramPassBinding {
3832 binding: 4,
3833 resource: phases_id,
3834 role: BufferRole::LookupTable,
3835 access: BufferAccess::ReadOnly,
3836 });
3837 }
3838 } else {
3839 program.passes[last_pass].bindings.push(ProgramPassBinding {
3840 binding: low_phase_binding,
3841 resource: phases_id,
3842 role: BufferRole::LookupTable,
3843 access: BufferAccess::ReadOnly,
3844 });
3845 }
3846 program.passes[0].name = format!("vkfft_dd_r2r_{label}_forced_rader_two_upload_1");
3847 program.passes[last_pass].name =
3848 format!("vkfft_dd_r2r_{label}_forced_rader_two_upload_0_recursive");
3849 program.name = format!(
3850 "vkfft_dd_r2r_{label}_forced_rader_two_upload_recursive_output_fused_{}_program",
3851 ir.length
3852 );
3853 program.validate()?;
3854 Ok(Some(program))
3855 }
3856
3857 fn double_double_r2r_fused_forced_rader_three_upload(
3858 ir: &DoubleDoubleR2rIr,
3859 elements: usize,
3860 external_scalar: ScalarType,
3861 external: ExternalBufferLayout,
3862 ) -> Result<Option<Self>> {
3863 let DoubleDoubleR2rAlgorithm::FftReduction { fft, phases, .. } = &ir.algorithm else {
3864 return Ok(None);
3865 };
3866 if !matches!(
3867 ir.effective_transform,
3868 crate::R2rTransform::Dct(crate::DctType::II | crate::DctType::III)
3869 | crate::R2rTransform::Dst(crate::DstType::II | crate::DstType::III)
3870 ) {
3871 return Ok(None);
3872 }
3873 let DoubleDoubleOneDimIr::Recursive(recursive) = fft.as_ref() else {
3874 return Ok(None);
3875 };
3876 if recursive.stockham_upload_schedule.is_some()
3877 || recursive.two_upload_four_step_plan.is_some()
3878 || recursive.three_upload_four_step_plan.is_some()
3879 || recursive.external_storage != crate::PrecisionStorage::DoubleDouble
3880 || recursive.zero_pad_pass.is_some()
3881 || recursive.normalize
3882 || recursive.logical_len != ir.length
3883 || recursive.batch_count != ir.batch_count
3884 || recursive.grouped_batch != ir.grouped_batch
3885 || phases.len() != ir.length
3886 {
3887 return Ok(None);
3888 }
3889 let Some(components) = recursive.forced_rader_three_upload_mapped_components()? else {
3890 return Ok(None);
3891 };
3892 if components.len() != 3 {
3893 return Ok(None);
3894 }
3895 let recursive_u2_pack_name = match components.first() {
3896 Some(
3897 crate::double_double_recursive_ir::DoubleDoubleForcedRaderThreeUploadComponentIr::Stockham {
3898 upload_id: 2,
3899 ..
3900 },
3901 ) => None,
3902 Some(
3903 crate::double_double_recursive_ir::DoubleDoubleForcedRaderThreeUploadComponentIr::Recursive {
3904 upload_id: 2,
3905 ir: crate::DoubleDoubleRecursiveFftNodeIr::CooleyTukey(cooley),
3906 },
3907 ) if matches!(
3908 cooley.pack_right.input_modifier,
3909 crate::double_double_recursive_ir::DoubleDoubleCooleyTukeyInputModifier::FourStepThreeUpload2(_)
3910 ) && cooley.pack_right.input_storage == crate::PrecisionStorage::DoubleDouble
3911 && cooley.pack_right.output_storage == crate::PrecisionStorage::DoubleDouble =>
3912 {
3913 Some(cooley.pack_right.name.clone())
3914 }
3915 _ => return Ok(None),
3916 };
3917 if !matches!(
3918 components.last(),
3919 Some(
3920 crate::double_double_recursive_ir::DoubleDoubleForcedRaderThreeUploadComponentIr::Stockham {
3921 upload_id: 0,
3922 ..
3923 }
3924 )
3925 ) {
3926 return Ok(None);
3927 }
3928
3929 let mut program = Self::double_double_recursive(recursive)?;
3930 let first_pass_matches = match recursive_u2_pack_name.as_deref() {
3931 Some(name) => program.passes.first().is_some_and(|pass| pass.name == name),
3932 None => program
3933 .passes
3934 .first()
3935 .is_some_and(|pass| pass.name.ends_with("forced_rader_three_upload_2")),
3936 };
3937 if program.passes.len() < 3
3938 || program.input_resource()?.scalar != ScalarType::DoubleDouble
3939 || program.output_resource()?.scalar != ScalarType::DoubleDouble
3940 || !first_pass_matches
3941 || !program
3942 .passes
3943 .last()
3944 .is_some_and(|pass| pass.name.ends_with("forced_rader_three_upload_0"))
3945 {
3946 return Ok(None);
3947 }
3948
3949 let external_output = ProgramResourceId(1);
3954 let exchange0 = ProgramResourceId(program.resources.len());
3955 program.resources.push(ProgramResource {
3956 id: exchange0,
3957 name: "double_double_r2r_forced_three_upload_exchange_0".to_owned(),
3958 kind: ProgramResourceKind::Scratch,
3959 scalar: ScalarType::DoubleDouble,
3960 elements,
3961 external_layout: None,
3962 initialization: ProgramResourceInitialization::Zeroed,
3963 });
3964 let last_pass = program.passes.len() - 1;
3965 for (index, pass) in program.passes.iter_mut().enumerate() {
3966 if index == last_pass {
3967 continue;
3968 }
3969 for binding in &mut pass.bindings {
3970 if binding.resource == external_output {
3971 binding.resource = exchange0;
3972 }
3973 }
3974 }
3975
3976 for resource in &mut program.resources {
3977 match resource.kind {
3978 ProgramResourceKind::Input => {
3979 resource.scalar = external_scalar;
3980 resource.elements = elements;
3981 resource.external_layout = Some(external);
3982 }
3983 ProgramResourceKind::Output => {
3984 resource.scalar = external_scalar;
3985 resource.elements = elements;
3986 resource.external_layout = Some(external);
3987 }
3988 _ => {}
3989 }
3990 }
3991 let phases_id = ProgramResourceId(program.resources.len());
3992 program.resources.push(ProgramResource {
3993 id: phases_id,
3994 name: "double_double_r2r_phases".to_owned(),
3995 kind: ProgramResourceKind::LookupTable,
3996 scalar: ScalarType::DoubleDouble,
3997 elements: phases.len(),
3998 external_layout: None,
3999 initialization: ProgramResourceInitialization::ComplexDoubleDouble(phases.clone()),
4000 });
4001 let label = match ir.effective_transform {
4002 crate::R2rTransform::Dct(crate::DctType::II) => "dct2",
4003 crate::R2rTransform::Dct(crate::DctType::III) => "dct3",
4004 crate::R2rTransform::Dst(crate::DstType::II) => "dst2",
4005 crate::R2rTransform::Dst(crate::DstType::III) => "dst3",
4006 _ => unreachable!("validated forced-three-upload DD Type-II/III fusion"),
4007 };
4008 let phase_binding = ProgramPassBinding {
4009 binding: if matches!(
4010 ir.effective_transform,
4011 crate::R2rTransform::Dct(crate::DctType::III)
4012 | crate::R2rTransform::Dst(crate::DstType::III)
4013 ) {
4014 if recursive_u2_pack_name.is_some() {
4015 2
4016 } else {
4017 4
4018 }
4019 } else {
4020 3
4021 },
4022 resource: phases_id,
4023 role: BufferRole::LookupTable,
4024 access: BufferAccess::ReadOnly,
4025 };
4026 if matches!(
4027 ir.effective_transform,
4028 crate::R2rTransform::Dct(crate::DctType::III)
4029 | crate::R2rTransform::Dst(crate::DstType::III)
4030 ) {
4031 program.passes[0].bindings.push(phase_binding);
4032 } else {
4033 program.passes[last_pass].bindings.push(phase_binding);
4034 }
4035 program.passes[0].name = format!("vkfft_dd_r2r_{label}_forced_rader_three_upload_2");
4036 program.passes[last_pass].name =
4037 format!("vkfft_dd_r2r_{label}_forced_rader_three_upload_0");
4038 program.name = format!(
4039 "vkfft_dd_r2r_{label}_forced_rader_three_upload_fused_{}_program",
4040 ir.length
4041 );
4042 program.validate()?;
4043 Ok(Some(program))
4044 }
4045
4046 fn double_double_r2r_fused_bluestein(
4047 ir: &DoubleDoubleR2rIr,
4048 elements: usize,
4049 external_scalar: ScalarType,
4050 external: ExternalBufferLayout,
4051 ) -> Result<Option<Self>> {
4052 let DoubleDoubleR2rAlgorithm::FftReduction { fft, phases, .. } = &ir.algorithm else {
4053 return Ok(None);
4054 };
4055 if !matches!(
4056 ir.effective_transform,
4057 crate::R2rTransform::Dct(crate::DctType::II | crate::DctType::III)
4058 | crate::R2rTransform::Dst(crate::DstType::II | crate::DstType::III)
4059 ) {
4060 return Ok(None);
4061 }
4062 let DoubleDoubleOneDimIr::Bluestein(bluestein) = fft.as_ref() else {
4063 return Ok(None);
4064 };
4065 if bluestein.logical_len != ir.length
4066 || bluestein.batch_count != ir.batch_count
4067 || bluestein.grouped_batch != ir.grouped_batch
4068 || bluestein.external_storage != crate::PrecisionStorage::DoubleDouble
4069 || bluestein.zero_padding.is_some()
4070 || phases.len() != ir.length
4071 {
4072 return Ok(None);
4073 }
4074
4075 let mut program = Self::double_double_bluestein(bluestein)?;
4076 if program.passes.len() < 2
4077 || program.input_resource()?.scalar != ScalarType::DoubleDouble
4078 || program.output_resource()?.scalar != ScalarType::DoubleDouble
4079 || !program
4080 .passes
4081 .first()
4082 .is_some_and(|pass| pass.name == format!("{}_preprocess", bluestein.name))
4083 || !program
4084 .passes
4085 .last()
4086 .is_some_and(|pass| pass.name == format!("{}_postprocess", bluestein.name))
4087 {
4088 return Ok(None);
4089 }
4090 for resource in &mut program.resources {
4091 match resource.kind {
4092 ProgramResourceKind::Input => {
4093 resource.scalar = external_scalar;
4094 resource.elements = elements;
4095 resource.external_layout = Some(external);
4096 }
4097 ProgramResourceKind::Output => {
4098 resource.scalar = external_scalar;
4099 resource.elements = elements;
4100 resource.external_layout = Some(external);
4101 }
4102 _ => {}
4103 }
4104 }
4105 let phases_id = ProgramResourceId(program.resources.len());
4106 program.resources.push(ProgramResource {
4107 id: phases_id,
4108 name: "double_double_r2r_phases".to_owned(),
4109 kind: ProgramResourceKind::LookupTable,
4110 scalar: ScalarType::DoubleDouble,
4111 elements: phases.len(),
4112 external_layout: None,
4113 initialization: ProgramResourceInitialization::ComplexDoubleDouble(phases.clone()),
4114 });
4115 let label = match ir.effective_transform {
4116 crate::R2rTransform::Dct(crate::DctType::II) => "dct2",
4117 crate::R2rTransform::Dct(crate::DctType::III) => "dct3",
4118 crate::R2rTransform::Dst(crate::DstType::II) => "dst2",
4119 crate::R2rTransform::Dst(crate::DstType::III) => "dst3",
4120 _ => unreachable!("validated DD Bluestein Type-II/III fusion"),
4121 };
4122 let phase_binding = ProgramPassBinding {
4123 binding: 3,
4124 resource: phases_id,
4125 role: BufferRole::LookupTable,
4126 access: BufferAccess::ReadOnly,
4127 };
4128 let last_pass = program.passes.len() - 1;
4129 if matches!(
4130 ir.effective_transform,
4131 crate::R2rTransform::Dct(crate::DctType::III)
4132 | crate::R2rTransform::Dst(crate::DstType::III)
4133 ) {
4134 program.passes[0].bindings.push(phase_binding);
4135 } else {
4136 program.passes[last_pass].bindings.push(phase_binding);
4137 }
4138 program.passes[0].name = format!("vkfft_dd_r2r_{label}_bluestein_preprocess");
4139 program.passes[last_pass].name = format!("vkfft_dd_r2r_{label}_bluestein_postprocess");
4140 program.name = format!("vkfft_dd_r2r_{label}_bluestein_fused_{}_program", ir.length);
4141 program.validate()?;
4142 Ok(Some(program))
4143 }
4144
4145 pub fn double_double_r2r(ir: &DoubleDoubleR2rIr) -> Result<Self> {
4146 ir.validate()?;
4147 let elements =
4148 ir.length
4149 .checked_mul(ir.batch_count)
4150 .ok_or(VkFftError::ArithmeticOverflow {
4151 operation: "double-double DCT/DST program element count",
4152 })?;
4153 let external_scalar = match ir.external_storage {
4154 crate::PrecisionStorage::DoubleDouble => ScalarType::DoubleDouble,
4155 crate::PrecisionStorage::F64 => ScalarType::F64,
4156 _ => {
4157 return Err(VkFftError::InvalidKernelIr(
4158 "double-double R2R ProgramIr requires DD or F64 external storage",
4159 ));
4160 }
4161 };
4162 let external = ExternalBufferLayout {
4163 logical_len: ir.length,
4164 physical_stride: ir.length,
4165 batch_count: ir.batch_count,
4166 element_shape: ProgramElementShape::Scalar,
4167 };
4168 if let Some(program) =
4169 Self::double_double_r2r_fused_bluestein(ir, elements, external_scalar, external)?
4170 {
4171 return Ok(program);
4172 }
4173 if let Some(program) = Self::double_double_r2r_fused_physical_stockham(
4174 ir,
4175 elements,
4176 external_scalar,
4177 external,
4178 )? {
4179 return Ok(program);
4180 }
4181 if let Some(program) = Self::double_double_r2r_fused_two_upload_four_step(
4182 ir,
4183 elements,
4184 external_scalar,
4185 external,
4186 )? {
4187 return Ok(program);
4188 }
4189 if let Some(program) = Self::double_double_r2r_fused_forced_rader_two_upload(
4190 ir,
4191 elements,
4192 external_scalar,
4193 external,
4194 )? {
4195 return Ok(program);
4196 }
4197 if let Some(program) =
4198 Self::double_double_r2r_fused_forced_rader_two_upload_recursive_output(
4199 ir,
4200 elements,
4201 external_scalar,
4202 external,
4203 )?
4204 {
4205 return Ok(program);
4206 }
4207 if let Some(program) = Self::double_double_r2r_fused_forced_rader_three_upload(
4208 ir,
4209 elements,
4210 external_scalar,
4211 external,
4212 )? {
4213 return Ok(program);
4214 }
4215 let mut resources = vec![
4216 ProgramResource {
4217 id: ProgramResourceId(0),
4218 name: "input".to_owned(),
4219 kind: ProgramResourceKind::Input,
4220 scalar: external_scalar,
4221 elements,
4222 external_layout: Some(external),
4223 initialization: ProgramResourceInitialization::ExternalInput,
4224 },
4225 ProgramResource {
4226 id: ProgramResourceId(1),
4227 name: "output".to_owned(),
4228 kind: ProgramResourceKind::Output,
4229 scalar: external_scalar,
4230 elements,
4231 external_layout: Some(external),
4232 initialization: ProgramResourceInitialization::Zeroed,
4233 },
4234 ];
4235 let mut passes = Vec::new();
4236 let dispatch = DispatchGeometry {
4237 x: u32::try_from(ir.batch_group_count()).map_err(|_| VkFftError::ValueOutOfRange {
4238 field: "double-double R2R boundary dispatch count",
4239 })?,
4240 y: 1,
4241 z: 1,
4242 };
4243 match &ir.algorithm {
4244 DoubleDoubleR2rAlgorithm::Direct { coefficients } => {
4245 let coefficients_id = ProgramResourceId(resources.len());
4246 resources.push(ProgramResource {
4247 id: coefficients_id,
4248 name: "double_double_r2r_coefficients".to_owned(),
4249 kind: ProgramResourceKind::LookupTable,
4250 scalar: ScalarType::DoubleDouble,
4251 elements: coefficients.len(),
4252 external_layout: None,
4253 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
4254 coefficients
4255 .iter()
4256 .copied()
4257 .map(|value| ComplexDoubleDouble::new(value, crate::DoubleDouble::ZERO))
4258 .collect(),
4259 ),
4260 });
4261 passes.push(ProgramPass {
4262 name: format!("vkfft_dd_r2r_direct_{}", ir.length),
4263 dispatch,
4264 bindings: vec![
4265 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
4266 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
4267 ProgramPassBinding {
4268 binding: 2,
4269 resource: coefficients_id,
4270 role: BufferRole::LookupTable,
4271 access: BufferAccess::ReadOnly,
4272 },
4273 ],
4274 });
4275 }
4276 DoubleDoubleR2rAlgorithm::FftReduction {
4277 fft_len,
4278 fft,
4279 phases,
4280 } => {
4281 let fft_elements =
4282 fft_len
4283 .checked_mul(ir.batch_count)
4284 .ok_or(VkFftError::ArithmeticOverflow {
4285 operation: "double-double R2R FFT program element count",
4286 })?;
4287 let fft_input = ProgramResourceId(resources.len());
4288 resources.push(ProgramResource {
4289 id: fft_input,
4290 name: "double_double_r2r_fft_input".to_owned(),
4291 kind: ProgramResourceKind::Scratch,
4292 scalar: ScalarType::DoubleDouble,
4293 elements: fft_elements,
4294 external_layout: None,
4295 initialization: ProgramResourceInitialization::Zeroed,
4296 });
4297 let fft_output = ProgramResourceId(resources.len());
4298 resources.push(ProgramResource {
4299 id: fft_output,
4300 name: "double_double_r2r_fft_output".to_owned(),
4301 kind: ProgramResourceKind::Scratch,
4302 scalar: ScalarType::DoubleDouble,
4303 elements: fft_elements,
4304 external_layout: None,
4305 initialization: ProgramResourceInitialization::Zeroed,
4306 });
4307 let phases_id = if phases.is_empty() {
4308 None
4309 } else {
4310 let phases_id = ProgramResourceId(resources.len());
4311 resources.push(ProgramResource {
4312 id: phases_id,
4313 name: "double_double_r2r_phases".to_owned(),
4314 kind: ProgramResourceKind::LookupTable,
4315 scalar: ScalarType::DoubleDouble,
4316 elements: phases.len(),
4317 external_layout: None,
4318 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
4319 phases.clone(),
4320 ),
4321 });
4322 Some(phases_id)
4323 };
4324 let child = Self::double_double_one_dim(fft)?;
4325 if child.scalar != ScalarType::DoubleDouble
4326 || child.input_resource()?.elements != fft_elements
4327 || child.output_resource()?.elements != fft_elements
4328 || child.input_resource()?.scalar != ScalarType::DoubleDouble
4329 || child.output_resource()?.scalar != ScalarType::DoubleDouble
4330 {
4331 return Err(VkFftError::InvalidKernelIr(
4332 "double-double R2R FFT child must expose full-DD boundaries",
4333 ));
4334 }
4335 let mut resource_map = vec![None; child.resources.len()];
4336 for resource in &child.resources {
4337 let mapped = match resource.kind {
4338 ProgramResourceKind::Input => fft_input,
4339 ProgramResourceKind::Output => fft_output,
4340 _ => {
4341 let new_id = ProgramResourceId(resources.len());
4342 let mut cloned = resource.clone();
4343 cloned.id = new_id;
4344 cloned.name = format!("double_double_r2r_{}", resource.name);
4345 cloned.external_layout = None;
4346 resources.push(cloned);
4347 new_id
4348 }
4349 };
4350 resource_map[resource.id.0] = Some(mapped);
4351 }
4352 let mut pre_bindings = vec![
4353 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
4354 ProgramPassBinding {
4355 binding: 1,
4356 resource: fft_input,
4357 role: BufferRole::Output,
4358 access: BufferAccess::WriteOnly,
4359 },
4360 ];
4361 if matches!(
4362 ir.effective_transform,
4363 crate::R2rTransform::Dct(crate::DctType::III | crate::DctType::IV)
4364 | crate::R2rTransform::Dst(crate::DstType::III | crate::DstType::IV)
4365 ) {
4366 pre_bindings.push(ProgramPassBinding {
4367 binding: 2,
4368 resource: phases_id.ok_or(VkFftError::InvalidKernelIr(
4369 "double-double R2R FFT preprocess is missing its phase LUT",
4370 ))?,
4371 role: BufferRole::LookupTable,
4372 access: BufferAccess::ReadOnly,
4373 });
4374 }
4375 passes.push(ProgramPass {
4376 name: match ir.effective_transform {
4377 crate::R2rTransform::Dct(crate::DctType::I) => {
4378 "vkfft_dd_r2r_fft_dct1_pre".to_owned()
4379 }
4380 crate::R2rTransform::Dst(crate::DstType::I) => {
4381 "vkfft_dd_r2r_fft_dst1_pre".to_owned()
4382 }
4383 crate::R2rTransform::Dct(crate::DctType::II) => {
4384 "vkfft_dd_r2r_fft_dct2_pre".to_owned()
4385 }
4386 crate::R2rTransform::Dct(crate::DctType::III) => {
4387 "vkfft_dd_r2r_fft_dct3_pre".to_owned()
4388 }
4389 crate::R2rTransform::Dst(crate::DstType::II) => {
4390 "vkfft_dd_r2r_fft_dst2_pre".to_owned()
4391 }
4392 crate::R2rTransform::Dst(crate::DstType::III) => {
4393 "vkfft_dd_r2r_fft_dst3_pre".to_owned()
4394 }
4395 crate::R2rTransform::Dct(crate::DctType::IV) => {
4396 "vkfft_dd_r2r_fft_dct4_pre".to_owned()
4397 }
4398 crate::R2rTransform::Dst(crate::DstType::IV) => {
4399 "vkfft_dd_r2r_fft_dst4_pre".to_owned()
4400 }
4401 },
4402 dispatch,
4403 bindings: pre_bindings,
4404 });
4405 for child_pass in &child.passes {
4406 let mut pass = child_pass.clone();
4407 pass.name = format!("vkfft_dd_r2r_{}", child_pass.name);
4408 for child_binding in &mut pass.bindings {
4409 child_binding.resource = resource_map
4410 .get(child_binding.resource.0)
4411 .and_then(|mapped| *mapped)
4412 .ok_or(VkFftError::InvalidKernelIr(
4413 "double-double R2R child resource mapping is incomplete",
4414 ))?;
4415 }
4416 passes.push(pass);
4417 }
4418 let mut post_bindings = vec![
4419 ProgramPassBinding {
4420 binding: 0,
4421 resource: fft_output,
4422 role: BufferRole::Input,
4423 access: BufferAccess::ReadOnly,
4424 },
4425 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
4426 ];
4427 if matches!(
4428 ir.effective_transform,
4429 crate::R2rTransform::Dct(crate::DctType::II | crate::DctType::IV)
4430 | crate::R2rTransform::Dst(crate::DstType::II | crate::DstType::IV)
4431 ) {
4432 post_bindings.push(ProgramPassBinding {
4433 binding: 2,
4434 resource: phases_id.ok_or(VkFftError::InvalidKernelIr(
4435 "double-double R2R FFT postprocess is missing its phase LUT",
4436 ))?,
4437 role: BufferRole::LookupTable,
4438 access: BufferAccess::ReadOnly,
4439 });
4440 }
4441 passes.push(ProgramPass {
4442 name: match ir.effective_transform {
4443 crate::R2rTransform::Dct(crate::DctType::I) => {
4444 "vkfft_dd_r2r_fft_dct1_post".to_owned()
4445 }
4446 crate::R2rTransform::Dst(crate::DstType::I) => {
4447 "vkfft_dd_r2r_fft_dst1_post".to_owned()
4448 }
4449 crate::R2rTransform::Dct(crate::DctType::II) => {
4450 "vkfft_dd_r2r_fft_dct2_post".to_owned()
4451 }
4452 crate::R2rTransform::Dct(crate::DctType::III) => {
4453 "vkfft_dd_r2r_fft_dct3_post".to_owned()
4454 }
4455 crate::R2rTransform::Dst(crate::DstType::II) => {
4456 "vkfft_dd_r2r_fft_dst2_post".to_owned()
4457 }
4458 crate::R2rTransform::Dst(crate::DstType::III) => {
4459 "vkfft_dd_r2r_fft_dst3_post".to_owned()
4460 }
4461 crate::R2rTransform::Dct(crate::DctType::IV) => {
4462 "vkfft_dd_r2r_fft_dct4_post".to_owned()
4463 }
4464 crate::R2rTransform::Dst(crate::DstType::IV) => {
4465 "vkfft_dd_r2r_fft_dst4_post".to_owned()
4466 }
4467 },
4468 dispatch,
4469 bindings: post_bindings,
4470 });
4471 }
4472 DoubleDoubleR2rAlgorithm::EvenTypeIvHalfSize {
4473 fft_len,
4474 fft,
4475 pack_phases,
4476 extract_phases,
4477 } => {
4478 let fft_elements =
4479 fft_len
4480 .checked_mul(ir.batch_count)
4481 .ok_or(VkFftError::ArithmeticOverflow {
4482 operation: "double-double even-IV half-size program element count",
4483 })?;
4484 let fft_input = ProgramResourceId(resources.len());
4485 resources.push(ProgramResource {
4486 id: fft_input,
4487 name: "double_double_r2r_even_iv_fft_input".to_owned(),
4488 kind: ProgramResourceKind::Scratch,
4489 scalar: ScalarType::DoubleDouble,
4490 elements: fft_elements,
4491 external_layout: None,
4492 initialization: ProgramResourceInitialization::Zeroed,
4493 });
4494 let fft_output = ProgramResourceId(resources.len());
4495 resources.push(ProgramResource {
4496 id: fft_output,
4497 name: "double_double_r2r_even_iv_fft_output".to_owned(),
4498 kind: ProgramResourceKind::Scratch,
4499 scalar: ScalarType::DoubleDouble,
4500 elements: fft_elements,
4501 external_layout: None,
4502 initialization: ProgramResourceInitialization::Zeroed,
4503 });
4504 let phases_id = ProgramResourceId(resources.len());
4505 let mut phases = Vec::with_capacity(pack_phases.len() + extract_phases.len());
4506 phases.extend_from_slice(pack_phases);
4507 phases.extend_from_slice(extract_phases);
4508 resources.push(ProgramResource {
4509 id: phases_id,
4510 name: "double_double_r2r_even_iv_phases".to_owned(),
4511 kind: ProgramResourceKind::LookupTable,
4512 scalar: ScalarType::DoubleDouble,
4513 elements: phases.len(),
4514 external_layout: None,
4515 initialization: ProgramResourceInitialization::ComplexDoubleDouble(phases),
4516 });
4517
4518 let child = Self::double_double_one_dim(fft)?;
4519 if child.scalar != ScalarType::DoubleDouble
4520 || child.input_resource()?.elements != fft_elements
4521 || child.output_resource()?.elements != fft_elements
4522 || child.input_resource()?.scalar != ScalarType::DoubleDouble
4523 || child.output_resource()?.scalar != ScalarType::DoubleDouble
4524 {
4525 return Err(VkFftError::InvalidKernelIr(
4526 "double-double even-IV child must expose full-DD boundaries",
4527 ));
4528 }
4529 let mut resource_map = vec![None; child.resources.len()];
4530 for resource in &child.resources {
4531 let mapped = match resource.kind {
4532 ProgramResourceKind::Input => fft_input,
4533 ProgramResourceKind::Output => fft_output,
4534 _ => {
4535 let new_id = ProgramResourceId(resources.len());
4536 let mut cloned = resource.clone();
4537 cloned.id = new_id;
4538 cloned.name = format!("double_double_r2r_even_iv_{}", resource.name);
4539 cloned.external_layout = None;
4540 resources.push(cloned);
4541 new_id
4542 }
4543 };
4544 resource_map[resource.id.0] = Some(mapped);
4545 }
4546
4547 passes.push(ProgramPass {
4548 name: "vkfft_dd_r2r_even_iv_pre".to_owned(),
4549 dispatch,
4550 bindings: vec![
4551 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
4552 ProgramPassBinding {
4553 binding: 1,
4554 resource: fft_input,
4555 role: BufferRole::Output,
4556 access: BufferAccess::WriteOnly,
4557 },
4558 ProgramPassBinding {
4559 binding: 2,
4560 resource: phases_id,
4561 role: BufferRole::LookupTable,
4562 access: BufferAccess::ReadOnly,
4563 },
4564 ],
4565 });
4566 for child_pass in &child.passes {
4567 let mut pass = child_pass.clone();
4568 pass.name = format!("vkfft_dd_r2r_even_iv_{}", child_pass.name);
4569 for child_binding in &mut pass.bindings {
4570 child_binding.resource = resource_map
4571 .get(child_binding.resource.0)
4572 .and_then(|mapped| *mapped)
4573 .ok_or(VkFftError::InvalidKernelIr(
4574 "double-double even-IV child resource mapping is incomplete",
4575 ))?;
4576 }
4577 passes.push(pass);
4578 }
4579 passes.push(ProgramPass {
4580 name: "vkfft_dd_r2r_even_iv_post".to_owned(),
4581 dispatch,
4582 bindings: vec![
4583 ProgramPassBinding {
4584 binding: 0,
4585 resource: fft_output,
4586 role: BufferRole::Input,
4587 access: BufferAccess::ReadOnly,
4588 },
4589 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
4590 ProgramPassBinding {
4591 binding: 2,
4592 resource: phases_id,
4593 role: BufferRole::LookupTable,
4594 access: BufferAccess::ReadOnly,
4595 },
4596 ],
4597 });
4598 }
4599 }
4600 let program = Self {
4601 name: format!("vkfft_dd_r2r_program_{}", ir.length),
4602 scalar: ScalarType::DoubleDouble,
4603 resources,
4604 passes,
4605 };
4606 program.validate()?;
4607 Ok(program)
4608 }
4609
4610 pub fn r2r(ir: &R2rIr) -> Result<Self> {
4611 ir.validate()?;
4612 let elements =
4613 ir.length
4614 .checked_mul(ir.batch_count)
4615 .ok_or(VkFftError::ArithmeticOverflow {
4616 operation: "DCT/DST program element count",
4617 })?;
4618 let external = ExternalBufferLayout {
4619 logical_len: ir.length,
4620 physical_stride: ir.length,
4621 batch_count: ir.batch_count,
4622 element_shape: ProgramElementShape::Complex,
4623 };
4624 let mut builder = RecursiveProgramBuilder {
4625 scalar: ir.scalar,
4626 resources: vec![
4627 ProgramResource {
4628 id: ProgramResourceId(0),
4629 name: "input".to_owned(),
4630 kind: ProgramResourceKind::Input,
4631 scalar: ir.external_scalar,
4632 elements,
4633 external_layout: Some(external),
4634 initialization: ProgramResourceInitialization::ExternalInput,
4635 },
4636 ProgramResource {
4637 id: ProgramResourceId(1),
4638 name: "output".to_owned(),
4639 kind: ProgramResourceKind::Output,
4640 scalar: ir.external_scalar,
4641 elements,
4642 external_layout: Some(external),
4643 initialization: ProgramResourceInitialization::Zeroed,
4644 },
4645 ],
4646 passes: Vec::new(),
4647 serial: 0,
4648 };
4649 builder.flatten_r2r(ir, ProgramResourceId(0), ProgramResourceId(1))?;
4650 let program = Self {
4651 name: format!("vkfft_r2r_program_{}", ir.length),
4652 scalar: ir.scalar,
4653 resources: builder.resources,
4654 passes: builder.passes,
4655 };
4656 program.validate()?;
4657 Ok(program)
4658 }
4659
4660 pub fn nd_r2r(ir: &NdR2rIr) -> Result<Self> {
4661 ir.validate()?;
4662 let dense_elements =
4663 ir.tensor_len
4664 .checked_mul(ir.batch_count)
4665 .ok_or(VkFftError::ArithmeticOverflow {
4666 operation: "multidimensional DCT/DST program element count",
4667 })?;
4668 let input_external_stride = if ir.input_formatted_copy.is_some() {
4669 ir.input_external_layout.batch_stride
4670 } else {
4671 ir.tensor_len
4672 };
4673 let output_external_stride = if ir.output_formatted_copy.is_some() {
4674 ir.output_external_layout.batch_stride
4675 } else {
4676 ir.tensor_len
4677 };
4678 let input_external = ExternalBufferLayout {
4679 logical_len: input_external_stride,
4680 physical_stride: input_external_stride,
4681 batch_count: ir.batch_count,
4682 element_shape: ProgramElementShape::Complex,
4683 };
4684 let output_external = ExternalBufferLayout {
4685 logical_len: output_external_stride,
4686 physical_stride: output_external_stride,
4687 batch_count: ir.batch_count,
4688 element_shape: ProgramElementShape::Complex,
4689 };
4690 let mut builder = RecursiveProgramBuilder {
4691 scalar: ir.scalar,
4692 resources: vec![
4693 ProgramResource {
4694 id: ProgramResourceId(0),
4695 name: "input".to_owned(),
4696 kind: ProgramResourceKind::Input,
4697 scalar: ir.external_scalar,
4698 elements: input_external.physical_elements()?,
4699 external_layout: Some(input_external),
4700 initialization: ProgramResourceInitialization::ExternalInput,
4701 },
4702 ProgramResource {
4703 id: ProgramResourceId(1),
4704 name: "output".to_owned(),
4705 kind: ProgramResourceKind::Output,
4706 scalar: ir.external_scalar,
4707 elements: output_external.physical_elements()?,
4708 external_layout: Some(output_external),
4709 initialization: ProgramResourceInitialization::Zeroed,
4710 },
4711 ],
4712 passes: Vec::new(),
4713 serial: 0,
4714 };
4715 let mut natural_input = if let Some(copy) = &ir.input_formatted_copy {
4716 let dense = builder.add_scratch("nd_r2r_formatted_input", dense_elements);
4717 builder.passes.push(two_buffer_pass(
4718 ©.name,
4719 copy.dispatch,
4720 ProgramResourceId(0).0,
4721 dense.0,
4722 ));
4723 dense
4724 } else {
4725 ProgramResourceId(0)
4726 };
4727 let formatted_output = if ir.output_formatted_copy.is_some() {
4728 Some(builder.add_scratch("nd_r2r_formatted_output", dense_elements))
4729 } else {
4730 None
4731 };
4732 for (axis_index, axis) in ir.axes.iter().enumerate() {
4733 let packed_input = builder.add_scratch("nd_r2r_axis_input", dense_elements);
4734 let packed_output = builder.add_scratch("nd_r2r_axis_output", dense_elements);
4735 let natural_output = if axis_index + 1 == ir.axes.len() {
4736 formatted_output.unwrap_or(ProgramResourceId(1))
4737 } else {
4738 builder.add_scratch("nd_r2r_natural_output", dense_elements)
4739 };
4740 builder.passes.push(two_buffer_pass(
4741 &axis.pack.name,
4742 axis.pack.dispatch,
4743 natural_input.0,
4744 packed_input.0,
4745 ));
4746 builder.flatten_r2r(&axis.transform, packed_input, packed_output)?;
4747 builder.passes.push(two_buffer_pass(
4748 &axis.scatter.name,
4749 axis.scatter.dispatch,
4750 packed_output.0,
4751 natural_output.0,
4752 ));
4753 natural_input = natural_output;
4754 }
4755 if let Some(copy) = &ir.output_formatted_copy {
4756 builder.passes.push(two_buffer_pass(
4757 ©.name,
4758 copy.dispatch,
4759 formatted_output.expect("formatted R2R output scratch").0,
4760 ProgramResourceId(1).0,
4761 ));
4762 }
4763 let program = Self {
4764 name: format!("vkfft_nd_r2r_program_{:?}", ir.dimensions),
4765 scalar: ir.scalar,
4766 resources: builder.resources,
4767 passes: builder.passes,
4768 };
4769 program.validate()?;
4770 Ok(program)
4771 }
4772
4773 pub fn nd_convolution(ir: &NdConvolutionIr) -> Result<Self> {
4774 ir.validate()?;
4775 let input_transform_batch_count = ir.batch_count.checked_mul(ir.coordinate_count).ok_or(
4776 VkFftError::ArithmeticOverflow {
4777 operation: "multidimensional performConvolution input coordinate-expanded batch count",
4778 },
4779 )?;
4780 let output_transform_batch_count = ir
4781 .output_batch_count()
4782 .checked_mul(ir.coordinate_count)
4783 .ok_or(VkFftError::ArithmeticOverflow {
4784 operation: "multidimensional performConvolution output coordinate-expanded batch count",
4785 })?;
4786 let input_elements = ir
4787 .tensor_len
4788 .checked_mul(input_transform_batch_count)
4789 .ok_or(VkFftError::ArithmeticOverflow {
4790 operation: "multidimensional performConvolution input program element count",
4791 })?;
4792 let output_elements = ir
4793 .tensor_len
4794 .checked_mul(output_transform_batch_count)
4795 .ok_or(VkFftError::ArithmeticOverflow {
4796 operation: "multidimensional performConvolution output program element count",
4797 })?;
4798 let input_external = ExternalBufferLayout {
4799 logical_len: ir.tensor_len,
4800 physical_stride: ir.tensor_len,
4801 batch_count: input_transform_batch_count,
4802 element_shape: ProgramElementShape::Complex,
4803 };
4804 let output_external = ExternalBufferLayout {
4805 logical_len: ir.tensor_len,
4806 physical_stride: ir.tensor_len,
4807 batch_count: output_transform_batch_count,
4808 element_shape: ProgramElementShape::Complex,
4809 };
4810 let mut resources = vec![
4811 ProgramResource {
4812 id: ProgramResourceId(0),
4813 name: "input".to_owned(),
4814 kind: ProgramResourceKind::Input,
4815 scalar: ir.scalar,
4816 elements: input_elements,
4817 external_layout: Some(input_external),
4818 initialization: ProgramResourceInitialization::ExternalInput,
4819 },
4820 ProgramResource {
4821 id: ProgramResourceId(1),
4822 name: "output".to_owned(),
4823 kind: ProgramResourceKind::Output,
4824 scalar: ir.scalar,
4825 elements: output_elements,
4826 external_layout: Some(output_external),
4827 initialization: ProgramResourceInitialization::Zeroed,
4828 },
4829 ProgramResource {
4830 id: ProgramResourceId(2),
4831 name: "nd_convolution_kernel_spectrum".to_owned(),
4832 kind: ProgramResourceKind::LookupTable,
4833 scalar: ir.scalar,
4834 elements: ir.kernel_spectrum().len(),
4835 external_layout: None,
4836 initialization: ProgramResourceInitialization::Complex64(
4837 ir.kernel_spectrum().to_vec(),
4838 ),
4839 },
4840 ProgramResource {
4841 id: ProgramResourceId(3),
4842 name: "nd_convolution_forward_spectrum".to_owned(),
4843 kind: ProgramResourceKind::Scratch,
4844 scalar: ir.scalar,
4845 elements: input_elements,
4846 external_layout: None,
4847 initialization: ProgramResourceInitialization::Zeroed,
4848 },
4849 ProgramResource {
4850 id: ProgramResourceId(4),
4851 name: "nd_convolution_multiplied_spectrum".to_owned(),
4852 kind: ProgramResourceKind::Scratch,
4853 scalar: ir.scalar,
4854 elements: output_elements,
4855 external_layout: None,
4856 initialization: ProgramResourceInitialization::Zeroed,
4857 },
4858 ];
4859 let mut passes = Vec::new();
4860 let append_child = |child: &NdFftIr,
4861 prefix: &str,
4862 input: ProgramResourceId,
4863 output: ProgramResourceId,
4864 resources: &mut Vec<ProgramResource>,
4865 passes: &mut Vec<ProgramPass>|
4866 -> Result<()> {
4867 let child = Self::nd_fft(child)?;
4868 let mut resource_map = vec![None; child.resources.len()];
4869 for resource in &child.resources {
4870 let mapped = match resource.kind {
4871 ProgramResourceKind::Input => input,
4872 ProgramResourceKind::Output => output,
4873 _ => {
4874 let id = ProgramResourceId(resources.len());
4875 let mut cloned = resource.clone();
4876 cloned.id = id;
4877 cloned.name = format!("{prefix}_{}", resource.name);
4878 cloned.external_layout = None;
4879 resources.push(cloned);
4880 id
4881 }
4882 };
4883 resource_map[resource.id.0] = Some(mapped);
4884 }
4885 for child_pass in &child.passes {
4886 let mut pass = child_pass.clone();
4887 pass.name = format!("{prefix}_{}", child_pass.name);
4888 for binding in &mut pass.bindings {
4889 binding.resource = resource_map
4890 .get(binding.resource.0)
4891 .and_then(|mapped| *mapped)
4892 .ok_or(VkFftError::InvalidKernelIr(
4893 "multidimensional convolution child resource mapping is incomplete",
4894 ))?;
4895 }
4896 passes.push(pass);
4897 }
4898 Ok(())
4899 };
4900 append_child(
4901 &ir.forward_fft,
4902 "vkfft_nd_convolution_forward",
4903 ProgramResourceId(0),
4904 ProgramResourceId(3),
4905 &mut resources,
4906 &mut passes,
4907 )?;
4908 passes.push(ProgramPass {
4909 name: ir.multiply.name.clone(),
4910 dispatch: ir.multiply.dispatch,
4911 bindings: vec![
4912 binding(0, 3, BufferRole::Input, BufferAccess::ReadOnly),
4913 binding(1, 4, BufferRole::Output, BufferAccess::WriteOnly),
4914 binding(2, 2, BufferRole::LookupTable, BufferAccess::ReadOnly),
4915 ],
4916 });
4917 append_child(
4918 &ir.inverse_fft,
4919 "vkfft_nd_convolution_inverse",
4920 ProgramResourceId(4),
4921 ProgramResourceId(1),
4922 &mut resources,
4923 &mut passes,
4924 )?;
4925 let program = Self {
4926 name: format!("vkfft_nd_convolution_program_{:?}", ir.dimensions),
4927 scalar: ir.scalar,
4928 resources,
4929 passes,
4930 };
4931 program.validate()?;
4932 Ok(program)
4933 }
4934
4935 pub fn nd_real_convolution(ir: &NdRealConvolutionIr) -> Result<Self> {
4936 ir.validate()?;
4937 let forward = Self::nd_real_fft(&ir.forward_r2c)?;
4938 let inverse = Self::nd_real_fft(&ir.inverse_c2r)?;
4939
4940 let child_resource =
4941 |program: &ProgramIr, kind: ProgramResourceKind| -> Result<ProgramResource> {
4942 program
4943 .resources
4944 .iter()
4945 .find(|resource| resource.kind == kind)
4946 .cloned()
4947 .ok_or(VkFftError::InvalidKernelIr(
4948 "multidimensional real convolution child is missing an external resource",
4949 ))
4950 };
4951 let mut input = child_resource(&forward, ProgramResourceKind::Input)?;
4952 input.id = ProgramResourceId(0);
4953 input.name = "input".to_owned();
4954 let mut output = child_resource(&inverse, ProgramResourceKind::Output)?;
4955 output.id = ProgramResourceId(1);
4956 output.name = "output".to_owned();
4957 let mut forward_spectrum = child_resource(&forward, ProgramResourceKind::Output)?;
4958 forward_spectrum.id = ProgramResourceId(3);
4959 forward_spectrum.name = "nd_real_convolution_forward_spectrum".to_owned();
4960 forward_spectrum.kind = ProgramResourceKind::Scratch;
4961 forward_spectrum.external_layout = None;
4962 forward_spectrum.initialization = ProgramResourceInitialization::Zeroed;
4963 let mut multiplied_spectrum = child_resource(&inverse, ProgramResourceKind::Input)?;
4964 multiplied_spectrum.id = ProgramResourceId(4);
4965 multiplied_spectrum.name = "nd_real_convolution_multiplied_spectrum".to_owned();
4966 multiplied_spectrum.kind = ProgramResourceKind::Scratch;
4967 multiplied_spectrum.external_layout = None;
4968 multiplied_spectrum.initialization = ProgramResourceInitialization::Zeroed;
4969
4970 let mut resources = vec![
4971 input,
4972 output,
4973 ProgramResource {
4974 id: ProgramResourceId(2),
4975 name: "nd_real_convolution_kernel_spectrum".to_owned(),
4976 kind: ProgramResourceKind::LookupTable,
4977 scalar: ir.scalar,
4978 elements: ir.kernel_spectrum().len(),
4979 external_layout: None,
4980 initialization: ProgramResourceInitialization::Complex64(
4981 ir.kernel_spectrum().to_vec(),
4982 ),
4983 },
4984 forward_spectrum,
4985 multiplied_spectrum,
4986 ];
4987 let mut passes = Vec::new();
4988 let append_child = |child: &ProgramIr,
4989 prefix: &str,
4990 input: ProgramResourceId,
4991 output: ProgramResourceId,
4992 resources: &mut Vec<ProgramResource>,
4993 passes: &mut Vec<ProgramPass>|
4994 -> Result<()> {
4995 let mut resource_map = vec![None; child.resources.len()];
4996 for resource in &child.resources {
4997 let mapped = match resource.kind {
4998 ProgramResourceKind::Input => input,
4999 ProgramResourceKind::Output => output,
5000 _ => {
5001 let id = ProgramResourceId(resources.len());
5002 let mut cloned = resource.clone();
5003 cloned.id = id;
5004 cloned.name = format!("{prefix}_{}", resource.name);
5005 cloned.external_layout = None;
5006 resources.push(cloned);
5007 id
5008 }
5009 };
5010 resource_map[resource.id.0] = Some(mapped);
5011 }
5012 for child_pass in &child.passes {
5013 let mut pass = child_pass.clone();
5014 pass.name = format!("{prefix}_{}", child_pass.name);
5015 for binding in &mut pass.bindings {
5016 binding.resource = resource_map
5017 .get(binding.resource.0)
5018 .and_then(|mapped| *mapped)
5019 .ok_or(VkFftError::InvalidKernelIr(
5020 "multidimensional real convolution child resource mapping is incomplete",
5021 ))?;
5022 }
5023 passes.push(pass);
5024 }
5025 Ok(())
5026 };
5027 append_child(
5028 &forward,
5029 "vkfft_nd_real_convolution_forward",
5030 ProgramResourceId(0),
5031 ProgramResourceId(3),
5032 &mut resources,
5033 &mut passes,
5034 )?;
5035 passes.push(ProgramPass {
5036 name: ir.multiply.name.clone(),
5037 dispatch: ir.multiply.dispatch,
5038 bindings: vec![
5039 binding(0, 3, BufferRole::Input, BufferAccess::ReadOnly),
5040 binding(1, 4, BufferRole::Output, BufferAccess::WriteOnly),
5041 binding(2, 2, BufferRole::LookupTable, BufferAccess::ReadOnly),
5042 ],
5043 });
5044 append_child(
5045 &inverse,
5046 "vkfft_nd_real_convolution_inverse",
5047 ProgramResourceId(4),
5048 ProgramResourceId(1),
5049 &mut resources,
5050 &mut passes,
5051 )?;
5052 let program = Self {
5053 name: format!("vkfft_nd_real_convolution_program_{:?}", ir.dimensions),
5054 scalar: ir.scalar,
5055 resources,
5056 passes,
5057 };
5058 program.validate()?;
5059 Ok(program)
5060 }
5061
5062 pub fn convolution(ir: &ConvolutionIr) -> Result<Self> {
5063 ir.validate()?;
5064 let input_transform_batch_count = ir.batch_count.checked_mul(ir.coordinate_count).ok_or(
5065 VkFftError::ArithmeticOverflow {
5066 operation: "performConvolution program input coordinate-expanded batch count",
5067 },
5068 )?;
5069 let output_transform_batch_count = ir
5070 .output_batch_count()
5071 .checked_mul(ir.coordinate_count)
5072 .ok_or(VkFftError::ArithmeticOverflow {
5073 operation: "performConvolution program output coordinate-expanded batch count",
5074 })?;
5075 let input_elements = ir
5076 .sequence_len
5077 .checked_mul(input_transform_batch_count)
5078 .ok_or(VkFftError::ArithmeticOverflow {
5079 operation: "performConvolution program input element count",
5080 })?;
5081 let output_elements = ir
5082 .sequence_len
5083 .checked_mul(output_transform_batch_count)
5084 .ok_or(VkFftError::ArithmeticOverflow {
5085 operation: "performConvolution program output element count",
5086 })?;
5087 let input_external = ExternalBufferLayout {
5088 logical_len: ir.sequence_len,
5089 physical_stride: ir.sequence_len,
5090 batch_count: input_transform_batch_count,
5091 element_shape: ProgramElementShape::Complex,
5092 };
5093 let output_external = ExternalBufferLayout {
5094 logical_len: ir.sequence_len,
5095 physical_stride: ir.sequence_len,
5096 batch_count: output_transform_batch_count,
5097 element_shape: ProgramElementShape::Complex,
5098 };
5099 let mut builder = RecursiveProgramBuilder {
5100 scalar: ir.scalar,
5101 resources: vec![
5102 ProgramResource {
5103 id: ProgramResourceId(0),
5104 name: "input".to_owned(),
5105 kind: ProgramResourceKind::Input,
5106 scalar: ir.external_scalar,
5107 elements: input_elements,
5108 external_layout: Some(input_external),
5109 initialization: ProgramResourceInitialization::ExternalInput,
5110 },
5111 ProgramResource {
5112 id: ProgramResourceId(1),
5113 name: "output".to_owned(),
5114 kind: ProgramResourceKind::Output,
5115 scalar: ir.external_scalar,
5116 elements: output_elements,
5117 external_layout: Some(output_external),
5118 initialization: ProgramResourceInitialization::Zeroed,
5119 },
5120 ],
5121 passes: Vec::new(),
5122 serial: 0,
5123 };
5124 let kernel = builder.add_lut("convolution_kernel_spectrum", ir.kernel_spectrum().to_vec());
5125 if let (Some(input_copy), Some(output_copy)) =
5126 (&ir.input_storage_copy, &ir.output_storage_copy)
5127 {
5128 let input_compute =
5129 builder.add_scratch("convolution_caller_input_compute", input_elements);
5130 let output_compute =
5131 builder.add_scratch("convolution_caller_output_compute", output_elements);
5132 let forward_spectrum =
5133 builder.add_scratch("convolution_forward_spectrum", input_elements);
5134 let multiplied_spectrum =
5135 builder.add_scratch("convolution_multiplied_spectrum", output_elements);
5136 builder.passes.push(two_buffer_pass(
5137 &input_copy.name,
5138 input_copy.dispatch,
5139 0,
5140 input_compute.0,
5141 ));
5142 builder.flatten_one_dim(&ir.forward_fft, input_compute, forward_spectrum)?;
5143 builder.passes.push(ProgramPass {
5144 name: ir.multiply.name.clone(),
5145 dispatch: ir.multiply.dispatch,
5146 bindings: vec![
5147 binding(
5148 0,
5149 forward_spectrum.0,
5150 BufferRole::Input,
5151 BufferAccess::ReadOnly,
5152 ),
5153 binding(
5154 1,
5155 multiplied_spectrum.0,
5156 BufferRole::Output,
5157 BufferAccess::WriteOnly,
5158 ),
5159 binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5160 ],
5161 });
5162 builder.flatten_one_dim(&ir.inverse_fft, multiplied_spectrum, output_compute)?;
5163 builder.passes.push(two_buffer_pass(
5164 &output_copy.name,
5165 output_copy.dispatch,
5166 output_compute.0,
5167 1,
5168 ));
5169 let program = Self {
5170 name: format!("vkfft_convolution_program_{}", ir.sequence_len),
5171 scalar: ir.scalar,
5172 resources: builder.resources,
5173 passes: builder.passes,
5174 };
5175 program.validate()?;
5176 return Ok(program);
5177 }
5178 if let Some(step) = &ir.fused_stockham_step {
5179 builder.passes.push(ProgramPass {
5180 name: step.name.clone(),
5181 dispatch: step.dispatch,
5182 bindings: vec![
5183 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
5184 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
5185 binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5186 ],
5187 });
5188 } else if let Some(step) = &ir.fused_multi_kernel_stockham_step {
5189 let stockham_twiddles = if step.requires_twiddle_lut()? {
5190 Some(builder.add_stockham_root_lut(step.sequence_len)?)
5191 } else {
5192 None
5193 };
5194 let mut bindings = vec![
5195 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
5196 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
5197 binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5198 ];
5199 if let Some(twiddles) = stockham_twiddles {
5200 bindings.push(binding(
5201 3,
5202 twiddles.0,
5203 BufferRole::TwiddleLookupTable,
5204 BufferAccess::ReadOnly,
5205 ));
5206 }
5207 builder.passes.push(ProgramPass {
5208 name: step.name.clone(),
5209 dispatch: step.dispatch,
5210 bindings,
5211 });
5212 } else if let Some(step) = &ir.fused_matrix_stockham_step {
5213 let stockham_twiddles = if step.requires_twiddle_lut()? {
5214 Some(builder.add_stockham_root_lut(step.sequence_len)?)
5215 } else {
5216 None
5217 };
5218 let mut bindings = vec![
5219 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
5220 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
5221 binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5222 ];
5223 if let Some(twiddles) = stockham_twiddles {
5224 bindings.push(binding(
5225 3,
5226 twiddles.0,
5227 BufferRole::TwiddleLookupTable,
5228 BufferAccess::ReadOnly,
5229 ));
5230 }
5231 builder.passes.push(ProgramPass {
5232 name: step.name.clone(),
5233 dispatch: step.dispatch,
5234 bindings,
5235 });
5236 } else if let Some(step) = &ir.fused_direct_rader_step {
5237 let rader_twiddles = builder.add_lut(
5238 "convolution_direct_rader_twiddles",
5239 step.forward.table.twiddles_by_generator_power.clone(),
5240 );
5241 builder.passes.push(ProgramPass {
5242 name: step.name.clone(),
5243 dispatch: step.dispatch,
5244 bindings: vec![
5245 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
5246 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
5247 binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5248 binding(
5249 3,
5250 rader_twiddles.0,
5251 BufferRole::TwiddleLookupTable,
5252 BufferAccess::ReadOnly,
5253 ),
5254 ],
5255 });
5256 } else if let Some(step) = &ir.fused_direct_rader_multi_kernel_step {
5257 let rader_twiddles = builder.add_lut(
5258 "convolution_direct_rader_multi_kernel_twiddles",
5259 step.forward.table.twiddles_by_generator_power.clone(),
5260 );
5261 builder.passes.push(ProgramPass {
5262 name: step.name.clone(),
5263 dispatch: step.dispatch,
5264 bindings: vec![
5265 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
5266 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
5267 binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5268 binding(
5269 3,
5270 rader_twiddles.0,
5271 BufferRole::TwiddleLookupTable,
5272 BufferAccess::ReadOnly,
5273 ),
5274 ],
5275 });
5276 } else if let Some(step) = &ir.fused_fft_rader_step {
5277 let forward_rader_spectrum = builder.add_lut(
5278 "convolution_fft_rader_forward_spectrum",
5279 step.forward.kernel_spectrum()?.to_vec(),
5280 );
5281 let inverse_rader_spectrum = builder.add_lut(
5282 "convolution_fft_rader_inverse_spectrum",
5283 step.inverse.kernel_spectrum()?.to_vec(),
5284 );
5285 let stockham_twiddles = if step.requires_twiddle_lut()? {
5286 Some(builder.add_stockham_root_lut(step.convolution_len)?)
5287 } else {
5288 None
5289 };
5290 let mut bindings = vec![
5291 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
5292 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
5293 binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5294 binding(
5295 3,
5296 forward_rader_spectrum.0,
5297 BufferRole::LookupTable,
5298 BufferAccess::ReadOnly,
5299 ),
5300 binding(
5301 4,
5302 inverse_rader_spectrum.0,
5303 BufferRole::LookupTable,
5304 BufferAccess::ReadOnly,
5305 ),
5306 ];
5307 if let Some(twiddles) = stockham_twiddles {
5308 bindings.push(binding(
5309 5,
5310 twiddles.0,
5311 BufferRole::TwiddleLookupTable,
5312 BufferAccess::ReadOnly,
5313 ));
5314 }
5315 builder.passes.push(ProgramPass {
5316 name: step.name.clone(),
5317 dispatch: step.dispatch,
5318 bindings,
5319 });
5320 } else if let Some(step) = &ir.fused_fft_rader_multi_kernel_step {
5321 let forward_rader_spectrum = builder.add_lut(
5322 "convolution_fft_rader_multi_kernel_forward_spectrum",
5323 step.forward.kernel_spectrum()?.to_vec(),
5324 );
5325 let inverse_rader_spectrum = builder.add_lut(
5326 "convolution_fft_rader_multi_kernel_inverse_spectrum",
5327 step.inverse.kernel_spectrum()?.to_vec(),
5328 );
5329 let stockham_twiddles = if step.requires_twiddle_lut()? {
5330 Some(builder.add_stockham_root_lut(step.convolution_len)?)
5331 } else {
5332 None
5333 };
5334 let mut bindings = vec![
5335 binding(0, 0, BufferRole::Input, BufferAccess::ReadOnly),
5336 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
5337 binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5338 binding(
5339 3,
5340 forward_rader_spectrum.0,
5341 BufferRole::LookupTable,
5342 BufferAccess::ReadOnly,
5343 ),
5344 binding(
5345 4,
5346 inverse_rader_spectrum.0,
5347 BufferRole::LookupTable,
5348 BufferAccess::ReadOnly,
5349 ),
5350 ];
5351 if let Some(twiddles) = stockham_twiddles {
5352 bindings.push(binding(
5353 5,
5354 twiddles.0,
5355 BufferRole::TwiddleLookupTable,
5356 BufferAccess::ReadOnly,
5357 ));
5358 }
5359 builder.passes.push(ProgramPass {
5360 name: step.name.clone(),
5361 dispatch: step.dispatch,
5362 bindings,
5363 });
5364 } else if let Some(two_upload) = &ir.fused_two_upload_multi_kernel_stockham {
5365 let forward_mid = builder.add_scratch(
5366 "convolution_two_upload_multi_kernel_forward_mid",
5367 input_elements,
5368 );
5369 let inverse_mid = builder.add_scratch(
5370 "convolution_two_upload_multi_kernel_inverse_mid",
5371 output_elements,
5372 );
5373 builder.push_stockham_pass(
5374 &two_upload.forward_high,
5375 ProgramResourceId(0),
5376 forward_mid,
5377 None,
5378 None,
5379 )?;
5380 let special_twiddle = match two_upload.special_twiddle_lut_len()? {
5381 Some(len) => Some(builder.add_stockham_root_lut(len)?),
5382 None => None,
5383 };
5384 let mut special_bindings = vec![
5385 binding(0, forward_mid.0, BufferRole::Input, BufferAccess::ReadOnly),
5386 binding(
5387 1,
5388 inverse_mid.0,
5389 BufferRole::Output,
5390 BufferAccess::WriteOnly,
5391 ),
5392 binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5393 ];
5394 if let Some(twiddle) = special_twiddle {
5395 special_bindings.push(binding(
5396 3,
5397 twiddle.0,
5398 BufferRole::TwiddleLookupTable,
5399 BufferAccess::ReadOnly,
5400 ));
5401 }
5402 builder.passes.push(ProgramPass {
5403 name: two_upload.name.clone(),
5404 dispatch: two_upload.special_dispatch,
5405 bindings: special_bindings,
5406 });
5407 builder.push_stockham_pass(
5408 &two_upload.inverse_high,
5409 inverse_mid,
5410 ProgramResourceId(1),
5411 None,
5412 None,
5413 )?;
5414 } else if let Some(two_upload) = &ir.fused_two_upload_stockham {
5415 let forward_mid =
5416 builder.add_scratch("convolution_two_upload_forward_mid", input_elements);
5417 let inverse_mid =
5418 builder.add_scratch("convolution_two_upload_inverse_mid", input_elements);
5419 builder.push_stockham_pass(
5420 &two_upload.forward_high,
5421 ProgramResourceId(0),
5422 forward_mid,
5423 None,
5424 None,
5425 )?;
5426 let special_twiddle = match two_upload.special_twiddle_lut_len()? {
5427 Some(len) => Some(builder.add_stockham_root_lut(len)?),
5428 None => None,
5429 };
5430 let mut special_bindings = vec![
5431 binding(0, forward_mid.0, BufferRole::Input, BufferAccess::ReadOnly),
5432 binding(
5433 1,
5434 inverse_mid.0,
5435 BufferRole::Output,
5436 BufferAccess::WriteOnly,
5437 ),
5438 binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5439 ];
5440 if let Some(twiddle) = special_twiddle {
5441 special_bindings.push(binding(
5442 3,
5443 twiddle.0,
5444 BufferRole::TwiddleLookupTable,
5445 BufferAccess::ReadOnly,
5446 ));
5447 }
5448 builder.passes.push(ProgramPass {
5449 name: two_upload.name.clone(),
5450 dispatch: two_upload.special_dispatch,
5451 bindings: special_bindings,
5452 });
5453 builder.push_stockham_pass(
5454 &two_upload.inverse_high,
5455 inverse_mid,
5456 ProgramResourceId(1),
5457 None,
5458 None,
5459 )?;
5460 } else if let Some(three_upload) = &ir.fused_three_upload_multi_kernel_stockham {
5461 let scratch_a = builder.add_scratch(
5462 "convolution_three_upload_multi_kernel_scratch_a",
5463 output_elements,
5464 );
5465 let scratch_b = builder.add_scratch(
5466 "convolution_three_upload_multi_kernel_scratch_b",
5467 output_elements,
5468 );
5469 builder.push_stockham_pass(
5470 &three_upload.forward_high,
5471 ProgramResourceId(0),
5472 scratch_a,
5473 None,
5474 None,
5475 )?;
5476 builder.push_stockham_pass(
5477 &three_upload.forward_middle,
5478 scratch_a,
5479 scratch_b,
5480 None,
5481 None,
5482 )?;
5483 builder.passes.push(ProgramPass {
5484 name: three_upload.name.clone(),
5485 dispatch: three_upload.special_dispatch,
5486 bindings: vec![
5487 binding(0, scratch_b.0, BufferRole::Input, BufferAccess::ReadOnly),
5488 binding(1, scratch_a.0, BufferRole::Output, BufferAccess::WriteOnly),
5489 binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5490 ],
5491 });
5492 builder.push_stockham_pass(
5493 &three_upload.inverse_middle,
5494 scratch_a,
5495 scratch_b,
5496 None,
5497 None,
5498 )?;
5499 builder.push_stockham_pass(
5500 &three_upload.inverse_high,
5501 scratch_b,
5502 ProgramResourceId(1),
5503 None,
5504 None,
5505 )?;
5506 } else if let Some(three_upload) = &ir.fused_three_upload_stockham {
5507 let scratch_a =
5508 builder.add_scratch("convolution_three_upload_scratch_a", input_elements);
5509 let scratch_b =
5510 builder.add_scratch("convolution_three_upload_scratch_b", input_elements);
5511 builder.push_stockham_pass(
5512 &three_upload.forward_high,
5513 ProgramResourceId(0),
5514 scratch_a,
5515 None,
5516 None,
5517 )?;
5518 builder.push_stockham_pass(
5519 &three_upload.forward_middle,
5520 scratch_a,
5521 scratch_b,
5522 None,
5523 None,
5524 )?;
5525 let special_twiddle = match three_upload.special_twiddle_lut_len()? {
5526 Some(len) => Some(builder.add_stockham_root_lut(len)?),
5527 None => None,
5528 };
5529 let mut special_bindings = vec![
5530 binding(0, scratch_b.0, BufferRole::Input, BufferAccess::ReadOnly),
5531 binding(1, scratch_a.0, BufferRole::Output, BufferAccess::WriteOnly),
5532 binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5533 ];
5534 if let Some(twiddle) = special_twiddle {
5535 special_bindings.push(binding(
5536 3,
5537 twiddle.0,
5538 BufferRole::TwiddleLookupTable,
5539 BufferAccess::ReadOnly,
5540 ));
5541 }
5542 builder.passes.push(ProgramPass {
5543 name: three_upload.name.clone(),
5544 dispatch: three_upload.special_dispatch,
5545 bindings: special_bindings,
5546 });
5547 builder.push_stockham_pass(
5548 &three_upload.inverse_middle,
5549 scratch_a,
5550 scratch_b,
5551 None,
5552 None,
5553 )?;
5554 builder.push_stockham_pass(
5555 &three_upload.inverse_high,
5556 scratch_b,
5557 ProgramResourceId(1),
5558 None,
5559 None,
5560 )?;
5561 } else {
5562 let forward_spectrum =
5563 builder.add_scratch("convolution_forward_spectrum", input_elements);
5564 builder.flatten_one_dim(&ir.forward_fft, ProgramResourceId(0), forward_spectrum)?;
5565 if let Some(fused_inverse) = &ir.fused_inverse_stockham {
5566 builder.push_stockham_pass(
5567 fused_inverse,
5568 forward_spectrum,
5569 ProgramResourceId(1),
5570 Some(kernel),
5571 None,
5572 )?;
5573 } else {
5574 let multiplied_spectrum =
5575 builder.add_scratch("convolution_multiplied_spectrum", output_elements);
5576 builder.passes.push(ProgramPass {
5577 name: ir.multiply.name.clone(),
5578 dispatch: ir.multiply.dispatch,
5579 bindings: vec![
5580 binding(
5581 0,
5582 forward_spectrum.0,
5583 BufferRole::Input,
5584 BufferAccess::ReadOnly,
5585 ),
5586 binding(
5587 1,
5588 multiplied_spectrum.0,
5589 BufferRole::Output,
5590 BufferAccess::WriteOnly,
5591 ),
5592 binding(2, kernel.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
5593 ],
5594 });
5595 builder.flatten_one_dim(
5596 &ir.inverse_fft,
5597 multiplied_spectrum,
5598 ProgramResourceId(1),
5599 )?;
5600 }
5601 }
5602 let program = Self {
5603 name: format!("vkfft_perform_convolution_program_{}", ir.sequence_len),
5604 scalar: ir.scalar,
5605 resources: builder.resources,
5606 passes: builder.passes,
5607 };
5608 program.validate()?;
5609 Ok(program)
5610 }
5611
5612 pub fn bluestein(pipeline: &BluesteinPipelineIr) -> Result<Self> {
5613 pipeline.validate()?;
5614 let logical_elements = pipeline
5615 .logical_len
5616 .checked_mul(pipeline.batch_count)
5617 .ok_or(VkFftError::ArithmeticOverflow {
5618 operation: "Bluestein program logical element count",
5619 })?;
5620 let convolution_elements = pipeline
5621 .convolution_len
5622 .checked_mul(pipeline.batch_count)
5623 .ok_or(VkFftError::ArithmeticOverflow {
5624 operation: "Bluestein program convolution element count",
5625 })?;
5626 let external = ExternalBufferLayout {
5627 logical_len: pipeline.logical_len,
5628 physical_stride: pipeline.logical_len,
5629 batch_count: pipeline.batch_count,
5630 element_shape: ProgramElementShape::Complex,
5631 };
5632 let lut = pipeline.kernel_spectrum()?.to_vec();
5633 let mut builder = RecursiveProgramBuilder {
5634 scalar: pipeline.scalar,
5635 resources: vec![
5636 ProgramResource {
5637 id: ProgramResourceId(0),
5638 name: "input".to_owned(),
5639 kind: ProgramResourceKind::Input,
5640 scalar: pipeline.external_storage_scalar(),
5641 elements: logical_elements,
5642 external_layout: Some(external),
5643 initialization: ProgramResourceInitialization::ExternalInput,
5644 },
5645 ProgramResource {
5646 id: ProgramResourceId(1),
5647 name: "output".to_owned(),
5648 kind: ProgramResourceKind::Output,
5649 scalar: pipeline.external_storage_scalar(),
5650 elements: logical_elements,
5651 external_layout: Some(external),
5652 initialization: ProgramResourceInitialization::Zeroed,
5653 },
5654 ProgramResource {
5655 id: ProgramResourceId(2),
5656 name: "kernel_spectrum".to_owned(),
5657 kind: ProgramResourceKind::LookupTable,
5658 scalar: pipeline.scalar,
5659 elements: pipeline.convolution_len,
5660 external_layout: None,
5661 initialization: ProgramResourceInitialization::Complex64(lut),
5662 },
5663 ProgramResource {
5664 id: ProgramResourceId(3),
5665 name: "scratch_a".to_owned(),
5666 kind: ProgramResourceKind::Scratch,
5667 scalar: pipeline.scalar,
5668 elements: convolution_elements,
5669 external_layout: None,
5670 initialization: ProgramResourceInitialization::Zeroed,
5671 },
5672 ProgramResource {
5673 id: ProgramResourceId(4),
5674 name: "scratch_b".to_owned(),
5675 kind: ProgramResourceKind::Scratch,
5676 scalar: pipeline.scalar,
5677 elements: convolution_elements,
5678 external_layout: None,
5679 initialization: ProgramResourceInitialization::Zeroed,
5680 },
5681 ],
5682 passes: vec![two_buffer_pass(
5683 &pipeline.preprocess.name,
5684 pipeline.preprocess.dispatch,
5685 0,
5686 3,
5687 )],
5688 serial: 0,
5689 };
5690 builder.flatten_recursive_ir(
5691 &pipeline.forward_fft,
5692 ProgramResourceId(3),
5693 ProgramResourceId(4),
5694 )?;
5695 let convolution_output =
5696 if let Some(fused_inverse) = pipeline.fused_inverse_stockham_kernel()? {
5697 builder.push_stockham_pass(
5698 &fused_inverse,
5699 ProgramResourceId(4),
5700 ProgramResourceId(3),
5701 Some(ProgramResourceId(2)),
5702 None,
5703 )?;
5704 ProgramResourceId(3)
5705 } else if pipeline.has_fused_recursive_inverse() {
5706 builder.flatten_node_with_boundary_resources(
5707 &pipeline.inverse_fft.root,
5708 ProgramResourceId(4),
5709 ProgramResourceId(3),
5710 Some(ProgramResourceId(2)),
5711 None,
5712 )?;
5713 ProgramResourceId(3)
5714 } else {
5715 builder.passes.push(ProgramPass {
5716 name: pipeline.multiply.name.clone(),
5717 dispatch: pipeline.multiply.dispatch,
5718 bindings: vec![
5719 binding(0, 4, BufferRole::Input, BufferAccess::ReadOnly),
5720 binding(1, 3, BufferRole::Output, BufferAccess::WriteOnly),
5721 binding(2, 2, BufferRole::LookupTable, BufferAccess::ReadOnly),
5722 ],
5723 });
5724 builder.flatten_recursive_ir(
5725 &pipeline.inverse_fft,
5726 ProgramResourceId(3),
5727 ProgramResourceId(4),
5728 )?;
5729 ProgramResourceId(4)
5730 };
5731 builder.passes.push(two_buffer_pass(
5732 &pipeline.postprocess.name,
5733 pipeline.postprocess.dispatch,
5734 convolution_output.0,
5735 1,
5736 ));
5737 let program = Self {
5738 name: format!("vkfft_bluestein_program_{}", pipeline.logical_len),
5739 scalar: pipeline.scalar,
5740 resources: builder.resources,
5741 passes: builder.passes,
5742 };
5743 program.validate()?;
5744 Ok(program)
5745 }
5746
5747 pub fn rader_fft(pipeline: &RaderFftPipelineIr) -> Result<Self> {
5748 pipeline.validate()?;
5749 let logical_elements = pipeline.prime.checked_mul(pipeline.batch_count).ok_or(
5750 VkFftError::ArithmeticOverflow {
5751 operation: "FFT Rader program logical element count",
5752 },
5753 )?;
5754 let convolution_elements = pipeline
5755 .convolution_len
5756 .checked_mul(pipeline.batch_count)
5757 .ok_or(VkFftError::ArithmeticOverflow {
5758 operation: "FFT Rader program convolution element count",
5759 })?;
5760 let external = ExternalBufferLayout {
5761 logical_len: pipeline.prime,
5762 physical_stride: pipeline.prime,
5763 batch_count: pipeline.batch_count,
5764 element_shape: ProgramElementShape::Complex,
5765 };
5766 let mut builder = RecursiveProgramBuilder {
5767 scalar: pipeline.scalar,
5768 resources: vec![
5769 ProgramResource {
5770 id: ProgramResourceId(0),
5771 name: "input".to_owned(),
5772 kind: ProgramResourceKind::Input,
5773 scalar: pipeline.input_storage_scalar,
5774 elements: logical_elements,
5775 external_layout: Some(external),
5776 initialization: ProgramResourceInitialization::ExternalInput,
5777 },
5778 ProgramResource {
5779 id: ProgramResourceId(1),
5780 name: "scratch_a".to_owned(),
5781 kind: ProgramResourceKind::Scratch,
5782 scalar: pipeline.scalar,
5783 elements: convolution_elements,
5784 external_layout: None,
5785 initialization: ProgramResourceInitialization::Zeroed,
5786 },
5787 ProgramResource {
5788 id: ProgramResourceId(2),
5789 name: "scratch_b".to_owned(),
5790 kind: ProgramResourceKind::Scratch,
5791 scalar: pipeline.scalar,
5792 elements: convolution_elements,
5793 external_layout: None,
5794 initialization: ProgramResourceInitialization::Zeroed,
5795 },
5796 ProgramResource {
5797 id: ProgramResourceId(3),
5798 name: "output".to_owned(),
5799 kind: ProgramResourceKind::Output,
5800 scalar: pipeline.output_storage_scalar,
5801 elements: logical_elements,
5802 external_layout: Some(external),
5803 initialization: ProgramResourceInitialization::Zeroed,
5804 },
5805 ProgramResource {
5806 id: ProgramResourceId(4),
5807 name: "kernel_spectrum".to_owned(),
5808 kind: ProgramResourceKind::LookupTable,
5809 scalar: pipeline.scalar,
5810 elements: pipeline.convolution_len,
5811 external_layout: None,
5812 initialization: ProgramResourceInitialization::Complex64(
5813 pipeline.kernel_spectrum()?.to_vec(),
5814 ),
5815 },
5816 ],
5817 passes: Vec::new(),
5818 serial: 0,
5819 };
5820 let forward_input = match pipeline.input_strategy {
5821 RaderFftInputStrategy::GatherReversePass => {
5822 builder.passes.push(two_buffer_pass(
5823 &pipeline.gather.name,
5824 pipeline.gather.dispatch,
5825 0,
5826 1,
5827 ));
5828 ProgramResourceId(1)
5829 }
5830 RaderFftInputStrategy::GeneratorOrderStockham
5831 | RaderFftInputStrategy::GeneratorOrderRecursive => ProgramResourceId(0),
5832 };
5833 builder.flatten_one_dim(&pipeline.forward_fft, forward_input, ProgramResourceId(2))?;
5834 if let Some(fused_inverse) = pipeline.fused_inverse_rader_kernel()? {
5835 builder.push_stockham_pass(
5836 &fused_inverse,
5837 ProgramResourceId(2),
5838 ProgramResourceId(3),
5839 Some(ProgramResourceId(4)),
5840 Some(ProgramResourceId(0)),
5841 )?;
5842 } else if pipeline.has_fused_recursive_inverse() {
5843 let inverse = pipeline
5844 .inverse_recursive()
5845 .ok_or(VkFftError::InvalidKernelIr(
5846 "fused recursive Rader program lost its recursive inverse child",
5847 ))?;
5848 builder.flatten_node_with_boundary_resources(
5849 &inverse.root,
5850 ProgramResourceId(2),
5851 ProgramResourceId(3),
5852 Some(ProgramResourceId(4)),
5853 Some(ProgramResourceId(0)),
5854 )?;
5855 } else {
5856 builder.passes.push(ProgramPass {
5857 name: pipeline.multiply.name.clone(),
5858 dispatch: pipeline.multiply.dispatch,
5859 bindings: vec![
5860 binding(0, 2, BufferRole::Input, BufferAccess::ReadOnly),
5861 binding(1, 1, BufferRole::Output, BufferAccess::WriteOnly),
5862 binding(2, 4, BufferRole::LookupTable, BufferAccess::ReadOnly),
5863 ],
5864 });
5865 builder.flatten_one_dim(
5866 &pipeline.inverse_fft,
5867 ProgramResourceId(1),
5868 ProgramResourceId(2),
5869 )?;
5870 builder.passes.push(ProgramPass {
5871 name: pipeline.scatter.name.clone(),
5872 dispatch: pipeline.scatter.dispatch,
5873 bindings: vec![
5874 binding(0, 2, BufferRole::Input, BufferAccess::ReadOnly),
5875 binding(1, 3, BufferRole::Output, BufferAccess::WriteOnly),
5876 binding(2, 0, BufferRole::Auxiliary, BufferAccess::ReadOnly),
5877 ],
5878 });
5879 }
5880 let program = Self {
5881 name: format!("vkfft_rader_fft_program_{}", pipeline.prime),
5882 scalar: pipeline.scalar,
5883 resources: builder.resources,
5884 passes: builder.passes,
5885 };
5886 program.validate()?;
5887 Ok(program)
5888 }
5889
5890 pub fn mixed_rader_stockham(ir: &MixedRaderStockhamIr) -> Result<Self> {
5891 ir.validate()?;
5892 match &ir.prime_stage {
5893 MixedPrimeRaderIr::FftConvolution(rader) => Self::mixed_rader_stockham_fft(ir, rader),
5894 MixedPrimeRaderIr::Direct(direct) => Self::mixed_rader_stockham_direct(ir, direct),
5895 }
5896 }
5897
5898 fn mixed_rader_stockham_fft(
5899 ir: &MixedRaderStockhamIr,
5900 rader: &RaderFftPipelineIr,
5901 ) -> Result<Self> {
5902 let logical_elements =
5903 ir.logical_len
5904 .checked_mul(ir.batch_count)
5905 .ok_or(VkFftError::ArithmeticOverflow {
5906 operation: "mixed program logical element count",
5907 })?;
5908 let convolution_elements = rader.convolution_len.checked_mul(rader.batch_count).ok_or(
5909 VkFftError::ArithmeticOverflow {
5910 operation: "mixed program Rader convolution element count",
5911 },
5912 )?;
5913 let external = ExternalBufferLayout {
5914 logical_len: ir.logical_len,
5915 physical_stride: ir.logical_len,
5916 batch_count: ir.batch_count,
5917 element_shape: ProgramElementShape::Complex,
5918 };
5919 let mut builder = RecursiveProgramBuilder {
5920 scalar: ir.scalar,
5921 resources: vec![
5922 ProgramResource {
5923 id: ProgramResourceId(0),
5924 name: "input".to_owned(),
5925 kind: ProgramResourceKind::Input,
5926 scalar: ir.scalar,
5927 elements: logical_elements,
5928 external_layout: Some(external),
5929 initialization: ProgramResourceInitialization::ExternalInput,
5930 },
5931 ProgramResource {
5932 id: ProgramResourceId(1),
5933 name: "prime_input".to_owned(),
5934 kind: ProgramResourceKind::Scratch,
5935 scalar: ir.scalar,
5936 elements: logical_elements,
5937 external_layout: None,
5938 initialization: ProgramResourceInitialization::Zeroed,
5939 },
5940 ProgramResource {
5941 id: ProgramResourceId(2),
5942 name: "rader_scratch_a".to_owned(),
5943 kind: ProgramResourceKind::Scratch,
5944 scalar: ir.scalar,
5945 elements: convolution_elements,
5946 external_layout: None,
5947 initialization: ProgramResourceInitialization::Zeroed,
5948 },
5949 ProgramResource {
5950 id: ProgramResourceId(3),
5951 name: "rader_scratch_b".to_owned(),
5952 kind: ProgramResourceKind::Scratch,
5953 scalar: ir.scalar,
5954 elements: convolution_elements,
5955 external_layout: None,
5956 initialization: ProgramResourceInitialization::Zeroed,
5957 },
5958 ProgramResource {
5959 id: ProgramResourceId(4),
5960 name: "prime_output".to_owned(),
5961 kind: ProgramResourceKind::Scratch,
5962 scalar: ir.scalar,
5963 elements: logical_elements,
5964 external_layout: None,
5965 initialization: ProgramResourceInitialization::Zeroed,
5966 },
5967 ProgramResource {
5968 id: ProgramResourceId(5),
5969 name: "rader_kernel_spectrum".to_owned(),
5970 kind: ProgramResourceKind::LookupTable,
5971 scalar: ir.scalar,
5972 elements: rader.convolution_len,
5973 external_layout: None,
5974 initialization: ProgramResourceInitialization::Complex64(
5975 rader.kernel_spectrum()?.to_vec(),
5976 ),
5977 },
5978 ProgramResource {
5979 id: ProgramResourceId(6),
5980 name: "stockham_input".to_owned(),
5981 kind: ProgramResourceKind::Scratch,
5982 scalar: ir.scalar,
5983 elements: logical_elements,
5984 external_layout: None,
5985 initialization: ProgramResourceInitialization::Zeroed,
5986 },
5987 ProgramResource {
5988 id: ProgramResourceId(7),
5989 name: "stockham_output".to_owned(),
5990 kind: ProgramResourceKind::Scratch,
5991 scalar: ir.scalar,
5992 elements: logical_elements,
5993 external_layout: None,
5994 initialization: ProgramResourceInitialization::Zeroed,
5995 },
5996 ProgramResource {
5997 id: ProgramResourceId(8),
5998 name: "output".to_owned(),
5999 kind: ProgramResourceKind::Output,
6000 scalar: ir.scalar,
6001 elements: logical_elements,
6002 external_layout: Some(external),
6003 initialization: ProgramResourceInitialization::Zeroed,
6004 },
6005 ],
6006 passes: vec![two_buffer_pass(
6007 &ir.pack_prime.name,
6008 ir.pack_prime.dispatch,
6009 0,
6010 1,
6011 )],
6012 serial: 0,
6013 };
6014 let forward_input = match rader.input_strategy {
6015 RaderFftInputStrategy::GatherReversePass => {
6016 builder.passes.push(two_buffer_pass(
6017 &rader.gather.name,
6018 rader.gather.dispatch,
6019 1,
6020 2,
6021 ));
6022 ProgramResourceId(2)
6023 }
6024 RaderFftInputStrategy::GeneratorOrderStockham
6025 | RaderFftInputStrategy::GeneratorOrderRecursive => ProgramResourceId(1),
6026 };
6027 builder.flatten_one_dim(&rader.forward_fft, forward_input, ProgramResourceId(3))?;
6028 if let Some(fused_inverse) = rader.fused_inverse_rader_kernel()? {
6029 builder.push_stockham_pass(
6030 &fused_inverse,
6031 ProgramResourceId(3),
6032 ProgramResourceId(4),
6033 Some(ProgramResourceId(5)),
6034 Some(ProgramResourceId(1)),
6035 )?;
6036 } else if rader.has_fused_recursive_inverse() {
6037 let inverse = rader
6038 .inverse_recursive()
6039 .ok_or(VkFftError::InvalidKernelIr(
6040 "mixed fused recursive Rader program lost its recursive inverse child",
6041 ))?;
6042 builder.flatten_node_with_boundary_resources(
6043 &inverse.root,
6044 ProgramResourceId(3),
6045 ProgramResourceId(4),
6046 Some(ProgramResourceId(5)),
6047 Some(ProgramResourceId(1)),
6048 )?;
6049 } else {
6050 builder.passes.push(ProgramPass {
6051 name: rader.multiply.name.clone(),
6052 dispatch: rader.multiply.dispatch,
6053 bindings: vec![
6054 binding(0, 3, BufferRole::Input, BufferAccess::ReadOnly),
6055 binding(1, 2, BufferRole::Output, BufferAccess::WriteOnly),
6056 binding(2, 5, BufferRole::LookupTable, BufferAccess::ReadOnly),
6057 ],
6058 });
6059 builder.flatten_one_dim(
6060 &rader.inverse_fft,
6061 ProgramResourceId(2),
6062 ProgramResourceId(3),
6063 )?;
6064 builder.passes.push(ProgramPass {
6065 name: rader.scatter.name.clone(),
6066 dispatch: rader.scatter.dispatch,
6067 bindings: vec![
6068 binding(0, 3, BufferRole::Input, BufferAccess::ReadOnly),
6069 binding(1, 4, BufferRole::Output, BufferAccess::WriteOnly),
6070 binding(2, 1, BufferRole::Auxiliary, BufferAccess::ReadOnly),
6071 ],
6072 });
6073 }
6074 builder.passes.push(two_buffer_pass(
6075 &ir.twiddle_transpose.name,
6076 ir.twiddle_transpose.dispatch,
6077 4,
6078 6,
6079 ));
6080 builder.passes.push(two_buffer_pass(
6081 &ir.stockham_stage.name,
6082 ir.stockham_stage.dispatch,
6083 6,
6084 7,
6085 ));
6086 builder.passes.push(two_buffer_pass(
6087 &ir.scatter_output.name,
6088 ir.scatter_output.dispatch,
6089 7,
6090 8,
6091 ));
6092 let program = Self {
6093 name: format!("vkfft_mixed_rader_stockham_program_{}", ir.logical_len),
6094 scalar: ir.scalar,
6095 resources: builder.resources,
6096 passes: builder.passes,
6097 };
6098 program.validate()?;
6099 Ok(program)
6100 }
6101
6102 fn mixed_rader_stockham_direct(
6103 ir: &MixedRaderStockhamIr,
6104 direct: &RaderDirectIr,
6105 ) -> Result<Self> {
6106 let logical_elements =
6107 ir.logical_len
6108 .checked_mul(ir.batch_count)
6109 .ok_or(VkFftError::ArithmeticOverflow {
6110 operation: "mixed direct-Rader program logical element count",
6111 })?;
6112 let external = ExternalBufferLayout {
6113 logical_len: ir.logical_len,
6114 physical_stride: ir.logical_len,
6115 batch_count: ir.batch_count,
6116 element_shape: ProgramElementShape::Complex,
6117 };
6118 let program = Self {
6119 name: format!(
6120 "vkfft_mixed_direct_rader_stockham_program_{}",
6121 ir.logical_len
6122 ),
6123 scalar: ir.scalar,
6124 resources: vec![
6125 ProgramResource {
6126 id: ProgramResourceId(0),
6127 name: "input".to_owned(),
6128 kind: ProgramResourceKind::Input,
6129 scalar: ir.scalar,
6130 elements: logical_elements,
6131 external_layout: Some(external),
6132 initialization: ProgramResourceInitialization::ExternalInput,
6133 },
6134 ProgramResource {
6135 id: ProgramResourceId(1),
6136 name: "prime_input".to_owned(),
6137 kind: ProgramResourceKind::Scratch,
6138 scalar: ir.scalar,
6139 elements: logical_elements,
6140 external_layout: None,
6141 initialization: ProgramResourceInitialization::Zeroed,
6142 },
6143 ProgramResource {
6144 id: ProgramResourceId(2),
6145 name: "prime_output".to_owned(),
6146 kind: ProgramResourceKind::Scratch,
6147 scalar: ir.scalar,
6148 elements: logical_elements,
6149 external_layout: None,
6150 initialization: ProgramResourceInitialization::Zeroed,
6151 },
6152 ProgramResource {
6153 id: ProgramResourceId(3),
6154 name: "rader_roots".to_owned(),
6155 kind: ProgramResourceKind::LookupTable,
6156 scalar: ir.scalar,
6157 elements: direct.twiddle_lut().len(),
6158 external_layout: None,
6159 initialization: ProgramResourceInitialization::Complex64(
6160 direct.twiddle_lut().to_vec(),
6161 ),
6162 },
6163 ProgramResource {
6164 id: ProgramResourceId(4),
6165 name: "stockham_input".to_owned(),
6166 kind: ProgramResourceKind::Scratch,
6167 scalar: ir.scalar,
6168 elements: logical_elements,
6169 external_layout: None,
6170 initialization: ProgramResourceInitialization::Zeroed,
6171 },
6172 ProgramResource {
6173 id: ProgramResourceId(5),
6174 name: "stockham_output".to_owned(),
6175 kind: ProgramResourceKind::Scratch,
6176 scalar: ir.scalar,
6177 elements: logical_elements,
6178 external_layout: None,
6179 initialization: ProgramResourceInitialization::Zeroed,
6180 },
6181 ProgramResource {
6182 id: ProgramResourceId(6),
6183 name: "output".to_owned(),
6184 kind: ProgramResourceKind::Output,
6185 scalar: ir.scalar,
6186 elements: logical_elements,
6187 external_layout: Some(external),
6188 initialization: ProgramResourceInitialization::Zeroed,
6189 },
6190 ],
6191 passes: vec![
6192 two_buffer_pass(&ir.pack_prime.name, ir.pack_prime.dispatch, 0, 1),
6193 ProgramPass {
6194 name: direct.name.clone(),
6195 dispatch: direct.dispatch,
6196 bindings: vec![
6197 binding(0, 1, BufferRole::Input, BufferAccess::ReadOnly),
6198 binding(1, 2, BufferRole::Output, BufferAccess::WriteOnly),
6199 binding(2, 3, BufferRole::LookupTable, BufferAccess::ReadOnly),
6200 ],
6201 },
6202 two_buffer_pass(
6203 &ir.twiddle_transpose.name,
6204 ir.twiddle_transpose.dispatch,
6205 2,
6206 4,
6207 ),
6208 two_buffer_pass(&ir.stockham_stage.name, ir.stockham_stage.dispatch, 4, 5),
6209 two_buffer_pass(&ir.scatter_output.name, ir.scatter_output.dispatch, 5, 6),
6210 ],
6211 };
6212 program.validate()?;
6213 Ok(program)
6214 }
6215
6216 pub fn recursive_fft(ir: &RecursiveFftIr) -> Result<Self> {
6217 ir.validate()?;
6218 let elements =
6219 ir.logical_len
6220 .checked_mul(ir.batch_count)
6221 .ok_or(VkFftError::ArithmeticOverflow {
6222 operation: "recursive FFT program element count",
6223 })?;
6224 let external = ExternalBufferLayout {
6225 logical_len: ir.logical_len,
6226 physical_stride: ir.logical_len,
6227 batch_count: ir.batch_count,
6228 element_shape: ProgramElementShape::Complex,
6229 };
6230 let external_scalar = ir.external_storage_scalar();
6231 let mut builder = RecursiveProgramBuilder {
6232 scalar: ir.scalar,
6233 resources: vec![
6234 ProgramResource {
6235 id: ProgramResourceId(0),
6236 name: "input".to_owned(),
6237 kind: ProgramResourceKind::Input,
6238 scalar: external_scalar,
6239 elements,
6240 external_layout: Some(external),
6241 initialization: ProgramResourceInitialization::ExternalInput,
6242 },
6243 ProgramResource {
6244 id: ProgramResourceId(1),
6245 name: "output".to_owned(),
6246 kind: ProgramResourceKind::Output,
6247 scalar: external_scalar,
6248 elements,
6249 external_layout: Some(external),
6250 initialization: ProgramResourceInitialization::Zeroed,
6251 },
6252 ],
6253 passes: Vec::new(),
6254 serial: 0,
6255 };
6256 builder.flatten_recursive_ir(ir, ProgramResourceId(0), ProgramResourceId(1))?;
6257 let program = Self {
6258 name: format!("vkfft_recursive_fft_program_{}", ir.logical_len),
6259 scalar: ir.scalar,
6260 resources: builder.resources,
6261 passes: builder.passes,
6262 };
6263 program.validate()?;
6264 Ok(program)
6265 }
6266
6267 pub fn one_dim_fft(ir: &OneDimFftIr) -> Result<Self> {
6268 ir.validate()?;
6269 if let OneDimFftIr::Recursive(recursive) = ir
6270 && ir.zero_pad_pass().is_none()
6271 {
6272 return Self::recursive_fft(recursive);
6273 }
6274 let elements = ir.logical_len().checked_mul(ir.batch_count()).ok_or(
6275 VkFftError::ArithmeticOverflow {
6276 operation: "generic one-dimensional FFT program element count",
6277 },
6278 )?;
6279 let external = ExternalBufferLayout {
6280 logical_len: ir.logical_len(),
6281 physical_stride: ir.logical_len(),
6282 batch_count: ir.batch_count(),
6283 element_shape: ProgramElementShape::Complex,
6284 };
6285 let external_scalar = ir.external_storage_scalar();
6286 let mut builder = RecursiveProgramBuilder {
6287 scalar: ir.scalar(),
6288 resources: vec![
6289 ProgramResource {
6290 id: ProgramResourceId(0),
6291 name: "input".to_owned(),
6292 kind: ProgramResourceKind::Input,
6293 scalar: external_scalar,
6294 elements,
6295 external_layout: Some(external),
6296 initialization: ProgramResourceInitialization::ExternalInput,
6297 },
6298 ProgramResource {
6299 id: ProgramResourceId(1),
6300 name: "output".to_owned(),
6301 kind: ProgramResourceKind::Output,
6302 scalar: external_scalar,
6303 elements,
6304 external_layout: Some(external),
6305 initialization: ProgramResourceInitialization::Zeroed,
6306 },
6307 ],
6308 passes: Vec::new(),
6309 serial: 0,
6310 };
6311 builder.flatten_one_dim(ir, ProgramResourceId(0), ProgramResourceId(1))?;
6312 let program = Self {
6313 name: format!("vkfft_1d_fft_program_{}", ir.logical_len()),
6314 scalar: ir.scalar(),
6315 resources: builder.resources,
6316 passes: builder.passes,
6317 };
6318 program.validate()?;
6319 Ok(program)
6320 }
6321
6322 pub fn nd_real_fft(ir: &NdRealFftIr) -> Result<Self> {
6323 ir.validate()?;
6324 let full_elements = ir.full_tensor_len.checked_mul(ir.batch_count).ok_or(
6325 VkFftError::ArithmeticOverflow {
6326 operation: "multidimensional real program full element count",
6327 },
6328 )?;
6329 let compact_elements = ir.compact_tensor_len.checked_mul(ir.batch_count).ok_or(
6330 VkFftError::ArithmeticOverflow {
6331 operation: "multidimensional real program compact element count",
6332 },
6333 )?;
6334 let input_tensor_len = ir.input_tensor_len();
6335 let output_tensor_len = ir.output_tensor_len();
6336 let input_layout = ExternalBufferLayout {
6337 logical_len: if ir.input_formatted_copy.is_some() {
6338 ir.input_external_layout.batch_stride
6339 } else {
6340 input_tensor_len
6341 },
6342 physical_stride: if ir.input_formatted_copy.is_some() {
6343 ir.input_external_layout.batch_stride
6344 } else {
6345 input_tensor_len
6346 },
6347 batch_count: ir.batch_count,
6348 element_shape: ProgramElementShape::Complex,
6349 };
6350 let output_layout = ExternalBufferLayout {
6351 logical_len: if ir.output_formatted_copy.is_some() {
6352 ir.output_external_layout.batch_stride
6353 } else {
6354 output_tensor_len
6355 },
6356 physical_stride: if ir.output_formatted_copy.is_some() {
6357 ir.output_external_layout.batch_stride
6358 } else {
6359 output_tensor_len
6360 },
6361 batch_count: ir.batch_count,
6362 element_shape: ProgramElementShape::Complex,
6363 };
6364 let input_elements = input_layout.physical_elements()?;
6365 let output_elements = output_layout.physical_elements()?;
6366 let mut builder = RecursiveProgramBuilder {
6367 scalar: ir.scalar,
6368 resources: vec![
6369 ProgramResource {
6370 id: ProgramResourceId(0),
6371 name: "input".to_owned(),
6372 kind: ProgramResourceKind::Input,
6373 scalar: if ir.input_boundary_compute_storage {
6374 ir.scalar
6375 } else {
6376 ir.external_scalar
6377 },
6378 elements: input_elements,
6379 external_layout: Some(input_layout),
6380 initialization: ProgramResourceInitialization::ExternalInput,
6381 },
6382 ProgramResource {
6383 id: ProgramResourceId(1),
6384 name: "output".to_owned(),
6385 kind: ProgramResourceKind::Output,
6386 scalar: if ir.output_boundary_compute_storage {
6387 ir.scalar
6388 } else {
6389 ir.external_scalar
6390 },
6391 elements: output_elements,
6392 external_layout: Some(output_layout),
6393 initialization: ProgramResourceInitialization::Zeroed,
6394 },
6395 ],
6396 passes: Vec::new(),
6397 serial: 0,
6398 };
6399
6400 let dense_input = if let Some(copy) = &ir.input_formatted_copy {
6401 let elements = input_tensor_len.checked_mul(ir.batch_count).ok_or(
6402 VkFftError::ArithmeticOverflow {
6403 operation: "multidimensional real formatted dense input size",
6404 },
6405 )?;
6406 let dense = builder.add_scratch("nd_real_formatted_input", elements);
6407 builder.passes.push(two_buffer_pass(
6408 ©.name,
6409 copy.dispatch,
6410 ProgramResourceId(0).0,
6411 dense.0,
6412 ));
6413 dense
6414 } else {
6415 ProgramResourceId(0)
6416 };
6417 let dense_output = if ir.output_formatted_copy.is_some() {
6418 let elements = output_tensor_len.checked_mul(ir.batch_count).ok_or(
6419 VkFftError::ArithmeticOverflow {
6420 operation: "multidimensional real formatted dense output size",
6421 },
6422 )?;
6423 Some(builder.add_scratch("nd_real_formatted_output", elements))
6424 } else {
6425 None
6426 };
6427
6428 match ir.kind {
6429 RealFftKind::RealToComplex => {
6430 let real_input = if let Some(zero_pad) = &ir.zero_pad_pass {
6431 let boundary = builder.add_scratch("nd_real_zero_padded_input", full_elements);
6432 builder.push_nd_zero_pad_pass(zero_pad, dense_input, boundary)?;
6433 boundary
6434 } else {
6435 dense_input
6436 };
6437 let final_output = dense_output.unwrap_or(ProgramResourceId(1));
6438 let mut natural = if ir.complex_axes.is_empty() {
6439 final_output
6440 } else {
6441 builder.add_scratch("nd_real_compact_natural", compact_elements)
6442 };
6443 builder.flatten_real_fft(&ir.real_axis, real_input, natural)?;
6444 for (axis_index, axis) in ir.complex_axes.iter().enumerate() {
6445 let packed = builder.add_scratch("nd_real_axis_input", compact_elements);
6446 let transformed = builder.add_scratch("nd_real_axis_output", compact_elements);
6447 let next = if axis_index + 1 == ir.complex_axes.len() {
6448 final_output
6449 } else {
6450 builder.add_scratch("nd_real_axis_natural", compact_elements)
6451 };
6452 builder.passes.push(two_buffer_pass(
6453 &axis.pack.name,
6454 axis.pack.dispatch,
6455 natural.0,
6456 packed.0,
6457 ));
6458 builder.flatten_one_dim(&axis.transform, packed, transformed)?;
6459 builder.passes.push(two_buffer_pass(
6460 &axis.scatter.name,
6461 axis.scatter.dispatch,
6462 transformed.0,
6463 next.0,
6464 ));
6465 natural = next;
6466 }
6467 if let Some(copy) = &ir.output_formatted_copy {
6468 builder.passes.push(two_buffer_pass(
6469 ©.name,
6470 copy.dispatch,
6471 final_output.0,
6472 ProgramResourceId(1).0,
6473 ));
6474 }
6475 }
6476 RealFftKind::ComplexToReal => {
6477 let mut natural = dense_input;
6478 for axis in &ir.complex_axes {
6479 let packed = builder.add_scratch("nd_real_axis_input", compact_elements);
6480 let transformed = builder.add_scratch("nd_real_axis_output", compact_elements);
6481 let next = builder.add_scratch("nd_real_axis_natural", compact_elements);
6482 builder.passes.push(two_buffer_pass(
6483 &axis.pack.name,
6484 axis.pack.dispatch,
6485 natural.0,
6486 packed.0,
6487 ));
6488 builder.flatten_one_dim(&axis.transform, packed, transformed)?;
6489 builder.passes.push(two_buffer_pass(
6490 &axis.scatter.name,
6491 axis.scatter.dispatch,
6492 transformed.0,
6493 next.0,
6494 ));
6495 natural = next;
6496 }
6497 let final_output = dense_output.unwrap_or(ProgramResourceId(1));
6498 let real_output = if ir.zero_pad_pass.is_some() {
6499 builder.add_scratch("nd_real_unpadded_output", full_elements)
6500 } else {
6501 final_output
6502 };
6503 builder.flatten_real_fft(&ir.real_axis, natural, real_output)?;
6504 let dense_final = if let Some(zero_pad) = &ir.zero_pad_pass {
6505 builder.push_nd_zero_pad_pass(zero_pad, real_output, final_output)?;
6506 final_output
6507 } else {
6508 real_output
6509 };
6510 if let Some(copy) = &ir.output_formatted_copy {
6511 builder.passes.push(two_buffer_pass(
6512 ©.name,
6513 copy.dispatch,
6514 dense_final.0,
6515 ProgramResourceId(1).0,
6516 ));
6517 }
6518 }
6519 }
6520 let program = Self {
6521 name: format!("vkfft_nd_real_program_{:?}_{:?}", ir.dimensions, ir.kind),
6522 scalar: ir.scalar,
6523 resources: builder.resources,
6524 passes: builder.passes,
6525 };
6526 program.validate()?;
6527 Ok(program)
6528 }
6529
6530 pub fn nd_fft(ir: &NdFftIr) -> Result<Self> {
6531 ir.validate()?;
6532 let elements =
6533 ir.tensor_len
6534 .checked_mul(ir.batch_count)
6535 .ok_or(VkFftError::ArithmeticOverflow {
6536 operation: "multidimensional FFT program element count",
6537 })?;
6538 let input_external_logical_len = if ir.has_formatted_input_tensor_strides()? {
6539 ir.input_buffer_batch_stride
6540 } else {
6541 ir.tensor_len
6542 };
6543 let output_external_logical_len = if ir.has_formatted_output_tensor_strides()? {
6544 ir.output_buffer_batch_stride
6545 } else {
6546 ir.tensor_len
6547 };
6548 let input_external = ExternalBufferLayout {
6549 logical_len: input_external_logical_len,
6550 physical_stride: ir.input_buffer_batch_stride,
6551 batch_count: ir.batch_count,
6552 element_shape: ProgramElementShape::Complex,
6553 };
6554 let output_external = ExternalBufferLayout {
6555 logical_len: output_external_logical_len,
6556 physical_stride: ir.output_buffer_batch_stride,
6557 batch_count: ir.batch_count,
6558 element_shape: ProgramElementShape::Complex,
6559 };
6560 let input_elements = input_external.physical_elements()?;
6561 let output_elements = output_external.physical_elements()?;
6562 let mut builder = RecursiveProgramBuilder {
6563 scalar: ir.scalar,
6564 resources: vec![
6565 ProgramResource {
6566 id: ProgramResourceId(0),
6567 name: "input".to_owned(),
6568 kind: ProgramResourceKind::Input,
6569 scalar: ir.external_scalar,
6570 elements: input_elements,
6571 external_layout: Some(input_external),
6572 initialization: ProgramResourceInitialization::ExternalInput,
6573 },
6574 ProgramResource {
6575 id: ProgramResourceId(1),
6576 name: "output".to_owned(),
6577 kind: ProgramResourceKind::Output,
6578 scalar: ir.external_scalar,
6579 elements: output_elements,
6580 external_layout: Some(output_external),
6581 initialization: ProgramResourceInitialization::Zeroed,
6582 },
6583 ],
6584 passes: Vec::new(),
6585 serial: 0,
6586 };
6587 let mut natural_input = ProgramResourceId(0);
6588 if let Some(copy) = &ir.input_formatted_copy {
6589 let dense = builder.add_scratch("nd_formatted_input", elements);
6590 builder.passes.push(two_buffer_pass(
6591 ©.name,
6592 copy.dispatch,
6593 natural_input.0,
6594 dense.0,
6595 ));
6596 natural_input = dense;
6597 }
6598 if let Some(pass) = &ir.zero_pad_pass
6599 && pass.operation.is_input_boundary()
6600 {
6601 let boundary = builder.add_scratch("nd_zero_pad_boundary", elements);
6602 builder.push_nd_zero_pad_pass(pass, natural_input, boundary)?;
6603 natural_input = boundary;
6604 }
6605 for (axis_index, axis) in ir.axes.iter().enumerate() {
6606 let packed_input = builder.add_scratch("nd_axis_input", elements);
6607 let packed_output = builder.add_scratch("nd_axis_output", elements);
6608 let natural_output = if axis_index + 1 == ir.axes.len() {
6609 if ir
6610 .zero_pad_pass
6611 .as_ref()
6612 .is_some_and(|pass| pass.operation.is_output_boundary())
6613 || ir.output_formatted_copy.is_some()
6614 {
6615 builder.add_scratch("nd_external_boundary", elements)
6616 } else {
6617 ProgramResourceId(1)
6618 }
6619 } else {
6620 builder.add_scratch("nd_natural_output", elements)
6621 };
6622 builder.passes.push(two_buffer_pass(
6623 &axis.pack.name,
6624 axis.pack.dispatch,
6625 natural_input.0,
6626 packed_input.0,
6627 ));
6628 builder.flatten_one_dim(&axis.transform, packed_input, packed_output)?;
6629 builder.passes.push(two_buffer_pass(
6630 &axis.scatter.name,
6631 axis.scatter.dispatch,
6632 packed_output.0,
6633 natural_output.0,
6634 ));
6635 natural_input = natural_output;
6636 }
6637 if let Some(pass) = &ir.zero_pad_pass
6638 && pass.operation.is_output_boundary()
6639 {
6640 let target = if ir.output_formatted_copy.is_some() {
6641 builder.add_scratch("nd_formatted_output", elements)
6642 } else {
6643 ProgramResourceId(1)
6644 };
6645 builder.push_nd_zero_pad_pass(pass, natural_input, target)?;
6646 natural_input = target;
6647 }
6648 if let Some(copy) = &ir.output_formatted_copy {
6649 builder.passes.push(two_buffer_pass(
6650 ©.name,
6651 copy.dispatch,
6652 natural_input.0,
6653 ProgramResourceId(1).0,
6654 ));
6655 }
6656 let program = Self {
6657 name: format!("vkfft_nd_fft_program_{:?}", ir.dimensions),
6658 scalar: ir.scalar,
6659 resources: builder.resources,
6660 passes: builder.passes,
6661 };
6662 program.validate()?;
6663 Ok(program)
6664 }
6665
6666 pub fn real_fft(ir: &RealFftIr) -> Result<Self> {
6667 ir.validate()?;
6668 let full_elements =
6669 ir.length
6670 .checked_mul(ir.batch_count)
6671 .ok_or(VkFftError::ArithmeticOverflow {
6672 operation: "real FFT program full element count",
6673 })?;
6674 let half_elements = ir.half_spectrum_len.checked_mul(ir.batch_count).ok_or(
6675 VkFftError::ArithmeticOverflow {
6676 operation: "real FFT program half-spectrum element count",
6677 },
6678 )?;
6679 let (input_elements, output_elements, input_layout, output_layout) = match ir.kind {
6680 RealFftKind::RealToComplex => (
6681 full_elements,
6682 half_elements,
6683 ExternalBufferLayout {
6684 logical_len: ir.length,
6685 physical_stride: ir.length,
6686 batch_count: ir.batch_count,
6687 element_shape: ProgramElementShape::Complex,
6688 },
6689 ExternalBufferLayout {
6690 logical_len: ir.half_spectrum_len,
6691 physical_stride: ir.half_spectrum_len,
6692 batch_count: ir.batch_count,
6693 element_shape: ProgramElementShape::Complex,
6694 },
6695 ),
6696 RealFftKind::ComplexToReal => (
6697 half_elements,
6698 full_elements,
6699 ExternalBufferLayout {
6700 logical_len: ir.half_spectrum_len,
6701 physical_stride: ir.half_spectrum_len,
6702 batch_count: ir.batch_count,
6703 element_shape: ProgramElementShape::Complex,
6704 },
6705 ExternalBufferLayout {
6706 logical_len: ir.length,
6707 physical_stride: ir.length,
6708 batch_count: ir.batch_count,
6709 element_shape: ProgramElementShape::Complex,
6710 },
6711 ),
6712 };
6713 let mut builder = RecursiveProgramBuilder {
6714 scalar: ir.scalar,
6715 resources: vec![
6716 ProgramResource {
6717 id: ProgramResourceId(0),
6718 name: "input".to_owned(),
6719 kind: ProgramResourceKind::Input,
6720 scalar: ir.input_storage_scalar,
6721 elements: input_elements,
6722 external_layout: Some(input_layout),
6723 initialization: ProgramResourceInitialization::ExternalInput,
6724 },
6725 ProgramResource {
6726 id: ProgramResourceId(1),
6727 name: "output".to_owned(),
6728 kind: ProgramResourceKind::Output,
6729 scalar: ir.output_storage_scalar,
6730 elements: output_elements,
6731 external_layout: Some(output_layout),
6732 initialization: ProgramResourceInitialization::Zeroed,
6733 },
6734 ],
6735 passes: Vec::new(),
6736 serial: 0,
6737 };
6738 let external_input = ProgramResourceId(0);
6739 let external_output = ProgramResourceId(1);
6740 let transform_input = if ir.kind == RealFftKind::RealToComplex {
6741 if let Some(zero_pad) = &ir.zero_pad_pass {
6742 let scratch = builder.add_scratch("real_zero_padded_input", full_elements);
6743 builder.passes.push(two_buffer_pass(
6744 &zero_pad.name,
6745 zero_pad.dispatch,
6746 external_input.0,
6747 scratch.0,
6748 ));
6749 scratch
6750 } else {
6751 external_input
6752 }
6753 } else {
6754 external_input
6755 };
6756 let transform_output =
6757 if ir.kind == RealFftKind::ComplexToReal && ir.zero_pad_pass.is_some() {
6758 builder.add_scratch("real_unpadded_output", full_elements)
6759 } else {
6760 external_output
6761 };
6762 builder.flatten_real_fft(ir, transform_input, transform_output)?;
6763 if ir.kind == RealFftKind::ComplexToReal
6764 && let Some(zero_pad) = &ir.zero_pad_pass
6765 {
6766 builder.passes.push(two_buffer_pass(
6767 &zero_pad.name,
6768 zero_pad.dispatch,
6769 transform_output.0,
6770 external_output.0,
6771 ));
6772 }
6773 let program = Self {
6774 name: format!(
6775 "vkfft_real_fft_program_{}_{}",
6776 ir.length,
6777 match ir.kind {
6778 RealFftKind::RealToComplex => "r2c",
6779 RealFftKind::ComplexToReal => "c2r",
6780 }
6781 ),
6782 scalar: ir.scalar,
6783 resources: builder.resources,
6784 passes: builder.passes,
6785 };
6786 program.validate()?;
6787 Ok(program)
6788 }
6789
6790 pub fn memory_plan(&self) -> Result<ProgramMemoryPlan> {
6796 self.validate()?;
6797 let mut resource_allocations = vec![ProgramAllocationId(usize::MAX); self.resources.len()];
6798 let mut allocations = Vec::<ProgramAllocation>::new();
6799
6800 let add_dedicated =
6801 |resource: &ProgramResource,
6802 kind: ProgramAllocationKind,
6803 allocations: &mut Vec<ProgramAllocation>,
6804 resource_allocations: &mut Vec<ProgramAllocationId>| {
6805 let id = ProgramAllocationId(allocations.len());
6806 allocations.push(ProgramAllocation {
6807 id,
6808 kind,
6809 scalar: resource.scalar,
6810 elements: resource.elements,
6811 resources: vec![resource.id],
6812 });
6813 resource_allocations[resource.id.0] = id;
6814 };
6815
6816 for resource in &self.resources {
6817 match resource.kind {
6818 ProgramResourceKind::Input => add_dedicated(
6819 resource,
6820 ProgramAllocationKind::ExternalInput,
6821 &mut allocations,
6822 &mut resource_allocations,
6823 ),
6824 ProgramResourceKind::Output => add_dedicated(
6825 resource,
6826 ProgramAllocationKind::ExternalOutput,
6827 &mut allocations,
6828 &mut resource_allocations,
6829 ),
6830 ProgramResourceKind::Scratch | ProgramResourceKind::LookupTable => {}
6831 }
6832 }
6833
6834 for resource in self
6835 .resources
6836 .iter()
6837 .filter(|resource| resource.kind == ProgramResourceKind::LookupTable)
6838 {
6839 let shared = self.resources[..resource.id.0].iter().find(|candidate| {
6840 candidate.kind == ProgramResourceKind::LookupTable
6841 && candidate.scalar == resource.scalar
6842 && candidate.elements == resource.elements
6843 && candidate.initialization == resource.initialization
6844 });
6845 if let Some(shared) = shared {
6846 let allocation_id = resource_allocations[shared.id.0];
6847 if allocation_id.0 == usize::MAX {
6848 return Err(VkFftError::InvalidKernelIr(
6849 "program LUT deduplication encountered an unassigned resource",
6850 ));
6851 }
6852 resource_allocations[resource.id.0] = allocation_id;
6853 allocations[allocation_id.0].resources.push(resource.id);
6854 } else {
6855 add_dedicated(
6856 resource,
6857 ProgramAllocationKind::LookupTable,
6858 &mut allocations,
6859 &mut resource_allocations,
6860 );
6861 }
6862 }
6863
6864 #[derive(Clone, Copy)]
6865 struct ScratchLifetime {
6866 resource: ProgramResourceId,
6867 first: usize,
6868 last: usize,
6869 first_access: BufferAccess,
6870 }
6871
6872 let mut scratch_lifetimes = Vec::<ScratchLifetime>::new();
6873 for resource in self
6874 .resources
6875 .iter()
6876 .filter(|resource| resource.kind == ProgramResourceKind::Scratch)
6877 {
6878 let mut first = None::<(usize, BufferAccess)>;
6879 let mut last = None::<usize>;
6880 for (pass_index, pass) in self.passes.iter().enumerate() {
6881 for binding in pass
6882 .bindings
6883 .iter()
6884 .filter(|binding| binding.resource == resource.id)
6885 {
6886 if first.is_none() {
6887 first = Some((pass_index, binding.access));
6888 }
6889 last = Some(pass_index);
6890 }
6891 }
6892 if let (Some((first, first_access)), Some(last)) = (first, last) {
6893 scratch_lifetimes.push(ScratchLifetime {
6894 resource: resource.id,
6895 first,
6896 last,
6897 first_access,
6898 });
6899 } else {
6900 add_dedicated(
6901 resource,
6902 ProgramAllocationKind::Scratch,
6903 &mut allocations,
6904 &mut resource_allocations,
6905 );
6906 }
6907 }
6908 scratch_lifetimes.sort_by_key(|lifetime| (lifetime.first, lifetime.last));
6909
6910 let mut scratch_slots = Vec::<(ProgramAllocationId, usize)>::new();
6911 for lifetime in scratch_lifetimes {
6912 let resource = &self.resources[lifetime.resource.0];
6913 let can_alias = lifetime.first_access == BufferAccess::WriteOnly
6914 && matches!(
6915 resource.initialization,
6916 ProgramResourceInitialization::Zeroed
6917 );
6918 let reusable = if can_alias {
6919 scratch_slots
6920 .iter()
6921 .enumerate()
6922 .filter(|(_, (allocation_id, last_pass))| {
6923 if *last_pass >= lifetime.first {
6924 return false;
6925 }
6926 allocations[allocation_id.0]
6927 .resources
6928 .first()
6929 .map(|resource_id| self.resources[resource_id.0].element_shape())
6930 == Some(resource.element_shape())
6931 })
6932 .min_by_key(|(_, (allocation_id, _))| {
6933 let allocation = &allocations[allocation_id.0];
6934 allocation.elements.max(resource.elements)
6935 })
6936 .map(|(index, _)| index)
6937 } else {
6938 None
6939 };
6940
6941 if let Some(slot_index) = reusable {
6942 let (allocation_id, last_pass) = &mut scratch_slots[slot_index];
6943 let allocation = &mut allocations[allocation_id.0];
6944 allocation.elements = allocation.elements.max(resource.elements);
6945 allocation.resources.push(resource.id);
6946 resource_allocations[resource.id.0] = *allocation_id;
6947 *last_pass = lifetime.last;
6948 } else {
6949 let id = ProgramAllocationId(allocations.len());
6950 allocations.push(ProgramAllocation {
6951 id,
6952 kind: ProgramAllocationKind::Scratch,
6953 scalar: resource.scalar,
6954 elements: resource.elements,
6955 resources: vec![resource.id],
6956 });
6957 resource_allocations[resource.id.0] = id;
6958 scratch_slots.push((id, lifetime.last));
6959 }
6960 }
6961
6962 if resource_allocations.iter().any(|id| id.0 == usize::MAX) {
6963 return Err(VkFftError::InvalidKernelIr(
6964 "program memory planner left a logical resource unassigned",
6965 ));
6966 }
6967 Ok(ProgramMemoryPlan {
6968 allocations,
6969 resource_allocations,
6970 })
6971 }
6972
6973 pub fn validate(&self) -> Result<()> {
6974 if self.resources.is_empty() || self.passes.is_empty() {
6975 return Err(VkFftError::InvalidKernelIr(
6976 "program requires at least one resource and one pass",
6977 ));
6978 }
6979 let mut input_count = 0usize;
6980 let mut output_count = 0usize;
6981 for (index, resource) in self.resources.iter().enumerate() {
6982 let external_mixed_scalar = matches!(
6983 resource.kind,
6984 ProgramResourceKind::Input | ProgramResourceKind::Output
6985 ) && matches!(
6986 (self.scalar, resource.scalar),
6987 (ScalarType::F64, ScalarType::F32)
6988 | (ScalarType::F32, ScalarType::F16)
6989 | (ScalarType::DoubleDouble, ScalarType::F64)
6990 );
6991 let boundary_mixed_scratch = resource.kind == ProgramResourceKind::Scratch
6992 && self.scalar == ScalarType::DoubleDouble
6993 && resource.scalar == ScalarType::F64;
6994 if resource.id.0 != index
6995 || resource.elements == 0
6996 || (resource.scalar != self.scalar
6997 && !external_mixed_scalar
6998 && !boundary_mixed_scratch)
6999 {
7000 return Err(VkFftError::InvalidKernelIr(
7001 "program resources must have contiguous IDs, non-zero sizes, and a compute-compatible scalar type",
7002 ));
7003 }
7004 match resource.kind {
7005 ProgramResourceKind::Input => input_count += 1,
7006 ProgramResourceKind::Output => output_count += 1,
7007 ProgramResourceKind::Scratch | ProgramResourceKind::LookupTable => {}
7008 }
7009 if let Some(layout) = resource.external_layout {
7010 layout.validate()?;
7011 if layout.physical_elements()? != resource.elements {
7012 return Err(VkFftError::InvalidKernelIr(
7013 "program external resource layout does not cover its physical allocation",
7014 ));
7015 }
7016 }
7017 match &resource.initialization {
7018 ProgramResourceInitialization::ExternalInput => {
7019 if resource.kind != ProgramResourceKind::Input
7020 || resource.external_layout.is_none()
7021 {
7022 return Err(VkFftError::InvalidKernelIr(
7023 "program external input initialization requires an external input resource",
7024 ));
7025 }
7026 }
7027 ProgramResourceInitialization::Zeroed => {}
7028 ProgramResourceInitialization::Complex64(values) => {
7029 if resource.kind != ProgramResourceKind::LookupTable
7030 || values.len() != resource.elements
7031 || values
7032 .iter()
7033 .any(|value| !value.re.is_finite() || !value.im.is_finite())
7034 {
7035 return Err(VkFftError::InvalidKernelIr(
7036 "program LUT initialization does not match the resource",
7037 ));
7038 }
7039 }
7040 ProgramResourceInitialization::ComplexDoubleDouble(values) => {
7041 if resource.kind != ProgramResourceKind::LookupTable
7042 || resource.scalar != ScalarType::DoubleDouble
7043 || values.len() != resource.elements
7044 || values.iter().any(|value| {
7045 !value.re.hi.is_finite()
7046 || !value.re.lo.is_finite()
7047 || !value.im.hi.is_finite()
7048 || !value.im.lo.is_finite()
7049 })
7050 {
7051 return Err(VkFftError::InvalidKernelIr(
7052 "program double-double LUT initialization does not match the resource",
7053 ));
7054 }
7055 }
7056 ProgramResourceInitialization::StockhamUnitRoots { len } => {
7057 if resource.kind != ProgramResourceKind::LookupTable
7058 || *len == 0
7059 || *len != resource.elements
7060 {
7061 return Err(VkFftError::InvalidKernelIr(
7062 "program Stockham unit-root initialization does not match the resource",
7063 ));
7064 }
7065 }
7066 }
7067 }
7068 if input_count != 1 || output_count != 1 {
7069 return Err(VkFftError::InvalidKernelIr(
7070 "program currently requires exactly one external input and one output resource",
7071 ));
7072 }
7073 let input_scalar = self.input_resource()?.scalar;
7074 let output_scalar = self.output_resource()?.scalar;
7075 let supports_external_scalar = |external_scalar| {
7076 external_scalar == self.scalar
7077 || matches!(
7078 (self.scalar, external_scalar),
7079 (ScalarType::F64, ScalarType::F32)
7080 | (ScalarType::F32, ScalarType::F16)
7081 | (ScalarType::DoubleDouble, ScalarType::F64)
7082 )
7083 };
7084 if !supports_external_scalar(input_scalar) || !supports_external_scalar(output_scalar) {
7085 return Err(VkFftError::InvalidKernelIr(
7086 "program external storage scalar is inconsistent with compute precision",
7087 ));
7088 }
7089
7090 let input_id = self
7091 .resources
7092 .iter()
7093 .find(|resource| resource.kind == ProgramResourceKind::Input)
7094 .map(|resource| resource.id)
7095 .expect("validated input resource");
7096 let output_id = self
7097 .resources
7098 .iter()
7099 .find(|resource| resource.kind == ProgramResourceKind::Output)
7100 .map(|resource| resource.id)
7101 .expect("validated output resource");
7102 let mut reads_external_input = false;
7103 let mut writes_external_output = false;
7104 for pass in &self.passes {
7105 if pass.name.is_empty()
7106 || pass.dispatch.x == 0
7107 || pass.dispatch.y == 0
7108 || pass.dispatch.z == 0
7109 || pass.bindings.len() < 2
7110 {
7111 return Err(VkFftError::InvalidKernelIr(
7112 "program pass metadata is incomplete",
7113 ));
7114 }
7115 let mut seen_bindings = Vec::with_capacity(pass.bindings.len());
7116 for pass_binding in &pass.bindings {
7117 if seen_bindings.contains(&pass_binding.binding) {
7118 return Err(VkFftError::InvalidKernelIr(
7119 "program pass has duplicate descriptor bindings",
7120 ));
7121 }
7122 seen_bindings.push(pass_binding.binding);
7123 let resource = self.resource(pass_binding.resource)?;
7124 if matches!(
7125 pass_binding.role,
7126 BufferRole::LookupTable | BufferRole::TwiddleLookupTable
7127 ) && pass_binding.access != BufferAccess::ReadOnly
7128 {
7129 return Err(VkFftError::InvalidKernelIr(
7130 "program immutable LUT bindings must be read-only",
7131 ));
7132 }
7133 if pass_binding.role == BufferRole::Auxiliary
7134 && !(pass_binding.access == BufferAccess::ReadOnly
7135 || (pass_binding.access == BufferAccess::ReadWrite
7136 && resource.kind == ProgramResourceKind::Scratch))
7137 {
7138 return Err(VkFftError::InvalidKernelIr(
7139 "program auxiliary bindings must be read-only or read-write scratch",
7140 ));
7141 }
7142 if pass_binding.resource == input_id
7143 && pass_binding.access == BufferAccess::ReadOnly
7144 {
7145 reads_external_input = true;
7146 }
7147 if pass_binding.resource == output_id
7148 && matches!(
7149 pass_binding.access,
7150 BufferAccess::WriteOnly | BufferAccess::ReadWrite
7151 )
7152 {
7153 writes_external_output = true;
7154 }
7155 if resource.kind == ProgramResourceKind::LookupTable
7156 && pass_binding.access != BufferAccess::ReadOnly
7157 {
7158 return Err(VkFftError::InvalidKernelIr(
7159 "program LUT resources are immutable",
7160 ));
7161 }
7162 }
7163 }
7164 if !reads_external_input || !writes_external_output {
7165 return Err(VkFftError::InvalidKernelIr(
7166 "program must read its external input and write its external output",
7167 ));
7168 }
7169 Ok(())
7170 }
7171
7172 pub fn resource(&self, id: ProgramResourceId) -> Result<&ProgramResource> {
7173 self.resources
7174 .get(id.0)
7175 .filter(|resource| resource.id == id)
7176 .ok_or(VkFftError::InvalidKernelIr(
7177 "program pass references an unknown resource",
7178 ))
7179 }
7180
7181 pub fn input_resource(&self) -> Result<&ProgramResource> {
7182 self.resources
7183 .iter()
7184 .find(|resource| resource.kind == ProgramResourceKind::Input)
7185 .ok_or(VkFftError::InvalidKernelIr("program has no input resource"))
7186 }
7187
7188 pub fn output_resource(&self) -> Result<&ProgramResource> {
7189 self.resources
7190 .iter()
7191 .find(|resource| resource.kind == ProgramResourceKind::Output)
7192 .ok_or(VkFftError::InvalidKernelIr(
7193 "program has no output resource",
7194 ))
7195 }
7196}
7197
7198fn add_double_double_recursive_scratch(
7199 resources: &mut Vec<ProgramResource>,
7200 serial: &mut usize,
7201 stem: &str,
7202 elements: usize,
7203) -> ProgramResourceId {
7204 *serial += 1;
7205 let id = ProgramResourceId(resources.len());
7206 resources.push(ProgramResource {
7207 id,
7208 name: format!("{stem}_{}", *serial),
7209 kind: ProgramResourceKind::Scratch,
7210 scalar: ScalarType::DoubleDouble,
7211 elements,
7212 external_layout: None,
7213 initialization: ProgramResourceInitialization::Zeroed,
7214 });
7215 id
7216}
7217
7218fn append_double_double_recursive_child(
7219 child: &ProgramIr,
7220 prefix: &str,
7221 input: ProgramResourceId,
7222 output: ProgramResourceId,
7223 resources: &mut Vec<ProgramResource>,
7224 passes: &mut Vec<ProgramPass>,
7225 allow_output_storage_override: bool,
7226) -> Result<()> {
7227 let parent_input_scalar = resources
7228 .get(input.0)
7229 .ok_or(VkFftError::InvalidKernelIr(
7230 "double-double recursive child input resource is missing",
7231 ))?
7232 .scalar;
7233 let parent_output_scalar = resources
7234 .get(output.0)
7235 .ok_or(VkFftError::InvalidKernelIr(
7236 "double-double recursive child output resource is missing",
7237 ))?
7238 .scalar;
7239 let child_output_scalar = child.output_resource()?.scalar;
7240 let output_matches = child_output_scalar == parent_output_scalar
7241 || (allow_output_storage_override
7242 && child_output_scalar == ScalarType::DoubleDouble
7243 && matches!(
7244 parent_output_scalar,
7245 ScalarType::DoubleDouble | ScalarType::F64
7246 ));
7247 if child.scalar != ScalarType::DoubleDouble
7248 || child.input_resource()?.scalar != parent_input_scalar
7249 || !output_matches
7250 {
7251 return Err(VkFftError::InvalidKernelIr(
7252 "double-double recursive leaf ProgramIr boundary storage does not match parent resources",
7253 ));
7254 }
7255 let mut resource_map = vec![None; child.resources.len()];
7256 for resource in &child.resources {
7257 let mapped = match resource.kind {
7258 ProgramResourceKind::Input => input,
7259 ProgramResourceKind::Output => output,
7260 _ => {
7261 let id = ProgramResourceId(resources.len());
7262 let mut cloned = resource.clone();
7263 cloned.id = id;
7264 cloned.name = format!("{prefix}_{}", resource.name);
7265 cloned.external_layout = None;
7266 resources.push(cloned);
7267 id
7268 }
7269 };
7270 resource_map[resource.id.0] = Some(mapped);
7271 }
7272 for child_pass in &child.passes {
7273 let mut pass = child_pass.clone();
7274 pass.name = format!("{prefix}_{}", child_pass.name);
7275 for binding in &mut pass.bindings {
7276 binding.resource = resource_map
7277 .get(binding.resource.0)
7278 .and_then(|mapped| *mapped)
7279 .ok_or(VkFftError::InvalidKernelIr(
7280 "double-double recursive child resource mapping is incomplete",
7281 ))?;
7282 }
7283 passes.push(pass);
7284 }
7285 Ok(())
7286}
7287
7288fn flatten_double_double_recursive_node(
7289 node: &DoubleDoubleRecursiveFftNodeIr,
7290 input: ProgramResourceId,
7291 output: ProgramResourceId,
7292 resources: &mut Vec<ProgramResource>,
7293 passes: &mut Vec<ProgramPass>,
7294 serial: &mut usize,
7295) -> Result<()> {
7296 node.validate()?;
7297 match node {
7298 DoubleDoubleRecursiveFftNodeIr::Stockham(ir) => {
7299 let child = ProgramIr::double_double_stockham(ir)?;
7300 *serial += 1;
7301 append_double_double_recursive_child(
7302 &child,
7303 &format!("vkfft_dd_recursive_leaf_{}", *serial),
7304 input,
7305 output,
7306 resources,
7307 passes,
7308 false,
7309 )?;
7310 }
7311 DoubleDoubleRecursiveFftNodeIr::DirectRader(ir) => {
7312 let child = ProgramIr::double_double_direct_rader(ir)?;
7313 *serial += 1;
7314 append_double_double_recursive_child(
7315 &child,
7316 &format!("vkfft_dd_recursive_leaf_{}", *serial),
7317 input,
7318 output,
7319 resources,
7320 passes,
7321 false,
7322 )?;
7323 }
7324 DoubleDoubleRecursiveFftNodeIr::FftRader(ir) => {
7325 let child = ProgramIr::double_double_fft_rader(ir)?;
7326 *serial += 1;
7327 append_double_double_recursive_child(
7328 &child,
7329 &format!("vkfft_dd_recursive_leaf_{}", *serial),
7330 input,
7331 output,
7332 resources,
7333 passes,
7334 false,
7335 )?;
7336 }
7337 DoubleDoubleRecursiveFftNodeIr::Bluestein(ir) => {
7338 let child = ProgramIr::double_double_bluestein(ir)?;
7339 *serial += 1;
7340 append_double_double_recursive_child(
7341 &child,
7342 &format!("vkfft_dd_recursive_leaf_{}", *serial),
7343 input,
7344 output,
7345 resources,
7346 passes,
7347 false,
7348 )?;
7349 }
7350 DoubleDoubleRecursiveFftNodeIr::CooleyTukey(ir) => {
7351 if let Some(fused) = ir.fused_small_direct_rader_stockham()? {
7352 let direct_lut_id = ProgramResourceId(resources.len());
7353 resources.push(ProgramResource {
7354 id: direct_lut_id,
7355 name: "double_double_composite_direct_rader_roots".to_owned(),
7356 kind: ProgramResourceKind::LookupTable,
7357 scalar: ScalarType::DoubleDouble,
7358 elements: fused.direct.table.twiddles_by_generator_power.len(),
7359 external_layout: None,
7360 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
7361 fused.direct.table.twiddles_by_generator_power.clone(),
7362 ),
7363 });
7364 let parent_roots_id = ProgramResourceId(resources.len());
7365 resources.push(ProgramResource {
7366 id: parent_roots_id,
7367 name: "double_double_composite_parent_roots".to_owned(),
7368 kind: ProgramResourceKind::LookupTable,
7369 scalar: ScalarType::DoubleDouble,
7370 elements: ir.twiddles.len(),
7371 external_layout: None,
7372 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
7373 ir.twiddles.clone(),
7374 ),
7375 });
7376 passes.push(ProgramPass {
7377 name: fused.name(),
7378 dispatch: ir.pack_right.dispatch,
7379 bindings: vec![
7380 binding(0, input.0, BufferRole::Input, BufferAccess::ReadOnly),
7381 binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
7382 ProgramPassBinding {
7383 binding: 2,
7384 resource: direct_lut_id,
7385 role: BufferRole::LookupTable,
7386 access: BufferAccess::ReadOnly,
7387 },
7388 ProgramPassBinding {
7389 binding: 3,
7390 resource: parent_roots_id,
7391 role: BufferRole::TwiddleLookupTable,
7392 access: BufferAccess::ReadOnly,
7393 },
7394 ],
7395 });
7396 return Ok(());
7397 }
7398 if let Some(fused) = ir.fused_small_fft_rader_stockham()? {
7399 let spectrum_id = ProgramResourceId(resources.len());
7400 resources.push(ProgramResource {
7401 id: spectrum_id,
7402 name: "double_double_composite_fft_rader_spectrum".to_owned(),
7403 kind: ProgramResourceKind::LookupTable,
7404 scalar: ScalarType::DoubleDouble,
7405 elements: fused.rader.kernel_spectrum.len(),
7406 external_layout: None,
7407 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
7408 fused.rader.kernel_spectrum.clone(),
7409 ),
7410 });
7411 let parent_roots_id = ProgramResourceId(resources.len());
7412 resources.push(ProgramResource {
7413 id: parent_roots_id,
7414 name: "double_double_composite_fft_rader_parent_roots".to_owned(),
7415 kind: ProgramResourceKind::LookupTable,
7416 scalar: ScalarType::DoubleDouble,
7417 elements: ir.twiddles.len(),
7418 external_layout: None,
7419 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
7420 ir.twiddles.clone(),
7421 ),
7422 });
7423 passes.push(ProgramPass {
7424 name: fused.name(),
7425 dispatch: ir.pack_right.dispatch,
7426 bindings: vec![
7427 binding(0, input.0, BufferRole::Input, BufferAccess::ReadOnly),
7428 binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
7429 ProgramPassBinding {
7430 binding: 2,
7431 resource: spectrum_id,
7432 role: BufferRole::LookupTable,
7433 access: BufferAccess::ReadOnly,
7434 },
7435 ProgramPassBinding {
7436 binding: 3,
7437 resource: parent_roots_id,
7438 role: BufferRole::TwiddleLookupTable,
7439 access: BufferAccess::ReadOnly,
7440 },
7441 ],
7442 });
7443 return Ok(());
7444 }
7445 if let Some(fused) = ir.fused_dual_fft_rader_supported()? {
7446 let right_spectrum_id = ProgramResourceId(resources.len());
7447 resources.push(ProgramResource {
7448 id: right_spectrum_id,
7449 name: "double_double_composite_dual_fft_rader_right_spectrum".to_owned(),
7450 kind: ProgramResourceKind::LookupTable,
7451 scalar: ScalarType::DoubleDouble,
7452 elements: fused.right.kernel_spectrum.len(),
7453 external_layout: None,
7454 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
7455 fused.right.kernel_spectrum.clone(),
7456 ),
7457 });
7458 let left_spectrum_id = ProgramResourceId(resources.len());
7459 resources.push(ProgramResource {
7460 id: left_spectrum_id,
7461 name: "double_double_composite_dual_fft_rader_left_spectrum".to_owned(),
7462 kind: ProgramResourceKind::LookupTable,
7463 scalar: ScalarType::DoubleDouble,
7464 elements: fused.left.kernel_spectrum.len(),
7465 external_layout: None,
7466 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
7467 fused.left.kernel_spectrum.clone(),
7468 ),
7469 });
7470 let parent_roots_id = ProgramResourceId(resources.len());
7471 resources.push(ProgramResource {
7472 id: parent_roots_id,
7473 name: "double_double_composite_dual_fft_rader_parent_roots".to_owned(),
7474 kind: ProgramResourceKind::LookupTable,
7475 scalar: ScalarType::DoubleDouble,
7476 elements: ir.twiddles.len(),
7477 external_layout: None,
7478 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
7479 ir.twiddles.clone(),
7480 ),
7481 });
7482 passes.push(ProgramPass {
7483 name: fused.name(),
7484 dispatch: ir.pack_right.dispatch,
7485 bindings: vec![
7486 binding(0, input.0, BufferRole::Input, BufferAccess::ReadOnly),
7487 binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
7488 ProgramPassBinding {
7489 binding: 2,
7490 resource: right_spectrum_id,
7491 role: BufferRole::LookupTable,
7492 access: BufferAccess::ReadOnly,
7493 },
7494 ProgramPassBinding {
7495 binding: 3,
7496 resource: left_spectrum_id,
7497 role: BufferRole::LookupTable,
7498 access: BufferAccess::ReadOnly,
7499 },
7500 ProgramPassBinding {
7501 binding: 4,
7502 resource: parent_roots_id,
7503 role: BufferRole::TwiddleLookupTable,
7504 access: BufferAccess::ReadOnly,
7505 },
7506 ],
7507 });
7508 return Ok(());
7509 }
7510 let elements = ir.logical_len.checked_mul(ir.batch_count).ok_or(
7511 VkFftError::ArithmeticOverflow {
7512 operation: "double-double recursive Cooley-Tukey program element count",
7513 },
7514 )?;
7515 let right_input = add_double_double_recursive_scratch(
7516 resources,
7517 serial,
7518 "vkfft_dd_recursive_right_input",
7519 elements,
7520 );
7521 let right_output = add_double_double_recursive_scratch(
7522 resources,
7523 serial,
7524 "vkfft_dd_recursive_right_output",
7525 elements,
7526 );
7527 let left_input = add_double_double_recursive_scratch(
7528 resources,
7529 serial,
7530 "vkfft_dd_recursive_left_input",
7531 elements,
7532 );
7533 let left_output = add_double_double_recursive_scratch(
7534 resources,
7535 serial,
7536 "vkfft_dd_recursive_left_output",
7537 elements,
7538 );
7539 passes.push(two_buffer_pass(
7540 &ir.pack_right.name,
7541 ir.pack_right.dispatch,
7542 input.0,
7543 right_input.0,
7544 ));
7545 flatten_double_double_recursive_node(
7546 &ir.right,
7547 right_input,
7548 right_output,
7549 resources,
7550 passes,
7551 serial,
7552 )?;
7553 *serial += 1;
7554 let twiddle_id = ProgramResourceId(resources.len());
7555 resources.push(ProgramResource {
7556 id: twiddle_id,
7557 name: format!("vkfft_dd_recursive_twiddles_{}", *serial),
7558 kind: ProgramResourceKind::LookupTable,
7559 scalar: ScalarType::DoubleDouble,
7560 elements: ir.twiddles.len(),
7561 external_layout: None,
7562 initialization: ProgramResourceInitialization::ComplexDoubleDouble(
7563 ir.twiddles.clone(),
7564 ),
7565 });
7566 passes.push(ProgramPass {
7567 name: ir.twiddle_transpose.name.clone(),
7568 dispatch: ir.twiddle_transpose.dispatch,
7569 bindings: vec![
7570 binding(0, right_output.0, BufferRole::Input, BufferAccess::ReadOnly),
7571 binding(1, left_input.0, BufferRole::Output, BufferAccess::WriteOnly),
7572 ProgramPassBinding {
7573 binding: 2,
7574 resource: twiddle_id,
7575 role: BufferRole::TwiddleLookupTable,
7576 access: BufferAccess::ReadOnly,
7577 },
7578 ],
7579 });
7580 flatten_double_double_recursive_node(
7581 &ir.left,
7582 left_input,
7583 left_output,
7584 resources,
7585 passes,
7586 serial,
7587 )?;
7588 let mut scatter_bindings = vec![
7589 binding(0, left_output.0, BufferRole::Input, BufferAccess::ReadOnly),
7590 binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
7591 ];
7592 let mapped_twiddle_period = match ir.scatter_output.output_modifier {
7593 DoubleDoubleCooleyTukeyOutputModifier::FourStepRight(mapping) => {
7594 Some(mapping.logical_len)
7595 }
7596 DoubleDoubleCooleyTukeyOutputModifier::FourStepThreeUpload2(mapping) => {
7597 Some(mapping.logical_len)
7598 }
7599 DoubleDoubleCooleyTukeyOutputModifier::FourStepThreeUpload1(mapping) => {
7600 let [a, b, _] = mapping.axis_split;
7601 Some(a.checked_mul(b).ok_or(VkFftError::ArithmeticOverflow {
7602 operation: "double-double recursive three-upload-1 twiddle period",
7603 })?)
7604 }
7605 _ => None,
7606 };
7607 if let Some(period) = mapped_twiddle_period {
7608 let values = crate::lut::unit_root_table_double_double(period, ir.direction)?;
7609 let twiddle_id = ProgramResourceId(resources.len());
7610 resources.push(ProgramResource {
7611 id: twiddle_id,
7612 name: format!("vkfft_dd_recursive_four_step_twiddles_{}", *serial),
7613 kind: ProgramResourceKind::LookupTable,
7614 scalar: ScalarType::DoubleDouble,
7615 elements: values.len(),
7616 external_layout: None,
7617 initialization: ProgramResourceInitialization::ComplexDoubleDouble(values),
7618 });
7619 scatter_bindings.push(ProgramPassBinding {
7620 binding: 2,
7621 resource: twiddle_id,
7622 role: BufferRole::TwiddleLookupTable,
7623 access: BufferAccess::ReadOnly,
7624 });
7625 }
7626 passes.push(ProgramPass {
7627 name: ir.scatter_output.name.clone(),
7628 dispatch: ir.scatter_output.dispatch,
7629 bindings: scatter_bindings,
7630 });
7631 }
7632 }
7633 Ok(())
7634}
7635
7636struct RecursiveProgramBuilder {
7637 scalar: ScalarType,
7638 resources: Vec<ProgramResource>,
7639 passes: Vec<ProgramPass>,
7640 serial: usize,
7641}
7642
7643impl RecursiveProgramBuilder {
7644 fn add_scratch(&mut self, stem: &str, elements: usize) -> ProgramResourceId {
7645 let id = ProgramResourceId(self.resources.len());
7646 self.serial += 1;
7647 self.resources.push(ProgramResource {
7648 id,
7649 name: format!("{stem}_{}", self.serial),
7650 kind: ProgramResourceKind::Scratch,
7651 scalar: self.scalar,
7652 elements,
7653 external_layout: None,
7654 initialization: ProgramResourceInitialization::Zeroed,
7655 });
7656 id
7657 }
7658
7659 fn add_lut(&mut self, stem: &str, values: Vec<Complex64>) -> ProgramResourceId {
7660 if let Some(existing) = self.resources.iter().find(|resource| {
7661 resource.kind == ProgramResourceKind::LookupTable
7662 && resource.scalar == self.scalar
7663 && matches!(
7664 &resource.initialization,
7665 ProgramResourceInitialization::Complex64(existing_values)
7666 if existing_values == &values
7667 )
7668 }) {
7669 return existing.id;
7670 }
7671 let id = ProgramResourceId(self.resources.len());
7672 self.serial += 1;
7673 self.resources.push(ProgramResource {
7674 id,
7675 name: format!("{stem}_{}", self.serial),
7676 kind: ProgramResourceKind::LookupTable,
7677 scalar: self.scalar,
7678 elements: values.len(),
7679 external_layout: None,
7680 initialization: ProgramResourceInitialization::Complex64(values),
7681 });
7682 id
7683 }
7684
7685 fn add_stockham_root_lut(&mut self, len: usize) -> Result<ProgramResourceId> {
7686 if len == 0 {
7687 return Err(VkFftError::InvalidKernelIr(
7688 "Stockham unit-root LUT requires a non-zero period",
7689 ));
7690 }
7691 let initialization = ProgramResourceInitialization::StockhamUnitRoots { len };
7692 if let Some(existing) = self.resources.iter().find(|resource| {
7693 resource.kind == ProgramResourceKind::LookupTable
7694 && resource.scalar == self.scalar
7695 && resource.elements == len
7696 && resource.initialization == initialization
7697 }) {
7698 return Ok(existing.id);
7699 }
7700 let id = ProgramResourceId(self.resources.len());
7701 self.serial += 1;
7702 self.resources.push(ProgramResource {
7703 id,
7704 name: format!("stockham_unit_roots_{}", self.serial),
7705 kind: ProgramResourceKind::LookupTable,
7706 scalar: self.scalar,
7707 elements: len,
7708 external_layout: None,
7709 initialization,
7710 });
7711 Ok(id)
7712 }
7713
7714 fn stockham_pass_bindings(
7715 &mut self,
7716 kernel: &KernelIr,
7717 input: ProgramResourceId,
7718 output: ProgramResourceId,
7719 lookup: Option<ProgramResourceId>,
7720 auxiliary: Option<ProgramResourceId>,
7721 ) -> Result<Vec<ProgramPassBinding>> {
7722 let twiddle = match kernel.twiddle_lut_len() {
7723 Some(len) => Some(self.add_stockham_root_lut(len)?),
7724 None => None,
7725 };
7726 kernel
7727 .bindings
7728 .iter()
7729 .map(|descriptor| {
7730 let resource = match descriptor.role {
7731 BufferRole::Input => input,
7732 BufferRole::Output => output,
7733 BufferRole::LookupTable => lookup.ok_or(VkFftError::InvalidKernelIr(
7734 "Stockham pass requires a lookup-table resource",
7735 ))?,
7736 BufferRole::TwiddleLookupTable => {
7737 twiddle.ok_or(VkFftError::InvalidKernelIr(
7738 "Stockham pass requires a twiddle-table resource",
7739 ))?
7740 }
7741 BufferRole::Auxiliary => auxiliary.ok_or(VkFftError::InvalidKernelIr(
7742 "Stockham pass requires an auxiliary resource",
7743 ))?,
7744 };
7745 Ok(binding(
7746 descriptor.binding,
7747 resource.0,
7748 descriptor.role,
7749 descriptor.access,
7750 ))
7751 })
7752 .collect()
7753 }
7754
7755 fn push_stockham_pass(
7756 &mut self,
7757 kernel: &KernelIr,
7758 input: ProgramResourceId,
7759 output: ProgramResourceId,
7760 lookup: Option<ProgramResourceId>,
7761 auxiliary: Option<ProgramResourceId>,
7762 ) -> Result<()> {
7763 let bindings = self.stockham_pass_bindings(kernel, input, output, lookup, auxiliary)?;
7764 self.passes.push(ProgramPass {
7765 name: kernel.name.clone(),
7766 dispatch: kernel.dispatch,
7767 bindings,
7768 });
7769 Ok(())
7770 }
7771
7772 fn flatten_bluestein(
7773 &mut self,
7774 pipeline: &BluesteinPipelineIr,
7775 input: ProgramResourceId,
7776 output: ProgramResourceId,
7777 ) -> Result<()> {
7778 pipeline.validate()?;
7779 let convolution_elements = pipeline
7780 .convolution_len
7781 .checked_mul(pipeline.batch_count)
7782 .ok_or(VkFftError::ArithmeticOverflow {
7783 operation: "generic program Bluestein convolution element count",
7784 })?;
7785 let scratch_a = self.add_scratch("bluestein_scratch_a", convolution_elements);
7786 let scratch_b = self.add_scratch("bluestein_scratch_b", convolution_elements);
7787 let lut = self.add_lut(
7788 "bluestein_kernel_spectrum",
7789 pipeline.kernel_spectrum()?.to_vec(),
7790 );
7791 self.passes.push(two_buffer_pass(
7792 &pipeline.preprocess.name,
7793 pipeline.preprocess.dispatch,
7794 input.0,
7795 scratch_a.0,
7796 ));
7797 self.flatten_recursive_ir(&pipeline.forward_fft, scratch_a, scratch_b)?;
7798 let convolution_output =
7799 if let Some(fused_inverse) = pipeline.fused_inverse_stockham_kernel()? {
7800 self.push_stockham_pass(&fused_inverse, scratch_b, scratch_a, Some(lut), None)?;
7801 scratch_a
7802 } else if pipeline.has_fused_recursive_inverse() {
7803 self.flatten_node_with_boundary_resources(
7804 &pipeline.inverse_fft.root,
7805 scratch_b,
7806 scratch_a,
7807 Some(lut),
7808 None,
7809 )?;
7810 scratch_a
7811 } else {
7812 self.passes.push(ProgramPass {
7813 name: pipeline.multiply.name.clone(),
7814 dispatch: pipeline.multiply.dispatch,
7815 bindings: vec![
7816 binding(0, scratch_b.0, BufferRole::Input, BufferAccess::ReadOnly),
7817 binding(1, scratch_a.0, BufferRole::Output, BufferAccess::WriteOnly),
7818 binding(2, lut.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
7819 ],
7820 });
7821 self.flatten_recursive_ir(&pipeline.inverse_fft, scratch_a, scratch_b)?;
7822 scratch_b
7823 };
7824 self.passes.push(two_buffer_pass(
7825 &pipeline.postprocess.name,
7826 pipeline.postprocess.dispatch,
7827 convolution_output.0,
7828 output.0,
7829 ));
7830 Ok(())
7831 }
7832
7833 fn push_zero_pad_pass(
7834 &mut self,
7835 pass: &crate::ZeroPadPassIr,
7836 input: ProgramResourceId,
7837 output: ProgramResourceId,
7838 ) -> Result<()> {
7839 pass.validate()?;
7840 self.passes.push(ProgramPass {
7841 name: pass.name.clone(),
7842 dispatch: pass.dispatch,
7843 bindings: vec![
7844 binding(0, input.0, BufferRole::Input, BufferAccess::ReadOnly),
7845 binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
7846 ],
7847 });
7848 Ok(())
7849 }
7850
7851 fn push_nd_zero_pad_pass(
7852 &mut self,
7853 pass: &crate::NdZeroPadPassIr,
7854 input: ProgramResourceId,
7855 output: ProgramResourceId,
7856 ) -> Result<()> {
7857 pass.validate()?;
7858 self.passes.push(ProgramPass {
7859 name: pass.name.clone(),
7860 dispatch: pass.dispatch,
7861 bindings: vec![
7862 binding(0, input.0, BufferRole::Input, BufferAccess::ReadOnly),
7863 binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
7864 ],
7865 });
7866 Ok(())
7867 }
7868
7869 fn flatten_recursive_ir(
7870 &mut self,
7871 ir: &RecursiveFftIr,
7872 input: ProgramResourceId,
7873 output: ProgramResourceId,
7874 ) -> Result<()> {
7875 if let Some(uploads) = ir.four_step_stockham_upload_kernels()? {
7876 let elements = ir.logical_len.checked_mul(ir.batch_count).ok_or(
7877 VkFftError::ArithmeticOverflow {
7878 operation: "recursive Four-step program element count",
7879 },
7880 )?;
7881 let scratch = self.add_scratch("four_step_exchange", elements);
7882 match uploads.as_slice() {
7883 [first, second] => {
7884 self.push_stockham_pass(first, input, scratch, None, None)?;
7885 self.push_stockham_pass(second, scratch, output, None, None)?;
7886 }
7887 [first, second, third] => {
7888 let output_scalar = self
7889 .resources
7890 .get(output.0)
7891 .ok_or(VkFftError::InvalidKernelIr(
7892 "recursive Four-step output resource is missing",
7893 ))?
7894 .scalar;
7895 if output_scalar == ir.scalar {
7896 self.push_stockham_pass(first, input, output, None, None)?;
7897 self.push_stockham_pass(second, output, scratch, None, None)?;
7898 self.push_stockham_pass(third, scratch, output, None, None)?;
7899 } else {
7900 let scratch_b = self.add_scratch("four_step_exchange_b", elements);
7901 self.push_stockham_pass(first, input, scratch, None, None)?;
7902 self.push_stockham_pass(second, scratch, scratch_b, None, None)?;
7903 self.push_stockham_pass(third, scratch_b, output, None, None)?;
7904 }
7905 }
7906 _ => {
7907 return Err(VkFftError::InvalidKernelIr(
7908 "Four-step program currently requires exactly two or three kernels",
7909 ));
7910 }
7911 }
7912 Ok(())
7913 } else if let Some(uploads) = ir.four_step_rader_upload_nodes()? {
7914 let elements = ir.logical_len.checked_mul(ir.batch_count).ok_or(
7915 VkFftError::ArithmeticOverflow {
7916 operation: "recursive Rader Four-step program element count",
7917 },
7918 )?;
7919 match uploads.as_slice() {
7920 [first, second] => {
7921 let scratch = self.add_scratch("rader_four_step_exchange", elements);
7922 self.flatten_node(first, input, scratch)?;
7923 self.flatten_node(second, scratch, output)
7924 }
7925 [first, second, third] => {
7926 let output_scalar = self
7927 .resources
7928 .get(output.0)
7929 .ok_or(VkFftError::InvalidKernelIr(
7930 "recursive Rader Four-step output resource is missing",
7931 ))?
7932 .scalar;
7933 if output_scalar == ir.scalar {
7934 let scratch = self.add_scratch("rader_four_step_exchange", elements);
7935 self.flatten_node(first, input, output)?;
7936 self.flatten_node(second, output, scratch)?;
7937 self.flatten_node(third, scratch, output)
7938 } else {
7939 let scratch_a = self.add_scratch("rader_four_step_exchange_a", elements);
7940 let scratch_b = self.add_scratch("rader_four_step_exchange_b", elements);
7941 self.flatten_node(first, input, scratch_a)?;
7942 self.flatten_node(second, scratch_a, scratch_b)?;
7943 self.flatten_node(third, scratch_b, output)
7944 }
7945 }
7946 _ => Err(VkFftError::InvalidKernelIr(
7947 "Rader Four-step program currently requires exactly two or three upload nodes",
7948 )),
7949 }
7950 } else {
7951 self.flatten_node(&ir.root, input, output)
7952 }
7953 }
7954
7955 fn flatten_one_dim(
7956 &mut self,
7957 ir: &OneDimFftIr,
7958 input: ProgramResourceId,
7959 output: ProgramResourceId,
7960 ) -> Result<()> {
7961 let Some(zero_pad) = ir.zero_pad_pass() else {
7962 return match ir {
7963 OneDimFftIr::Recursive(recursive) => {
7964 self.flatten_recursive_ir(recursive, input, output)
7965 }
7966 OneDimFftIr::Bluestein(pipeline) => self.flatten_bluestein(pipeline, input, output),
7967 };
7968 };
7969 let elements = ir.logical_len().checked_mul(ir.batch_count()).ok_or(
7970 VkFftError::ArithmeticOverflow {
7971 operation: "zero-padded one-dimensional program element count",
7972 },
7973 )?;
7974 let boundary = self.add_scratch("zero_pad_boundary", elements);
7975 if zero_pad.operation.is_input_boundary() {
7976 self.push_zero_pad_pass(zero_pad, input, boundary)?;
7977 match ir {
7978 OneDimFftIr::Recursive(recursive) => {
7979 self.flatten_recursive_ir(recursive, boundary, output)
7980 }
7981 OneDimFftIr::Bluestein(pipeline) => {
7982 self.flatten_bluestein(pipeline, boundary, output)
7983 }
7984 }
7985 } else {
7986 match ir {
7987 OneDimFftIr::Recursive(recursive) => {
7988 self.flatten_recursive_ir(recursive, input, boundary)?
7989 }
7990 OneDimFftIr::Bluestein(pipeline) => {
7991 self.flatten_bluestein(pipeline, input, boundary)?
7992 }
7993 }
7994 self.push_zero_pad_pass(zero_pad, boundary, output)
7995 }
7996 }
7997
7998 fn flatten_r2r(
7999 &mut self,
8000 ir: &R2rIr,
8001 input: ProgramResourceId,
8002 output: ProgramResourceId,
8003 ) -> Result<()> {
8004 ir.validate()?;
8005 if let Some(reduction) = ir.fft_reduction.as_deref() {
8006 let fft_elements = reduction.fft_len.checked_mul(reduction.batch_count).ok_or(
8007 VkFftError::ArithmeticOverflow {
8008 operation: "R2R FFT reduction program element count",
8009 },
8010 )?;
8011 let fft_input = self.add_scratch("r2r_fft_input", fft_elements);
8012 let fft_output = self.add_scratch("r2r_fft_output", fft_elements);
8013 self.passes.push(two_buffer_pass(
8014 &reduction.preprocess.name,
8015 reduction.preprocess.dispatch,
8016 input.0,
8017 fft_input.0,
8018 ));
8019 self.flatten_one_dim(&reduction.fft, fft_input, fft_output)?;
8020 self.passes.push(two_buffer_pass(
8021 &reduction.postprocess.name,
8022 reduction.postprocess.dispatch,
8023 fft_output.0,
8024 output.0,
8025 ));
8026 } else {
8027 self.passes.push(two_buffer_pass(
8028 &format!("vkfft_r2r_{:?}_{:?}", ir.effective_transform, ir.direction),
8029 ir.dispatch,
8030 input.0,
8031 output.0,
8032 ));
8033 }
8034 Ok(())
8035 }
8036
8037 fn flatten_real_fft(
8038 &mut self,
8039 ir: &RealFftIr,
8040 input: ProgramResourceId,
8041 output: ProgramResourceId,
8042 ) -> Result<()> {
8043 ir.validate()?;
8044 if let Some(fused) = ir.fused_even_input_stockham_kernel()? {
8045 let transform_elements = ir.transform_len().checked_mul(ir.batch_count).ok_or(
8046 VkFftError::ArithmeticOverflow {
8047 operation: "fused real FFT internal transform element count",
8048 },
8049 )?;
8050 let fuses_postprocess = matches!(
8051 fused.output_modifier,
8052 crate::StockhamOutputModifier::RealEvenPostprocess(_)
8053 | crate::StockhamOutputModifier::RealEvenUnpack(_)
8054 );
8055 let transform_output = if ir.postprocess.is_some() && !fuses_postprocess {
8056 self.add_scratch("real_transform", transform_elements)
8057 } else {
8058 output
8059 };
8060 let auxiliary = if matches!(
8061 fused.output_modifier,
8062 crate::StockhamOutputModifier::RealEvenPostprocess(_)
8063 ) {
8064 Some(self.add_scratch("real_postprocess_scratch", transform_elements))
8065 } else {
8066 None
8067 };
8068 self.push_stockham_pass(&fused, input, transform_output, None, auxiliary)?;
8069 if !fuses_postprocess && let Some(postprocess) = &ir.postprocess {
8070 self.passes.push(two_buffer_pass(
8071 &postprocess.name,
8072 postprocess.dispatch,
8073 transform_output.0,
8074 output.0,
8075 ));
8076 }
8077 return Ok(());
8078 }
8079 let transform_elements = ir.transform_len().checked_mul(ir.batch_count).ok_or(
8080 VkFftError::ArithmeticOverflow {
8081 operation: "real FFT program internal transform element count",
8082 },
8083 )?;
8084 if let Some(fused_recursive) = ir.fused_even_recursive_ir()? {
8085 self.flatten_node_with_boundary_resources(
8086 &fused_recursive.root,
8087 input,
8088 output,
8089 None,
8090 None,
8091 )?;
8092 return Ok(());
8093 }
8094 if let Some(fused_bluestein) = ir.fused_even_bluestein_ir()? {
8095 self.flatten_bluestein(&fused_bluestein, input, output)?;
8096 return Ok(());
8097 }
8098
8099 let transform_input = if let Some(preprocess) = &ir.preprocess {
8100 let scratch = self.add_scratch("real_preprocess", transform_elements);
8101 self.passes.push(two_buffer_pass(
8102 &preprocess.name,
8103 preprocess.dispatch,
8104 input.0,
8105 scratch.0,
8106 ));
8107 scratch
8108 } else {
8109 input
8110 };
8111 let transform_output = if ir.postprocess.is_some() {
8112 self.add_scratch("real_transform", transform_elements)
8113 } else {
8114 output
8115 };
8116 self.flatten_one_dim(&ir.transform, transform_input, transform_output)?;
8117 if let Some(postprocess) = &ir.postprocess {
8118 self.passes.push(two_buffer_pass(
8119 &postprocess.name,
8120 postprocess.dispatch,
8121 transform_output.0,
8122 output.0,
8123 ));
8124 }
8125 Ok(())
8126 }
8127
8128 fn flatten_node_with_boundary_resources(
8129 &mut self,
8130 node: &RecursiveFftNodeIr,
8131 input: ProgramResourceId,
8132 output: ProgramResourceId,
8133 lookup: Option<ProgramResourceId>,
8134 auxiliary: Option<ProgramResourceId>,
8135 ) -> Result<()> {
8136 node.validate()?;
8137 let RecursiveFftNodeIr::CooleyTukey(ir) = node else {
8138 return Err(VkFftError::UnsupportedKernelPath(
8139 "recursive boundary resources require a Cooley-Tukey root",
8140 ));
8141 };
8142 let elements =
8143 ir.logical_len
8144 .checked_mul(ir.batch_count)
8145 .ok_or(VkFftError::ArithmeticOverflow {
8146 operation: "recursive program fused Cooley-Tukey element count",
8147 })?;
8148 let right_input = self.add_scratch("recursive_right_input", elements);
8149 let right_output = self.add_scratch("recursive_right_output", elements);
8150 let left_input = self.add_scratch("recursive_left_input", elements);
8151 let left_output = self.add_scratch("recursive_left_output", elements);
8152
8153 let mut pack_bindings = vec![
8154 binding(0, input.0, BufferRole::Input, BufferAccess::ReadOnly),
8155 binding(
8156 1,
8157 right_input.0,
8158 BufferRole::Output,
8159 BufferAccess::WriteOnly,
8160 ),
8161 ];
8162 if ir.pack_right.input_modifier == CooleyTukeyInputModifier::MultiplyLookupTable {
8163 let lookup = lookup.ok_or(VkFftError::InvalidKernelIr(
8164 "recursive fused Cooley-Tukey pack requires a lookup resource",
8165 ))?;
8166 pack_bindings.push(binding(
8167 2,
8168 lookup.0,
8169 BufferRole::LookupTable,
8170 BufferAccess::ReadOnly,
8171 ));
8172 }
8173 self.passes.push(ProgramPass {
8174 name: ir.pack_right.name.clone(),
8175 dispatch: ir.pack_right.dispatch,
8176 bindings: pack_bindings,
8177 });
8178 self.flatten_node(&ir.right, right_input, right_output)?;
8179 self.passes.push(two_buffer_pass(
8180 &ir.twiddle_transpose.name,
8181 ir.twiddle_transpose.dispatch,
8182 right_output.0,
8183 left_input.0,
8184 ));
8185 self.flatten_node(&ir.left, left_input, left_output)?;
8186
8187 let mut scatter_bindings = vec![
8188 binding(0, left_output.0, BufferRole::Input, BufferAccess::ReadOnly),
8189 binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
8190 ];
8191 if matches!(
8192 ir.scatter_output.output_modifier,
8193 CooleyTukeyOutputModifier::RaderScatter(_)
8194 ) {
8195 let auxiliary = auxiliary.ok_or(VkFftError::InvalidKernelIr(
8196 "recursive fused Cooley-Tukey scatter requires an auxiliary resource",
8197 ))?;
8198 scatter_bindings.push(binding(
8199 2,
8200 auxiliary.0,
8201 BufferRole::Auxiliary,
8202 BufferAccess::ReadOnly,
8203 ));
8204 }
8205 self.passes.push(ProgramPass {
8206 name: ir.scatter_output.name.clone(),
8207 dispatch: ir.scatter_output.dispatch,
8208 bindings: scatter_bindings,
8209 });
8210 Ok(())
8211 }
8212
8213 fn flatten_node(
8214 &mut self,
8215 node: &RecursiveFftNodeIr,
8216 input: ProgramResourceId,
8217 output: ProgramResourceId,
8218 ) -> Result<()> {
8219 node.validate()?;
8220 match node {
8221 RecursiveFftNodeIr::Stockham(kernel) => {
8222 self.push_stockham_pass(kernel, input, output, None, None)?;
8223 }
8224 RecursiveFftNodeIr::DirectRader(direct) => {
8225 let lut = self.add_lut("rader_roots", direct.twiddle_lut().to_vec());
8226 self.passes.push(ProgramPass {
8227 name: direct.name.clone(),
8228 dispatch: direct.dispatch,
8229 bindings: vec![
8230 binding(0, input.0, BufferRole::Input, BufferAccess::ReadOnly),
8231 binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
8232 binding(2, lut.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
8233 ],
8234 });
8235 }
8236 RecursiveFftNodeIr::FftRader(rader) => {
8237 let convolution_elements = rader
8238 .convolution_len
8239 .checked_mul(rader.batch_count)
8240 .ok_or(VkFftError::ArithmeticOverflow {
8241 operation: "recursive program Rader convolution element count",
8242 })?;
8243 let scratch_a = self.add_scratch("rader_scratch_a", convolution_elements);
8244 let scratch_b = self.add_scratch("rader_scratch_b", convolution_elements);
8245 let lut = self.add_lut("rader_kernel_spectrum", rader.kernel_spectrum()?.to_vec());
8246 let forward_input = match rader.input_strategy {
8247 RaderFftInputStrategy::GatherReversePass => {
8248 self.passes.push(two_buffer_pass(
8249 &rader.gather.name,
8250 rader.gather.dispatch,
8251 input.0,
8252 scratch_a.0,
8253 ));
8254 scratch_a
8255 }
8256 RaderFftInputStrategy::GeneratorOrderStockham
8257 | RaderFftInputStrategy::GeneratorOrderRecursive => input,
8258 };
8259 self.flatten_one_dim(&rader.forward_fft, forward_input, scratch_b)?;
8260 if let Some(fused_inverse) = rader.fused_inverse_rader_kernel()? {
8261 self.push_stockham_pass(
8262 &fused_inverse,
8263 scratch_b,
8264 output,
8265 Some(lut),
8266 Some(input),
8267 )?;
8268 } else if rader.has_fused_recursive_inverse() {
8269 let inverse = rader
8270 .inverse_recursive()
8271 .ok_or(VkFftError::InvalidKernelIr(
8272 "recursive fused Rader program lost its recursive inverse child",
8273 ))?;
8274 self.flatten_node_with_boundary_resources(
8275 &inverse.root,
8276 scratch_b,
8277 output,
8278 Some(lut),
8279 Some(input),
8280 )?;
8281 } else {
8282 self.passes.push(ProgramPass {
8283 name: rader.multiply.name.clone(),
8284 dispatch: rader.multiply.dispatch,
8285 bindings: vec![
8286 binding(0, scratch_b.0, BufferRole::Input, BufferAccess::ReadOnly),
8287 binding(1, scratch_a.0, BufferRole::Output, BufferAccess::WriteOnly),
8288 binding(2, lut.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
8289 ],
8290 });
8291 self.flatten_one_dim(&rader.inverse_fft, scratch_a, scratch_b)?;
8292 self.passes.push(ProgramPass {
8293 name: rader.scatter.name.clone(),
8294 dispatch: rader.scatter.dispatch,
8295 bindings: vec![
8296 binding(0, scratch_b.0, BufferRole::Input, BufferAccess::ReadOnly),
8297 binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
8298 binding(2, input.0, BufferRole::Auxiliary, BufferAccess::ReadOnly),
8299 ],
8300 });
8301 }
8302 }
8303 RecursiveFftNodeIr::CooleyTukey(ir) => {
8304 if let Some(fused) = ir.fused_small_direct_rader_stockham()? {
8305 let lut = self.add_lut(
8306 "composite_direct_rader_roots",
8307 fused.direct.twiddle_lut().to_vec(),
8308 );
8309 self.passes.push(ProgramPass {
8310 name: fused.name(),
8311 dispatch: ir.pack_right.dispatch,
8312 bindings: vec![
8313 binding(0, input.0, BufferRole::Input, BufferAccess::ReadOnly),
8314 binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
8315 binding(2, lut.0, BufferRole::LookupTable, BufferAccess::ReadOnly),
8316 ],
8317 });
8318 return Ok(());
8319 }
8320 if let Some(fused) = ir.fused_small_fft_rader_stockham()? {
8321 let spectrum = self.add_lut(
8322 "composite_fft_rader_spectrum",
8323 fused.rader.kernel_spectrum()?.to_vec(),
8324 );
8325 let mut bindings = vec![
8326 binding(0, input.0, BufferRole::Input, BufferAccess::ReadOnly),
8327 binding(1, output.0, BufferRole::Output, BufferAccess::WriteOnly),
8328 binding(
8329 2,
8330 spectrum.0,
8331 BufferRole::LookupTable,
8332 BufferAccess::ReadOnly,
8333 ),
8334 ];
8335 if let Some(len) = fused.forward.twiddle_lut_len() {
8336 let twiddle = self.add_stockham_root_lut(len)?;
8337 bindings.push(binding(
8338 3,
8339 twiddle.0,
8340 BufferRole::TwiddleLookupTable,
8341 BufferAccess::ReadOnly,
8342 ));
8343 }
8344 self.passes.push(ProgramPass {
8345 name: fused.name(),
8346 dispatch: ir.pack_right.dispatch,
8347 bindings,
8348 });
8349 return Ok(());
8350 }
8351 if let Some((mapped_right, mapped_left)) = ir.fused_fft_rader_cooley_boundaries()? {
8352 let elements = ir.logical_len.checked_mul(ir.batch_count).ok_or(
8353 VkFftError::ArithmeticOverflow {
8354 operation: "recursive program fused Cooley-right element count",
8355 },
8356 )?;
8357 let right_output = self.add_scratch("recursive_right_output", elements);
8358 let mapped_right = RecursiveFftNodeIr::FftRader(Box::new(mapped_right));
8359 self.flatten_node(&mapped_right, input, right_output)?;
8360 self.push_stockham_pass(&mapped_left, right_output, output, None, None)?;
8361 return Ok(());
8362 }
8363 if let Some(mapped_left) = ir.fused_fft_rader_left_stockham()? {
8364 let elements = ir.logical_len.checked_mul(ir.batch_count).ok_or(
8365 VkFftError::ArithmeticOverflow {
8366 operation: "recursive program fused Cooley-left element count",
8367 },
8368 )?;
8369 let right_input = self.add_scratch("recursive_right_input", elements);
8370 let right_output = self.add_scratch("recursive_right_output", elements);
8371 self.passes.push(two_buffer_pass(
8372 &ir.pack_right.name,
8373 ir.pack_right.dispatch,
8374 input.0,
8375 right_input.0,
8376 ));
8377 self.flatten_node(&ir.right, right_input, right_output)?;
8378 self.push_stockham_pass(&mapped_left, right_output, output, None, None)?;
8379 return Ok(());
8380 }
8381 if ir.pack_right.input_modifier == CooleyTukeyInputModifier::MultiplyLookupTable
8382 || !matches!(
8383 ir.scatter_output.output_modifier,
8384 CooleyTukeyOutputModifier::None
8385 | CooleyTukeyOutputModifier::FourStepRight(_)
8386 | CooleyTukeyOutputModifier::FourStepLeft(_)
8387 | CooleyTukeyOutputModifier::FourStepThreeUpload2(_)
8388 | CooleyTukeyOutputModifier::FourStepThreeUpload1(_)
8389 | CooleyTukeyOutputModifier::FourStepThreeUpload0(_)
8390 )
8391 {
8392 return Err(VkFftError::InvalidKernelIr(
8393 "modified recursive Cooley-Tukey boundaries require explicit resources",
8394 ));
8395 }
8396 let elements = ir.logical_len.checked_mul(ir.batch_count).ok_or(
8397 VkFftError::ArithmeticOverflow {
8398 operation: "recursive program Cooley-Tukey element count",
8399 },
8400 )?;
8401 let right_input = self.add_scratch("recursive_right_input", elements);
8402 let right_output = self.add_scratch("recursive_right_output", elements);
8403 let left_input = self.add_scratch("recursive_left_input", elements);
8404 let left_output = self.add_scratch("recursive_left_output", elements);
8405 self.passes.push(two_buffer_pass(
8406 &ir.pack_right.name,
8407 ir.pack_right.dispatch,
8408 input.0,
8409 right_input.0,
8410 ));
8411 self.flatten_node(&ir.right, right_input, right_output)?;
8412 self.passes.push(two_buffer_pass(
8413 &ir.twiddle_transpose.name,
8414 ir.twiddle_transpose.dispatch,
8415 right_output.0,
8416 left_input.0,
8417 ));
8418 self.flatten_node(&ir.left, left_input, left_output)?;
8419 self.passes.push(two_buffer_pass(
8420 &ir.scatter_output.name,
8421 ir.scatter_output.dispatch,
8422 left_output.0,
8423 output.0,
8424 ));
8425 }
8426 }
8427 Ok(())
8428 }
8429}
8430
8431fn double_double_stockham_input_lookup_eligible(ir: &DoubleDoubleStockhamIr) -> bool {
8432 ir.sequence_len > 64
8433 && ir.external_storage == crate::PrecisionStorage::DoubleDouble
8434 && ir.zero_pad_pass.is_none()
8435 && !ir.stages.is_empty()
8436}
8437
8438fn attach_double_double_stockham_input_lookup(
8439 pass: &mut ProgramPass,
8440 lookup: ProgramResourceId,
8441) -> Result<()> {
8442 if pass.bindings.iter().any(|binding| binding.binding == 3) {
8443 return Err(VkFftError::InvalidKernelIr(
8444 "fused DD Stockham input lookup binding 3 is already occupied",
8445 ));
8446 }
8447 pass.bindings.push(ProgramPassBinding {
8448 binding: 3,
8449 resource: lookup,
8450 role: BufferRole::LookupTable,
8451 access: BufferAccess::ReadOnly,
8452 });
8453 pass.name.push_str("_input_mul_lut");
8454 Ok(())
8455}
8456
8457fn binding(
8458 binding: u32,
8459 resource: usize,
8460 role: BufferRole,
8461 access: BufferAccess,
8462) -> ProgramPassBinding {
8463 ProgramPassBinding {
8464 binding,
8465 resource: ProgramResourceId(resource),
8466 role,
8467 access,
8468 }
8469}
8470
8471fn two_buffer_pass(
8472 name: &str,
8473 dispatch: DispatchGeometry,
8474 input: usize,
8475 output: usize,
8476) -> ProgramPass {
8477 ProgramPass {
8478 name: name.to_owned(),
8479 dispatch,
8480 bindings: vec![
8481 binding(0, input, BufferRole::Input, BufferAccess::ReadOnly),
8482 binding(1, output, BufferRole::Output, BufferAccess::WriteOnly),
8483 ],
8484 }
8485}
8486
8487#[cfg(test)]
8488mod tests {
8489 use super::*;
8490 use crate::config::{
8491 Backend, DeviceProfile, Direction, GpuVendor, Precision, TransformKind, ZeroPaddingDomain,
8492 };
8493 use crate::{FftConfig, FftPlan};
8494
8495 fn device() -> DeviceProfile {
8496 DeviceProfile {
8497 shared_memory_bytes: 128 * 1024,
8498 shared_memory_pow2_bytes: 128 * 1024,
8499 ..DeviceProfile::generic(Backend::Vulkan, GpuVendor::Nvidia)
8500 }
8501 }
8502
8503 #[test]
8504 fn double_double_even_real_program_uses_half_size_child_and_root_lut() {
8505 for precision in [Precision::DoubleDouble, Precision::DoubleDoubleF64Storage] {
8506 for transform in [TransformKind::RealToComplex, TransformKind::ComplexToReal] {
8507 let plan = FftPlan::build(
8508 FftConfig::new(vec![16])
8509 .with_batch_count(2)
8510 .with_transform(transform)
8511 .with_precision(precision)
8512 .with_inverse_normalization(transform == TransformKind::ComplexToReal),
8513 )
8514 .unwrap();
8515 let ir = crate::DoubleDoubleRealFftIr::build(&plan).unwrap();
8516 assert!(ir.even_half_size);
8517 assert_eq!(ir.transform.sequence_len(), 8);
8518 assert_eq!(ir.even_roots.len(), 9);
8519 let program = ProgramIr::double_double_real(&ir).unwrap();
8520 assert_eq!(program.resources[2].elements, 16);
8521 assert_eq!(program.resources[3].elements, 16);
8522 let roots = program
8523 .resources
8524 .iter()
8525 .find(|resource| resource.name == "double_double_real_even_roots")
8526 .expect("even DD real ProgramIr must own a root LUT");
8527 assert_eq!(roots.scalar, ScalarType::DoubleDouble);
8528 assert_eq!(roots.elements, 9);
8529 let ProgramResourceInitialization::ComplexDoubleDouble(values) =
8530 &roots.initialization
8531 else {
8532 panic!("even DD real roots must preserve DD words");
8533 };
8534 assert_eq!(values, &ir.even_roots);
8535 let first = program.passes.first().unwrap();
8536 let last = program.passes.last().unwrap();
8537 match transform {
8538 TransformKind::RealToComplex => {
8539 assert_eq!(first.bindings.len(), 2);
8540 assert_eq!(last.bindings.len(), 3);
8541 assert!(last.bindings.iter().any(|binding| {
8542 binding.binding == 2
8543 && binding.resource == roots.id
8544 && binding.role == BufferRole::LookupTable
8545 }));
8546 }
8547 TransformKind::ComplexToReal => {
8548 assert_eq!(first.bindings.len(), 3);
8549 assert_eq!(last.bindings.len(), 2);
8550 assert!(first.bindings.iter().any(|binding| {
8551 binding.binding == 2
8552 && binding.resource == roots.id
8553 && binding.role == BufferRole::LookupTable
8554 }));
8555 }
8556 _ => unreachable!(),
8557 }
8558 }
8559 }
8560 }
8561
8562 #[test]
8563 fn double_double_real_program_separates_real_compact_and_full_dd_resources() {
8564 for precision in [Precision::DoubleDouble, Precision::DoubleDoubleF64Storage] {
8565 let external_scalar = if precision == Precision::DoubleDouble {
8566 ScalarType::DoubleDouble
8567 } else {
8568 ScalarType::F64
8569 };
8570 for (transform, kind, input_len, output_len) in [
8571 (
8572 TransformKind::RealToComplex,
8573 crate::RealFftKind::RealToComplex,
8574 15usize,
8575 8usize,
8576 ),
8577 (
8578 TransformKind::ComplexToReal,
8579 crate::RealFftKind::ComplexToReal,
8580 8usize,
8581 15usize,
8582 ),
8583 ] {
8584 let plan = FftPlan::build(
8585 FftConfig::new(vec![15])
8586 .with_batch_count(2)
8587 .with_transform(transform)
8588 .with_precision(precision)
8589 .with_inverse_normalization(transform == TransformKind::ComplexToReal),
8590 )
8591 .unwrap();
8592 let ir = crate::DoubleDoubleRealFftIr::build(&plan).unwrap();
8593 assert_eq!(ir.kind, kind);
8594 let child = ProgramIr::double_double_one_dim(&ir.transform).unwrap();
8595 let program = ProgramIr::double_double_real(&ir).unwrap();
8596 let input = program.input_resource().unwrap();
8597 let output = program.output_resource().unwrap();
8598 assert_eq!(input.scalar, external_scalar);
8599 assert_eq!(output.scalar, external_scalar);
8600 assert_eq!(input.external_layout.unwrap().logical_len, input_len);
8601 assert_eq!(output.external_layout.unwrap().logical_len, output_len);
8602 assert_eq!(program.resources[2].scalar, ScalarType::DoubleDouble);
8603 assert_eq!(program.resources[2].elements, 30);
8604 assert_eq!(program.resources[3].scalar, ScalarType::DoubleDouble);
8605 assert_eq!(program.resources[3].elements, 30);
8606 assert_eq!(program.passes.len(), child.passes.len() + 2);
8607 assert_eq!(
8608 program.passes.first().unwrap().bindings[0].resource,
8609 ProgramResourceId(0)
8610 );
8611 assert_eq!(
8612 program.passes.first().unwrap().bindings[1].resource,
8613 ProgramResourceId(2)
8614 );
8615 assert_eq!(
8616 program.passes.last().unwrap().bindings[0].resource,
8617 ProgramResourceId(3)
8618 );
8619 assert_eq!(
8620 program.passes.last().unwrap().bindings[1].resource,
8621 ProgramResourceId(1)
8622 );
8623 for child_resource in child.resources.iter().filter(|resource| {
8624 !matches!(
8625 resource.kind,
8626 ProgramResourceKind::Input | ProgramResourceKind::Output
8627 )
8628 }) {
8629 let name = format!("double_double_real_{}", child_resource.name);
8630 let mapped = program
8631 .resources
8632 .iter()
8633 .find(|resource| resource.name == name)
8634 .expect("DD real child resource must be cloned into parent");
8635 assert_eq!(mapped.scalar, ScalarType::DoubleDouble);
8636 assert_eq!(mapped.initialization, child_resource.initialization);
8637 assert!(mapped.external_layout.is_none());
8638 }
8639 }
8640 }
8641 }
8642
8643 #[test]
8644 fn double_double_real_padding_stays_fused_in_true_real_boundaries() {
8645 for length in [15usize, 16usize] {
8646 for precision in [Precision::DoubleDouble, Precision::DoubleDoubleF64Storage] {
8647 let external_scalar = if precision == Precision::DoubleDouble {
8648 ScalarType::DoubleDouble
8649 } else {
8650 ScalarType::F64
8651 };
8652 for transform in [TransformKind::RealToComplex, TransformKind::ComplexToReal] {
8653 let config = FftConfig::new(vec![length])
8654 .with_batch_count(7)
8655 .with_transform(transform)
8656 .with_precision(precision)
8657 .with_inverse_normalization(transform == TransformKind::ComplexToReal)
8658 .with_grouped_batch(0, 3)
8659 .unwrap()
8660 .with_zero_padding(0, 3, 7)
8661 .unwrap();
8662 let ir = crate::DoubleDoubleRealFftIr::build(&FftPlan::build(config).unwrap())
8663 .unwrap();
8664 assert!(ir.has_spatial_zero_padding());
8665 assert_eq!(ir.grouped_batch, 3);
8666 assert_eq!(ir.transform.grouped_batch(), 3);
8667 assert_eq!(ir.batch_group_count(), 3);
8668 let child = ProgramIr::double_double_one_dim(&ir.transform).unwrap();
8669 let program = ProgramIr::double_double_real(&ir).unwrap();
8670 assert!(program.passes.iter().all(|pass| pass.dispatch.x == 3));
8671 assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
8672 assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
8673 assert_eq!(program.passes.len(), child.passes.len() + 2);
8674 assert!(
8675 !program
8676 .passes
8677 .iter()
8678 .any(|pass| pass.name.contains("zero_pad"))
8679 );
8680 assert_eq!(
8681 program.passes.first().unwrap().bindings[0].resource,
8682 ProgramResourceId(0)
8683 );
8684 assert_eq!(
8685 program.passes.last().unwrap().bindings[1].resource,
8686 ProgramResourceId(1)
8687 );
8688 assert!(
8689 program
8690 .resources
8691 .iter()
8692 .skip(2)
8693 .all(|resource| { resource.scalar == ScalarType::DoubleDouble })
8694 );
8695 assert_eq!(ir.even_half_size, length.is_multiple_of(2));
8696 if ir.even_half_size {
8697 assert_eq!(ir.transform.sequence_len(), length / 2);
8698 } else {
8699 assert_eq!(ir.transform.sequence_len(), length);
8700 }
8701 }
8702 }
8703 }
8704 }
8705
8706 #[test]
8707 fn double_double_nd_real_program_keeps_scalar_full_and_complex_compact_storage() {
8708 for last_len in [7usize, 8usize] {
8709 for precision in [Precision::DoubleDouble, Precision::DoubleDoubleF64Storage] {
8710 let external_scalar = if precision == Precision::DoubleDouble {
8711 ScalarType::DoubleDouble
8712 } else {
8713 ScalarType::F64
8714 };
8715 for transform in [TransformKind::RealToComplex, TransformKind::ComplexToReal] {
8716 let plan = FftPlan::build(
8717 FftConfig::new(vec![3, last_len])
8718 .with_batch_count(7)
8719 .with_precision(precision)
8720 .with_transform(transform)
8721 .with_inverse_normalization(transform == TransformKind::ComplexToReal)
8722 .with_grouped_batch(0, 3)
8723 .unwrap()
8724 .with_grouped_batch(1, 3)
8725 .unwrap(),
8726 )
8727 .unwrap();
8728 let ir = crate::DoubleDoubleNdRealFftIr::build(&plan).unwrap();
8729 let program = ProgramIr::double_double_nd_real(&ir).unwrap();
8730 let compact_last = last_len / 2 + 1;
8731 assert_eq!(ir.real_grouped_batch, 3);
8732 assert_eq!(ir.real_axis.grouped_batch, 3);
8733 assert_eq!(ir.batch_group_count(), 3);
8734 assert_eq!(ir.complex_axes.len(), 1);
8735 assert_eq!(ir.complex_axes[0].grouped_batch, 3);
8736 assert_eq!(ir.complex_axes[0].transform.grouped_batch(), 3);
8737 assert!(program.passes.iter().any(|pass| pass.dispatch.x == 3));
8738 assert!(program.passes.iter().any(|pass| pass.dispatch.x == 7));
8739 let outer_groups = (7 * compact_last).div_ceil(3);
8740 assert!(
8741 program
8742 .passes
8743 .iter()
8744 .any(|pass| pass.dispatch.x == outer_groups as u32)
8745 );
8746 let input = program.input_resource().unwrap();
8747 let output = program.output_resource().unwrap();
8748 assert_eq!(input.scalar, external_scalar);
8749 assert_eq!(output.scalar, external_scalar);
8750 assert_eq!(
8751 input.element_shape(),
8752 if transform == TransformKind::RealToComplex {
8753 ProgramElementShape::Scalar
8754 } else {
8755 ProgramElementShape::Complex
8756 }
8757 );
8758 assert_eq!(
8759 output.element_shape(),
8760 if transform == TransformKind::RealToComplex {
8761 ProgramElementShape::Complex
8762 } else {
8763 ProgramElementShape::Scalar
8764 }
8765 );
8766 assert_eq!(program.resources[2].scalar, ScalarType::DoubleDouble);
8767 assert_eq!(
8768 program.resources[2].element_shape(),
8769 ProgramElementShape::Scalar
8770 );
8771 assert_eq!(program.resources[2].element_bytes(), 16);
8772 for resource in program.resources.iter().skip(3) {
8773 assert_eq!(resource.scalar, ScalarType::DoubleDouble);
8774 }
8775 assert!(
8776 program
8777 .passes
8778 .first()
8779 .unwrap()
8780 .name
8781 .contains("external_input_promote")
8782 );
8783 assert!(
8784 program
8785 .passes
8786 .last()
8787 .unwrap()
8788 .name
8789 .contains("external_output_finalize")
8790 );
8791 assert!(
8792 program
8793 .passes
8794 .iter()
8795 .any(|pass| pass.name.contains("last_axis_vkfft_dd_real_"))
8796 );
8797 assert!(
8798 program
8799 .passes
8800 .iter()
8801 .any(|pass| pass.name.contains("axis_0_pack"))
8802 );
8803 let memory = program.memory_plan().unwrap();
8804 for allocation in &memory.allocations {
8805 let Some(first) = allocation.resources.first().copied() else {
8806 continue;
8807 };
8808 let shape = program.resources[first.0].element_shape();
8809 assert!(
8810 allocation
8811 .resources
8812 .iter()
8813 .all(|resource| program.resources[resource.0].element_shape()
8814 == shape)
8815 );
8816 }
8817 }
8818 }
8819 }
8820 }
8821
8822 #[test]
8823 fn double_double_nd_real_padding_stays_fused_in_full_real_boundary() {
8824 for (precision, external_scalar) in [
8825 (Precision::DoubleDouble, ScalarType::DoubleDouble),
8826 (Precision::DoubleDoubleF64Storage, ScalarType::F64),
8827 ] {
8828 for transform in [TransformKind::RealToComplex, TransformKind::ComplexToReal] {
8829 let config = FftConfig::new(vec![3, 8])
8830 .with_batch_count(7)
8831 .with_precision(precision)
8832 .with_transform(transform)
8833 .with_inverse_normalization(transform == TransformKind::ComplexToReal)
8834 .with_grouped_batch(0, 3)
8835 .unwrap()
8836 .with_grouped_batch(1, 3)
8837 .unwrap()
8838 .with_zero_padding(0, 1, 2)
8839 .unwrap()
8840 .with_zero_padding(1, 2, 4)
8841 .unwrap();
8842 let ir = crate::DoubleDoubleNdRealFftIr::build(&FftPlan::build(config).unwrap())
8843 .unwrap();
8844 assert!(ir.has_spatial_zero_padding());
8845 let program = ProgramIr::double_double_nd_real(&ir).unwrap();
8846 assert_eq!(ir.real_grouped_batch, 3);
8847 assert_eq!(ir.real_axis.grouped_batch, 3);
8848 assert_eq!(ir.complex_axes[0].transform.grouped_batch(), 3);
8849 assert_eq!(ir.batch_group_count(), 3);
8850 assert!(program.passes.iter().any(|pass| pass.dispatch.x == 3));
8851 assert!(program.passes.iter().any(|pass| pass.dispatch.x == 7));
8852 assert!(program.passes.iter().any(|pass| pass.dispatch.x == 12));
8853 assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
8854 assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
8855 assert!(
8856 program
8857 .passes
8858 .first()
8859 .unwrap()
8860 .name
8861 .contains("external_input_promote")
8862 );
8863 assert!(
8864 program
8865 .passes
8866 .last()
8867 .unwrap()
8868 .name
8869 .contains("external_output_finalize")
8870 );
8871 assert!(
8872 !program
8873 .passes
8874 .iter()
8875 .any(|pass| pass.name.contains("zero_pad"))
8876 );
8877 assert_eq!(
8878 program.resources[2].element_shape(),
8879 ProgramElementShape::Scalar
8880 );
8881 assert_eq!(program.resources[2].element_bytes(), 16);
8882 assert!(program.resources.iter().skip(2).all(|resource| {
8883 resource.scalar == ScalarType::DoubleDouble
8884 && resource.kind != ProgramResourceKind::Input
8885 && resource.kind != ProgramResourceKind::Output
8886 }));
8887 }
8888 }
8889 }
8890
8891 #[test]
8892 fn double_double_nd_padding_stays_fused_in_external_pack_scatter() {
8893 for (precision, external_scalar) in [
8894 (Precision::DoubleDouble, ScalarType::DoubleDouble),
8895 (Precision::DoubleDoubleF64Storage, ScalarType::F64),
8896 ] {
8897 for direction in [Direction::Forward, Direction::Inverse] {
8898 let config = FftConfig::new(vec![3, 4])
8899 .with_batch_count(2)
8900 .with_precision(precision)
8901 .with_inverse_normalization(direction == Direction::Inverse)
8902 .with_zero_padding(1, 1, 3)
8903 .unwrap();
8904 let ir =
8905 crate::DoubleDoubleNdFftIr::build(&FftPlan::build(config).unwrap(), direction)
8906 .unwrap();
8907 assert!(ir.has_spatial_zero_padding());
8908 let program = ProgramIr::double_double_nd(&ir).unwrap();
8909 assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
8910 assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
8911 assert!(
8912 !program
8913 .passes
8914 .iter()
8915 .any(|pass| pass.name.contains("zero_pad"))
8916 );
8917 assert_eq!(
8918 program.passes.first().unwrap().bindings[0].resource,
8919 ProgramResourceId(0)
8920 );
8921 assert_eq!(
8922 program.passes.last().unwrap().bindings[1].resource,
8923 ProgramResourceId(1)
8924 );
8925 assert!(
8926 program
8927 .resources
8928 .iter()
8929 .skip(2)
8930 .all(|resource| { resource.scalar == ScalarType::DoubleDouble })
8931 );
8932 }
8933 }
8934 }
8935
8936 #[test]
8937 fn double_double_nd_program_maps_fft_rader_child_resources_and_boundaries() {
8938 for (precision, external_scalar) in [
8939 (Precision::DoubleDouble, ScalarType::DoubleDouble),
8940 (Precision::DoubleDoubleF64Storage, ScalarType::F64),
8941 ] {
8942 let plan =
8943 FftPlan::build(FftConfig::new(vec![3, 17]).with_precision(precision)).unwrap();
8944 let ir = crate::DoubleDoubleNdFftIr::build(&plan, Direction::Forward).unwrap();
8945 let crate::DoubleDoubleOneDimIr::FftRader(rader) = &ir.axes[0].transform else {
8946 panic!("DD ND [3,17] fastest axis must use FFT Rader");
8947 };
8948 assert_eq!(rader.prime, 17);
8949 let child = ProgramIr::double_double_one_dim(&ir.axes[0].transform).unwrap();
8950 let program = ProgramIr::double_double_nd(&ir).unwrap();
8951 assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
8952 assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
8953 assert!(
8954 program
8955 .resources
8956 .iter()
8957 .skip(2)
8958 .all(|resource| { resource.scalar == ScalarType::DoubleDouble })
8959 );
8960 for child_resource in child.resources.iter().filter(|resource| {
8961 !matches!(
8962 resource.kind,
8963 ProgramResourceKind::Input | ProgramResourceKind::Output
8964 )
8965 }) {
8966 let name = format!("double_double_nd_axis_1_{}", child_resource.name);
8967 let mapped = program
8968 .resources
8969 .iter()
8970 .find(|resource| resource.name == name)
8971 .expect("FFT-Rader child resource must be cloned into ND parent");
8972 assert_eq!(mapped.scalar, ScalarType::DoubleDouble);
8973 assert_eq!(mapped.elements, child_resource.elements);
8974 assert_eq!(mapped.initialization, child_resource.initialization);
8975 assert!(mapped.external_layout.is_none());
8976 }
8977 let gather = program
8978 .passes
8979 .iter()
8980 .find(|pass| pass.name.contains("axis_1_") && pass.name.contains("_gather"))
8981 .expect("ND parent must include FFT-Rader gather pass");
8982 assert_eq!(gather.bindings[0].resource, ProgramResourceId(4));
8983 let rader_scatter = program
8984 .passes
8985 .iter()
8986 .find(|pass| pass.name.contains("axis_1_") && pass.name.contains("_scatter"))
8987 .expect("ND parent must include FFT-Rader scatter pass");
8988 assert_eq!(rader_scatter.bindings[0].resource, ProgramResourceId(4));
8989 assert_eq!(rader_scatter.bindings[1].resource, ProgramResourceId(5));
8990 assert!(rader_scatter.bindings.iter().any(|binding| {
8991 binding.role == BufferRole::Auxiliary
8992 && binding.resource != ProgramResourceId(4)
8993 && binding.resource != ProgramResourceId(5)
8994 }));
8995 assert_eq!(
8996 program.passes.first().unwrap().bindings[0].resource,
8997 ProgramResourceId(0)
8998 );
8999 assert_eq!(
9000 program.passes.last().unwrap().bindings[1].resource,
9001 ProgramResourceId(1)
9002 );
9003 }
9004 }
9005
9006 #[test]
9007 fn double_double_nd_program_keeps_only_tensor_boundaries_external() {
9008 for (precision, external_scalar) in [
9009 (Precision::DoubleDouble, ScalarType::DoubleDouble),
9010 (Precision::DoubleDoubleF64Storage, ScalarType::F64),
9011 ] {
9012 let plan = FftPlan::build(
9013 FftConfig::new(vec![3, 4])
9014 .with_batch_count(2)
9015 .with_precision(precision),
9016 )
9017 .unwrap();
9018 let ir = crate::DoubleDoubleNdFftIr::build(&plan, Direction::Forward).unwrap();
9019 let program = ProgramIr::double_double_nd(&ir).unwrap();
9020 assert_eq!(program.scalar, ScalarType::DoubleDouble);
9021 assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
9022 assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
9023 let child_programs = ir
9024 .axes
9025 .iter()
9026 .map(|axis| ProgramIr::double_double_one_dim(&axis.transform).unwrap())
9027 .collect::<Vec<_>>();
9028 let expected_internal_resources = child_programs
9029 .iter()
9030 .map(|child| {
9031 child
9032 .resources
9033 .iter()
9034 .filter(|resource| {
9035 !matches!(
9036 resource.kind,
9037 ProgramResourceKind::Input | ProgramResourceKind::Output
9038 )
9039 })
9040 .count()
9041 })
9042 .sum::<usize>();
9043 assert_eq!(program.resources.len(), 6 + expected_internal_resources);
9044 for resource in &program.resources[2..] {
9045 assert_eq!(resource.scalar, ScalarType::DoubleDouble);
9046 }
9047 let expected_passes = child_programs
9048 .iter()
9049 .map(|child| child.passes.len() + 2)
9050 .sum::<usize>();
9051 assert_eq!(program.passes.len(), expected_passes);
9052 assert_eq!(
9053 program.passes.first().unwrap().bindings[0].resource,
9054 ProgramResourceId(0)
9055 );
9056 assert_eq!(
9057 program.passes.last().unwrap().bindings[1].resource,
9058 ProgramResourceId(1)
9059 );
9060 for (axis, child) in ir.axes.iter().zip(&child_programs) {
9061 for child_resource in child.resources.iter().filter(|resource| {
9062 !matches!(
9063 resource.kind,
9064 ProgramResourceKind::Input | ProgramResourceKind::Output
9065 )
9066 }) {
9067 let expected_name = format!(
9068 "double_double_nd_axis_{}_{}",
9069 axis.axis, child_resource.name
9070 );
9071 let cloned = program
9072 .resources
9073 .iter()
9074 .find(|resource| resource.name == expected_name)
9075 .expect("DD ND child internal resource must be cloned into parent");
9076 assert_eq!(cloned.kind, child_resource.kind);
9077 assert_eq!(cloned.scalar, child_resource.scalar);
9078 assert_eq!(cloned.elements, child_resource.elements);
9079 assert_eq!(cloned.initialization, child_resource.initialization);
9080 assert!(cloned.external_layout.is_none());
9081 }
9082 }
9083 }
9084 }
9085
9086 #[test]
9087 fn double_double_stockham_large_program_uses_dd_ping_pong_stages() {
9088 for (precision, external_scalar) in [
9089 (Precision::DoubleDouble, ScalarType::DoubleDouble),
9090 (Precision::DoubleDoubleF64Storage, ScalarType::F64),
9091 ] {
9092 let plan = FftPlan::build(
9093 FftConfig::new(vec![1024])
9094 .with_batch_count(2)
9095 .with_precision(precision),
9096 )
9097 .unwrap();
9098 let ir = crate::DoubleDoubleStockhamIr::build(&plan, Direction::Forward).unwrap();
9099 assert!(ir.stages.len() >= 2);
9100 let program = ProgramIr::double_double_stockham(&ir).unwrap();
9101 assert_eq!(program.resources.len(), 5);
9102 assert_eq!(program.passes.len(), ir.stages.len() + 2);
9103 assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
9104 assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
9105 assert_eq!(program.resources[2].scalar, ScalarType::DoubleDouble);
9106 assert_eq!(program.resources[3].scalar, ScalarType::DoubleDouble);
9107 assert_eq!(program.resources[4].scalar, ScalarType::DoubleDouble);
9108 assert_eq!(program.passes.first().unwrap().bindings.len(), 2);
9109 assert_eq!(program.passes.last().unwrap().bindings.len(), 2);
9110 for pass in &program.passes[1..program.passes.len() - 1] {
9111 assert_eq!(pass.bindings.len(), 3);
9112 assert_eq!(pass.bindings[2].role, BufferRole::TwiddleLookupTable);
9113 }
9114 let ProgramResourceInitialization::ComplexDoubleDouble(values) =
9115 &program.resources[4].initialization
9116 else {
9117 panic!("large DD Stockham twiddle resource must preserve DD words");
9118 };
9119 assert_eq!(values, &ir.twiddles.packed_values());
9120 }
9121 }
9122
9123 #[test]
9124 fn double_double_stockham_program_separates_compute_and_external_storage() {
9125 for (precision, external_scalar) in [
9126 (Precision::DoubleDouble, ScalarType::DoubleDouble),
9127 (Precision::DoubleDoubleF64Storage, ScalarType::F64),
9128 ] {
9129 let plan = FftPlan::build(
9130 FftConfig::new(vec![15])
9131 .with_batch_count(2)
9132 .with_precision(precision),
9133 )
9134 .unwrap();
9135 let ir = crate::DoubleDoubleStockhamIr::build(&plan, Direction::Forward).unwrap();
9136 let program = ProgramIr::double_double_stockham(&ir).unwrap();
9137 assert_eq!(program.scalar, ScalarType::DoubleDouble);
9138 assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
9139 assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
9140 assert_eq!(program.passes.len(), 1);
9141 assert_eq!(program.passes[0].bindings.len(), 3);
9142 assert_eq!(
9143 program.passes[0].bindings[2].role,
9144 BufferRole::TwiddleLookupTable
9145 );
9146
9147 let lut = program
9148 .resources
9149 .iter()
9150 .find(|resource| resource.kind == ProgramResourceKind::LookupTable)
9151 .unwrap();
9152 assert_eq!(lut.scalar, ScalarType::DoubleDouble);
9153 let ProgramResourceInitialization::ComplexDoubleDouble(values) = &lut.initialization
9154 else {
9155 panic!("double-double twiddle resource must retain DD words");
9156 };
9157 assert_eq!(values.len(), lut.elements);
9158 assert_eq!(values, &ir.twiddles.packed_values());
9159
9160 let memory = program.memory_plan().unwrap();
9161 let input_allocation = memory.allocation_for(ProgramResourceId(0)).unwrap();
9162 let output_allocation = memory.allocation_for(ProgramResourceId(1)).unwrap();
9163 let lut_allocation = memory.allocation_for(lut.id).unwrap();
9164 assert_eq!(
9165 memory.allocations[input_allocation.0].scalar,
9166 external_scalar
9167 );
9168 assert_eq!(
9169 memory.allocations[output_allocation.0].scalar,
9170 external_scalar
9171 );
9172 assert_eq!(
9173 memory.allocations[lut_allocation.0].scalar,
9174 ScalarType::DoubleDouble
9175 );
9176 assert_eq!(ScalarType::DoubleDouble.complex_bytes(), 32);
9177 assert_eq!(
9178 memory.allocations[lut_allocation.0].elements
9179 * ScalarType::DoubleDouble.complex_bytes(),
9180 lut.elements * 32
9181 );
9182 let expected_external_bytes = if external_scalar == ScalarType::DoubleDouble {
9183 15 * 2 * 32
9184 } else {
9185 15 * 2 * 16
9186 };
9187 assert_eq!(
9188 memory.allocations[input_allocation.0].elements * external_scalar.complex_bytes(),
9189 expected_external_bytes
9190 );
9191 }
9192 }
9193
9194 #[test]
9195 fn double_double_direct_rader_program_keeps_lut_at_compute_precision() {
9196 for (precision, external_scalar) in [
9197 (Precision::DoubleDouble, ScalarType::DoubleDouble),
9198 (Precision::DoubleDoubleF64Storage, ScalarType::F64),
9199 ] {
9200 let plan = FftPlan::build(
9201 FftConfig::new(vec![47])
9202 .with_batch_count(2)
9203 .with_precision(precision),
9204 )
9205 .unwrap();
9206 let ir = crate::DoubleDoubleDirectRaderIr::build(&plan, Direction::Forward).unwrap();
9207 let program = ProgramIr::double_double_direct_rader(&ir).unwrap();
9208 assert_eq!(program.scalar, ScalarType::DoubleDouble);
9209 assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
9210 assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
9211 assert_eq!(program.resources.len(), 3);
9212 assert_eq!(program.passes.len(), 1);
9213 assert_eq!(program.passes[0].bindings.len(), 3);
9214 assert_eq!(program.passes[0].bindings[2].role, BufferRole::LookupTable);
9215 let lut = &program.resources[2];
9216 assert_eq!(lut.kind, ProgramResourceKind::LookupTable);
9217 assert_eq!(lut.scalar, ScalarType::DoubleDouble);
9218 assert_eq!(lut.elements, 46);
9219 let ProgramResourceInitialization::ComplexDoubleDouble(values) = &lut.initialization
9220 else {
9221 panic!("DD direct-Rader roots must keep four-word initialization");
9222 };
9223 assert_eq!(values, &ir.table.twiddles_by_generator_power);
9224 let memory = program.memory_plan().unwrap();
9225 assert_eq!(
9226 memory.allocations[memory.allocation_for(lut.id).unwrap().0].scalar,
9227 ScalarType::DoubleDouble
9228 );
9229 }
9230 }
9231
9232 #[test]
9233 fn double_double_bluestein_program_keeps_chirp_and_convolution_at_dd_precision() {
9234 for (precision, external_scalar) in [
9235 (Precision::DoubleDouble, ScalarType::DoubleDouble),
9236 (Precision::DoubleDoubleF64Storage, ScalarType::F64),
9237 ] {
9238 let mut tuning = crate::PlannerTuning::portable();
9239 tuning.max_rader_fft_prime = 100;
9240 let plan = FftPlan::build(
9241 FftConfig::new(vec![103])
9242 .with_batch_count(7)
9243 .with_precision(precision)
9244 .with_tuning(tuning)
9245 .with_grouped_batch(0, 3)
9246 .unwrap(),
9247 )
9248 .unwrap();
9249 let ir = crate::DoubleDoubleBluesteinIr::build(&plan, Direction::Forward).unwrap();
9250 assert_eq!(ir.convolution_len, 210);
9251 assert_eq!(ir.batch_count, 7);
9252 assert_eq!(ir.grouped_batch, 3);
9253 assert_eq!(ir.batch_group_count(), 3);
9254 assert_eq!(ir.forward_fft.grouped_batch(), 3);
9255 assert_eq!(ir.inverse_fft.grouped_batch(), 3);
9256 let program = ProgramIr::double_double_bluestein(&ir).unwrap();
9257 assert_eq!(program.scalar, ScalarType::DoubleDouble);
9258 assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
9259 assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
9260 let parent_scratch = [
9261 "double_double_bluestein_forward_input",
9262 "double_double_bluestein_forward_output",
9263 "double_double_bluestein_inverse_output",
9264 ]
9265 .map(|name| {
9266 program
9267 .resources
9268 .iter()
9269 .find(|resource| resource.name == name)
9270 .unwrap_or_else(|| panic!("missing fused DD Bluestein scratch {name}"))
9271 });
9272 for resource in parent_scratch {
9273 assert_eq!(resource.kind, ProgramResourceKind::Scratch);
9274 assert_eq!(resource.scalar, ScalarType::DoubleDouble);
9275 assert_eq!(resource.elements, 210 * 7);
9276 }
9277 assert!(
9278 !program
9279 .resources
9280 .iter()
9281 .any(|resource| { resource.name == "double_double_bluestein_inverse_input" })
9282 );
9283 let forward_input = program
9284 .resources
9285 .iter()
9286 .find(|resource| resource.name == "double_double_bluestein_forward_input")
9287 .unwrap();
9288 let forward_output = program
9289 .resources
9290 .iter()
9291 .find(|resource| resource.name == "double_double_bluestein_forward_output")
9292 .unwrap();
9293 let inverse_output = program
9294 .resources
9295 .iter()
9296 .find(|resource| resource.name == "double_double_bluestein_inverse_output")
9297 .unwrap();
9298 let chirp = program
9299 .resources
9300 .iter()
9301 .find(|resource| resource.name == "double_double_bluestein_chirp")
9302 .unwrap();
9303 let kernel = program
9304 .resources
9305 .iter()
9306 .find(|resource| resource.name == "double_double_bluestein_kernel_spectrum")
9307 .unwrap();
9308 assert!(program.resources.len() > 8);
9309 assert!(program.passes.len() > 3);
9310 assert!(program.passes.iter().all(|pass| pass.dispatch.x == 3));
9311 assert!(
9312 program
9313 .passes
9314 .iter()
9315 .all(|pass| !pass.name.ends_with("_multiply"))
9316 );
9317 let inverse_stage0 = program
9318 .passes
9319 .iter()
9320 .find(|pass| {
9321 pass.name.contains("vkfft_dd_bluestein_inverse")
9322 && pass.name.contains("stage_0")
9323 })
9324 .expect("DD Bluestein inverse Stockham stage0");
9325 assert!(inverse_stage0.name.ends_with("_input_mul_lut"));
9326 assert!(inverse_stage0.bindings.iter().any(|binding| {
9327 binding.binding == 3
9328 && binding.role == BufferRole::LookupTable
9329 && binding.access == BufferAccess::ReadOnly
9330 }));
9331 let preprocess = &program.passes[0];
9332 assert_eq!(preprocess.bindings.len(), 3);
9333 assert_eq!(preprocess.bindings[0].resource, ProgramResourceId(0));
9334 assert_eq!(preprocess.bindings[1].resource, forward_input.id);
9335 assert_eq!(preprocess.bindings[2].resource, chirp.id);
9336 assert_eq!(preprocess.bindings[2].role, BufferRole::LookupTable);
9337 let postprocess = program.passes.last().unwrap();
9338 assert_eq!(postprocess.bindings[0].role, BufferRole::Input);
9339 assert_eq!(postprocess.bindings[0].resource, inverse_output.id);
9340 assert_eq!(postprocess.bindings[1].resource, ProgramResourceId(1));
9341 assert_eq!(postprocess.bindings[2].resource, chirp.id);
9342 for resource in program
9343 .resources
9344 .iter()
9345 .filter(|resource| resource.kind == ProgramResourceKind::LookupTable)
9346 {
9347 assert_eq!(resource.scalar, ScalarType::DoubleDouble);
9348 let ProgramResourceInitialization::ComplexDoubleDouble(values) =
9349 &resource.initialization
9350 else {
9351 panic!("DD Bluestein immutable tables must retain all four words");
9352 };
9353 assert_eq!(values.len(), resource.elements);
9354 assert!(
9355 values
9356 .iter()
9357 .any(|value| value.re.lo != 0.0 || value.im.lo != 0.0)
9358 );
9359 }
9360 assert_eq!(chirp.elements, 103);
9361 assert_eq!(kernel.elements, 210);
9362 let memory = program.memory_plan().unwrap();
9363 let parent_scratch_allocations =
9364 [forward_input.id, forward_output.id, inverse_output.id]
9365 .map(|resource| memory.allocation_for(resource).unwrap());
9366 assert!(
9367 parent_scratch_allocations
9368 .iter()
9369 .all(|allocation| *allocation == parent_scratch_allocations[0])
9370 );
9371 let scratch_allocations = memory
9372 .allocations
9373 .iter()
9374 .filter(|allocation| allocation.kind == ProgramAllocationKind::Scratch)
9375 .collect::<Vec<_>>();
9376 assert_eq!(scratch_allocations.len(), 4);
9377 assert!(scratch_allocations.iter().all(|allocation| {
9378 allocation.scalar == ScalarType::DoubleDouble && allocation.elements == 210 * 7
9379 }));
9380 for resource in &program.resources[2..] {
9381 assert_eq!(
9382 memory.allocations[memory.allocation_for(resource.id).unwrap().0].scalar,
9383 ScalarType::DoubleDouble
9384 );
9385 }
9386 }
9387 }
9388
9389 #[test]
9390 fn double_double_fft_rader_program_keeps_all_convolution_tables_at_dd_precision() {
9391 for (precision, external_scalar) in [
9392 (Precision::DoubleDouble, ScalarType::DoubleDouble),
9393 (Precision::DoubleDoubleF64Storage, ScalarType::F64),
9394 ] {
9395 let plan = FftPlan::build(
9396 FftConfig::new(vec![257])
9397 .with_batch_count(2)
9398 .with_precision(precision),
9399 )
9400 .unwrap();
9401 let ir = crate::DoubleDoubleFftRaderIr::build(&plan, Direction::Forward).unwrap();
9402 let program = ProgramIr::double_double_fft_rader(&ir).unwrap();
9403 assert_eq!(program.scalar, ScalarType::DoubleDouble);
9404 assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
9405 assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
9406 let child_resources = |child: &crate::DoubleDoubleBluesteinConvolutionIr| match child {
9407 crate::DoubleDoubleBluesteinConvolutionIr::Stockham(child) => {
9408 ProgramIr::double_double_stockham(child)
9409 .unwrap()
9410 .resources
9411 .len()
9412 - 2
9413 }
9414 crate::DoubleDoubleBluesteinConvolutionIr::Recursive(child) => {
9415 ProgramIr::double_double_recursive(child)
9416 .unwrap()
9417 .resources
9418 .len()
9419 - 2
9420 }
9421 crate::DoubleDoubleBluesteinConvolutionIr::Bluestein(child) => {
9422 ProgramIr::double_double_bluestein(child)
9423 .unwrap()
9424 .resources
9425 .len()
9426 - 2
9427 }
9428 };
9429 assert_eq!(
9430 program.resources.len(),
9431 5 + child_resources(&ir.forward_fft) + child_resources(&ir.inverse_fft)
9432 );
9433 assert_eq!(program.resources[2].kind, ProgramResourceKind::Scratch);
9434 assert_eq!(program.resources[3].kind, ProgramResourceKind::Scratch);
9435 assert_eq!(program.resources[2].elements, 256 * 2);
9436 assert_eq!(program.resources[3].elements, 256 * 2);
9437 let child_passes = |child: &crate::DoubleDoubleBluesteinConvolutionIr| match child {
9438 crate::DoubleDoubleBluesteinConvolutionIr::Stockham(child) => {
9439 ProgramIr::double_double_stockham_internal(child)
9440 .unwrap()
9441 .passes
9442 .len()
9443 }
9444 crate::DoubleDoubleBluesteinConvolutionIr::Recursive(child) => {
9445 ProgramIr::double_double_recursive(child)
9446 .unwrap()
9447 .passes
9448 .len()
9449 }
9450 crate::DoubleDoubleBluesteinConvolutionIr::Bluestein(child) => {
9451 ProgramIr::double_double_bluestein(child)
9452 .unwrap()
9453 .passes
9454 .len()
9455 }
9456 };
9457 let expected_passes = 2 + child_passes(&ir.forward_fft) + child_passes(&ir.inverse_fft);
9458 assert_eq!(program.passes.len(), expected_passes);
9459 assert!(
9460 program
9461 .passes
9462 .iter()
9463 .all(|pass| !pass.name.ends_with("_multiply"))
9464 );
9465 assert_eq!(program.passes[0].bindings.len(), 2);
9466 assert_eq!(program.passes[0].bindings[0].resource, ProgramResourceId(0));
9467 assert_eq!(program.passes[0].bindings[1].resource, ProgramResourceId(2));
9468 let kernel_resource = program
9469 .resources
9470 .iter()
9471 .find(|resource| resource.name == "double_double_rader_kernel_spectrum")
9472 .expect("DD FFT-Rader kernel spectrum resource");
9473 assert_eq!(kernel_resource.elements, 256);
9474 let inverse_stage0 = program
9475 .passes
9476 .iter()
9477 .find(|pass| {
9478 pass.name.contains("vkfft_dd_fft_rader_inverse")
9479 && pass.name.contains("stage_0")
9480 })
9481 .expect("DD FFT-Rader inverse Stockham stage0");
9482 assert!(inverse_stage0.name.ends_with("_input_mul_lut"));
9483 assert!(inverse_stage0.bindings.iter().any(|binding| {
9484 binding.binding == 3
9485 && binding.resource == kernel_resource.id
9486 && binding.role == BufferRole::LookupTable
9487 && binding.access == BufferAccess::ReadOnly
9488 }));
9489 assert!(program.passes.iter().any(|pass| {
9490 pass.bindings.iter().any(|binding| {
9491 binding.resource == kernel_resource.id
9492 && binding.role == BufferRole::LookupTable
9493 })
9494 }));
9495 assert!(
9496 program
9497 .passes
9498 .iter()
9499 .filter(|pass| {
9500 pass.bindings
9501 .iter()
9502 .any(|binding| binding.role == BufferRole::TwiddleLookupTable)
9503 })
9504 .count()
9505 >= 2
9506 );
9507 let scatter = program.passes.last().unwrap();
9508 assert_eq!(scatter.bindings.len(), 3);
9509 assert_eq!(scatter.bindings[0].resource, ProgramResourceId(0));
9510 assert_eq!(scatter.bindings[1].resource, ProgramResourceId(1));
9511 assert_eq!(scatter.bindings[2].role, BufferRole::Auxiliary);
9512 assert!(matches!(scatter.bindings[2].resource.0, 2 | 3));
9513 for resource in program
9514 .resources
9515 .iter()
9516 .filter(|resource| resource.kind == ProgramResourceKind::LookupTable)
9517 {
9518 assert_eq!(resource.scalar, ScalarType::DoubleDouble);
9519 let ProgramResourceInitialization::ComplexDoubleDouble(values) =
9520 &resource.initialization
9521 else {
9522 panic!("DD FFT-Rader immutable tables must keep four-word initialization");
9523 };
9524 assert_eq!(values.len(), resource.elements);
9525 assert!(
9526 values
9527 .iter()
9528 .any(|value| value.re.lo != 0.0 || value.im.lo != 0.0)
9529 );
9530 }
9531 let memory = program.memory_plan().unwrap();
9532 for resource in &program.resources[2..] {
9533 assert_eq!(
9534 memory.allocations[memory.allocation_for(resource.id).unwrap().0].scalar,
9535 ScalarType::DoubleDouble
9536 );
9537 }
9538 }
9539 }
9540
9541 #[test]
9542 fn bluestein_program_describes_ping_pong_and_lut_resources() {
9543 let mut bluestein_tuning = crate::PlannerTuning::portable();
9544 bluestein_tuning.max_rader_fft_prime = 100;
9545 let plan = FftPlan::build(
9546 FftConfig::new(vec![103])
9547 .with_batch_count(2)
9548 .with_tuning(bluestein_tuning),
9549 )
9550 .unwrap();
9551 let pipeline = BluesteinPipelineIr::build(&plan, Direction::Forward, device()).unwrap();
9552 let program = ProgramIr::bluestein(&pipeline).unwrap();
9553 assert_eq!(program.resources.len(), 5);
9554 assert_eq!(program.passes.len(), 4);
9555 assert_eq!(
9556 program.input_resource().unwrap().elements,
9557 pipeline.logical_len * 2
9558 );
9559 assert_eq!(
9560 program.output_resource().unwrap().elements,
9561 pipeline.logical_len * 2
9562 );
9563 assert_eq!(program.resources[2].kind, ProgramResourceKind::LookupTable);
9564 assert_eq!(program.resources[3].elements, pipeline.convolution_len * 2);
9565 assert_eq!(program.resources[4].elements, pipeline.convolution_len * 2);
9566 assert_eq!(program.passes[2].bindings[2].role, BufferRole::LookupTable);
9567 assert_eq!(program.passes[2].bindings[2].resource, ProgramResourceId(2));
9568 assert!(
9569 !program
9570 .passes
9571 .iter()
9572 .any(|pass| pass.name == pipeline.multiply.name)
9573 );
9574 assert_eq!(program.passes[0].bindings[0].resource, ProgramResourceId(0));
9575 assert_eq!(program.passes[0].bindings[1].resource, ProgramResourceId(3));
9576 assert_eq!(program.passes[3].bindings[0].resource, ProgramResourceId(3));
9577 assert_eq!(program.passes[3].bindings[1].resource, ProgramResourceId(1));
9578 }
9579
9580 #[test]
9581 fn grouped_bluestein_program_keeps_parent_tail_and_single_kernel_lut() {
9582 let mut tuning = crate::PlannerTuning::portable();
9583 tuning.max_rader_fft_prime = 100;
9584 let config = FftConfig::new(vec![103])
9585 .with_batch_count(7)
9586 .with_grouped_batch(0, 3)
9587 .unwrap()
9588 .with_precision(Precision::F16StorageF32Compute)
9589 .with_tuning(tuning)
9590 .with_zero_padding(0, 19, 31)
9591 .unwrap();
9592 let plan = FftPlan::build(config).unwrap();
9593 let ir = crate::OneDimFftIr::build(&plan, Direction::Forward, device()).unwrap();
9594 let crate::OneDimFftIr::Bluestein(pipeline) = &ir else {
9595 panic!("forced p103 grouped plan should use Bluestein");
9596 };
9597 assert_eq!(pipeline.grouped_batch, 3);
9598 let program = ProgramIr::one_dim_fft(&ir).unwrap();
9599 assert!(
9600 program.passes.iter().all(|pass| pass.dispatch.x == 3),
9601 "grouped Bluestein dispatches: {:?}",
9602 program
9603 .passes
9604 .iter()
9605 .map(|pass| (pass.name.as_str(), pass.dispatch.x))
9606 .collect::<Vec<_>>()
9607 );
9608 let kernel_luts = program
9609 .resources
9610 .iter()
9611 .filter(|resource| {
9612 resource.kind == ProgramResourceKind::LookupTable
9613 && resource.name.contains("bluestein_kernel_spectrum")
9614 })
9615 .collect::<Vec<_>>();
9616 assert_eq!(kernel_luts.len(), 1);
9617 assert_eq!(kernel_luts[0].elements, pipeline.convolution_len);
9618 assert_eq!(program.input_resource().unwrap().elements, 103 * 7);
9619 assert_eq!(program.output_resource().unwrap().elements, 103 * 7);
9620 assert!(
9621 program
9622 .passes
9623 .first()
9624 .unwrap()
9625 .name
9626 .contains("zero_pad_forward_input")
9627 );
9628 program.validate().unwrap();
9629 }
9630
9631 #[test]
9632 fn multidimensional_real_and_r2r_programs_use_axis_composition() {
9633 let r2c_plan = FftPlan::build(
9634 FftConfig::new(vec![3, 8]).with_transform(crate::TransformKind::RealToComplex),
9635 )
9636 .unwrap();
9637 let r2c = crate::NdRealFftIr::build(&r2c_plan, device()).unwrap();
9638 let r2c_program = ProgramIr::nd_real_fft(&r2c).unwrap();
9639 assert_eq!(r2c_program.input_resource().unwrap().elements, 24);
9640 assert_eq!(r2c_program.output_resource().unwrap().elements, 15);
9641 assert_eq!(r2c_program.passes.len(), 4);
9642
9643 let r2r_plan = FftPlan::build(
9644 FftConfig::new(vec![3, 4])
9645 .with_transform(crate::TransformKind::Dct(crate::DctType::II)),
9646 )
9647 .unwrap();
9648 let r2r = crate::NdR2rIr::build(&r2r_plan, Direction::Forward, device()).unwrap();
9649 let r2r_program = ProgramIr::nd_r2r(&r2r).unwrap();
9650 assert_eq!(r2r_program.input_resource().unwrap().elements, 12);
9651 assert_eq!(r2r_program.output_resource().unwrap().elements, 12);
9652 assert_eq!(r2r_program.passes.len(), 10);
9653 assert!(r2r_program.passes[0].name.contains("nd_r2r_pack_axis_1"));
9654 }
9655
9656 #[test]
9657 fn r2r_fft_reduction_programs_cover_all_i_through_iv_shapes() {
9658 for (transform, label, length, fft_elements) in [
9659 (
9660 crate::TransformKind::Dct(crate::DctType::I),
9661 "dct1",
9662 9usize,
9663 32usize,
9664 ),
9665 (
9666 crate::TransformKind::Dst(crate::DstType::I),
9667 "dst1",
9668 9usize,
9669 40usize,
9670 ),
9671 (
9672 crate::TransformKind::Dct(crate::DctType::II),
9673 "dct2",
9674 9usize,
9675 18usize,
9676 ),
9677 (
9678 crate::TransformKind::Dct(crate::DctType::III),
9679 "dct3",
9680 9usize,
9681 18usize,
9682 ),
9683 (
9684 crate::TransformKind::Dst(crate::DstType::II),
9685 "dst2",
9686 9usize,
9687 18usize,
9688 ),
9689 (
9690 crate::TransformKind::Dst(crate::DstType::III),
9691 "dst3",
9692 9usize,
9693 18usize,
9694 ),
9695 (
9696 crate::TransformKind::Dct(crate::DctType::IV),
9697 "dct4_even",
9698 8usize,
9699 8usize,
9700 ),
9701 (
9702 crate::TransformKind::Dst(crate::DstType::IV),
9703 "dst4_even",
9704 8usize,
9705 8usize,
9706 ),
9707 (
9708 crate::TransformKind::Dct(crate::DctType::IV),
9709 "dct4_odd",
9710 9usize,
9711 36usize,
9712 ),
9713 (
9714 crate::TransformKind::Dst(crate::DstType::IV),
9715 "dst4_odd",
9716 9usize,
9717 36usize,
9718 ),
9719 ] {
9720 let plan = FftPlan::build(
9721 FftConfig::new(vec![length])
9722 .with_batch_count(2)
9723 .with_transform(transform),
9724 )
9725 .unwrap();
9726 let ir = crate::R2rIr::build(&plan, Direction::Forward, device()).unwrap();
9727 assert!(ir.fft_reduction.is_some());
9728 let program = ProgramIr::r2r(&ir).unwrap();
9729 assert_eq!(program.resources.len(), 4);
9730 assert_eq!(program.passes.len(), 3);
9731 assert_eq!(program.input_resource().unwrap().elements, 2 * length);
9732 assert_eq!(program.output_resource().unwrap().elements, 2 * length);
9733 assert_eq!(program.resources[2].elements, fft_elements);
9734 assert_eq!(program.resources[3].elements, fft_elements);
9735 assert!(
9736 program
9737 .passes
9738 .first()
9739 .unwrap()
9740 .name
9741 .contains(&format!("r2r_fft_{label}_pre"))
9742 );
9743 assert!(
9744 program
9745 .passes
9746 .last()
9747 .unwrap()
9748 .name
9749 .contains(&format!("r2r_fft_{label}_post"))
9750 );
9751 }
9752 }
9753
9754 #[test]
9755 fn one_dim_zero_padding_wraps_forward_and_inverse_at_external_boundaries() {
9756 let config = FftConfig::new(vec![64])
9757 .with_zero_padding(0, 32, 64)
9758 .unwrap();
9759 let plan = FftPlan::build(config).unwrap();
9760 let forward = OneDimFftIr::build(&plan, Direction::Forward, device()).unwrap();
9761 let forward_program = ProgramIr::one_dim_fft(&forward).unwrap();
9762 assert!(
9763 forward_program
9764 .passes
9765 .first()
9766 .unwrap()
9767 .name
9768 .contains("zero_pad_forward_input")
9769 );
9770 assert_eq!(
9771 forward_program.passes.first().unwrap().bindings[0].resource,
9772 ProgramResourceId(0)
9773 );
9774 assert_eq!(
9775 forward_program
9776 .resource(forward_program.passes.first().unwrap().bindings[1].resource)
9777 .unwrap()
9778 .kind,
9779 ProgramResourceKind::Scratch
9780 );
9781
9782 let config = FftConfig::new(vec![64])
9783 .with_inverse_normalization(true)
9784 .with_zero_padding(0, 32, 64)
9785 .unwrap();
9786 let plan = FftPlan::build(config).unwrap();
9787 let inverse = OneDimFftIr::build(&plan, Direction::Inverse, device()).unwrap();
9788 let inverse_program = ProgramIr::one_dim_fft(&inverse).unwrap();
9789 assert!(
9790 inverse_program
9791 .passes
9792 .last()
9793 .unwrap()
9794 .name
9795 .contains("zero_pad_inverse_output")
9796 );
9797 assert_eq!(
9798 inverse_program.passes.last().unwrap().bindings[1].resource,
9799 ProgramResourceId(1)
9800 );
9801 assert_eq!(
9802 inverse_program
9803 .resource(inverse_program.passes.last().unwrap().bindings[0].resource)
9804 .unwrap()
9805 .kind,
9806 ProgramResourceKind::Scratch
9807 );
9808 }
9809
9810 #[test]
9811 fn frequency_zero_padding_reverses_mixed_storage_program_boundary_order() {
9812 let build = |direction| {
9813 let config = FftConfig::new(vec![64])
9814 .with_batch_count(2)
9815 .with_precision(Precision::F16StorageF32Compute)
9816 .with_inverse_normalization(direction == Direction::Inverse)
9817 .with_zero_padding(0, 16, 32)
9818 .unwrap()
9819 .with_zero_padding_domain(ZeroPaddingDomain::Frequency);
9820 let plan = FftPlan::build(config).unwrap();
9821 let ir = OneDimFftIr::build(&plan, direction, device()).unwrap();
9822 ProgramIr::one_dim_fft(&ir).unwrap()
9823 };
9824
9825 let forward = build(Direction::Forward);
9826 let forward_pad = forward.passes.last().unwrap();
9827 assert!(forward_pad.name.contains("zero_pad_forward_output"));
9828 assert_eq!(forward_pad.bindings[1].resource, ProgramResourceId(1));
9829 assert_eq!(
9830 forward
9831 .resource(forward_pad.bindings[0].resource)
9832 .unwrap()
9833 .kind,
9834 ProgramResourceKind::Scratch
9835 );
9836 assert_eq!(
9837 forward
9838 .resource(forward_pad.bindings[0].resource)
9839 .unwrap()
9840 .scalar,
9841 ScalarType::F32
9842 );
9843 assert_eq!(forward.output_resource().unwrap().scalar, ScalarType::F16);
9844
9845 let inverse = build(Direction::Inverse);
9846 let inverse_pad = inverse.passes.first().unwrap();
9847 assert!(inverse_pad.name.contains("zero_pad_inverse_input"));
9848 assert_eq!(inverse_pad.bindings[0].resource, ProgramResourceId(0));
9849 assert_eq!(inverse.input_resource().unwrap().scalar, ScalarType::F16);
9850 assert_eq!(
9851 inverse
9852 .resource(inverse_pad.bindings[1].resource)
9853 .unwrap()
9854 .kind,
9855 ProgramResourceKind::Scratch
9856 );
9857 assert_eq!(
9858 inverse
9859 .resource(inverse_pad.bindings[1].resource)
9860 .unwrap()
9861 .scalar,
9862 ScalarType::F32
9863 );
9864 }
9865
9866 #[test]
9867 fn generic_one_dim_program_covers_bluestein_with_compact_external_buffers() {
9868 let mut bluestein_tuning = crate::PlannerTuning::portable();
9869 bluestein_tuning.max_rader_fft_prime = 100;
9870 let plan = FftPlan::build(
9871 FftConfig::new(vec![103])
9872 .with_batch_count(2)
9873 .with_tuning(bluestein_tuning),
9874 )
9875 .unwrap();
9876 let ir = crate::OneDimFftIr::build(&plan, Direction::Forward, device()).unwrap();
9877 assert!(matches!(ir, crate::OneDimFftIr::Bluestein(_)));
9878 let program = ProgramIr::one_dim_fft(&ir).unwrap();
9879 assert_eq!(program.input_resource().unwrap().elements, 206);
9880 assert_eq!(program.output_resource().unwrap().elements, 206);
9881 assert_eq!(program.passes.len(), 4);
9882 assert_eq!(program.passes[2].bindings.len(), 3);
9883 assert_eq!(program.passes[2].bindings[2].role, BufferRole::LookupTable);
9884 assert!(
9885 program
9886 .resources
9887 .iter()
9888 .any(|resource| resource.kind == ProgramResourceKind::LookupTable)
9889 );
9890 let memory = program.memory_plan().unwrap();
9891 assert!(memory.allocations.len() <= program.resources.len());
9892 }
9893
9894 #[test]
9895 fn mixed_program_flattens_outer_and_nested_fft_rader_passes() {
9896 for length in [34usize, 514] {
9897 let plan = FftPlan::build(FftConfig::new(vec![length]).with_batch_count(2)).unwrap();
9898 let ir =
9899 crate::MixedRaderStockhamIr::build(&plan, Direction::Forward, device()).unwrap();
9900 let program = ProgramIr::mixed_rader_stockham(&ir).unwrap();
9901 assert_eq!(program.resources.len(), 9);
9902 assert_eq!(program.passes.len(), 6);
9903 assert_eq!(program.input_resource().unwrap().elements, 2 * length);
9904 assert_eq!(program.output_resource().unwrap().elements, 2 * length);
9905 assert_eq!(program.resources[5].kind, ProgramResourceKind::LookupTable);
9906 assert_eq!(program.passes[2].bindings[2].role, BufferRole::LookupTable);
9907 assert_eq!(program.passes[2].bindings[2].resource, ProgramResourceId(5));
9908 assert_eq!(program.passes[2].bindings[3].role, BufferRole::Auxiliary);
9909 assert_eq!(program.passes[2].bindings[3].resource, ProgramResourceId(1));
9910 if length == 514 {
9911 assert_eq!(program.passes[1].dispatch.x, 2);
9914 assert_eq!(program.passes[2].dispatch.x, 2);
9915 }
9916 }
9917 }
9918
9919 #[test]
9920 fn p257_rader_transpose_fuses_generator_gather_into_forward_stockham() {
9921 let plan = FftPlan::build(FftConfig::new(vec![2056])).unwrap();
9922 let ir = crate::MixedRaderStockhamIr::build(&plan, Direction::Forward, device()).unwrap();
9923 let MixedPrimeRaderIr::FftConvolution(rader) = &ir.prime_stage else {
9924 panic!("2056 should use FFT-convolution Rader");
9925 };
9926 assert_eq!(
9927 rader.input_strategy,
9928 RaderFftInputStrategy::GeneratorOrderStockham
9929 );
9930 let program = ProgramIr::mixed_rader_stockham(&ir).unwrap();
9931 assert_eq!(program.resources.len(), 9);
9932 assert_eq!(program.passes.len(), 6);
9933 assert!(
9934 !program
9935 .passes
9936 .iter()
9937 .any(|pass| pass.name == rader.gather.name)
9938 );
9939 assert_eq!(program.passes[1].bindings[0].resource, ProgramResourceId(1));
9940 assert_eq!(program.passes[1].bindings[0].role, BufferRole::Input);
9941 assert_eq!(program.passes[2].bindings[2].role, BufferRole::LookupTable);
9942 assert_eq!(program.passes[2].bindings[2].resource, ProgramResourceId(5));
9943 assert_eq!(program.passes[2].bindings[3].role, BufferRole::Auxiliary);
9944 assert_eq!(program.passes[2].bindings[3].resource, ProgramResourceId(1));
9945 assert_eq!(program.passes[2].bindings[1].resource, ProgramResourceId(4));
9946 assert!(
9947 !program
9948 .passes
9949 .iter()
9950 .any(|pass| pass.name == rader.scatter.name)
9951 );
9952 }
9953
9954 #[test]
9955 fn mixed_program_flattens_direct_rader_stage() {
9956 let plan = FftPlan::build(FftConfig::new(vec![94]).with_batch_count(2)).unwrap();
9957 let ir = crate::MixedRaderStockhamIr::build(&plan, Direction::Forward, device()).unwrap();
9958 let program = ProgramIr::mixed_rader_stockham(&ir).unwrap();
9959 assert_eq!(program.resources.len(), 7);
9960 assert_eq!(program.passes.len(), 5);
9961 assert_eq!(program.resources[3].kind, ProgramResourceKind::LookupTable);
9962 assert_eq!(program.passes[1].bindings[2].resource, ProgramResourceId(3));
9963 }
9964
9965 #[test]
9966 fn fft_rader_program_preserves_prime_input_for_scatter() {
9967 let plan = FftPlan::build(FftConfig::new(vec![17]).with_batch_count(2)).unwrap();
9968 let pipeline = RaderFftPipelineIr::build(&plan, Direction::Forward, device()).unwrap();
9969 let program = ProgramIr::rader_fft(&pipeline).unwrap();
9970 assert_eq!(program.resources.len(), 5);
9971 assert_eq!(program.passes.len(), 2);
9972 assert_eq!(
9973 pipeline.input_strategy,
9974 RaderFftInputStrategy::GeneratorOrderStockham
9975 );
9976 let fused_inverse = &program.passes[1];
9977 assert_eq!(fused_inverse.bindings[2].role, BufferRole::LookupTable);
9978 assert_eq!(fused_inverse.bindings[2].resource, ProgramResourceId(4));
9979 assert_eq!(fused_inverse.bindings[3].role, BufferRole::Auxiliary);
9980 assert_eq!(fused_inverse.bindings[3].resource, ProgramResourceId(0));
9981 assert_eq!(fused_inverse.bindings[1].resource, ProgramResourceId(3));
9982 assert!(
9983 !program
9984 .passes
9985 .iter()
9986 .any(|pass| pass.name == pipeline.scatter.name)
9987 );
9988 assert_eq!(program.resources[4].kind, ProgramResourceKind::LookupTable);
9989 }
9990
9991 #[test]
9992 fn recursive_program_flattens_repeated_and_multi_prime_rader_trees() {
9993 for (length, expected_passes, expected_luts) in
9994 [(289usize, 7usize, 1usize), (323, 7, 2), (578, 8, 1)]
9995 {
9996 let plan = FftPlan::build(FftConfig::new(vec![length])).unwrap();
9997 let ir = crate::RecursiveFftIr::build(&plan, Direction::Forward, device()).unwrap();
9998 let program = ProgramIr::recursive_fft(&ir).unwrap();
9999 assert_eq!(program.passes.len(), expected_passes);
10000 assert_eq!(program.input_resource().unwrap().elements, length);
10001 assert_eq!(program.output_resource().unwrap().elements, length);
10002 assert_eq!(
10003 program
10004 .resources
10005 .iter()
10006 .filter(|resource| resource.kind == ProgramResourceKind::LookupTable)
10007 .count(),
10008 expected_luts
10009 );
10010 assert!(program.passes.iter().any(|pass| {
10011 pass.bindings
10012 .iter()
10013 .any(|binding| binding.role == BufferRole::Auxiliary)
10014 }));
10015 if length == 578 {
10016 assert!(
10017 !program
10018 .passes
10019 .iter()
10020 .any(|pass| pass.name == "vkfft_recursive_twiddle_34_forward")
10021 );
10022 assert!(
10023 !program
10024 .passes
10025 .iter()
10026 .any(|pass| pass.name.contains("recursive_scatter_34"))
10027 );
10028 assert!(
10029 !program
10030 .passes
10031 .iter()
10032 .any(|pass| pass.name.contains("recursive_pack_34"))
10033 );
10034 }
10035 }
10036 }
10037
10038 #[test]
10039 fn real_fft_program_uses_compact_external_layouts() {
10040 let length = 17usize;
10041 let batch_count = 2usize;
10042 let r2c_plan = FftPlan::build(
10043 FftConfig::new(vec![length])
10044 .with_batch_count(batch_count)
10045 .with_transform(crate::TransformKind::RealToComplex),
10046 )
10047 .unwrap();
10048 let r2c = crate::RealFftIr::build(&r2c_plan, device()).unwrap();
10049 let r2c_program = ProgramIr::real_fft(&r2c).unwrap();
10050 assert_eq!(
10051 r2c_program.input_resource().unwrap().elements,
10052 length * batch_count
10053 );
10054 assert_eq!(
10055 r2c_program.output_resource().unwrap().elements,
10056 (length / 2 + 1) * batch_count
10057 );
10058 assert!(r2c_program.passes.last().unwrap().name.contains("real_r2c"));
10059
10060 let c2r_plan = FftPlan::build(
10061 FftConfig::new(vec![length])
10062 .with_batch_count(batch_count)
10063 .with_transform(crate::TransformKind::ComplexToReal)
10064 .with_inverse_normalization(true),
10065 )
10066 .unwrap();
10067 let c2r = crate::RealFftIr::build(&c2r_plan, device()).unwrap();
10068 let c2r_program = ProgramIr::real_fft(&c2r).unwrap();
10069 assert_eq!(
10070 c2r_program.input_resource().unwrap().elements,
10071 (length / 2 + 1) * batch_count
10072 );
10073 assert_eq!(
10074 c2r_program.output_resource().unwrap().elements,
10075 length * batch_count
10076 );
10077 assert!(
10078 c2r_program
10079 .passes
10080 .first()
10081 .unwrap()
10082 .name
10083 .contains("real_c2r")
10084 );
10085 }
10086
10087 #[test]
10088 fn real_zero_padding_wraps_only_external_spatial_boundaries() {
10089 let r2c_plan = FftPlan::build(
10090 FftConfig::new(vec![34])
10091 .with_transform(crate::TransformKind::RealToComplex)
10092 .with_zero_padding(0, 17, 34)
10093 .unwrap(),
10094 )
10095 .unwrap();
10096 let r2c = crate::RealFftIr::build(&r2c_plan, device()).unwrap();
10097 let r2c_program = ProgramIr::real_fft(&r2c).unwrap();
10098 assert!(
10099 r2c_program
10100 .passes
10101 .first()
10102 .unwrap()
10103 .name
10104 .contains("zero_pad_forward_input")
10105 );
10106 assert_eq!(
10107 r2c_program.passes.first().unwrap().bindings[0].resource,
10108 ProgramResourceId(0)
10109 );
10110 assert_eq!(
10111 r2c_program
10112 .resource(r2c_program.passes.first().unwrap().bindings[1].resource)
10113 .unwrap()
10114 .kind,
10115 ProgramResourceKind::Scratch
10116 );
10117
10118 let c2r_plan = FftPlan::build(
10119 FftConfig::new(vec![34])
10120 .with_transform(crate::TransformKind::ComplexToReal)
10121 .with_inverse_normalization(true)
10122 .with_zero_padding(0, 17, 34)
10123 .unwrap(),
10124 )
10125 .unwrap();
10126 let c2r = crate::RealFftIr::build(&c2r_plan, device()).unwrap();
10127 let c2r_program = ProgramIr::real_fft(&c2r).unwrap();
10128 assert!(
10129 c2r_program
10130 .passes
10131 .last()
10132 .unwrap()
10133 .name
10134 .contains("zero_pad_inverse_output")
10135 );
10136 assert_eq!(
10137 c2r_program.passes.last().unwrap().bindings[1].resource,
10138 ProgramResourceId(1)
10139 );
10140
10141 let nd_plan = FftPlan::build(
10142 FftConfig::new(vec![3, 8])
10143 .with_transform(crate::TransformKind::RealToComplex)
10144 .with_zero_padding(0, 2, 3)
10145 .unwrap()
10146 .with_zero_padding(1, 6, 8)
10147 .unwrap(),
10148 )
10149 .unwrap();
10150 let nd = crate::NdRealFftIr::build(&nd_plan, device()).unwrap();
10151 assert!(nd.real_axis.zero_pad_pass.is_none());
10152 let nd_program = ProgramIr::nd_real_fft(&nd).unwrap();
10153 assert!(
10154 nd_program
10155 .passes
10156 .first()
10157 .unwrap()
10158 .name
10159 .contains("nd_zero_pad_forward_input")
10160 );
10161 assert_eq!(
10162 nd_program.passes.first().unwrap().bindings[0].resource,
10163 ProgramResourceId(0)
10164 );
10165 }
10166
10167 #[test]
10168 fn even_real_stockham_program_fuses_io_passes() {
10169 let length = 16usize;
10170 let batch_count = 2usize;
10171 let r2c_plan = FftPlan::build(
10172 FftConfig::new(vec![length])
10173 .with_batch_count(batch_count)
10174 .with_transform(crate::TransformKind::RealToComplex),
10175 )
10176 .unwrap();
10177 let r2c = crate::RealFftIr::build(&r2c_plan, device()).unwrap();
10178 let fused_r2c = r2c
10179 .fused_even_input_stockham_kernel()
10180 .unwrap()
10181 .expect("16-point R2C should have a single half-size Stockham root");
10182 assert!(matches!(
10183 fused_r2c.io_mapping,
10184 crate::StockhamIoMapping::RealEvenPack(_)
10185 ));
10186 assert!(matches!(
10187 fused_r2c.output_modifier,
10188 crate::StockhamOutputModifier::RealEvenPostprocess(_)
10189 ));
10190 let r2c_program = ProgramIr::real_fft(&r2c).unwrap();
10191 assert_eq!(r2c_program.passes.len(), 1);
10192 assert!(r2c_program.passes[0].name.contains("stockham"));
10193 assert_eq!(r2c_program.passes[0].bindings.len(), 3);
10194 assert_eq!(
10195 r2c_program.passes[0].bindings[2].role,
10196 BufferRole::Auxiliary
10197 );
10198 assert_eq!(
10199 r2c_program.passes[0].bindings[2].access,
10200 BufferAccess::ReadWrite
10201 );
10202 assert_eq!(
10203 r2c_program
10204 .resource(r2c_program.passes[0].bindings[2].resource)
10205 .unwrap()
10206 .kind,
10207 ProgramResourceKind::Scratch
10208 );
10209
10210 let c2r_plan = FftPlan::build(
10211 FftConfig::new(vec![length])
10212 .with_batch_count(batch_count)
10213 .with_transform(crate::TransformKind::ComplexToReal)
10214 .with_inverse_normalization(true),
10215 )
10216 .unwrap();
10217 let c2r = crate::RealFftIr::build(&c2r_plan, device()).unwrap();
10218 let fused_c2r = c2r
10219 .fused_even_input_stockham_kernel()
10220 .unwrap()
10221 .expect("16-point C2R should have a single half-size Stockham root");
10222 assert!(matches!(
10223 fused_c2r.io_mapping,
10224 crate::StockhamIoMapping::RealEvenInversePreprocess(_)
10225 ));
10226 assert!(matches!(
10227 fused_c2r.output_modifier,
10228 crate::StockhamOutputModifier::RealEvenUnpack(_)
10229 ));
10230 let c2r_program = ProgramIr::real_fft(&c2r).unwrap();
10231 assert_eq!(c2r_program.passes.len(), 1);
10232 assert_eq!(
10233 c2r_program.passes[0].bindings[0].resource,
10234 ProgramResourceId(0)
10235 );
10236 assert_eq!(
10237 c2r_program.passes[0].bindings[1].resource,
10238 ProgramResourceId(1)
10239 );
10240 }
10241
10242 #[test]
10243 fn recursive_even_real_program_fuses_root_boundaries() {
10244 let length = 578usize;
10245 for transform in [
10246 crate::TransformKind::RealToComplex,
10247 crate::TransformKind::ComplexToReal,
10248 ] {
10249 let plan = FftPlan::build(
10250 FftConfig::new(vec![length])
10251 .with_transform(transform)
10252 .with_inverse_normalization(transform == crate::TransformKind::ComplexToReal),
10253 )
10254 .unwrap();
10255 let ir = crate::RealFftIr::build(&plan, device()).unwrap();
10256 assert!(ir.fused_even_input_stockham_kernel().unwrap().is_none());
10257 assert!(ir.fused_even_recursive_ir().unwrap().is_some());
10258 let program = ProgramIr::real_fft(&ir).unwrap();
10259 assert!(
10260 program
10261 .resources
10262 .iter()
10263 .all(|resource| !resource.name.contains("real_recursive_transform"))
10264 );
10265 assert!(program.passes.iter().all(|pass| {
10266 !pass.name.contains("pack_even_odd")
10267 && !pass.name.contains("preprocess_even_half")
10268 && !pass.name.contains("postprocess_even_half")
10269 && !pass.name.contains("unpack_even_odd")
10270 }));
10271 match transform {
10272 crate::TransformKind::RealToComplex => {
10273 assert!(
10274 program
10275 .passes
10276 .first()
10277 .unwrap()
10278 .name
10279 .contains("real_even_pack")
10280 );
10281 assert!(
10282 program
10283 .passes
10284 .last()
10285 .unwrap()
10286 .name
10287 .contains("real_even_postprocess")
10288 );
10289 }
10290 crate::TransformKind::ComplexToReal => {
10291 assert!(
10292 program
10293 .passes
10294 .first()
10295 .unwrap()
10296 .name
10297 .contains("real_even_inverse_preprocess")
10298 );
10299 assert!(
10300 program
10301 .passes
10302 .last()
10303 .unwrap()
10304 .name
10305 .contains("real_even_unpack")
10306 );
10307 }
10308 _ => unreachable!(),
10309 }
10310 }
10311 }
10312
10313 #[test]
10314 fn bluestein_even_real_program_fuses_outer_boundaries() {
10315 let length = 206usize;
10316 let mut tuning = crate::PlannerTuning::portable();
10317 tuning.max_rader_fft_prime = 100;
10318 for transform in [
10319 crate::TransformKind::RealToComplex,
10320 crate::TransformKind::ComplexToReal,
10321 ] {
10322 let plan = FftPlan::build(
10323 FftConfig::new(vec![length])
10324 .with_transform(transform)
10325 .with_inverse_normalization(transform == crate::TransformKind::ComplexToReal)
10326 .with_tuning(tuning),
10327 )
10328 .unwrap();
10329 let ir = crate::RealFftIr::build(&plan, device()).unwrap();
10330 assert!(ir.fused_even_input_stockham_kernel().unwrap().is_none());
10331 assert!(ir.fused_even_recursive_ir().unwrap().is_none());
10332 assert!(ir.fused_even_bluestein_ir().unwrap().is_some());
10333 let program = ProgramIr::real_fft(&ir).unwrap();
10334 assert!(program.passes.iter().all(|pass| {
10335 !pass.name.contains("pack_even_odd")
10336 && !pass.name.contains("preprocess_even_half")
10337 && !pass.name.contains("postprocess_even_half")
10338 && !pass.name.contains("unpack_even_odd")
10339 }));
10340 assert!(program.resources.iter().all(|resource| {
10341 !resource.name.contains("real_preprocess")
10342 && !resource.name.contains("real_transform")
10343 }));
10344 match transform {
10345 crate::TransformKind::RealToComplex => {
10346 assert!(
10347 program
10348 .passes
10349 .first()
10350 .unwrap()
10351 .name
10352 .contains("real_even_pack")
10353 );
10354 assert!(
10355 program
10356 .passes
10357 .last()
10358 .unwrap()
10359 .name
10360 .contains("real_even_postprocess")
10361 );
10362 }
10363 crate::TransformKind::ComplexToReal => {
10364 assert!(
10365 program
10366 .passes
10367 .first()
10368 .unwrap()
10369 .name
10370 .contains("real_even_inverse_preprocess")
10371 );
10372 assert!(
10373 program
10374 .passes
10375 .last()
10376 .unwrap()
10377 .name
10378 .contains("real_even_unpack")
10379 );
10380 }
10381 _ => unreachable!(),
10382 }
10383 assert!(
10384 program
10385 .resources
10386 .iter()
10387 .any(|resource| resource.name.contains("bluestein_scratch"))
10388 );
10389 }
10390 }
10391
10392 #[test]
10393 fn multidimensional_program_flattens_axis_pack_transform_scatter() {
10394 let plan = FftPlan::build(FftConfig::new(vec![3, 4]).with_batch_count(2)).unwrap();
10395 let ir = crate::NdFftIr::build(&plan, Direction::Forward, device()).unwrap();
10396 let program = ProgramIr::nd_fft(&ir).unwrap();
10397 assert_eq!(program.passes.len(), 6);
10398 assert_eq!(program.input_resource().unwrap().elements, 24);
10399 assert_eq!(program.output_resource().unwrap().elements, 24);
10400 assert!(program.passes[0].name.contains("nd_pack_axis_1"));
10401 assert!(program.passes[2].name.contains("nd_scatter_axis_1"));
10402 assert!(program.passes[3].name.contains("nd_pack_axis_0"));
10403 assert_eq!(program.passes[4].dispatch.x, 2);
10404 let outer = ir
10405 .axes
10406 .iter()
10407 .find(|axis| axis.axis == 0)
10408 .expect("outer ND axis");
10409 let crate::OneDimFftIr::Recursive(outer_recursive) = &outer.transform else {
10410 panic!("outer ND Stockham axis unexpectedly selected Bluestein");
10411 };
10412 let crate::RecursiveFftNodeIr::Stockham(outer_kernel) = &outer_recursive.root else {
10413 panic!("outer ND Stockham axis unexpectedly selected Rader");
10414 };
10415 assert_eq!(
10416 [outer_kernel.workgroup_size.x, outer_kernel.workgroup_size.y],
10417 [4, 1]
10418 );
10419 assert_eq!(outer_kernel.dispatch.x, 2);
10420 let memory = program.memory_plan().unwrap();
10421 assert!(memory.allocations.len() < program.resources.len());
10422 }
10423
10424 #[test]
10425 fn omit_dimension_removes_only_selected_axis_from_program_graph() {
10426 let plan = FftPlan::build(
10427 FftConfig::new(vec![3, 4])
10428 .with_batch_count(2)
10429 .with_omit_dimension(1, true)
10430 .unwrap(),
10431 )
10432 .unwrap();
10433 let ir = crate::NdFftIr::build(&plan, Direction::Forward, device()).unwrap();
10434 let program = ProgramIr::nd_fft(&ir).unwrap();
10435 assert_eq!(
10436 ir.axes.iter().map(|axis| axis.axis).collect::<Vec<_>>(),
10437 vec![0]
10438 );
10439 assert_eq!(program.input_resource().unwrap().elements, 24);
10440 assert_eq!(program.output_resource().unwrap().elements, 24);
10441 assert!(
10442 program
10443 .passes
10444 .iter()
10445 .any(|pass| pass.name.contains("nd_pack_axis_0"))
10446 );
10447 assert!(
10448 program
10449 .passes
10450 .iter()
10451 .any(|pass| pass.name.contains("nd_scatter_axis_0"))
10452 );
10453 assert!(
10454 !program
10455 .passes
10456 .iter()
10457 .any(|pass| pass.name.contains("axis_1"))
10458 );
10459
10460 let dd_plan = FftPlan::build(
10461 FftConfig::new(vec![3, 4])
10462 .with_batch_count(2)
10463 .with_precision(Precision::DoubleDouble)
10464 .with_omit_dimension(1, true)
10465 .unwrap(),
10466 )
10467 .unwrap();
10468 let dd_ir = crate::DoubleDoubleNdFftIr::build(&dd_plan, Direction::Forward).unwrap();
10469 let dd_program = ProgramIr::double_double_nd(&dd_ir).unwrap();
10470 assert_eq!(
10471 dd_ir.axes.iter().map(|axis| axis.axis).collect::<Vec<_>>(),
10472 vec![0]
10473 );
10474 assert_eq!(dd_program.input_resource().unwrap().elements, 24);
10475 assert_eq!(dd_program.output_resource().unwrap().elements, 24);
10476 assert!(
10477 !dd_program
10478 .passes
10479 .iter()
10480 .any(|pass| pass.name.contains("axis_1"))
10481 );
10482 }
10483
10484 #[test]
10485 fn formatted_batch_stride_nd_expands_only_external_allocations() {
10486 let plan = FftPlan::build(
10487 FftConfig::new(vec![3, 4])
10488 .with_batch_count(2)
10489 .with_input_buffer_batch_stride(17)
10490 .with_output_buffer_batch_stride(19),
10491 )
10492 .unwrap();
10493 let ir = crate::NdFftIr::build(&plan, Direction::Forward, device()).unwrap();
10494 let program = ProgramIr::nd_fft(&ir).unwrap();
10495 let input = program.input_resource().unwrap();
10496 let output = program.output_resource().unwrap();
10497 assert_eq!(input.elements, 34);
10498 assert_eq!(output.elements, 38);
10499 assert_eq!(input.external_layout.unwrap().logical_len, 12);
10500 assert_eq!(input.external_layout.unwrap().physical_stride, 17);
10501 assert_eq!(output.external_layout.unwrap().logical_len, 12);
10502 assert_eq!(output.external_layout.unwrap().physical_stride, 19);
10503 assert!(
10504 program
10505 .resources
10506 .iter()
10507 .filter(|resource| resource.kind == ProgramResourceKind::Scratch)
10508 .all(|resource| resource.elements == 24
10509 || resource.kind == ProgramResourceKind::LookupTable)
10510 );
10511 program.validate().unwrap();
10512 }
10513
10514 #[test]
10515 fn formatted_tensor_stride_nd_exposes_physical_external_batches_and_dense_scratch() {
10516 let plan = FftPlan::build(
10517 FftConfig::new(vec![2, 3, 4])
10518 .with_batch_count(2)
10519 .with_input_buffer_axis_stride(1, 6)
10520 .unwrap()
10521 .with_input_buffer_axis_stride(0, 20)
10522 .unwrap()
10523 .with_output_buffer_axis_stride(1, 7)
10524 .unwrap()
10525 .with_output_buffer_axis_stride(0, 24)
10526 .unwrap(),
10527 )
10528 .unwrap();
10529 let ir = crate::NdFftIr::build(&plan, Direction::Forward, device()).unwrap();
10530 let program = ProgramIr::nd_fft(&ir).unwrap();
10531 let input = program.input_resource().unwrap();
10532 let output = program.output_resource().unwrap();
10533 assert_eq!(input.elements, 80);
10534 assert_eq!(output.elements, 96);
10535 assert_eq!(input.external_layout.unwrap().logical_len, 40);
10536 assert_eq!(input.external_layout.unwrap().physical_stride, 40);
10537 assert_eq!(output.external_layout.unwrap().logical_len, 48);
10538 assert_eq!(output.external_layout.unwrap().physical_stride, 48);
10539 assert!(
10540 program
10541 .resources
10542 .iter()
10543 .filter(|resource| resource.kind == ProgramResourceKind::Scratch)
10544 .all(|resource| resource.elements == 48)
10545 );
10546 program.validate().unwrap();
10547 }
10548
10549 #[test]
10550 fn formatted_zero_padding_nd_program_keeps_padding_on_dense_compute_boundaries() {
10551 let mut device = device();
10552 device.supports_f64 = true;
10553 for (domain, input_boundary) in [
10554 (ZeroPaddingDomain::Spatial, true),
10555 (ZeroPaddingDomain::Frequency, false),
10556 ] {
10557 let plan = FftPlan::build(
10558 FftConfig::new(vec![3, 4])
10559 .with_batch_count(2)
10560 .with_precision(Precision::F64ComputeF32Storage)
10561 .with_input_buffer_axis_stride(0, 7)
10562 .unwrap()
10563 .with_output_buffer_axis_stride(0, 9)
10564 .unwrap()
10565 .with_zero_padding(1, 1, 3)
10566 .unwrap()
10567 .with_zero_padding_domain(domain),
10568 )
10569 .unwrap();
10570 let ir = crate::NdFftIr::build(&plan, Direction::Forward, device).unwrap();
10571 let program = ProgramIr::nd_fft(&ir).unwrap();
10572 assert_eq!(program.input_resource().unwrap().scalar, ScalarType::F32);
10573 assert_eq!(program.output_resource().unwrap().scalar, ScalarType::F32);
10574 assert_eq!(program.input_resource().unwrap().elements, 42);
10575 assert_eq!(program.output_resource().unwrap().elements, 54);
10576 let names = program
10577 .passes
10578 .iter()
10579 .map(|pass| pass.name.as_str())
10580 .collect::<Vec<_>>();
10581 if input_boundary {
10582 assert_eq!(names[0], "vkfft_nd_gather_formatted_input_before_zero_pad");
10583 assert!(names[1].contains("nd_zero_pad_forward_input"));
10584 assert!(ir.input_formatted_copy.is_some());
10585 assert!(ir.output_formatted_copy.is_none());
10586 } else {
10587 assert!(names[names.len() - 2].contains("nd_zero_pad_forward_output"));
10588 assert_eq!(
10589 names[names.len() - 1],
10590 "vkfft_nd_scatter_formatted_output_after_zero_pad"
10591 );
10592 assert!(ir.input_formatted_copy.is_none());
10593 assert!(ir.output_formatted_copy.is_some());
10594 }
10595 let zero_pad = ir.zero_pad_pass.as_ref().unwrap();
10596 assert_eq!(zero_pad.input_storage_scalar, ScalarType::F64);
10597 assert_eq!(zero_pad.output_storage_scalar, ScalarType::F64);
10598 assert!(program.resources.iter().any(|resource| {
10599 resource.kind == ProgramResourceKind::Scratch
10600 && resource.scalar == ScalarType::F64
10601 && resource.elements == 24
10602 && resource.name.contains("formatted")
10603 }));
10604 program.validate().unwrap();
10605 }
10606 }
10607
10608 #[test]
10609 fn formatted_real_tensor_strides_insert_physical_gather_and_scatter_around_dense_program() {
10610 for (transform, input_stride, output_stride, input_elements, output_elements) in [
10611 (
10612 TransformKind::RealToComplex,
10613 11usize,
10614 7usize,
10615 66usize,
10616 42usize,
10617 ),
10618 (
10619 TransformKind::ComplexToReal,
10620 7usize,
10621 11usize,
10622 42usize,
10623 66usize,
10624 ),
10625 ] {
10626 let plan = FftPlan::build(
10627 FftConfig::new(vec![3usize, 8])
10628 .with_batch_count(2)
10629 .with_transform(transform)
10630 .with_inverse_normalization(transform == TransformKind::ComplexToReal)
10631 .with_input_buffer_axis_stride(0, input_stride)
10632 .unwrap()
10633 .with_output_buffer_axis_stride(0, output_stride)
10634 .unwrap(),
10635 )
10636 .unwrap();
10637 let ir = crate::NdRealFftIr::build(&plan, device()).unwrap();
10638 let program = ProgramIr::nd_real_fft(&ir).unwrap();
10639 let input = program.input_resource().unwrap();
10640 let output = program.output_resource().unwrap();
10641 assert_eq!(input.elements, input_elements);
10642 assert_eq!(output.elements, output_elements);
10643 assert_eq!(
10644 input.external_layout.unwrap().logical_len,
10645 ir.input_external_layout.batch_stride
10646 );
10647 assert_eq!(
10648 output.external_layout.unwrap().logical_len,
10649 ir.output_external_layout.batch_stride
10650 );
10651 assert!(
10652 program
10653 .passes
10654 .first()
10655 .unwrap()
10656 .name
10657 .contains("nd_real_gather_formatted_input")
10658 );
10659 assert!(
10660 program
10661 .passes
10662 .last()
10663 .unwrap()
10664 .name
10665 .contains("nd_real_scatter_formatted_output")
10666 );
10667 assert!(program.resources.iter().any(|resource| {
10668 resource.name.contains("nd_real_formatted_input")
10669 && resource.elements == ir.input_tensor_len() * ir.batch_count
10670 }));
10671 assert!(program.resources.iter().any(|resource| {
10672 resource.name.contains("nd_real_formatted_output")
10673 && resource.elements == ir.output_tensor_len() * ir.batch_count
10674 }));
10675 program.validate().unwrap();
10676 }
10677 }
10678
10679 #[test]
10680 fn formatted_real_zero_padding_program_keeps_pad_between_dense_copies() {
10681 let mut device = device();
10682 device.supports_f64 = true;
10683 for (transform, input_elements, output_elements, pad_at_start) in [
10684 (TransformKind::RealToComplex, 66usize, 42usize, true),
10685 (TransformKind::ComplexToReal, 42usize, 66usize, false),
10686 ] {
10687 let (input_stride, output_stride) = match transform {
10688 TransformKind::RealToComplex => (11usize, 7usize),
10689 TransformKind::ComplexToReal => (7usize, 11usize),
10690 _ => unreachable!(),
10691 };
10692 let plan = FftPlan::build(
10693 FftConfig::new(vec![3usize, 8])
10694 .with_batch_count(2)
10695 .with_transform(transform)
10696 .with_precision(Precision::F64ComputeF32Storage)
10697 .with_inverse_normalization(transform == TransformKind::ComplexToReal)
10698 .with_input_buffer_axis_stride(0, input_stride)
10699 .unwrap()
10700 .with_output_buffer_axis_stride(0, output_stride)
10701 .unwrap()
10702 .with_zero_padding(0, 1, 2)
10703 .unwrap()
10704 .with_zero_padding(1, 2, 4)
10705 .unwrap(),
10706 )
10707 .unwrap();
10708 let ir = crate::NdRealFftIr::build(&plan, device).unwrap();
10709 let program = ProgramIr::nd_real_fft(&ir).unwrap();
10710 assert_eq!(program.input_resource().unwrap().elements, input_elements);
10711 assert_eq!(program.output_resource().unwrap().elements, output_elements);
10712 assert_eq!(program.input_resource().unwrap().scalar, ScalarType::F32);
10713 assert_eq!(program.output_resource().unwrap().scalar, ScalarType::F32);
10714 let zero_pad = ir.zero_pad_pass.as_ref().unwrap();
10715 assert_eq!(zero_pad.input_storage_scalar, ScalarType::F64);
10716 assert_eq!(zero_pad.output_storage_scalar, ScalarType::F64);
10717 let names = program
10718 .passes
10719 .iter()
10720 .map(|pass| pass.name.as_str())
10721 .collect::<Vec<_>>();
10722 assert_eq!(
10723 names.first().copied(),
10724 Some("vkfft_nd_real_gather_formatted_input")
10725 );
10726 assert_eq!(
10727 names.last().copied(),
10728 Some("vkfft_nd_real_scatter_formatted_output")
10729 );
10730 if pad_at_start {
10731 assert!(names[1].contains("nd_zero_pad_forward_input"));
10732 } else {
10733 assert!(names[names.len() - 2].contains("nd_zero_pad_inverse_output"));
10734 }
10735 assert!(program.resources.iter().any(|resource| {
10736 resource.kind == ProgramResourceKind::Scratch
10737 && resource.scalar == ScalarType::F64
10738 && resource.name.contains("formatted")
10739 }));
10740 program.validate().unwrap();
10741 }
10742 }
10743
10744 #[test]
10745 fn formatted_nd_r2r_program_wraps_dense_pipeline_with_physical_copies() {
10746 let plan = FftPlan::build(
10747 FftConfig::new(vec![2usize, 3, 4])
10748 .with_batch_count(2)
10749 .with_transform(TransformKind::Dct(crate::DctType::II))
10750 .with_input_buffer_axis_stride(1, 6)
10751 .unwrap()
10752 .with_input_buffer_axis_stride(0, 20)
10753 .unwrap()
10754 .with_output_buffer_axis_stride(1, 7)
10755 .unwrap()
10756 .with_output_buffer_axis_stride(0, 24)
10757 .unwrap(),
10758 )
10759 .unwrap();
10760 let ir = crate::NdR2rIr::build(&plan, Direction::Forward, device()).unwrap();
10761 let program = ProgramIr::nd_r2r(&ir).unwrap();
10762 let input = program.input_resource().unwrap();
10763 let output = program.output_resource().unwrap();
10764 assert_eq!(input.elements, 80);
10765 assert_eq!(output.elements, 96);
10766 assert_eq!(input.external_layout.unwrap().physical_stride, 40);
10767 assert_eq!(output.external_layout.unwrap().physical_stride, 48);
10768 assert_eq!(
10769 program.passes.first().unwrap().name,
10770 "vkfft_nd_r2r_gather_formatted_input"
10771 );
10772 assert_eq!(
10773 program.passes.last().unwrap().name,
10774 "vkfft_nd_r2r_scatter_formatted_output"
10775 );
10776 let formatted_input = program
10777 .resources
10778 .iter()
10779 .find(|resource| resource.name.starts_with("nd_r2r_formatted_input"))
10780 .unwrap();
10781 let formatted_output = program
10782 .resources
10783 .iter()
10784 .find(|resource| resource.name.starts_with("nd_r2r_formatted_output"))
10785 .unwrap();
10786 assert_eq!(formatted_input.kind, ProgramResourceKind::Scratch);
10787 assert_eq!(formatted_output.kind, ProgramResourceKind::Scratch);
10788 assert_eq!(formatted_input.elements, 48);
10789 assert_eq!(formatted_output.elements, 48);
10790 assert!(formatted_input.external_layout.is_none());
10791 assert!(formatted_output.external_layout.is_none());
10792 program.validate().unwrap();
10793 }
10794
10795 #[test]
10796 fn formatted_double_double_nd_c2c_program_keeps_dense_dd_scratch() {
10797 for (precision, external_scalar) in [
10798 (Precision::DoubleDouble, ScalarType::DoubleDouble),
10799 (Precision::DoubleDoubleF64Storage, ScalarType::F64),
10800 ] {
10801 let plan = FftPlan::build(
10802 FftConfig::new(vec![2usize, 3, 4])
10803 .with_batch_count(2)
10804 .with_precision(precision)
10805 .with_input_buffer_axis_stride(1, 6)
10806 .unwrap()
10807 .with_input_buffer_axis_stride(0, 20)
10808 .unwrap()
10809 .with_output_buffer_axis_stride(1, 7)
10810 .unwrap()
10811 .with_output_buffer_axis_stride(0, 24)
10812 .unwrap(),
10813 )
10814 .unwrap();
10815 let ir = crate::DoubleDoubleNdFftIr::build(&plan, Direction::Forward).unwrap();
10816 let program = ProgramIr::double_double_nd(&ir).unwrap();
10817 let input = program.input_resource().unwrap();
10818 let output = program.output_resource().unwrap();
10819 assert_eq!(input.scalar, external_scalar);
10820 assert_eq!(output.scalar, external_scalar);
10821 assert_eq!(input.elements, 80);
10822 assert_eq!(output.elements, 96);
10823 assert_eq!(input.external_layout.unwrap().physical_stride, 40);
10824 assert_eq!(output.external_layout.unwrap().physical_stride, 48);
10825 assert_eq!(
10826 program.passes.first().unwrap().name,
10827 "vkfft_dd_nd_gather_formatted_input"
10828 );
10829 assert_eq!(
10830 program.passes.last().unwrap().name,
10831 "vkfft_dd_nd_scatter_formatted_output"
10832 );
10833 for name in ["double_double_nd_tensor_a", "double_double_nd_tensor_b"] {
10834 let scratch = program
10835 .resources
10836 .iter()
10837 .find(|resource| resource.name == name)
10838 .unwrap();
10839 assert_eq!(scratch.kind, ProgramResourceKind::Scratch);
10840 assert_eq!(scratch.scalar, ScalarType::DoubleDouble);
10841 assert_eq!(scratch.elements, 48);
10842 assert!(scratch.external_layout.is_none());
10843 }
10844 program.validate().unwrap();
10845 }
10846 }
10847
10848 #[test]
10849 fn formatted_double_double_real_and_r2r_programs_keep_dense_dd_scratch() {
10850 for (precision, external_scalar) in [
10851 (Precision::DoubleDouble, ScalarType::DoubleDouble),
10852 (Precision::DoubleDoubleF64Storage, ScalarType::F64),
10853 ] {
10854 let real_plan = FftPlan::build(
10855 FftConfig::new(vec![2usize, 3, 8])
10856 .with_batch_count(2)
10857 .with_transform(TransformKind::RealToComplex)
10858 .with_precision(precision)
10859 .with_input_buffer_axis_stride(1, 11)
10860 .unwrap()
10861 .with_input_buffer_axis_stride(0, 40)
10862 .unwrap()
10863 .with_output_buffer_axis_stride(1, 7)
10864 .unwrap()
10865 .with_output_buffer_axis_stride(0, 24)
10866 .unwrap(),
10867 )
10868 .unwrap();
10869 let real_ir = crate::DoubleDoubleNdRealFftIr::build(&real_plan).unwrap();
10870 let real_program = ProgramIr::double_double_nd_real(&real_ir).unwrap();
10871 let real_input = real_program.input_resource().unwrap();
10872 let real_output = real_program.output_resource().unwrap();
10873 assert_eq!(real_input.scalar, external_scalar);
10874 assert_eq!(real_output.scalar, external_scalar);
10875 assert_eq!(real_input.elements, 160);
10876 assert_eq!(real_output.elements, 96);
10877 assert_eq!(real_input.external_layout.unwrap().physical_stride, 80);
10878 assert_eq!(real_output.external_layout.unwrap().physical_stride, 48);
10879 assert_eq!(
10880 real_program.passes.first().unwrap().name,
10881 "vkfft_dd_nd_real_gather_formatted_input"
10882 );
10883 assert_eq!(
10884 real_program.passes.last().unwrap().name,
10885 "vkfft_dd_nd_real_scatter_formatted_output"
10886 );
10887 let full_scratch = real_program
10888 .resources
10889 .iter()
10890 .find(|resource| resource.name == "double_double_nd_real_full_scalar")
10891 .unwrap();
10892 assert_eq!(full_scratch.scalar, ScalarType::DoubleDouble);
10893 assert_eq!(full_scratch.elements, 96);
10894 let compact_scratch = real_program
10895 .resources
10896 .iter()
10897 .find(|resource| resource.name == "double_double_nd_real_compact_a")
10898 .unwrap();
10899 assert_eq!(compact_scratch.scalar, ScalarType::DoubleDouble);
10900 assert_eq!(compact_scratch.elements, 60);
10901 real_program.validate().unwrap();
10902
10903 let r2r_plan = FftPlan::build(
10904 FftConfig::new(vec![2usize, 3, 4])
10905 .with_batch_count(2)
10906 .with_transform(TransformKind::Dct(crate::DctType::II))
10907 .with_precision(precision)
10908 .with_input_buffer_axis_stride(1, 6)
10909 .unwrap()
10910 .with_input_buffer_axis_stride(0, 20)
10911 .unwrap()
10912 .with_output_buffer_axis_stride(1, 7)
10913 .unwrap()
10914 .with_output_buffer_axis_stride(0, 24)
10915 .unwrap(),
10916 )
10917 .unwrap();
10918 let r2r_ir = crate::DoubleDoubleNdR2rIr::build(&r2r_plan, Direction::Forward).unwrap();
10919 let r2r_program = ProgramIr::double_double_nd_r2r(&r2r_ir).unwrap();
10920 let r2r_input = r2r_program.input_resource().unwrap();
10921 let r2r_output = r2r_program.output_resource().unwrap();
10922 assert_eq!(r2r_input.scalar, external_scalar);
10923 assert_eq!(r2r_output.scalar, external_scalar);
10924 assert_eq!(r2r_input.elements, 80);
10925 assert_eq!(r2r_output.elements, 96);
10926 assert_eq!(r2r_input.external_layout.unwrap().physical_stride, 40);
10927 assert_eq!(r2r_output.external_layout.unwrap().physical_stride, 48);
10928 assert_eq!(
10929 r2r_program.passes.first().unwrap().name,
10930 "vkfft_dd_nd_r2r_gather_formatted_input"
10931 );
10932 assert_eq!(
10933 r2r_program.passes.last().unwrap().name,
10934 "vkfft_dd_nd_r2r_scatter_formatted_output"
10935 );
10936 for name in [
10937 "double_double_nd_r2r_tensor_a",
10938 "double_double_nd_r2r_tensor_b",
10939 ] {
10940 let scratch = r2r_program
10941 .resources
10942 .iter()
10943 .find(|resource| resource.name == name)
10944 .unwrap();
10945 assert_eq!(scratch.scalar, ScalarType::DoubleDouble);
10946 assert_eq!(scratch.elements, 48);
10947 }
10948 r2r_program.validate().unwrap();
10949 }
10950 }
10951
10952 #[test]
10953 fn double_double_omit_dimension_programs_drop_omitted_axis_passes() {
10954 let real_plan = FftPlan::build(
10955 FftConfig::new(vec![3usize, 8])
10956 .with_transform(TransformKind::RealToComplex)
10957 .with_precision(Precision::DoubleDouble)
10958 .with_omit_dimension(0, true)
10959 .unwrap(),
10960 )
10961 .unwrap();
10962 let real_ir = crate::DoubleDoubleNdRealFftIr::build(&real_plan).unwrap();
10963 assert!(real_ir.complex_axes.is_empty());
10964 let real_program = ProgramIr::double_double_nd_real(&real_ir).unwrap();
10965 assert!(
10966 !real_program
10967 .passes
10968 .iter()
10969 .any(|pass| pass.name.contains("nd_real_axis_0"))
10970 );
10971 assert!(
10972 real_program
10973 .passes
10974 .iter()
10975 .any(|pass| pass.name.contains("nd_real_last_axis"))
10976 );
10977 real_program.validate().unwrap();
10978
10979 let r2r_plan = FftPlan::build(
10980 FftConfig::new(vec![3usize, 4])
10981 .with_transform(TransformKind::Dct(crate::DctType::II))
10982 .with_precision(Precision::DoubleDouble)
10983 .with_omit_dimension(1, true)
10984 .unwrap(),
10985 )
10986 .unwrap();
10987 let r2r_ir = crate::DoubleDoubleNdR2rIr::build(&r2r_plan, Direction::Forward).unwrap();
10988 assert_eq!(
10989 r2r_ir.axes.iter().map(|axis| axis.axis).collect::<Vec<_>>(),
10990 vec![0]
10991 );
10992 let r2r_program = ProgramIr::double_double_nd_r2r(&r2r_ir).unwrap();
10993 assert!(
10994 !r2r_program
10995 .passes
10996 .iter()
10997 .any(|pass| pass.name.contains("nd_r2r_axis_1"))
10998 );
10999 assert!(
11000 r2r_program
11001 .passes
11002 .iter()
11003 .any(|pass| pass.name.contains("nd_r2r_axis_0"))
11004 );
11005 r2r_program.validate().unwrap();
11006 }
11007
11008 #[test]
11009 fn f64_four_step_uses_one_lazy_full_period_twiddle_lut() {
11010 let length = 1_048_576usize;
11011 let mut profile = DeviceProfile::generic(Backend::Vulkan, GpuVendor::Nvidia);
11012 profile.shared_memory_bytes = 48 * 1024;
11013 profile.shared_memory_pow2_bytes = 48 * 1024;
11014 profile.max_threads_per_block = 1024;
11015 profile.supports_f64 = true;
11016 let plan =
11017 FftPlan::build(FftConfig::new(vec![length]).with_precision(Precision::F64)).unwrap();
11018 let ir = crate::RecursiveFftIr::build(&plan, Direction::Forward, profile).unwrap();
11019 assert!(ir.four_step_plan.is_some());
11020 let kernels = ir.four_step_stockham_upload_kernels().unwrap().unwrap();
11021 assert!(
11022 kernels
11023 .iter()
11024 .all(|kernel| kernel.twiddle_lut_len() == Some(length))
11025 );
11026
11027 let program = ProgramIr::recursive_fft(&ir).unwrap();
11028 let root_luts = program
11029 .resources
11030 .iter()
11031 .filter(|resource| {
11032 matches!(
11033 resource.initialization,
11034 ProgramResourceInitialization::StockhamUnitRoots { len } if len == length
11035 )
11036 })
11037 .collect::<Vec<_>>();
11038 assert_eq!(root_luts.len(), 1);
11039 assert_eq!(root_luts[0].elements, length);
11040 assert!(program.passes.iter().all(|pass| {
11041 pass.bindings
11042 .iter()
11043 .any(|binding| binding.role == BufferRole::TwiddleLookupTable)
11044 }));
11045 assert!(!program.resources.iter().any(|resource| {
11046 matches!(
11047 &resource.initialization,
11048 ProgramResourceInitialization::Complex64(values) if values.len() == length
11049 )
11050 }));
11051 }
11052
11053 #[test]
11054 fn two_upload_four_step_program_collapses_root_to_two_fft_passes() {
11055 let length = 1_048_576usize;
11056 let mut profile = DeviceProfile::generic(Backend::Vulkan, GpuVendor::Nvidia);
11057 profile.shared_memory_bytes = 48 * 1024;
11058 profile.shared_memory_pow2_bytes = 48 * 1024;
11059 profile.max_threads_per_block = 1024;
11060 let plan = FftPlan::build(FftConfig::new(vec![length])).unwrap();
11061 let ir = crate::RecursiveFftIr::build(&plan, Direction::Forward, profile).unwrap();
11062 assert!(ir.four_step_plan.is_some());
11063
11064 let program = ProgramIr::recursive_fft(&ir).unwrap();
11065 assert_eq!(program.passes.len(), 2);
11066 assert_eq!(program.resources.len(), 3);
11067 assert_eq!(
11068 program
11069 .resources
11070 .iter()
11071 .filter(|resource| resource.kind == ProgramResourceKind::Scratch)
11072 .count(),
11073 1
11074 );
11075 assert!(program.passes[0].name.ends_with("four_step_upload_1"));
11076 assert!(program.passes[1].name.ends_with("four_step_upload_0"));
11077 assert_eq!(program.passes[0].dispatch.x, 256);
11078 assert_eq!(program.passes[1].dispatch.x, 256);
11079 program.validate().unwrap();
11080 }
11081
11082 #[test]
11083 fn three_upload_four_step_program_collapses_root_to_three_fft_passes() {
11084 let length = 8_388_608usize;
11085 let mut profile = DeviceProfile::generic(Backend::Vulkan, GpuVendor::Nvidia);
11086 profile.shared_memory_bytes = 48 * 1024;
11087 profile.shared_memory_pow2_bytes = 48 * 1024;
11088 profile.max_threads_per_block = 1024;
11089 let plan = FftPlan::build(FftConfig::new(vec![length])).unwrap();
11090 let ir = crate::RecursiveFftIr::build(&plan, Direction::Forward, profile).unwrap();
11091 assert!(ir.four_step_plan.is_some());
11092
11093 let program = ProgramIr::recursive_fft(&ir).unwrap();
11094 assert_eq!(program.passes.len(), 3);
11095 assert_eq!(program.resources.len(), 3);
11096 let scratch = program
11097 .resources
11098 .iter()
11099 .find(|resource| resource.kind == ProgramResourceKind::Scratch)
11100 .unwrap()
11101 .id;
11102 assert_eq!(scratch, ProgramResourceId(2));
11103 assert!(program.passes[0].name.ends_with("four_step_upload_2"));
11104 assert!(program.passes[1].name.ends_with("four_step_upload_1"));
11105 assert!(program.passes[2].name.ends_with("four_step_upload_0"));
11106 assert_eq!(
11107 program
11108 .passes
11109 .iter()
11110 .map(|pass| pass.dispatch.x)
11111 .collect::<Vec<_>>(),
11112 vec![2048, 4096, 2048]
11113 );
11114 assert_eq!(program.passes[0].bindings[0].resource, ProgramResourceId(0));
11115 assert_eq!(program.passes[0].bindings[1].resource, ProgramResourceId(1));
11116 assert_eq!(program.passes[1].bindings[0].resource, ProgramResourceId(1));
11117 assert_eq!(program.passes[1].bindings[1].resource, scratch);
11118 assert_eq!(program.passes[2].bindings[0].resource, scratch);
11119 assert_eq!(program.passes[2].bindings[1].resource, ProgramResourceId(1));
11120 program.validate().unwrap();
11121 }
11122
11123 #[test]
11124 fn rader_three_upload_reuses_output_only_for_same_scalar_storage() {
11125 let length = 1_922usize;
11126 let base_profile = DeviceProfile {
11127 shared_memory_bytes: 1024,
11128 shared_memory_pow2_bytes: 1024,
11129 max_threads_per_block: 32,
11130 max_workgroup_size: [32, 32, 64],
11131 ..DeviceProfile::generic(Backend::Vulkan, GpuVendor::Nvidia)
11132 };
11133
11134 for (precision, expected_exchange_scratch, expected_output_scalar) in [
11135 (Precision::F32, 1usize, ScalarType::F32),
11136 (Precision::F16StorageF32Compute, 2usize, ScalarType::F16),
11137 ] {
11138 let mut profile = base_profile;
11139 if precision == Precision::F16StorageF32Compute {
11140 profile.shared_memory_bytes = 2 * 1024;
11141 profile.shared_memory_pow2_bytes = 2 * 1024;
11142 }
11143 let plan =
11144 FftPlan::build(FftConfig::new(vec![length]).with_precision(precision)).unwrap();
11145 let ir = crate::RecursiveFftIr::build(&plan, Direction::Forward, profile).unwrap();
11146 assert_eq!(ir.scalar, ScalarType::F32);
11147 assert_eq!(ir.external_scalar, expected_output_scalar);
11148 assert_eq!(
11149 ir.rader_forced_upload_schedule
11150 .as_ref()
11151 .expect("N1922 must retain the forced-Rader schedule")
11152 .axis_split,
11153 vec![2, 31, 31]
11154 );
11155 assert_eq!(
11156 ir.four_step_rader_upload_nodes()
11157 .unwrap()
11158 .expect("N1922 must materialize three Rader Four-step uploads")
11159 .len(),
11160 3
11161 );
11162
11163 let program = ProgramIr::recursive_fft(&ir).unwrap();
11164 let output = program.output_resource().unwrap();
11165 assert_eq!(output.scalar, expected_output_scalar);
11166 let exchange = program
11167 .resources
11168 .iter()
11169 .filter(|resource| {
11170 resource.kind == ProgramResourceKind::Scratch
11171 && resource.name.starts_with("rader_four_step_exchange")
11172 })
11173 .collect::<Vec<_>>();
11174 assert_eq!(exchange.len(), expected_exchange_scratch, "{precision:?}");
11175 if precision == Precision::F32 {
11176 assert!(program.passes.iter().any(|pass| {
11177 pass.bindings
11178 .iter()
11179 .any(|binding| binding.resource == output.id)
11180 }));
11181 } else {
11182 assert_ne!(exchange[0].id, exchange[1].id);
11183 assert!(
11184 exchange
11185 .iter()
11186 .all(|resource| resource.scalar == ScalarType::F32)
11187 );
11188 }
11189 program.validate().unwrap();
11190 }
11191 }
11192
11193 #[test]
11194 fn double_double_three_upload_reuses_output_only_for_dd_storage() {
11195 let stockham_device = DeviceProfile {
11196 shared_memory_bytes: 1024,
11197 shared_memory_pow2_bytes: 1024,
11198 max_threads_per_block: 1024,
11199 max_workgroup_size: [1024, 1024, 64],
11200 supports_f64: true,
11201 ..DeviceProfile::generic(Backend::Vulkan, GpuVendor::Nvidia)
11202 };
11203 let forced_rader_device = DeviceProfile {
11204 shared_memory_bytes: 8 * 1024,
11205 shared_memory_pow2_bytes: 8 * 1024,
11206 max_threads_per_block: 1024,
11207 max_workgroup_size: [1024, 1024, 64],
11208 supports_f64: true,
11209 ..DeviceProfile::generic(Backend::Vulkan, GpuVendor::Nvidia)
11210 };
11211
11212 for (precision, expected_scratch, expected_output_scalar) in [
11213 (Precision::DoubleDouble, 1usize, ScalarType::DoubleDouble),
11214 (Precision::DoubleDoubleF64Storage, 2usize, ScalarType::F64),
11215 ] {
11216 let stockham_plan = FftPlan::build_for_device(
11217 FftConfig::new(vec![8_192]).with_precision(precision),
11218 stockham_device,
11219 )
11220 .unwrap();
11221 let stockham_ir = crate::DoubleDoubleRecursiveFftIr::build_for_device(
11222 &stockham_plan,
11223 Direction::Forward,
11224 stockham_device,
11225 )
11226 .unwrap();
11227 assert_eq!(
11228 stockham_ir
11229 .three_upload_four_step_plan
11230 .expect("constrained DD N8192 must keep three-upload Four-step metadata")
11231 .axis_split,
11232 [32, 16, 16]
11233 );
11234 let stockham_program = ProgramIr::double_double_recursive(&stockham_ir).unwrap();
11235 assert_eq!(
11236 stockham_program.output_resource().unwrap().scalar,
11237 expected_output_scalar
11238 );
11239 let stockham_exchange = stockham_program
11240 .resources
11241 .iter()
11242 .filter(|resource| {
11243 resource.kind == ProgramResourceKind::Scratch
11244 && resource
11245 .name
11246 .starts_with("double_double_three_upload_four_step_scratch")
11247 })
11248 .collect::<Vec<_>>();
11249 assert_eq!(stockham_exchange.len(), expected_scratch, "{precision:?}");
11250 assert!(
11251 stockham_exchange
11252 .iter()
11253 .all(|resource| resource.scalar == ScalarType::DoubleDouble)
11254 );
11255 stockham_program.validate().unwrap();
11256
11257 let forced_length = 17usize * 47 * 128;
11258 let forced_plan =
11259 FftPlan::build(FftConfig::new(vec![forced_length]).with_precision(precision))
11260 .unwrap();
11261 let forced_ir = crate::DoubleDoubleRecursiveFftIr::build_for_device(
11262 &forced_plan,
11263 Direction::Forward,
11264 forced_rader_device,
11265 )
11266 .unwrap();
11267 let forced_schedule = forced_ir
11268 .rader_forced_upload_schedule
11269 .as_ref()
11270 .expect("DD p47 composite must retain forced-Rader upload metadata");
11271 assert_eq!(forced_schedule.axis_split, vec![64, 47, 34]);
11272 assert_eq!(forced_schedule.upload_count, 3);
11273 let forced_program = ProgramIr::double_double_recursive(&forced_ir).unwrap();
11274 assert_eq!(
11275 forced_program.output_resource().unwrap().scalar,
11276 expected_output_scalar
11277 );
11278 let forced_exchange = forced_program
11279 .resources
11280 .iter()
11281 .filter(|resource| {
11282 resource.kind == ProgramResourceKind::Scratch
11283 && resource
11284 .name
11285 .starts_with("double_double_forced_rader_three_upload_scratch")
11286 })
11287 .collect::<Vec<_>>();
11288 assert_eq!(forced_exchange.len(), expected_scratch, "{precision:?}");
11289 assert!(
11290 forced_exchange
11291 .iter()
11292 .all(|resource| resource.scalar == ScalarType::DoubleDouble)
11293 );
11294 forced_program.validate().unwrap();
11295 }
11296 }
11297
11298 #[test]
11299 fn memory_plan_reuses_dead_scratch_and_preserves_external_buffers() {
11300 let plan = FftPlan::build(FftConfig::new(vec![578])).unwrap();
11301 let ir = crate::RecursiveFftIr::build(&plan, Direction::Forward, device()).unwrap();
11302 let program = ProgramIr::recursive_fft(&ir).unwrap();
11303 let memory = program.memory_plan().unwrap();
11304 let input_allocation = memory
11305 .allocation_for(program.input_resource().unwrap().id)
11306 .unwrap();
11307 let output_allocation = memory
11308 .allocation_for(program.output_resource().unwrap().id)
11309 .unwrap();
11310 assert_ne!(input_allocation, output_allocation);
11311 assert_eq!(
11312 memory
11313 .allocations
11314 .iter()
11315 .filter(|allocation| allocation.kind == ProgramAllocationKind::LookupTable)
11316 .count(),
11317 1
11318 );
11319 assert!(memory.allocations.len() < program.resources.len());
11320 let logical_elements = program
11321 .resources
11322 .iter()
11323 .map(|resource| resource.elements)
11324 .sum::<usize>();
11325 assert!(memory.allocated_elements().unwrap() < logical_elements);
11326 assert!(memory.allocations.iter().any(|allocation| allocation.kind
11327 == ProgramAllocationKind::Scratch
11328 && allocation.resources.len() > 1));
11329 }
11330
11331 #[test]
11332 fn double_double_r2r_program_uses_scalar_boundaries_and_dd_internal_resources() {
11333 for precision in [Precision::DoubleDouble, Precision::DoubleDoubleF64Storage] {
11334 let external_scalar = if precision == Precision::DoubleDouble {
11335 ScalarType::DoubleDouble
11336 } else {
11337 ScalarType::F64
11338 };
11339 for transform in [
11340 TransformKind::Dct(crate::DctType::I),
11341 TransformKind::Dst(crate::DstType::I),
11342 TransformKind::Dst(crate::DstType::II),
11343 TransformKind::Dst(crate::DstType::III),
11344 TransformKind::Dct(crate::DctType::II),
11345 TransformKind::Dct(crate::DctType::III),
11346 TransformKind::Dct(crate::DctType::IV),
11347 TransformKind::Dst(crate::DstType::IV),
11348 ] {
11349 let plan = FftPlan::build(
11350 FftConfig::new(vec![9])
11351 .with_batch_count(7)
11352 .with_precision(precision)
11353 .with_transform(transform)
11354 .with_grouped_batch(0, 3)
11355 .unwrap(),
11356 )
11357 .unwrap();
11358 let ir = crate::DoubleDoubleR2rIr::build(&plan, Direction::Forward).unwrap();
11359 let crate::DoubleDoubleR2rAlgorithm::FftReduction { fft, fft_len, .. } =
11360 &ir.algorithm
11361 else {
11362 panic!("all DD DCT/DST I-IV families must use FFT reduction");
11363 };
11364 assert_eq!(
11365 *fft_len,
11366 match transform {
11367 TransformKind::Dct(crate::DctType::I) => 16,
11368 TransformKind::Dst(crate::DstType::I) => 20,
11369 TransformKind::Dct(crate::DctType::II | crate::DctType::III)
11370 | TransformKind::Dst(crate::DstType::II | crate::DstType::III) => 9,
11371 TransformKind::Dct(crate::DctType::IV)
11372 | TransformKind::Dst(crate::DstType::IV) => 18,
11373 _ => unreachable!(),
11374 }
11375 );
11376 assert_eq!(ir.grouped_batch, 3);
11377 assert_eq!(fft.grouped_batch(), 3);
11378 assert_eq!(ir.batch_group_count(), 3);
11379 let child = ProgramIr::double_double_one_dim(fft).unwrap();
11380 let program = ProgramIr::double_double_r2r(&ir).unwrap();
11381 assert_eq!(program.passes.len(), child.passes.len() + 2);
11382 assert!(program.passes.iter().all(|pass| pass.dispatch.x == 3));
11383 for resource in [
11384 program.input_resource().unwrap(),
11385 program.output_resource().unwrap(),
11386 ] {
11387 assert_eq!(resource.scalar, external_scalar);
11388 assert_eq!(
11389 resource.external_layout.unwrap().element_shape,
11390 ProgramElementShape::Scalar
11391 );
11392 }
11393 let phases = program
11394 .resources
11395 .iter()
11396 .find(|resource| resource.name == "double_double_r2r_phases");
11397 if matches!(
11398 transform,
11399 TransformKind::Dct(
11400 crate::DctType::II | crate::DctType::III | crate::DctType::IV
11401 ) | TransformKind::Dst(
11402 crate::DstType::II | crate::DstType::III | crate::DstType::IV
11403 )
11404 ) {
11405 let phases = phases.unwrap();
11406 assert_eq!(phases.scalar, ScalarType::DoubleDouble);
11407 assert_eq!(phases.elements, 9);
11408 assert!(matches!(
11409 phases.initialization,
11410 ProgramResourceInitialization::ComplexDoubleDouble(_)
11411 ));
11412 } else {
11413 assert!(phases.is_none());
11414 }
11415 assert!(
11416 program
11417 .resources
11418 .iter()
11419 .skip(2)
11420 .all(|resource| { resource.scalar == ScalarType::DoubleDouble })
11421 );
11422 }
11423 }
11424 }
11425
11426 #[test]
11427 fn double_double_nd_r2r_program_preserves_scalar_tensor_and_complex_child_shapes() {
11428 for (precision, external_scalar) in [
11429 (Precision::DoubleDouble, ScalarType::DoubleDouble),
11430 (Precision::DoubleDoubleF64Storage, ScalarType::F64),
11431 ] {
11432 let plan = FftPlan::build(
11433 FftConfig::new(vec![3, 4])
11434 .with_batch_count(7)
11435 .with_precision(precision)
11436 .with_transform(TransformKind::Dct(crate::DctType::II))
11437 .with_grouped_batch(0, 3)
11438 .unwrap()
11439 .with_grouped_batch(1, 3)
11440 .unwrap(),
11441 )
11442 .unwrap();
11443 let ir = crate::double_double_ir::DoubleDoubleNdR2rIr::build(&plan, Direction::Forward)
11444 .unwrap();
11445 let program = ProgramIr::double_double_nd_r2r(&ir).unwrap();
11446 assert!(ir.axes.iter().all(|axis| axis.grouped_batch == 3));
11447 assert_eq!(ir.axes[0].transform.grouped_batch, 3);
11448 assert_eq!(ir.axes[1].transform.grouped_batch, 3);
11449 assert!(program.passes.iter().any(|pass| pass.dispatch.x == 3));
11450 assert!(program.passes.iter().any(|pass| pass.dispatch.x == 7));
11451 assert!(program.passes.iter().any(|pass| pass.dispatch.x == 10));
11452 assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
11453 assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
11454 assert_eq!(
11455 program.input_resource().unwrap().element_shape(),
11456 ProgramElementShape::Scalar
11457 );
11458 assert_eq!(
11459 program.output_resource().unwrap().element_shape(),
11460 ProgramElementShape::Scalar
11461 );
11462 for resource_id in 2..=5 {
11463 let resource = &program.resources[resource_id];
11464 assert_eq!(resource.scalar, ScalarType::DoubleDouble);
11465 assert_eq!(resource.element_shape(), ProgramElementShape::Scalar);
11466 assert_eq!(resource.element_bytes(), 16);
11467 }
11468 assert!(program.resources.iter().any(
11469 |resource| resource.name.contains("r2r_fft_input")
11470 && resource.element_shape() == ProgramElementShape::Complex
11471 && resource.element_bytes() == 32
11472 ));
11473 assert!(program.passes.first().unwrap().name.contains("axis_1_pack"));
11474 assert!(
11475 program
11476 .passes
11477 .last()
11478 .unwrap()
11479 .name
11480 .contains("axis_0_scatter")
11481 );
11482 assert!(
11483 program
11484 .passes
11485 .iter()
11486 .any(|pass| pass.name.contains("axis_1_vkfft_dd_r2r_fft_dct2_pre"))
11487 );
11488 let memory = program.memory_plan().unwrap();
11489 for resource_id in 2..=5 {
11490 let allocation_id = memory
11491 .allocation_for(ProgramResourceId(resource_id))
11492 .unwrap();
11493 let allocation = &memory.allocations[allocation_id.0];
11494 let first = allocation.resources.first().copied().unwrap();
11495 assert_eq!(
11496 program.resources[first.0].element_shape(),
11497 ProgramElementShape::Scalar
11498 );
11499 }
11500 for allocation in &memory.allocations {
11501 let first_shape = allocation
11502 .resources
11503 .first()
11504 .map(|resource| program.resources[resource.0].element_shape())
11505 .unwrap();
11506 assert!(
11507 allocation
11508 .resources
11509 .iter()
11510 .all(
11511 |resource| program.resources[resource.0].element_shape() == first_shape
11512 ),
11513 "scratch alias mixed scalar and complex shapes"
11514 );
11515 }
11516 }
11517 }
11518
11519 #[test]
11520 fn double_double_nd_r2r_padding_stays_fused_in_scalar_tensor_boundaries() {
11521 for (precision, external_scalar) in [
11522 (Precision::DoubleDouble, ScalarType::DoubleDouble),
11523 (Precision::DoubleDoubleF64Storage, ScalarType::F64),
11524 ] {
11525 for direction in [Direction::Forward, Direction::Inverse] {
11526 let config = FftConfig::new(vec![3, 4])
11527 .with_batch_count(7)
11528 .with_precision(precision)
11529 .with_transform(TransformKind::Dct(crate::DctType::II))
11530 .with_inverse_normalization(direction == Direction::Inverse)
11531 .with_grouped_batch(0, 3)
11532 .unwrap()
11533 .with_grouped_batch(1, 3)
11534 .unwrap()
11535 .with_zero_padding(0, 1, 2)
11536 .unwrap()
11537 .with_zero_padding(1, 1, 3)
11538 .unwrap();
11539 let ir =
11540 crate::DoubleDoubleNdR2rIr::build(&FftPlan::build(config).unwrap(), direction)
11541 .unwrap();
11542 assert!(ir.has_spatial_zero_padding());
11543 let program = ProgramIr::double_double_nd_r2r(&ir).unwrap();
11544 assert!(ir.axes.iter().all(|axis| axis.grouped_batch == 3));
11545 assert!(ir.axes.iter().all(|axis| axis.transform.grouped_batch == 3));
11546 assert!(program.passes.iter().any(|pass| pass.dispatch.x == 3));
11547 assert!(program.passes.iter().any(|pass| pass.dispatch.x == 7));
11548 assert!(program.passes.iter().any(|pass| pass.dispatch.x == 10));
11549 assert_eq!(program.input_resource().unwrap().scalar, external_scalar);
11550 assert_eq!(program.output_resource().unwrap().scalar, external_scalar);
11551 assert!(
11552 !program
11553 .passes
11554 .iter()
11555 .any(|pass| pass.name.contains("zero_pad"))
11556 );
11557 for resource_id in 2..=5 {
11558 let resource = &program.resources[resource_id];
11559 assert_eq!(resource.scalar, ScalarType::DoubleDouble);
11560 assert_eq!(resource.element_shape(), ProgramElementShape::Scalar);
11561 assert_eq!(resource.element_bytes(), 16);
11562 }
11563 assert!(program.resources.iter().any(|resource| {
11564 resource.name.contains("r2r_fft_input")
11565 && resource.scalar == ScalarType::DoubleDouble
11566 && resource.element_shape() == ProgramElementShape::Complex
11567 && resource.element_bytes() == 32
11568 }));
11569 assert_eq!(
11570 program.passes.first().unwrap().bindings[0].resource,
11571 ProgramResourceId(0)
11572 );
11573 assert_eq!(
11574 program.passes.last().unwrap().bindings[1].resource,
11575 ProgramResourceId(1)
11576 );
11577 }
11578 }
11579 }
11580}