1pub mod application;
42pub mod backend;
43pub mod binary16;
44pub mod bluestein_ir;
45pub mod complex;
46pub mod config;
47pub mod convolution_ir;
48pub mod double_double;
49pub mod double_double_ir;
50pub mod double_double_recursive_ir;
51pub mod error;
52pub mod kernel_ir;
53pub mod lut;
54pub mod mixed_ir;
55pub mod nd_ir;
56pub mod nd_real_ir;
57pub mod one_dim_ir;
58pub mod planner;
59pub mod precision;
60pub mod program_ir;
61pub mod r2r_ir;
62pub mod rader_ir;
63pub mod real_ir;
64pub mod recursive_ir;
65pub mod reference;
66pub mod scheduler;
67pub mod scheduler_snapshot;
68pub mod zero_pad_ir;
69
70pub use application::TransformIr;
71pub use binary16::{Binary16, Complex16, decode_complex16_native, encode_complex16_native};
72pub use bluestein_ir::{
73 BluesteinPassIr, BluesteinPassOperation, BluesteinPipelineIr, execute_bluestein_ir,
74};
75pub use complex::{Complex, Complex32, Complex64};
76pub use config::{
77 Backend, ConvolutionConjugation, DctType, DeviceProfile, Direction, DstType, FftConfig,
78 GpuVendor, PLANNER_TUNING_PROFILE_VERSION, PlannerTuning, Precision, PrecisionCompute,
79 PrecisionLayout, PrecisionStorage, SubgroupProfile, TransformKind, ZeroPaddingDomain,
80 ZeroPaddingRange,
81};
82pub use convolution_ir::{
83 ConvolutionDirectRaderMultiKernelStepIr, ConvolutionFftRaderMultiKernelStepIr,
84 ConvolutionFftRaderStepIr, ConvolutionIr, ConvolutionMatrixLayout,
85 ConvolutionMatrixStockhamStepIr, ConvolutionMultiKernelStockhamStepIr, ConvolutionMultiplyIr,
86 ConvolutionMultiplyPolicy, ConvolutionThreeUploadMultiKernelStockhamIr,
87 ConvolutionTwoUploadMultiKernelStockhamIr, NdConvolutionIr, NdRealConvolutionIr,
88 execute_convolution_ir, execute_nd_convolution_ir, execute_nd_real_convolution_ir,
89};
90pub use double_double::{
91 ComplexDoubleDouble, DoubleDouble, decode_complex_double_double, dft as double_double_dft,
92 encode_complex_double_double, quick_two_sum, two_prod, two_sum,
93 unit_root as double_double_unit_root,
94};
95pub use double_double_ir::{
96 DoubleDoubleBluesteinConvolutionIr, DoubleDoubleBluesteinIr, DoubleDoubleDirectRaderIr,
97 DoubleDoubleFftRaderIr, DoubleDoubleNdAxisIr, DoubleDoubleNdFftIr, DoubleDoubleNdR2rAxisIr,
98 DoubleDoubleNdR2rIr, DoubleDoubleNdRealFftIr, DoubleDoubleOneDimIr, DoubleDoubleR2rAlgorithm,
99 DoubleDoubleR2rIr, DoubleDoubleRealFftIr, DoubleDoubleStockhamIr,
100 execute_double_double_bluestein_ir, execute_double_double_bluestein_ir_f64_storage,
101 execute_double_double_c2r_ir, execute_double_double_c2r_ir_f64_storage,
102 execute_double_double_direct_rader_ir, execute_double_double_direct_rader_ir_f64_storage,
103 execute_double_double_fft_rader_ir, execute_double_double_fft_rader_ir_f64_storage,
104 execute_double_double_nd_c2r_ir, execute_double_double_nd_c2r_ir_f64_storage,
105 execute_double_double_nd_ir, execute_double_double_nd_ir_f64_storage,
106 execute_double_double_nd_r2c_ir, execute_double_double_nd_r2c_ir_f64_storage,
107 execute_double_double_nd_r2r_ir, execute_double_double_nd_r2r_ir_f64_storage,
108 execute_double_double_one_dim_ir, execute_double_double_one_dim_ir_f64_storage,
109 execute_double_double_r2c_ir, execute_double_double_r2c_ir_f64_storage,
110 execute_double_double_r2r_ir, execute_double_double_r2r_ir_f64_storage,
111 execute_double_double_stockham_ir, execute_double_double_stockham_ir_f64_storage,
112};
113pub use double_double_recursive_ir::{
114 DoubleDoubleCooleyTukeyPassIr, DoubleDoubleCooleyTukeyPassOperation,
115 DoubleDoubleRecursiveCooleyTukeyIr, DoubleDoubleRecursiveFftIr, DoubleDoubleRecursiveFftNodeIr,
116 DoubleDoubleThreeUploadFourStepPlanIr, DoubleDoubleTwoUploadFourStepPlanIr,
117 execute_double_double_recursive_ir, execute_double_double_recursive_ir_f64_storage,
118};
119pub use error::{Result, VkFftError};
120pub use kernel_ir::{
121 BufferAccess, BufferBinding, BufferRole, DispatchGeometry, FourStepMapping, KernelIr,
122 KernelOperation, RaderGeneratorMapping, RaderScatterMapping, RealEvenInversePreprocessMapping,
123 RealEvenPackMapping, RealEvenUnpackMapping, RegisterStageBoundary,
124 RegisterStageBoundaryResidency, RegisterStockhamStage, RegisterSubgroupBoundary,
125 RegisterSubgroupLaneModel, RegisterSubgroupSource, ScalarType, SharedBuffer, SharedMemoryPlan,
126 StockhamExecutionLayout, StockhamInputModifier, StockhamIoMapping, StockhamOutputModifier,
127 StockhamSharedMemoryLayout, StockhamStage, StockhamWorkgroupGrouping,
128 ThreeUploadFourStepMapping, WorkgroupSize, execute_stockham_ir,
129};
130pub use lut::{
131 BluesteinTable, DoubleDoubleBluesteinTable, DoubleDoubleRaderTable,
132 DoubleDoubleStockhamTwiddleStage, DoubleDoubleStockhamTwiddleTable, RaderTable,
133 StockhamTwiddleStage, StockhamTwiddleTable, stockham_root_table_double_double,
134};
135pub use mixed_ir::{
136 MixedPassIr, MixedPassOperation, MixedPrimeRaderIr, MixedRaderStockhamIr,
137 execute_mixed_rader_stockham_ir,
138};
139pub use nd_ir::{
140 NdAxisIr, NdExternalTensorLayout, NdFftIr, NdFormattedCopyOperation, NdFormattedCopyPassIr,
141 NdPassIr, NdPassOperation, execute_nd_fft_ir,
142};
143pub use nd_real_ir::{NdRealFftIr, execute_nd_c2r_ir, execute_nd_r2c_ir};
144pub use one_dim_ir::{OneDimFftIr, execute_one_dim_fft_ir};
145pub use planner::{
146 AlgorithmKind, AxisAlgorithm, AxisPlan, FftPlan, RaderMode, RaderPrimePlan, RadixPlan,
147};
148pub use precision::{
149 PRECISION_REPORT_SCHEMA_VERSION, PrecisionCaseReport, PrecisionMetrics,
150 PrecisionTransformFamily, complex_precision_metrics, real_precision_metrics,
151};
152pub use program_ir::{
153 ExternalBufferLayout, ProgramAllocation, ProgramAllocationId, ProgramAllocationKind, ProgramIr,
154 ProgramMemoryPlan, ProgramPass, ProgramPassBinding, ProgramResource, ProgramResourceId,
155 ProgramResourceInitialization, ProgramResourceKind,
156};
157pub use r2r_ir::{
158 NdR2rAxisIr, NdR2rIr, R2rFftPassIr, R2rFftPassOperation, R2rFftReductionIr, R2rIr, R2rNdPassIr,
159 R2rNdPassOperation, R2rTransform, execute_nd_r2r_ir, execute_r2r_ir,
160};
161pub use rader_ir::{
162 RaderDirectIr, RaderFftInputStrategy, RaderFftPassIr, RaderFftPassOperation,
163 RaderFftPipelineIr, execute_rader_direct_ir, execute_rader_fft_ir,
164};
165pub use real_ir::{
166 RealFftAlgorithm, RealFftIr, RealFftKind, RealPassIr, RealPassOperation, execute_c2r_ir,
167 execute_r2c_ir,
168};
169pub use recursive_ir::{
170 CooleyTukeyPassIr, CooleyTukeyPassOperation, FourStepInputLayout, FourStepOutputLayout,
171 FourStepPlanIr, FourStepTwiddlePlacement, FourStepUploadIr, FusedFftRaderStaticResourceReport,
172 RecursiveCooleyTukeyIr, RecursiveFftIr, RecursiveFftNodeIr, execute_recursive_fft_ir,
173};
174pub use scheduler::{
175 DoubleDoubleStockhamUploadSchedule, GpuSchedulerPolicy, NvidiaVulkanSchedulerTuning,
176 RaderFftRegisterSchedule, RaderFftStageLaneLayout, RaderFftStageLaneSchedule,
177 RaderFftTransposeSchedule, RadixRegisterSchedule, StockhamUploadSchedule,
178 VKFFT_RADIX_TABLE_LEN, has_specialized_gpu_scheduler_policy,
179 plan_gpu_double_double_stockham_uploads_for_batches, plan_gpu_force_rader_two_upload,
180 plan_gpu_power_of_two_radix_registers, plan_gpu_power_of_two_stockham_uploads,
181 plan_gpu_power_of_two_stockham_uploads_for_batches, plan_gpu_rader_fft_registers,
182 plan_gpu_rader_fft_registers_for_containers, plan_gpu_scheduler_policy,
183 plan_gpu_small_mixed_radix_registers, plan_gpu_smooth_stockham_uploads,
184 plan_gpu_smooth_stockham_uploads_for_batches, plan_gpu_stockham_twiddle_source,
185 plan_nvidia_vulkan_power_of_two_radix_registers,
186 plan_nvidia_vulkan_power_of_two_stockham_uploads,
187 plan_nvidia_vulkan_power_of_two_stockham_uploads_for_batches,
188 plan_nvidia_vulkan_rader_fft_registers, plan_nvidia_vulkan_rader_fft_registers_for_containers,
189 plan_nvidia_vulkan_small_mixed_radix_registers, plan_nvidia_vulkan_smooth_stockham_uploads,
190 plan_nvidia_vulkan_smooth_stockham_uploads_for_batches,
191 plan_nvidia_vulkan_two_three_radix_registers,
192};
193pub use scheduler_snapshot::{
194 SCHEDULER_SNAPSHOT_SCHEMA_VERSION, SchedulerAxisClassification,
195 SchedulerPhysicalAxisProbeContext, SchedulerRaderUploadProbe, SchedulerSnapshotMismatch,
196 SchedulerSnapshotPayload, SchedulerSnapshotRecord, compare_upstream_scheduler_snapshot_jsonl,
197 fixed_upstream_scheduler_snapshot_corpus, scheduler_axis_classification_probe,
198 scheduler_rader_upload_axis_block_probe_records,
199 scheduler_rader_upload_axis_block_probe_records_with_context,
200 scheduler_rader_upload_probe_record,
201};
202
203pub use zero_pad_ir::{
204 NdZeroPadPassIr, ZeroPadPassIr, ZeroPadPassOperation, execute_nd_zero_pad_pass,
205 execute_zero_pad_pass,
206};
207
208pub const UPSTREAM_VKFFT_VERSION: &str = "1.3.4";
210
211pub const UPSTREAM_VKFFT_COMMIT: &str = "066a17c17068c0f11c9298d848c2976c71fad1c1";