Skip to main content

vkfft_rs/
lib.rs

1//! Pure-Rust reimplementation of VkFFT.
2//!
3//! `vkfft-rs` ports VkFFT's Application -> Plan -> Code architecture into typed Rust state:
4//! configuration and device-aware planning, Stockham/Rader/Bluestein and real/R2R composition,
5//! backend-neutral program/resource IR, generated GPU kernels, and correctness-first execution
6//! runtimes. The fixed porting baseline is VkFFT 1.3.4 at upstream commit
7//! `066a17c17068c0f11c9298d848c2976c71fad1c1`.
8//!
9//! # Default build
10//!
11//! The default feature set has no GPU runtime loader. It provides the planner, CPU reference
12//! paths, typed IR, scheduler policies, LUT generation, and backend source/SPIR-V generation.
13//! This makes planning and code generation usable without a CUDA, HIP, OpenCL, Level Zero,
14//! Metal, or Vulkan runtime installed on the host.
15//!
16//! # Runtime features
17//!
18//! - `vulkan-runtime`: Vulkan 1.1 execution through `ash`, including explicit device selection,
19//!   async tickets, pooled resources, resident program chaining, and versioned pipeline caches.
20//! - `cuda-runtime`: CUDA Driver + NVRTC execution loaded dynamically at runtime.
21//! - `hip-runtime`: HIP + HIPRTC/AMD compiler execution loaded dynamically at runtime.
22//! - `opencl-runtime`: OpenCL execution loaded dynamically at runtime.
23//! - `level-zero-runtime`: Intel Level Zero execution loaded dynamically at runtime. It can be
24//!   combined with `opencl-runtime` to use the validated Intel native-binary compiler fallback
25//!   when `ocloc` is unavailable.
26//! - `metal-runtime`: correctness-first Metal F32/F16 execution on macOS; the runtime surface is
27//!   implemented, while real Apple-GPU validation remains an explicit hardware gate.
28//!
29//! # Planning example
30//!
31//! ```rust
32//! use vkfft_rs::{AlgorithmKind, FftConfig, FftPlan};
33//!
34//! let plan = FftPlan::build(FftConfig::new(vec![1024, 17, 65537]))?;
35//! assert_eq!(plan.axes[0].algorithm.kind(), AlgorithmKind::Stockham);
36//! assert_eq!(plan.axes[1].algorithm.kind(), AlgorithmKind::Rader);
37//! assert_eq!(plan.axes[2].algorithm.kind(), AlgorithmKind::Bluestein);
38//! # Ok::<(), vkfft_rs::VkFftError>(())
39//! ```
40
41pub mod application;
42pub mod backend;
43pub mod binary16;
44pub mod bluestein_ir;
45pub mod complex;
46pub mod config;
47pub mod convolution_ir;
48pub mod double_double;
49pub mod double_double_ir;
50pub mod double_double_recursive_ir;
51pub mod error;
52pub mod kernel_ir;
53pub mod lut;
54pub mod mixed_ir;
55pub mod nd_ir;
56pub mod nd_real_ir;
57pub mod one_dim_ir;
58pub mod planner;
59pub mod precision;
60pub mod program_ir;
61pub mod r2r_ir;
62pub mod rader_ir;
63pub mod real_ir;
64pub mod recursive_ir;
65pub mod reference;
66pub mod scheduler;
67pub mod scheduler_snapshot;
68pub mod zero_pad_ir;
69
70pub use application::TransformIr;
71pub use binary16::{Binary16, Complex16, decode_complex16_native, encode_complex16_native};
72pub use bluestein_ir::{
73    BluesteinPassIr, BluesteinPassOperation, BluesteinPipelineIr, execute_bluestein_ir,
74};
75pub use complex::{Complex, Complex32, Complex64};
76pub use config::{
77    Backend, ConvolutionConjugation, DctType, DeviceProfile, Direction, DstType, FftConfig,
78    GpuVendor, PLANNER_TUNING_PROFILE_VERSION, PlannerTuning, Precision, PrecisionCompute,
79    PrecisionLayout, PrecisionStorage, SubgroupProfile, TransformKind, ZeroPaddingDomain,
80    ZeroPaddingRange,
81};
82pub use convolution_ir::{
83    ConvolutionDirectRaderMultiKernelStepIr, ConvolutionFftRaderMultiKernelStepIr,
84    ConvolutionFftRaderStepIr, ConvolutionIr, ConvolutionMatrixLayout,
85    ConvolutionMatrixStockhamStepIr, ConvolutionMultiKernelStockhamStepIr, ConvolutionMultiplyIr,
86    ConvolutionMultiplyPolicy, ConvolutionThreeUploadMultiKernelStockhamIr,
87    ConvolutionTwoUploadMultiKernelStockhamIr, NdConvolutionIr, NdRealConvolutionIr,
88    execute_convolution_ir, execute_nd_convolution_ir, execute_nd_real_convolution_ir,
89};
90pub use double_double::{
91    ComplexDoubleDouble, DoubleDouble, decode_complex_double_double, dft as double_double_dft,
92    encode_complex_double_double, quick_two_sum, two_prod, two_sum,
93    unit_root as double_double_unit_root,
94};
95pub use double_double_ir::{
96    DoubleDoubleBluesteinConvolutionIr, DoubleDoubleBluesteinIr, DoubleDoubleDirectRaderIr,
97    DoubleDoubleFftRaderIr, DoubleDoubleNdAxisIr, DoubleDoubleNdFftIr, DoubleDoubleNdR2rAxisIr,
98    DoubleDoubleNdR2rIr, DoubleDoubleNdRealFftIr, DoubleDoubleOneDimIr, DoubleDoubleR2rAlgorithm,
99    DoubleDoubleR2rIr, DoubleDoubleRealFftIr, DoubleDoubleStockhamIr,
100    execute_double_double_bluestein_ir, execute_double_double_bluestein_ir_f64_storage,
101    execute_double_double_c2r_ir, execute_double_double_c2r_ir_f64_storage,
102    execute_double_double_direct_rader_ir, execute_double_double_direct_rader_ir_f64_storage,
103    execute_double_double_fft_rader_ir, execute_double_double_fft_rader_ir_f64_storage,
104    execute_double_double_nd_c2r_ir, execute_double_double_nd_c2r_ir_f64_storage,
105    execute_double_double_nd_ir, execute_double_double_nd_ir_f64_storage,
106    execute_double_double_nd_r2c_ir, execute_double_double_nd_r2c_ir_f64_storage,
107    execute_double_double_nd_r2r_ir, execute_double_double_nd_r2r_ir_f64_storage,
108    execute_double_double_one_dim_ir, execute_double_double_one_dim_ir_f64_storage,
109    execute_double_double_r2c_ir, execute_double_double_r2c_ir_f64_storage,
110    execute_double_double_r2r_ir, execute_double_double_r2r_ir_f64_storage,
111    execute_double_double_stockham_ir, execute_double_double_stockham_ir_f64_storage,
112};
113pub use double_double_recursive_ir::{
114    DoubleDoubleCooleyTukeyPassIr, DoubleDoubleCooleyTukeyPassOperation,
115    DoubleDoubleRecursiveCooleyTukeyIr, DoubleDoubleRecursiveFftIr, DoubleDoubleRecursiveFftNodeIr,
116    DoubleDoubleThreeUploadFourStepPlanIr, DoubleDoubleTwoUploadFourStepPlanIr,
117    execute_double_double_recursive_ir, execute_double_double_recursive_ir_f64_storage,
118};
119pub use error::{Result, VkFftError};
120pub use kernel_ir::{
121    BufferAccess, BufferBinding, BufferRole, DispatchGeometry, FourStepMapping, KernelIr,
122    KernelOperation, RaderGeneratorMapping, RaderScatterMapping, RealEvenInversePreprocessMapping,
123    RealEvenPackMapping, RealEvenUnpackMapping, RegisterStageBoundary,
124    RegisterStageBoundaryResidency, RegisterStockhamStage, RegisterSubgroupBoundary,
125    RegisterSubgroupLaneModel, RegisterSubgroupSource, ScalarType, SharedBuffer, SharedMemoryPlan,
126    StockhamExecutionLayout, StockhamInputModifier, StockhamIoMapping, StockhamOutputModifier,
127    StockhamSharedMemoryLayout, StockhamStage, StockhamWorkgroupGrouping,
128    ThreeUploadFourStepMapping, WorkgroupSize, execute_stockham_ir,
129};
130pub use lut::{
131    BluesteinTable, DoubleDoubleBluesteinTable, DoubleDoubleRaderTable,
132    DoubleDoubleStockhamTwiddleStage, DoubleDoubleStockhamTwiddleTable, RaderTable,
133    StockhamTwiddleStage, StockhamTwiddleTable, stockham_root_table_double_double,
134};
135pub use mixed_ir::{
136    MixedPassIr, MixedPassOperation, MixedPrimeRaderIr, MixedRaderStockhamIr,
137    execute_mixed_rader_stockham_ir,
138};
139pub use nd_ir::{
140    NdAxisIr, NdExternalTensorLayout, NdFftIr, NdFormattedCopyOperation, NdFormattedCopyPassIr,
141    NdPassIr, NdPassOperation, execute_nd_fft_ir,
142};
143pub use nd_real_ir::{NdRealFftIr, execute_nd_c2r_ir, execute_nd_r2c_ir};
144pub use one_dim_ir::{OneDimFftIr, execute_one_dim_fft_ir};
145pub use planner::{
146    AlgorithmKind, AxisAlgorithm, AxisPlan, FftPlan, RaderMode, RaderPrimePlan, RadixPlan,
147};
148pub use precision::{
149    PRECISION_REPORT_SCHEMA_VERSION, PrecisionCaseReport, PrecisionMetrics,
150    PrecisionTransformFamily, complex_precision_metrics, real_precision_metrics,
151};
152pub use program_ir::{
153    ExternalBufferLayout, ProgramAllocation, ProgramAllocationId, ProgramAllocationKind, ProgramIr,
154    ProgramMemoryPlan, ProgramPass, ProgramPassBinding, ProgramResource, ProgramResourceId,
155    ProgramResourceInitialization, ProgramResourceKind,
156};
157pub use r2r_ir::{
158    NdR2rAxisIr, NdR2rIr, R2rFftPassIr, R2rFftPassOperation, R2rFftReductionIr, R2rIr, R2rNdPassIr,
159    R2rNdPassOperation, R2rTransform, execute_nd_r2r_ir, execute_r2r_ir,
160};
161pub use rader_ir::{
162    RaderDirectIr, RaderFftInputStrategy, RaderFftPassIr, RaderFftPassOperation,
163    RaderFftPipelineIr, execute_rader_direct_ir, execute_rader_fft_ir,
164};
165pub use real_ir::{
166    RealFftAlgorithm, RealFftIr, RealFftKind, RealPassIr, RealPassOperation, execute_c2r_ir,
167    execute_r2c_ir,
168};
169pub use recursive_ir::{
170    CooleyTukeyPassIr, CooleyTukeyPassOperation, FourStepInputLayout, FourStepOutputLayout,
171    FourStepPlanIr, FourStepTwiddlePlacement, FourStepUploadIr, FusedFftRaderStaticResourceReport,
172    RecursiveCooleyTukeyIr, RecursiveFftIr, RecursiveFftNodeIr, execute_recursive_fft_ir,
173};
174pub use scheduler::{
175    DoubleDoubleStockhamUploadSchedule, GpuSchedulerPolicy, NvidiaVulkanSchedulerTuning,
176    RaderFftRegisterSchedule, RaderFftStageLaneLayout, RaderFftStageLaneSchedule,
177    RaderFftTransposeSchedule, RadixRegisterSchedule, StockhamUploadSchedule,
178    VKFFT_RADIX_TABLE_LEN, has_specialized_gpu_scheduler_policy,
179    plan_gpu_double_double_stockham_uploads_for_batches, plan_gpu_force_rader_two_upload,
180    plan_gpu_power_of_two_radix_registers, plan_gpu_power_of_two_stockham_uploads,
181    plan_gpu_power_of_two_stockham_uploads_for_batches, plan_gpu_rader_fft_registers,
182    plan_gpu_rader_fft_registers_for_containers, plan_gpu_scheduler_policy,
183    plan_gpu_small_mixed_radix_registers, plan_gpu_smooth_stockham_uploads,
184    plan_gpu_smooth_stockham_uploads_for_batches, plan_gpu_stockham_twiddle_source,
185    plan_nvidia_vulkan_power_of_two_radix_registers,
186    plan_nvidia_vulkan_power_of_two_stockham_uploads,
187    plan_nvidia_vulkan_power_of_two_stockham_uploads_for_batches,
188    plan_nvidia_vulkan_rader_fft_registers, plan_nvidia_vulkan_rader_fft_registers_for_containers,
189    plan_nvidia_vulkan_small_mixed_radix_registers, plan_nvidia_vulkan_smooth_stockham_uploads,
190    plan_nvidia_vulkan_smooth_stockham_uploads_for_batches,
191    plan_nvidia_vulkan_two_three_radix_registers,
192};
193pub use scheduler_snapshot::{
194    SCHEDULER_SNAPSHOT_SCHEMA_VERSION, SchedulerAxisClassification,
195    SchedulerPhysicalAxisProbeContext, SchedulerRaderUploadProbe, SchedulerSnapshotMismatch,
196    SchedulerSnapshotPayload, SchedulerSnapshotRecord, compare_upstream_scheduler_snapshot_jsonl,
197    fixed_upstream_scheduler_snapshot_corpus, scheduler_axis_classification_probe,
198    scheduler_rader_upload_axis_block_probe_records,
199    scheduler_rader_upload_axis_block_probe_records_with_context,
200    scheduler_rader_upload_probe_record,
201};
202
203pub use zero_pad_ir::{
204    NdZeroPadPassIr, ZeroPadPassIr, ZeroPadPassOperation, execute_nd_zero_pad_pass,
205    execute_zero_pad_pass,
206};
207
208/// VkFFT release encoded by the upstream header used as the initial porting baseline.
209pub const UPSTREAM_VKFFT_VERSION: &str = "1.3.4";
210
211/// Upstream VkFFT commit used as the initial porting baseline.
212pub const UPSTREAM_VKFFT_COMMIT: &str = "066a17c17068c0f11c9298d848c2976c71fad1c1";