Skip to main content

leto_ops/
lib.rs

1#![warn(missing_docs)]
2//! Leto Operations contains performance-tuned math and reduction kernels for Leto arrays.
3
4/// Application-level operation entry points.
5pub mod application;
6/// Operation scalar and strategy contracts.
7pub mod domain;
8/// SIMD and parallel execution infrastructure.
9pub mod infrastructure;
10
11pub use domain::real::RealScalar;
12pub use domain::rng::Xorshift64;
13pub use domain::scalar::Scalar;
14pub use domain::strategy::{ExecutionStrategy, ScalarStrategy};
15pub use infrastructure::cache::{
16    cache_geometry, cached_cache_geometry, CacheGeometry, MatmulTilePolicy,
17};
18
19pub use domain::strategy::SimdStrategy;
20
21#[cfg(feature = "parallel")]
22pub use domain::strategy::ParallelStrategy;
23
24#[cfg(feature = "parallel")]
25pub use infrastructure::parallel::{for_each_task_partition_mut, for_each_task_partition_mut_with};
26
27pub use application::attention::{
28    scaled_dot_product_attention_backward_accumulate, scaled_dot_product_attention_into,
29    AttentionError, AttentionGradients, AttentionMask, AttentionOperand, AttentionResult,
30    GroupedKeepMask,
31};
32pub use application::convolution::{
33    convolution_backward_accumulate, convolution_forward_into,
34    convolution_transposed_backward_accumulate, convolution_transposed_forward_into,
35    ConvolutionParameters, TransposedConvolutionGradients, TransposedConvolutionParameters,
36};
37pub use application::embedding::{embedding_gather, embedding_gather_into};
38pub use application::linalg::{
39    bidiagonalize,
40    bunch_kaufman,
41    cholesky_decompose,
42    cholesky_det,
43    cholesky_inv,
44    cholesky_solve,
45    col_piv_qr,
46    // ── Complex linear algebra ────────────────────────────────────────────────
47    complex_inv,
48    complex_solve,
49    det,
50    eigenvalues,
51    full_piv_lu,
52    // ── Hermitian eigensolvers ─────────────────────────────────────────────────
53    hermitian_eigen_jacobi,
54    hermitian_eigen_qr,
55    hessenberg,
56    inv,
57    kron,
58    l2_normalize,
59    l2_normalize_into,
60    lu_decompose,
61    matexp,
62    matpow,
63    matrix_rank,
64    matrix_rank_with_tolerance,
65    nnls,
66    norm,
67    norm_l1,
68    norm_l2,
69    norm_max,
70    pinv,
71    qr_decompose,
72    schur,
73    singular_values,
74    solve,
75    solve_least_squares,
76    svd_decompose,
77    symmetric_eigen_jacobi,
78    symmetric_eigen_jacobi_with_tolerance,
79    symmetric_eigen_qr,
80    symmetric_eigenvalues_jacobi,
81    symmetric_eigenvalues_jacobi_with_tolerance,
82    take_dbdsqr_counts,
83    take_francis_steps,
84    take_ql_sweeps,
85    trace,
86    udu_decompose,
87    AsMatrixView,
88
89    BidiagonalDecomposition,
90    BunchKaufmanDecomposition,
91    CholeskyDecomposition,
92    ColPivQrDecomposition,
93
94    FullPivLuDecomposition,
95    HermitianEigenConfig,
96    HermitianEigenResult,
97    HessenbergDecomposition,
98
99    LuDecomposition,
100    MatrixDecompose,
101    MatrixFunction,
102    MatrixNorm,
103    MatrixProduct,
104    MatrixProperties,
105    MatrixSolve,
106    NnlsConfig,
107    NnlsResult,
108    NormKind,
109    NormL1,
110    NormL2,
111    NormMax,
112
113    QrDecomposition,
114    RealSchur,
115
116    SvdDecomposition,
117    SymmetricEigenDecomposition,
118    SymmetricEigenWorkspace,
119    UduDecomposition,
120};
121pub use application::loss::{
122    cross_entropy_backward_accumulate, cross_entropy_forward_into, ctc, CrossEntropyError,
123    CrossEntropyOperand, CrossEntropyResult,
124};
125pub use application::map::{
126    add, binary_map, binary_map_with_cache_geometry, div, mul, scalar_map, scalar_map_into, sub,
127    sum, AddOp, BinaryOp, DivOp, EqOp, GeOp, GtOp, LeOp, LtOp, MulOp, NeOp, SubOp,
128};
129pub use application::matrix::{batched_matmul, matmul, matmul_accumulate, matmul_with_tile_policy};
130pub use application::nonlinear::{AndersonAccelerator, AndersonConfig, AndersonMethod};
131pub use application::optimization::{minimize, LbfgsConfig, LbfgsMemory, LbfgsResult};
132pub use application::pooling::{pooling_backward_accumulate, pooling_forward_into, PoolingMode};
133pub use application::random::{
134    normal_with_seed, normal_with_seed_into, uniform_with_seed, uniform_with_seed_into,
135};
136pub use application::reduction::{
137    max, max_axis, max_axis_into, mean_axis, mean_axis_into, min, min_axis, min_axis_into,
138    product_axis, product_axis_into, reduce_all, reduce_axis, reduce_axis_into, sum_axis,
139    sum_axis_into, topk_axis, topk_axis_into, AxisReduction, MaxAxis, MeanAxis, MinAxis,
140    ProductAxis, SumAxis, Topk,
141};
142pub use application::scan::{
143    cumsum, cumsum_into, scan_axis, scan_axis_into, CumProdOp, CumSumOp, ScanDirection, ScanOp,
144};
145pub use application::signal::{blackman, hamming, hann, tukey, wrap_to_pi};
146pub use application::sliding_window::{fold_into, unfold_into};
147pub use application::sparse::{
148    csc_spmv, csc_spmv_into, csc_spmv_view_into, csr_to_dense, factor_numeric, factor_symbolic,
149    factor_symbolic_with_ordering, solve_csr_via_dense_lu, sparse_lu_solve, spgemm, spmm,
150    spmm_into, spmm_view_into, spmv, spmv_into, spmv_view_into, CooMatrix, CscColumn, CscMatrix,
151    CscView, CsrMatrix, CsrRow, CsrView, NumericLu, OrderingStrategy, OwnedNumericLu,
152    SparseLuSolver, SymbolicLu, DENSE_LIMIT_DEFAULT,
153};
154/// Special mathematical functions (sinc, erf, Bessel J₀/J₁/Jₙ, Legendre polynomials).
155pub use application::special::{bessel_k0, erf, j0, j1, jn, sinc};
156pub use application::special_legendre::{legendre_poly, legendre_poly_and_deriv};
157pub use application::stateful_update::{
158    stateful_update, AdaGrad, AdaGradParameters, Adam, AdamParameters, AdamW, AdamWParameters,
159    RmsProp, RmsPropParameters, Sgd, SgdParameters, StatefulUpdateRule,
160};
161pub use application::statistics::{
162    normalized_rmse, nrmse, pearson, percentile_range, phase_error_degrees_for_correlation,
163    phase_shift_correlation_curve, psnr, rmse, validation_psnr_from_relative_rmse,
164};
165pub use application::stencil::laplacian_2d_into;
166pub use application::triangular::{triangular, triangular_into, triangular_keeps, TriangularMode};
167pub use application::unary::{
168    map, map_inplace, map_into, map_into_with_cache_geometry, mapv, unary_map, unary_map_into,
169    AbsOp, AcosOp, AcoshOp, AsinOp, AsinhOp, AtanOp, AtanhOp, CeilOp, CosOp, CoshOp, EluGradOp,
170    EluOp, ErfOp, ErfcOp, Exp2Op, ExpNegOp, ExpOp, Expm1Op, FloorOp, GeluGradOp, GeluOp,
171    GeluTanhGradOp, GeluTanhOp, HardsigmoidGradOp, HardsigmoidOp, HardswishGradOp, HardswishOp,
172    J0Op, J1Op, K0Op, LgammaOp, LnOp, Log10Op, Log1pOp, Log2Op, MishGradOp, MishOp, NegOp, PowfOp,
173    RecipOp, ReluGradOp, ReluOp, RoundOp, SigmoidGradOp, SigmoidOp, SignOp, SiluGradOp, SiluOp,
174    SinOp, SincOp, SinhOp, SoftplusGradOp, SoftplusOp, SoftsignGradOp, SoftsignOp, SqrtOp, TanOp,
175    TanhGradOp, TanhOp, TruncOp, UnaryOp,
176};
177pub use application::vector::{cross_into, dot, hamming_distance, jaccard_distance, matvec};
178pub use application::volume::{ray_line_integrals, ray_line_integrals_into, RAY_STRIDE};
179pub use application::zip::{
180    coordinate_map_inplace, coordinate_map_plan, coordinate_map_plan_inplace, indexed_fold,
181    indexed_fold_fortran, indexed_map4_inplace, indexed_map_inplace, indexed_zip_mut_with,
182    zip_fold, zip_mut_with, CoordinateMapPlan, IndexedZipMutOutputs, ZipMutOutputs, ZipSources,
183};
184pub use domain::layout::{ComplexLayout, SquareTransposeError};
185
186// ── Interpolation (SSOT) ──────────────────────────────────────────────────────
187/// 2-D and 3-D spatial interpolation (bilinear, trilinear) in index space.
188pub use application::interpolation::{
189    bilinear, bilinear_index_space, trilinear, trilinear_index_space,
190};
191/// 1-D interpolation trait and implementations.
192pub use application::interpolation::{
193    CubicSplineInterpolation, Interpolation1D, LagrangeInterpolation, LinearInterpolation,
194};
195
196// ── Finite-difference differentiation (SSOT) ─────────────────────────────────
197/// Derived staggered/collocated first-derivative taps to arbitrary even order.
198pub use application::diff::{
199    central_first_derivative_coefficients, staggered_first_derivative_coefficients,
200    TapCoefficients, MAX_HALF_ORDER,
201};
202/// Arbitrary-even-order staggered gradient/divergence pair for a Yee leapfrog.
203pub use application::diff::{Axis, StaggeredLeapfrog3D};
204/// Generic 1-D finite-difference operator.
205pub use application::diff::{FiniteDifference, FiniteDifferenceScheme};
206/// Generic 3-D finite-difference operator (provider-SSOT for kwavers/CFDrs/helios
207/// first-derivative kernels: central 2nd/4th/6th + Yee staggered forward/backward).
208pub use application::diff::{FiniteDifference3D, FiniteDifference3DScheme};
209/// Fields holding only a window of a grid's x-planes, for fused passes run a
210/// slab at a time.
211pub use application::diff::{PlaneWindow, PlaneWindowMut};
212
213// ── Quadrature (SSOT) ─────────────────────────────────────────────────────────
214/// Numerical quadrature (integration) rules.
215pub use application::quadrature::{
216    gauss_legendre_nodes_weights, CompositeQuadrature, GaussLegendre2, GaussLegendre3,
217    GaussLegendre5, GaussLegendreN, Quadrature, SimpsonsRule, TrapezoidalRule, GL3_NODES,
218    GL3_NODES_UNIT, GL3_WEIGHTS, GL3_WEIGHTS_UNIT,
219};