#![allow(
clippy::eq_op,
clippy::erasing_op,
clippy::identity_op,
clippy::len_without_is_empty
)]
use crate::spectral::{deterministic_unit_vector, tridiag_eigenvalue, RitzBounds};
use cubecl::{client::ComputeClient, prelude::*, server::Handle};
const WG: u32 = 256;
const WG_USIZE: usize = 256;
const GROUPS_PER_ROW: usize = 16;
const MAX_BASIS: usize = 32;
pub const EVENT_BREAKDOWN: i32 = 0;
pub const EVENT_NONFINITE_ALPHA: i32 = -1;
pub const EVENT_NONFINITE_BETA: i32 = -2;
#[derive(Clone)]
struct Vec64 {
handle: Handle,
len: usize,
}
#[derive(Clone)]
struct Scalar64 {
handle: Handle,
}
#[derive(Debug, Default, Clone, Copy)]
pub struct RobustStats {
pub scalar_reduction_calls: usize,
pub scalar_reduction_dispatches: usize,
pub basis_store_dispatches: usize,
pub gram_partial_dispatches: usize,
pub gram_finish_dispatches: usize,
pub vector_dispatches: usize,
pub control_dispatches: usize,
pub scalar_readbacks: usize,
pub batch_readbacks: usize,
}
impl RobustStats {
pub fn total_dispatches(&self) -> usize {
self.scalar_reduction_dispatches
+ self.basis_store_dispatches
+ self.gram_partial_dispatches
+ self.gram_finish_dispatches
+ self.vector_dispatches
+ self.control_dispatches
}
}
#[cube(launch_unchecked)]
fn diagonal_f32_f64(d: &Array<f32>, x: &Array<f64>, y: &mut Array<f64>) {
let i = ABSOLUTE_POS;
if i < y.len() {
y[i] = f64::cast_from(d[i]) * x[i];
}
}
#[cube(launch_unchecked)]
fn csr_f32_f64(
row_ptr: &mut Array<u32>,
col_idx: &mut Array<u32>,
values: &Array<f32>,
x: &Array<f64>,
y: &mut Array<f64>,
) {
let row = ABSOLUTE_POS;
if row < y.len() {
let begin = usize::cast_from(row_ptr[row]);
let end = usize::cast_from(row_ptr[row + 1usize]);
let mut sum = 0.0_f64;
let mut k = begin;
while k < end {
let col = usize::cast_from(col_idx[k]);
sum += f64::cast_from(values[k]) * x[col];
k += 1usize;
}
y[row] = sum;
}
}
#[cube(launch_unchecked)]
fn sub_scaled_f64(alpha: &Array<f64>, x: &Array<f64>, y: &mut Array<f64>) {
let i = ABSOLUTE_POS;
if i < y.len() {
y[i] -= alpha[0usize] * x[i];
}
}
#[cube(launch_unchecked)]
fn sub_scaled_store_f64(
alpha: &Array<f64>,
x: &Array<f64>,
y: &mut Array<f64>,
index: &mut Array<u32>,
coeff: &mut Array<f64>,
) {
let i = ABSOLUTE_POS;
if i < y.len() {
y[i] -= alpha[0usize] * x[i];
if i < 1usize {
coeff[usize::cast_from(index[0usize])] = alpha[0usize];
}
}
}
#[cube(launch_unchecked)]
fn store_basis_f64_to_f32(q: &Array<f64>, row_arg: &mut Array<u32>, basis: &mut Array<f32>) {
let i = ABSOLUTE_POS;
if i < q.len() {
let row = usize::cast_from(row_arg[0usize]);
basis[row * q.len() + i] = f32::cast_from(q[i]);
}
}
#[cube(launch_unchecked)]
fn dot_radix16_f64(x: &Array<f64>, y: &Array<f64>, output: &mut Array<f64>) {
if ABSOLUTE_POS < output.len() {
let base = ABSOLUTE_POS * 16usize;
let mut v0 = 0.0_f64;
let i0 = base + 0usize;
if i0 < x.len() {
v0 = x[i0] * y[i0];
}
let mut v1 = 0.0_f64;
let i1 = base + 1usize;
if i1 < x.len() {
v1 = x[i1] * y[i1];
}
let mut v2 = 0.0_f64;
let i2 = base + 2usize;
if i2 < x.len() {
v2 = x[i2] * y[i2];
}
let mut v3 = 0.0_f64;
let i3 = base + 3usize;
if i3 < x.len() {
v3 = x[i3] * y[i3];
}
let mut v4 = 0.0_f64;
let i4 = base + 4usize;
if i4 < x.len() {
v4 = x[i4] * y[i4];
}
let mut v5 = 0.0_f64;
let i5 = base + 5usize;
if i5 < x.len() {
v5 = x[i5] * y[i5];
}
let mut v6 = 0.0_f64;
let i6 = base + 6usize;
if i6 < x.len() {
v6 = x[i6] * y[i6];
}
let mut v7 = 0.0_f64;
let i7 = base + 7usize;
if i7 < x.len() {
v7 = x[i7] * y[i7];
}
let mut v8 = 0.0_f64;
let i8 = base + 8usize;
if i8 < x.len() {
v8 = x[i8] * y[i8];
}
let mut v9 = 0.0_f64;
let i9 = base + 9usize;
if i9 < x.len() {
v9 = x[i9] * y[i9];
}
let mut v10 = 0.0_f64;
let i10 = base + 10usize;
if i10 < x.len() {
v10 = x[i10] * y[i10];
}
let mut v11 = 0.0_f64;
let i11 = base + 11usize;
if i11 < x.len() {
v11 = x[i11] * y[i11];
}
let mut v12 = 0.0_f64;
let i12 = base + 12usize;
if i12 < x.len() {
v12 = x[i12] * y[i12];
}
let mut v13 = 0.0_f64;
let i13 = base + 13usize;
if i13 < x.len() {
v13 = x[i13] * y[i13];
}
let mut v14 = 0.0_f64;
let i14 = base + 14usize;
if i14 < x.len() {
v14 = x[i14] * y[i14];
}
let mut v15 = 0.0_f64;
let i15 = base + 15usize;
if i15 < x.len() {
v15 = x[i15] * y[i15];
}
let a0 = v0 + v1;
let a1 = v2 + v3;
let a2 = v4 + v5;
let a3 = v6 + v7;
let a4 = v8 + v9;
let a5 = v10 + v11;
let a6 = v12 + v13;
let a7 = v14 + v15;
let b0 = a0 + a1;
let b1 = a2 + a3;
let b2 = a4 + a5;
let b3 = a6 + a7;
let c0 = b0 + b1;
let c1 = b2 + b3;
output[ABSOLUTE_POS] = c0 + c1;
}
}
#[cube(launch_unchecked)]
fn reduce_radix16_f64(input: &Array<f64>, output: &mut Array<f64>) {
if ABSOLUTE_POS < output.len() {
let base = ABSOLUTE_POS * 16usize;
let mut v0 = 0.0_f64;
let i0 = base + 0usize;
if i0 < input.len() {
v0 = input[i0];
}
let mut v1 = 0.0_f64;
let i1 = base + 1usize;
if i1 < input.len() {
v1 = input[i1];
}
let mut v2 = 0.0_f64;
let i2 = base + 2usize;
if i2 < input.len() {
v2 = input[i2];
}
let mut v3 = 0.0_f64;
let i3 = base + 3usize;
if i3 < input.len() {
v3 = input[i3];
}
let mut v4 = 0.0_f64;
let i4 = base + 4usize;
if i4 < input.len() {
v4 = input[i4];
}
let mut v5 = 0.0_f64;
let i5 = base + 5usize;
if i5 < input.len() {
v5 = input[i5];
}
let mut v6 = 0.0_f64;
let i6 = base + 6usize;
if i6 < input.len() {
v6 = input[i6];
}
let mut v7 = 0.0_f64;
let i7 = base + 7usize;
if i7 < input.len() {
v7 = input[i7];
}
let mut v8 = 0.0_f64;
let i8 = base + 8usize;
if i8 < input.len() {
v8 = input[i8];
}
let mut v9 = 0.0_f64;
let i9 = base + 9usize;
if i9 < input.len() {
v9 = input[i9];
}
let mut v10 = 0.0_f64;
let i10 = base + 10usize;
if i10 < input.len() {
v10 = input[i10];
}
let mut v11 = 0.0_f64;
let i11 = base + 11usize;
if i11 < input.len() {
v11 = input[i11];
}
let mut v12 = 0.0_f64;
let i12 = base + 12usize;
if i12 < input.len() {
v12 = input[i12];
}
let mut v13 = 0.0_f64;
let i13 = base + 13usize;
if i13 < input.len() {
v13 = input[i13];
}
let mut v14 = 0.0_f64;
let i14 = base + 14usize;
if i14 < input.len() {
v14 = input[i14];
}
let mut v15 = 0.0_f64;
let i15 = base + 15usize;
if i15 < input.len() {
v15 = input[i15];
}
let a0 = v0 + v1;
let a1 = v2 + v3;
let a2 = v4 + v5;
let a3 = v6 + v7;
let a4 = v8 + v9;
let a5 = v10 + v11;
let a6 = v12 + v13;
let a7 = v14 + v15;
let b0 = a0 + a1;
let b1 = a2 + a3;
let b2 = a4 + a5;
let b3 = a6 + a7;
let c0 = b0 + b1;
let c1 = b2 + b3;
output[ABSOLUTE_POS] = c0 + c1;
}
}
#[cube(launch_unchecked)]
fn tile2_gram_mixed_partials_f64(
basis: &Array<f32>,
z: &Array<f64>,
rows_arg: &mut Array<u32>,
partials: &mut Array<f64>,
) {
let group = ABSOLUTE_POS / WG_USIZE;
let lane = ABSOLUTE_POS - group * WG_USIZE;
let tile = group / GROUPS_PER_ROW;
let group_in_tile = group - tile * GROUPS_PER_ROW;
let row0 = tile * 2usize;
let rows = usize::cast_from(rows_arg[0usize]);
let n = z.len();
let mut acc0 = 0.0_f64;
let mut acc1 = 0.0_f64;
let mut i = group_in_tile * WG_USIZE + lane;
while i < n {
let zv = z[i];
if row0 < rows {
acc0 += f64::cast_from(basis[row0 * n + i]) * zv;
}
if row0 + 1usize < rows {
acc1 += f64::cast_from(basis[(row0 + 1usize) * n + i]) * zv;
}
i += GROUPS_PER_ROW * WG_USIZE;
}
let mut shared = SharedMemory::<f64>::new(512usize);
shared[lane] = acc0;
shared[WG_USIZE + lane] = acc1;
sync_cube();
if lane < 128usize {
shared[lane] = shared[lane] + shared[lane + 128usize];
shared[WG_USIZE + lane] = shared[WG_USIZE + lane] + shared[WG_USIZE + lane + 128usize];
}
sync_cube();
if lane < 64usize {
shared[lane] = shared[lane] + shared[lane + 64usize];
shared[WG_USIZE + lane] = shared[WG_USIZE + lane] + shared[WG_USIZE + lane + 64usize];
}
sync_cube();
if lane < 32usize {
shared[lane] = shared[lane] + shared[lane + 32usize];
shared[WG_USIZE + lane] = shared[WG_USIZE + lane] + shared[WG_USIZE + lane + 32usize];
}
sync_cube();
if lane < 16usize {
shared[lane] = shared[lane] + shared[lane + 16usize];
shared[WG_USIZE + lane] = shared[WG_USIZE + lane] + shared[WG_USIZE + lane + 16usize];
}
sync_cube();
if lane < 8usize {
shared[lane] = shared[lane] + shared[lane + 8usize];
shared[WG_USIZE + lane] = shared[WG_USIZE + lane] + shared[WG_USIZE + lane + 8usize];
}
sync_cube();
if lane < 4usize {
shared[lane] = shared[lane] + shared[lane + 4usize];
shared[WG_USIZE + lane] = shared[WG_USIZE + lane] + shared[WG_USIZE + lane + 4usize];
}
sync_cube();
if lane < 2usize {
shared[lane] = shared[lane] + shared[lane + 2usize];
shared[WG_USIZE + lane] = shared[WG_USIZE + lane] + shared[WG_USIZE + lane + 2usize];
}
sync_cube();
if lane < 1usize {
if row0 < rows {
partials[row0 * GROUPS_PER_ROW + group_in_tile] = shared[0usize] + shared[1usize];
}
if row0 + 1usize < rows {
partials[(row0 + 1usize) * GROUPS_PER_ROW + group_in_tile] =
shared[WG_USIZE] + shared[WG_USIZE + 1usize];
}
}
}
#[cube(launch_unchecked)]
fn tile2_finish_apply_mixed_f64(
partials: &Array<f64>,
basis: &Array<f32>,
rows_arg: &mut Array<u32>,
z: &mut Array<f64>,
) {
let group = ABSOLUTE_POS / WG_USIZE;
let lane = ABSOLUTE_POS - group * WG_USIZE;
let rows = usize::cast_from(rows_arg[0usize]);
let mut coeff = SharedMemory::<f64>::new(MAX_BASIS);
if lane < rows {
let base = lane * GROUPS_PER_ROW;
let v0 = partials[base];
let v1 = partials[base + 1usize];
let v2 = partials[base + 2usize];
let v3 = partials[base + 3usize];
let v4 = partials[base + 4usize];
let v5 = partials[base + 5usize];
let v6 = partials[base + 6usize];
let v7 = partials[base + 7usize];
let v8 = partials[base + 8usize];
let v9 = partials[base + 9usize];
let v10 = partials[base + 10usize];
let v11 = partials[base + 11usize];
let v12 = partials[base + 12usize];
let v13 = partials[base + 13usize];
let v14 = partials[base + 14usize];
let v15 = partials[base + 15usize];
let a0 = v0 + v1;
let a1 = v2 + v3;
let a2 = v4 + v5;
let a3 = v6 + v7;
let a4 = v8 + v9;
let a5 = v10 + v11;
let a6 = v12 + v13;
let a7 = v14 + v15;
let b0 = a0 + a1;
let b1 = a2 + a3;
let b2 = a4 + a5;
let b3 = a6 + a7;
coeff[lane] = (b0 + b1) + (b2 + b3);
}
sync_cube();
let i = ABSOLUTE_POS;
if i < z.len() {
let n = z.len();
let mut v = z[i];
if rows > 0usize {
v -= coeff[0usize] * f64::cast_from(basis[0usize * n + i]);
}
if rows > 1usize {
v -= coeff[1usize] * f64::cast_from(basis[1usize * n + i]);
}
if rows > 2usize {
v -= coeff[2usize] * f64::cast_from(basis[2usize * n + i]);
}
if rows > 3usize {
v -= coeff[3usize] * f64::cast_from(basis[3usize * n + i]);
}
if rows > 4usize {
v -= coeff[4usize] * f64::cast_from(basis[4usize * n + i]);
}
if rows > 5usize {
v -= coeff[5usize] * f64::cast_from(basis[5usize * n + i]);
}
if rows > 6usize {
v -= coeff[6usize] * f64::cast_from(basis[6usize * n + i]);
}
if rows > 7usize {
v -= coeff[7usize] * f64::cast_from(basis[7usize * n + i]);
}
if rows > 8usize {
v -= coeff[8usize] * f64::cast_from(basis[8usize * n + i]);
}
if rows > 9usize {
v -= coeff[9usize] * f64::cast_from(basis[9usize * n + i]);
}
if rows > 10usize {
v -= coeff[10usize] * f64::cast_from(basis[10usize * n + i]);
}
if rows > 11usize {
v -= coeff[11usize] * f64::cast_from(basis[11usize * n + i]);
}
if rows > 12usize {
v -= coeff[12usize] * f64::cast_from(basis[12usize * n + i]);
}
if rows > 13usize {
v -= coeff[13usize] * f64::cast_from(basis[13usize * n + i]);
}
if rows > 14usize {
v -= coeff[14usize] * f64::cast_from(basis[14usize * n + i]);
}
if rows > 15usize {
v -= coeff[15usize] * f64::cast_from(basis[15usize * n + i]);
}
if rows > 16usize {
v -= coeff[16usize] * f64::cast_from(basis[16usize * n + i]);
}
if rows > 17usize {
v -= coeff[17usize] * f64::cast_from(basis[17usize * n + i]);
}
if rows > 18usize {
v -= coeff[18usize] * f64::cast_from(basis[18usize * n + i]);
}
if rows > 19usize {
v -= coeff[19usize] * f64::cast_from(basis[19usize * n + i]);
}
if rows > 20usize {
v -= coeff[20usize] * f64::cast_from(basis[20usize * n + i]);
}
if rows > 21usize {
v -= coeff[21usize] * f64::cast_from(basis[21usize * n + i]);
}
if rows > 22usize {
v -= coeff[22usize] * f64::cast_from(basis[22usize * n + i]);
}
if rows > 23usize {
v -= coeff[23usize] * f64::cast_from(basis[23usize * n + i]);
}
if rows > 24usize {
v -= coeff[24usize] * f64::cast_from(basis[24usize * n + i]);
}
if rows > 25usize {
v -= coeff[25usize] * f64::cast_from(basis[25usize * n + i]);
}
if rows > 26usize {
v -= coeff[26usize] * f64::cast_from(basis[26usize * n + i]);
}
if rows > 27usize {
v -= coeff[27usize] * f64::cast_from(basis[27usize * n + i]);
}
if rows > 28usize {
v -= coeff[28usize] * f64::cast_from(basis[28usize * n + i]);
}
if rows > 29usize {
v -= coeff[29usize] * f64::cast_from(basis[29usize * n + i]);
}
if rows > 30usize {
v -= coeff[30usize] * f64::cast_from(basis[30usize * n + i]);
}
if rows > 31usize {
v -= coeff[31usize] * f64::cast_from(basis[31usize * n + i]);
}
z[i] = v;
}
}
#[cube(launch_unchecked)]
fn finalize_beta_normalize_f64(
norm_sq: &Array<f64>,
index: &mut Array<u32>,
beta_prev: &Array<f64>,
active_prev: &Array<f64>,
beta_next: &mut Array<f64>,
active_next: &mut Array<f64>,
coeff: &mut Array<f64>,
z: &Array<f64>,
q_next: &mut Array<f64>,
) {
let i = ABSOLUTE_POS;
if i < q_next.len() {
let ai = usize::cast_from(index[0usize]);
let bi = usize::cast_from(index[1usize]);
let si = usize::cast_from(index[2usize]);
if active_prev[0usize] > 0.5_f64 {
let alpha = coeff[ai];
let bp = beta_prev[0usize];
let beta = norm_sq[0usize].sqrt();
let aa = if alpha < 0.0_f64 { -alpha } else { alpha };
let ba = if beta < 0.0_f64 { -beta } else { beta };
let bpa = if bp < 0.0_f64 { -bp } else { bp };
if i < 1usize {
coeff[bi] = beta;
}
if alpha != alpha || aa > 1.7976931348623157e308_f64 {
q_next[i] = 0.0_f64;
if i < 1usize {
coeff[si] = -1.0_f64;
beta_next[0usize] = 0.0_f64;
active_next[0usize] = 0.0_f64;
}
} else if beta != beta || ba > 1.7976931348623157e308_f64 {
q_next[i] = 0.0_f64;
if i < 1usize {
coeff[si] = -2.0_f64;
beta_next[0usize] = 0.0_f64;
active_next[0usize] = 0.0_f64;
}
} else {
let scale = if aa > bpa { aa } else { bpa };
let tol = 256.0_f64 * 1.1920928955078125e-7_f64 * scale;
if beta <= tol {
q_next[i] = 0.0_f64;
if i < 1usize {
coeff[si] = 0.0_f64;
beta_next[0usize] = 0.0_f64;
active_next[0usize] = 0.0_f64;
}
} else {
q_next[i] = z[i] / beta;
if i < 1usize {
coeff[si] = 1.0_f64;
beta_next[0usize] = beta;
active_next[0usize] = 1.0_f64;
}
}
}
} else {
q_next[i] = 0.0_f64;
if i < 1usize {
coeff[bi] = 0.0_f64;
coeff[si] = 9.0_f64;
beta_next[0usize] = 0.0_f64;
active_next[0usize] = 0.0_f64;
}
}
}
}
#[cube(launch_unchecked)]
fn project_free_mask_f64(mask: &mut Array<u32>, x: &mut Array<f64>) {
let i = ABSOLUTE_POS;
if i < x.len() && mask[i] == 0u32 {
x[i] = 0.0_f64;
}
}
pub trait ResidentF64Operator<R: Runtime> {
fn len(&self) -> usize;
fn apply(&mut self, client: &ComputeClient<R>, x: &Handle, y: &Handle);
fn dispatches_per_apply(&self) -> usize {
1
}
fn name(&self) -> &'static str;
}
pub struct DiagonalF32Operator<R: Runtime> {
d: Handle,
n: usize,
_runtime: core::marker::PhantomData<R>,
}
impl<R: Runtime> DiagonalF32Operator<R> {
pub fn new(client: &ComputeClient<R>, d32: &[f32]) -> Self {
Self {
d: client.create_from_slice(bytemuck::cast_slice(d32)),
n: d32.len(),
_runtime: core::marker::PhantomData,
}
}
}
impl<R: Runtime> ResidentF64Operator<R> for DiagonalF32Operator<R> {
fn len(&self) -> usize {
self.n
}
fn apply(&mut self, client: &ComputeClient<R>, x: &Handle, y: &Handle) {
unsafe {
diagonal_f32_f64::launch_unchecked::<R>(
client,
CubeCount::Static((self.n as u32).div_ceil(WG), 1, 1),
CubeDim::new_1d(WG),
ArrayArg::from_raw_parts(self.d.clone(), self.n),
ArrayArg::from_raw_parts(x.clone(), self.n),
ArrayArg::from_raw_parts(y.clone(), self.n),
);
}
}
fn name(&self) -> &'static str {
"diagonal-f32-to-f64"
}
}
pub struct CsrF32Operator<R: Runtime> {
row_ptr: Handle,
col_idx: Handle,
values: Handle,
n: usize,
nnz: usize,
_runtime: core::marker::PhantomData<R>,
}
fn validate_csr_f32(row_ptr: &[u32], col_idx: &[u32], values: &[f32]) -> Result<usize, String> {
if row_ptr.len() < 2 {
return Err("krylovscope: CSR row_ptr too short".into());
}
if row_ptr[0] != 0 {
return Err("krylovscope: CSR row_ptr[0] must be zero".into());
}
let n = row_ptr.len() - 1;
if n > u32::MAX as usize {
return Err("krylovscope: CSR dimension exceeds u32".into());
}
if col_idx.len() != values.len() || row_ptr[n] as usize != values.len() {
return Err("krylovscope: CSR structure mismatch".into());
}
for row in 0..n {
let begin = row_ptr[row] as usize;
let end = row_ptr[row + 1] as usize;
if begin > end || end > values.len() {
return Err(format!("krylovscope: CSR bad row range at {row}"));
}
for k in begin..end {
if col_idx[k] as usize >= n {
return Err(format!("krylovscope: CSR column out of range at row {row}"));
}
if !values[k].is_finite() {
return Err(format!(
"krylovscope: CSR nonfinite value at row {row}, entry {k}"
));
}
}
}
Ok(n)
}
impl<R: Runtime> CsrF32Operator<R> {
pub fn new(
client: &ComputeClient<R>,
row_ptr: &[u32],
col_idx: &[u32],
values: &[f32],
) -> Result<Self, String> {
let n = validate_csr_f32(row_ptr, col_idx, values)?;
Ok(Self {
row_ptr: client.create_from_slice(bytemuck::cast_slice(row_ptr)),
col_idx: client.create_from_slice(bytemuck::cast_slice(col_idx)),
values: client.create_from_slice(bytemuck::cast_slice(values)),
n,
nnz: values.len(),
_runtime: core::marker::PhantomData,
})
}
pub fn nnz(&self) -> usize {
self.nnz
}
}
impl<R: Runtime> ResidentF64Operator<R> for CsrF32Operator<R> {
fn len(&self) -> usize {
self.n
}
fn apply(&mut self, client: &ComputeClient<R>, x: &Handle, y: &Handle) {
unsafe {
csr_f32_f64::launch_unchecked::<R>(
client,
CubeCount::Static((self.n as u32).div_ceil(WG), 1, 1),
CubeDim::new_1d(WG),
ArrayArg::from_raw_parts(self.row_ptr.clone(), self.n + 1),
ArrayArg::from_raw_parts(self.col_idx.clone(), self.nnz),
ArrayArg::from_raw_parts(self.values.clone(), self.nnz),
ArrayArg::from_raw_parts(x.clone(), self.n),
ArrayArg::from_raw_parts(y.clone(), self.n),
);
}
}
fn name(&self) -> &'static str {
"csr-f32-to-f64"
}
}
pub struct RobustRunner<R: Runtime, O: ResidentF64Operator<R>> {
client: ComputeClient<R>,
operator: O,
basis: Handle,
partials: Handle,
scratch: Vec<Handle>,
scratch_lengths: Vec<usize>,
n: usize,
max_steps: usize,
free_mask: Option<(Vec<u32>, Handle)>,
stats: RobustStats,
}
impl<R: Runtime, O: ResidentF64Operator<R>> RobustRunner<R, O> {
pub fn new(device: &R::Device, operator: O, max_steps: usize) -> Self {
Self::try_new(device, operator, max_steps).unwrap_or_else(|e| panic!("{e}"))
}
pub fn try_new(device: &R::Device, operator: O, max_steps: usize) -> Result<Self, String> {
let n = operator.len();
if n == 0 {
return Err("krylovscope: operator dimension must be nonzero".into());
}
if n > u32::MAX as usize {
return Err("krylovscope: operator dimension exceeds u32 index space".into());
}
if max_steps == 0 || max_steps > MAX_BASIS {
return Err(format!(
"krylovscope: max_steps must be in 1..={MAX_BASIS}, got {max_steps}"
));
}
let basis_bytes = max_steps
.checked_mul(n)
.and_then(|x| x.checked_mul(core::mem::size_of::<f32>()))
.ok_or("krylovscope: basis allocation size overflow")?;
let partial_bytes = max_steps
.checked_mul(GROUPS_PER_ROW)
.and_then(|x| x.checked_mul(core::mem::size_of::<f64>()))
.ok_or("krylovscope: Gram partial allocation size overflow")?;
let client = R::client(device);
let basis = client.empty(basis_bytes);
let partials = client.empty(partial_bytes);
let mut lengths = Vec::new();
let mut k = n;
while k > 1 {
k = k.div_ceil(16);
lengths.push(k);
}
if lengths.is_empty() {
lengths.push(1);
}
let scratch = lengths
.iter()
.map(|&len| client.empty(len * core::mem::size_of::<f64>()))
.collect();
Ok(Self {
client,
operator,
basis,
partials,
scratch,
scratch_lengths: lengths,
n,
max_steps,
free_mask: None,
stats: RobustStats::default(),
})
}
fn launch_1d(&self, len: usize) -> CubeCount {
CubeCount::Static((len as u32).div_ceil(WG), 1, 1)
}
fn vec_from(&self, x: &[f64]) -> Vec64 {
Vec64 {
handle: self.client.create_from_slice(bytemuck::cast_slice(x)),
len: x.len(),
}
}
fn zeros(&self) -> Vec64 {
Vec64 {
handle: self.client.empty(self.n * core::mem::size_of::<f64>()),
len: self.n,
}
}
fn scalar(&self, x: f64) -> Scalar64 {
Scalar64 {
handle: self.client.create_from_slice(bytemuck::cast_slice(&[x])),
}
}
fn series(&self, len: usize) -> Handle {
self.client.empty(len * core::mem::size_of::<f64>())
}
fn reset_stats(&mut self) {
self.stats = RobustStats::default();
}
pub fn operator_name(&self) -> &'static str {
self.operator.name()
}
pub fn set_free_mask(&mut self, free_mask: &[u32]) -> Result<(), String> {
if free_mask.len() != self.n {
return Err(format!(
"krylovscope: free-mask length {} != {}",
free_mask.len(),
self.n
));
}
if !free_mask.iter().any(|&x| x != 0) {
return Err("krylovscope: free-mask contains no active DOF".into());
}
let host = free_mask.to_vec();
let handle = self.client.create_from_slice(bytemuck::cast_slice(&host));
self.free_mask = Some((host, handle));
Ok(())
}
pub fn clear_free_mask(&mut self) {
self.free_mask = None;
}
fn dot(&mut self, x: &Vec64, y: &Vec64) -> Scalar64 {
unsafe {
let out0 = self.scratch_lengths[0];
dot_radix16_f64::launch_unchecked::<R>(
&self.client,
self.launch_1d(out0),
CubeDim::new_1d(WG),
ArrayArg::from_raw_parts(x.handle.clone(), x.len),
ArrayArg::from_raw_parts(y.handle.clone(), y.len),
ArrayArg::from_raw_parts(self.scratch[0].clone(), out0),
);
for s in 1..self.scratch_lengths.len() {
let il = self.scratch_lengths[s - 1];
let ol = self.scratch_lengths[s];
reduce_radix16_f64::launch_unchecked::<R>(
&self.client,
self.launch_1d(ol),
CubeDim::new_1d(WG),
ArrayArg::from_raw_parts(self.scratch[s - 1].clone(), il),
ArrayArg::from_raw_parts(self.scratch[s].clone(), ol),
);
}
}
self.stats.scalar_reduction_calls += 1;
self.stats.scalar_reduction_dispatches += self.scratch_lengths.len();
Scalar64 {
handle: self.scratch.last().unwrap().clone(),
}
}
fn push_basis(&mut self, q: &Vec64, row: usize) {
let rh = self
.client
.create_from_slice(bytemuck::cast_slice(&[row as u32]));
unsafe {
store_basis_f64_to_f32::launch_unchecked::<R>(
&self.client,
self.launch_1d(self.n),
CubeDim::new_1d(WG),
ArrayArg::from_raw_parts(q.handle.clone(), self.n),
ArrayArg::from_raw_parts(rh, 1),
ArrayArg::from_raw_parts(self.basis.clone(), self.max_steps * self.n),
);
}
self.stats.basis_store_dispatches += 1;
}
fn reorth_one_pass(&mut self, z: &mut Vec64, rows: usize) {
let rh = self
.client
.create_from_slice(bytemuck::cast_slice(&[rows as u32]));
let partial_len = rows * GROUPS_PER_ROW;
let workgroups = rows.div_ceil(2) * GROUPS_PER_ROW;
unsafe {
tile2_gram_mixed_partials_f64::launch_unchecked::<R>(
&self.client,
CubeCount::Static(workgroups as u32, 1, 1),
CubeDim::new_1d(WG),
ArrayArg::from_raw_parts(self.basis.clone(), rows * self.n),
ArrayArg::from_raw_parts(z.handle.clone(), self.n),
ArrayArg::from_raw_parts(rh.clone(), 1),
ArrayArg::from_raw_parts(self.partials.clone(), partial_len),
);
tile2_finish_apply_mixed_f64::launch_unchecked::<R>(
&self.client,
self.launch_1d(self.n),
CubeDim::new_1d(WG),
ArrayArg::from_raw_parts(self.partials.clone(), partial_len),
ArrayArg::from_raw_parts(self.basis.clone(), rows * self.n),
ArrayArg::from_raw_parts(rh, 1),
ArrayArg::from_raw_parts(z.handle.clone(), self.n),
);
}
self.stats.gram_partial_dispatches += 1;
self.stats.gram_finish_dispatches += 1;
}
pub fn run(&mut self, requested: usize, seed: u64) -> Result<Pass64, String> {
let start = deterministic_unit_vector(self.n, seed);
self.run_from_start(requested, &start)
}
pub fn run_masked(
&mut self,
requested: usize,
seed: u64,
free_mask: &[u32],
) -> Result<Pass64, String> {
self.set_free_mask(free_mask)?;
self.run(requested, seed)
}
pub fn run_from_start(&mut self, requested: usize, start: &[f64]) -> Result<Pass64, String> {
if requested == 0 || requested > self.max_steps || requested > MAX_BASIS {
return Err("krylovscope: invalid steps".into());
}
if start.len() != self.n {
return Err(format!(
"krylovscope: start length {} != {}",
start.len(),
self.n
));
}
if start.iter().any(|x| !x.is_finite()) {
return Err("krylovscope: start vector contains non-finite values".into());
}
let mut normalized_start = start.to_vec();
if let Some((mask, _)) = &self.free_mask {
for (x, &active) in normalized_start.iter_mut().zip(mask) {
if active == 0 {
*x = 0.0;
}
}
}
let scale = normalized_start
.iter()
.map(|x| x.abs())
.fold(0.0_f64, f64::max);
if scale == 0.0 {
let message = if self.free_mask.is_some() {
"krylovscope: start vector has zero norm in the free-DOF subspace"
} else {
"krylovscope: start vector has zero norm"
};
return Err(message.into());
}
let scaled_norm_sq: f64 = normalized_start
.iter()
.map(|x| {
let y = *x / scale;
y * y
})
.sum();
if !(scaled_norm_sq > 0.0 && scaled_norm_sq.is_finite()) {
return Err("krylovscope: start vector normalization failed".into());
}
let inv_scaled_norm = scaled_norm_sq.sqrt().recip();
for x in &mut normalized_start {
*x = (*x / scale) * inv_scaled_norm;
}
self.reset_stats();
let steps = requested.min(self.n);
let mut q = self.vec_from(&normalized_start);
let mut q_prev = self.zeros();
let mut q_next = self.zeros();
let mut z = self.zeros();
let coeff = self.series(3 * steps);
let mut beta_prev = self.scalar(0.0);
let mut beta_next = self.scalar(0.0);
let mut active_prev = self.scalar(1.0);
let mut active_next = self.scalar(1.0);
for j in 0..steps {
self.push_basis(&q, j);
{
let client = &self.client;
self.operator.apply(client, &q.handle, &z.handle);
}
self.stats.vector_dispatches += self.operator.dispatches_per_apply();
if let Some((_, mask)) = &self.free_mask {
unsafe {
project_free_mask_f64::launch_unchecked::<R>(
&self.client,
self.launch_1d(self.n),
CubeDim::new_1d(WG),
ArrayArg::from_raw_parts(mask.clone(), self.n),
ArrayArg::from_raw_parts(z.handle.clone(), self.n),
);
}
self.stats.vector_dispatches += 1;
}
if j > 0 {
unsafe {
sub_scaled_f64::launch_unchecked::<R>(
&self.client,
self.launch_1d(self.n),
CubeDim::new_1d(WG),
ArrayArg::from_raw_parts(beta_prev.handle.clone(), 1),
ArrayArg::from_raw_parts(q_prev.handle.clone(), self.n),
ArrayArg::from_raw_parts(z.handle.clone(), self.n),
);
}
self.stats.vector_dispatches += 1;
}
let alpha = self.dot(&q, &z);
let ih = self
.client
.create_from_slice(bytemuck::cast_slice(&[j as u32]));
unsafe {
sub_scaled_store_f64::launch_unchecked::<R>(
&self.client,
self.launch_1d(self.n),
CubeDim::new_1d(WG),
ArrayArg::from_raw_parts(alpha.handle.clone(), 1),
ArrayArg::from_raw_parts(q.handle.clone(), self.n),
ArrayArg::from_raw_parts(z.handle.clone(), self.n),
ArrayArg::from_raw_parts(ih, 1),
ArrayArg::from_raw_parts(coeff.clone(), 3 * steps),
);
}
self.stats.vector_dispatches += 1;
self.reorth_one_pass(&mut z, j + 1);
let beta_sq = self.dot(&z, &z);
let idx = self.client.create_from_slice(bytemuck::cast_slice(&[
j as u32,
(steps + j) as u32,
(2 * steps + j) as u32,
]));
unsafe {
finalize_beta_normalize_f64::launch_unchecked::<R>(
&self.client,
self.launch_1d(self.n),
CubeDim::new_1d(WG),
ArrayArg::from_raw_parts(beta_sq.handle.clone(), 1),
ArrayArg::from_raw_parts(idx, 3),
ArrayArg::from_raw_parts(beta_prev.handle.clone(), 1),
ArrayArg::from_raw_parts(active_prev.handle.clone(), 1),
ArrayArg::from_raw_parts(beta_next.handle.clone(), 1),
ArrayArg::from_raw_parts(active_next.handle.clone(), 1),
ArrayArg::from_raw_parts(coeff.clone(), 3 * steps),
ArrayArg::from_raw_parts(z.handle.clone(), self.n),
ArrayArg::from_raw_parts(q_next.handle.clone(), self.n),
);
}
self.stats.vector_dispatches += 1;
self.stats.control_dispatches += 1;
std::mem::swap(&mut q_prev, &mut q);
std::mem::swap(&mut q, &mut q_next);
std::mem::swap(&mut beta_prev, &mut beta_next);
std::mem::swap(&mut active_prev, &mut active_next);
}
let bytes = self
.client
.read_one(coeff.clone())
.map_err(|e| format!("krylovscope: coefficient readback: {e:?}"))?;
self.stats.batch_readbacks += 1;
let packed: &[f64] = bytemuck::cast_slice(&bytes);
if packed.len() != 3 * steps {
return Err(format!(
"krylovscope: packed len {} != {}",
packed.len(),
3 * steps
));
}
let alpha = &packed[..steps];
let beta = &packed[steps..2 * steps];
let status = &packed[2 * steps..];
let first = status.iter().position(|&v| v != 1.0_f64);
let actual = first.map(|j| j + 1).unwrap_or(steps);
let event = first.map(|j| status[j] as i32);
let nonfinite = matches!(
event,
Some(EVENT_NONFINITE_ALPHA) | Some(EVENT_NONFINITE_BETA)
);
if nonfinite || actual == 0 {
return Ok(Pass64 {
ritz: None,
steps: actual,
event,
stats: self.stats,
});
}
let off = &beta[..actual.saturating_sub(1)];
let ritz = RitzBounds {
lambda_min: tridiag_eigenvalue(&alpha[..actual], off, 0),
lambda_max: tridiag_eigenvalue(&alpha[..actual], off, actual - 1),
steps: actual,
tail_beta: beta[actual - 1],
};
Ok(Pass64 {
ritz: Some(ritz),
steps: actual,
event,
stats: self.stats,
})
}
}
#[derive(Debug, Clone, Copy)]
pub struct Pass64 {
pub ritz: Option<RitzBounds>,
pub steps: usize,
pub event: Option<i32>,
pub stats: RobustStats,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn csr_validation_rejects_bad_row_ranges() {
let error = validate_csr_f32(&[0, 2, 1], &[0], &[1.0]).unwrap_err();
assert!(error.contains("bad row range"));
}
#[test]
fn csr_validation_rejects_out_of_range_columns() {
let error = validate_csr_f32(&[0, 1], &[1], &[1.0]).unwrap_err();
assert!(error.contains("column out of range"));
}
#[test]
fn total_dispatches_includes_control_dispatches() {
let stats = RobustStats {
scalar_reduction_dispatches: 2,
basis_store_dispatches: 3,
gram_partial_dispatches: 5,
gram_finish_dispatches: 7,
vector_dispatches: 11,
control_dispatches: 13,
..RobustStats::default()
};
assert_eq!(stats.total_dispatches(), 41);
}
}