#![allow(clippy::assign_op_pattern)]
use crate::cuda::{CsrF32Operator, ResidentF64Operator};
use cubecl::{client::ComputeClient, prelude::*, server::Handle};
use std::collections::HashMap;
const WG: u32 = 256;
const PDSM_WINDOW: usize = 32;
#[derive(Clone, Copy, Debug)]
pub enum HostSparseOperator<'a> {
CsrF32 {
row_ptr: &'a [u32],
col_idx: &'a [u32],
values: &'a [f32],
},
Pdsm32F32 {
base_col: &'a [u32],
bitmap: &'a [u32],
row_value_offset: &'a [u32],
values: &'a [f32],
},
}
#[derive(Clone, Copy, Debug)]
pub struct ValidationPolicy {
pub require_sorted_columns: bool,
pub require_diagonal: bool,
pub require_symmetric: bool,
pub symmetry_tol: f64,
}
impl Default for ValidationPolicy {
fn default() -> Self {
Self {
require_sorted_columns: true,
require_diagonal: true,
require_symmetric: true,
symmetry_tol: 2.0e-6,
}
}
}
#[derive(Clone, Debug)]
pub struct ValidationReport {
pub kind: &'static str,
pub n: usize,
pub nnz: usize,
pub sorted_columns: bool,
pub diagonal_complete: bool,
pub symmetric_checked: bool,
pub max_symmetry_abs: f64,
}
fn key(row: usize, col: usize) -> u64 {
((row as u64) << 32) | col as u64
}
fn check_symmetry(n: usize, entries: &[(usize, usize, f32)], tol: f64) -> Result<f64, String> {
if n > u32::MAX as usize {
return Err("krylovscope: dimension exceeds u32 index space".into());
}
let mut map = HashMap::<u64, f32>::with_capacity(entries.len() * 2);
for &(r, c, v) in entries {
if map.insert(key(r, c), v).is_some() {
return Err(format!("krylovscope: duplicate matrix entry at ({r},{c})"));
}
}
let mut worst = 0.0_f64;
for &(r, c, v) in entries {
let Some(&vt) = map.get(&key(c, r)) else {
return Err(format!(
"krylovscope: symmetry partner missing for ({r},{c})"
));
};
let d = (v as f64 - vt as f64).abs();
worst = worst.max(d);
let scale = (v as f64).abs().max((vt as f64).abs());
if d > tol * scale {
return Err(format!(
"krylovscope: symmetry mismatch at ({r},{c}): abs={d:.3e}"
));
}
}
Ok(worst)
}
pub fn validate_host_operator(
op: HostSparseOperator<'_>,
policy: ValidationPolicy,
) -> Result<ValidationReport, String> {
if policy.require_symmetric && (!policy.symmetry_tol.is_finite() || policy.symmetry_tol < 0.0) {
return Err(format!(
"krylovscope: symmetry_tol must be finite and nonnegative, got {}",
policy.symmetry_tol
));
}
match op {
HostSparseOperator::CsrF32 {
row_ptr,
col_idx,
values,
} => {
if row_ptr.len() < 2 {
return Err("krylovscope: CSR row_ptr too short".into());
}
if row_ptr[0] != 0 {
return Err("krylovscope: CSR row_ptr[0] must be zero".into());
}
let n = row_ptr.len() - 1;
if n > u32::MAX as usize {
return Err("krylovscope: CSR dimension exceeds u32".into());
}
if col_idx.len() != values.len() || row_ptr[n] as usize != values.len() {
return Err("krylovscope: CSR nnz/row_ptr mismatch".into());
}
let mut diag = vec![false; n];
let mut entries = Vec::with_capacity(values.len());
let mut sorted = true;
for r in 0..n {
let b = row_ptr[r] as usize;
let e = row_ptr[r + 1] as usize;
if b > e || e > values.len() {
return Err(format!("krylovscope: CSR bad row range at {r}"));
}
let mut prev = None;
for k in b..e {
let c = col_idx[k] as usize;
if c >= n {
return Err(format!("krylovscope: CSR column out of range at row {r}"));
}
if let Some(p) = prev {
if c <= p {
sorted = false;
}
}
prev = Some(c);
if c == r {
diag[r] = true;
}
if !values[k].is_finite() {
return Err(format!("krylovscope: CSR nonfinite value at ({r},{c})"));
}
entries.push((r, c, values[k]));
}
}
if policy.require_sorted_columns && !sorted {
return Err("krylovscope: CSR columns must be strictly sorted per row".into());
}
let diag_ok = diag.iter().all(|&x| x);
if policy.require_diagonal && !diag_ok {
return Err("krylovscope: CSR diagonal is incomplete".into());
}
let max_sym = if policy.require_symmetric {
check_symmetry(n, &entries, policy.symmetry_tol)?
} else {
0.0
};
Ok(ValidationReport {
kind: "csr-f32",
n,
nnz: values.len(),
sorted_columns: sorted,
diagonal_complete: diag_ok,
symmetric_checked: policy.require_symmetric,
max_symmetry_abs: max_sym,
})
}
HostSparseOperator::Pdsm32F32 {
base_col,
bitmap,
row_value_offset,
values,
} => {
let n = base_col.len();
if n == 0 || bitmap.len() != n || row_value_offset.len() != n + 1 {
return Err("krylovscope: malformed PDSM32 metadata".into());
}
if row_value_offset[0] != 0 || row_value_offset[n] as usize != values.len() {
return Err("krylovscope: PDSM32 packed offset mismatch".into());
}
if n > u32::MAX as usize {
return Err("krylovscope: PDSM32 dimension exceeds u32".into());
}
let mut diag = vec![false; n];
let mut entries = Vec::with_capacity(values.len());
for r in 0..n {
let b = row_value_offset[r] as usize;
let e = row_value_offset[r + 1] as usize;
if b > e || e > values.len() {
return Err(format!("krylovscope: PDSM32 bad row offset at {r}"));
}
let base = base_col[r] as usize;
let mask = bitmap[r];
if mask.count_ones() as usize != e - b {
return Err(format!(
"krylovscope: PDSM32 bitmap/value count mismatch at row {r}"
));
}
let mut k = b;
for slot in 0..PDSM_WINDOW {
if (mask & (1u32 << slot)) != 0 {
let c = base
.checked_add(slot)
.ok_or("krylovscope: PDSM32 column overflow")?;
if c >= n {
return Err(format!(
"krylovscope: PDSM32 column out of range at row {r}"
));
}
let v = values[k];
if !v.is_finite() {
return Err(format!(
"krylovscope: PDSM32 nonfinite value at ({r},{c})"
));
}
if c == r {
diag[r] = true;
}
entries.push((r, c, v));
k += 1;
}
}
if k != e {
return Err(format!(
"krylovscope: PDSM32 packed scan mismatch at row {r}"
));
}
}
let diag_ok = diag.iter().all(|&x| x);
if policy.require_diagonal && !diag_ok {
return Err("krylovscope: PDSM32 diagonal is incomplete".into());
}
let max_sym = if policy.require_symmetric {
check_symmetry(n, &entries, policy.symmetry_tol)?
} else {
0.0
};
Ok(ValidationReport {
kind: "pdsm32-f32",
n,
nnz: values.len(),
sorted_columns: true,
diagonal_complete: diag_ok,
symmetric_checked: policy.require_symmetric,
max_symmetry_abs: max_sym,
})
}
}
}
#[cube(launch_unchecked)]
fn pdsm32_f32_f64(
base_col: &mut Array<u32>,
bitmap: &mut Array<u32>,
row_value_offset: &mut Array<u32>,
values: &Array<f32>,
x: &Array<f64>,
y: &mut Array<f64>,
) {
let row = ABSOLUTE_POS;
if row < y.len() {
let base = usize::cast_from(base_col[row]);
let mask = bitmap[row];
let mut k = usize::cast_from(row_value_offset[row]);
let mut slot = 0usize;
let mut bit = 1u32;
let mut sum = 0.0_f64;
while slot < PDSM_WINDOW {
if (mask & bit) != 0u32 {
sum += f64::cast_from(values[k]) * x[base + slot];
k += 1usize;
}
slot += 1usize;
bit = bit * 2u32;
}
y[row] = sum;
}
}
pub struct Pdsm32F32Operator<R: Runtime> {
base_col: Handle,
bitmap: Handle,
row_value_offset: Handle,
values: Handle,
n: usize,
nnz: usize,
_runtime: core::marker::PhantomData<R>,
}
impl<R: Runtime> Pdsm32F32Operator<R> {
pub(crate) fn upload(
client: &ComputeClient<R>,
base_col: &[u32],
bitmap: &[u32],
row_value_offset: &[u32],
values: &[f32],
) -> Self {
Self {
base_col: client.create_from_slice(bytemuck::cast_slice(base_col)),
bitmap: client.create_from_slice(bytemuck::cast_slice(bitmap)),
row_value_offset: client.create_from_slice(bytemuck::cast_slice(row_value_offset)),
values: client.create_from_slice(bytemuck::cast_slice(values)),
n: base_col.len(),
nnz: values.len(),
_runtime: core::marker::PhantomData,
}
}
pub fn nnz(&self) -> usize {
self.nnz
}
}
impl<R: Runtime> ResidentF64Operator<R> for Pdsm32F32Operator<R> {
fn len(&self) -> usize {
self.n
}
fn apply(&mut self, client: &ComputeClient<R>, x: &Handle, y: &Handle) {
unsafe {
pdsm32_f32_f64::launch_unchecked::<R>(
client,
CubeCount::Static((self.n as u32).div_ceil(WG), 1, 1),
CubeDim::new_1d(WG),
ArrayArg::from_raw_parts(self.base_col.clone(), self.n),
ArrayArg::from_raw_parts(self.bitmap.clone(), self.n),
ArrayArg::from_raw_parts(self.row_value_offset.clone(), self.n + 1),
ArrayArg::from_raw_parts(self.values.clone(), self.nnz),
ArrayArg::from_raw_parts(x.clone(), self.n),
ArrayArg::from_raw_parts(y.clone(), self.n),
);
}
}
fn name(&self) -> &'static str {
"pdsm32-f32-to-f64"
}
}
pub enum UploadedResidentOperator<R: Runtime> {
Csr(CsrF32Operator<R>),
Pdsm32(Pdsm32F32Operator<R>),
}
impl<R: Runtime> ResidentF64Operator<R> for UploadedResidentOperator<R> {
fn len(&self) -> usize {
match self {
Self::Csr(x) => x.len(),
Self::Pdsm32(x) => x.len(),
}
}
fn apply(&mut self, client: &ComputeClient<R>, x: &Handle, y: &Handle) {
match self {
Self::Csr(op) => op.apply(client, x, y),
Self::Pdsm32(op) => op.apply(client, x, y),
}
}
fn dispatches_per_apply(&self) -> usize {
1
}
fn name(&self) -> &'static str {
match self {
Self::Csr(_) => "csr-f32-to-f64",
Self::Pdsm32(_) => "pdsm32-f32-to-f64",
}
}
}
pub fn validate_and_upload<R: Runtime>(
client: &ComputeClient<R>,
op: HostSparseOperator<'_>,
policy: ValidationPolicy,
) -> Result<(UploadedResidentOperator<R>, ValidationReport), String> {
let report = validate_host_operator(op, policy)?;
let uploaded = match op {
HostSparseOperator::CsrF32 {
row_ptr,
col_idx,
values,
} => UploadedResidentOperator::Csr(CsrF32Operator::new(client, row_ptr, col_idx, values)?),
HostSparseOperator::Pdsm32F32 {
base_col,
bitmap,
row_value_offset,
values,
} => UploadedResidentOperator::Pdsm32(Pdsm32F32Operator::upload(
client,
base_col,
bitmap,
row_value_offset,
values,
)),
};
Ok((uploaded, report))
}
#[cfg(test)]
mod tests {
use super::*;
fn asymmetric_csr<'a>(
row_ptr: &'a [u32],
col_idx: &'a [u32],
values: &'a [f32],
) -> HostSparseOperator<'a> {
HostSparseOperator::CsrF32 {
row_ptr,
col_idx,
values,
}
}
#[test]
fn nonfinite_symmetry_tolerance_is_rejected() {
let row_ptr = [0, 2, 4];
let col_idx = [0, 1, 0, 1];
let values = [1.0, 2.0, 3.0, 1.0];
for tol in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
let policy = ValidationPolicy {
symmetry_tol: tol,
..ValidationPolicy::default()
};
let error = validate_host_operator(asymmetric_csr(&row_ptr, &col_idx, &values), policy)
.unwrap_err();
assert!(error.contains("symmetry_tol"));
}
}
#[test]
fn negative_symmetry_tolerance_is_rejected() {
let row_ptr = [0, 2, 4];
let col_idx = [0, 1, 0, 1];
let values = [1.0, 2.0, 3.0, 1.0];
let policy = ValidationPolicy {
symmetry_tol: -1.0,
..ValidationPolicy::default()
};
let error = validate_host_operator(asymmetric_csr(&row_ptr, &col_idx, &values), policy)
.unwrap_err();
assert!(error.contains("symmetry_tol"));
}
#[test]
fn symmetry_tolerance_is_relative_at_small_scale() {
let row_ptr = [0, 2, 4];
let col_idx = [0, 1, 0, 1];
let values = [1.0e-8, 2.0e-8, 3.0e-8, 1.0e-8];
let error = validate_host_operator(
asymmetric_csr(&row_ptr, &col_idx, &values),
ValidationPolicy::default(),
)
.unwrap_err();
assert!(error.contains("symmetry mismatch"));
}
}