use super::{
LANES, add_squared_diff_scaled, all_finite, col_slice_checked, col_slice_mut_checked,
finite_slice, load4, rows_checked, store4, unit_row_stride,
};
use crate::error::GprError;
use crate::kernel::KernelScalar;
use crate::kernel::dist::{ArdBlocks, BlockState, GATE_DIMS, Gate};
use crate::kernel::dist::{ArdSqDiff, LowerRuns, Runs, col_chunk, par_lower_cols, worker_count};
use crate::kernel::{Triangle, finite_dist};
use crate::math::KernelMath;
use faer::linalg::matmul::matmul;
use faer::reborrow::ReborrowMut;
use faer::{Accum, MatMut, MatRef, Par};
use rayon::prelude::*;
use wide::f64x4;
pub(crate) fn try_apply_cross<M: KernelMath>(
x: MatRef<'_, f64>,
xs: MatRef<'_, f64>,
mut out: MatMut<'_, f64>,
inv_ell_sq: &[f64],
) -> Result<bool, GprError> {
let n = x.nrows();
let m = xs.nrows();
let d = inv_ell_sq.len();
if x.ncols() != d || xs.ncols() != d || out.nrows() != n || out.ncols() != m {
return Ok(false);
}
if !unit_row_stride(x) || !unit_row_stride(xs) || !unit_row_stride(out.as_ref()) {
return Ok(false);
}
let column = |query: usize, dest: &mut [f64]| {
dest.fill(0.0);
for (dim, &w) in inv_ell_sq.iter().enumerate() {
let x0 = col_slice_checked(xs, dim)?[query];
add_squared_diff_scaled(col_slice_checked(x, dim)?, x0, w, dest);
}
exp_half_in_place::<M>(dest)
};
if m == 1 {
column(0, col_slice_mut_checked(out, 0)?)?;
return Ok(true);
}
let n_parts = worker_count();
out.rb_mut()
.par_col_partition_mut(n_parts)
.enumerate()
.try_for_each(|(chunk_idx, mut part)| {
let (start, len) = col_chunk(m, chunk_idx, n_parts);
for local in 0..len {
column(start + local, col_slice_mut_checked(part.rb_mut(), local)?)?;
}
Ok::<(), GprError>(())
})?;
Ok(true)
}
pub(crate) fn try_apply_cross_from_blocks<M: KernelMath, T: KernelScalar, S: BlockState>(
blocks: ArdBlocks<'_, T, S>,
mut out: MatMut<'_, f64>,
inv_ell_sq: &[f64],
) -> Result<bool, GprError> {
let m = out.ncols();
let d = inv_ell_sq.len();
if !unit_row_stride(out.as_ref())
|| blocks.d() != d
|| blocks.rows() != out.nrows()
|| blocks.cols() != m
|| !blocks.dense_f64()
{
return Ok(false);
}
let chunk = |start: usize, mut part: MatMut<'_, f64>| -> Result<(), GprError> {
let len = part.ncols();
if len == 0 {
return Ok(());
}
if d > GATE_DIMS {
for local in 0..len {
let dest = col_slice_mut_checked(part.rb_mut(), local)?;
dest.fill(0.0);
for (dim, &w) in inv_ell_sq.iter().enumerate() {
let run = blocks.column(dim, start + local)?.ok_or_else(not_dense)?;
scale_add_checked(run, w, dest);
}
exp_half_in_place::<M>(dest)?;
}
return Ok(());
}
let gates = blocks.gates().ok_or_else(not_dense)?;
let mut gate = gates.gate(start);
weighted_sum(
&mut gate,
inv_ell_sq,
col_slice_mut_checked(part.rb_mut(), 0)?,
);
gate.verdict().map_err(|err| blocks.locate(err))?;
for local in 0..len {
let finite = if local + 1 < len {
let mut gate = gates.gate(start + local + 1);
let (left, right) = part.rb_mut().split_at_col_mut(local + 1);
let finite = exp_half_then_sum::<M, S>(
col_slice_mut_checked(left, local)?,
&mut gate,
inv_ell_sq,
col_slice_mut_checked(right, 0)?,
);
gate.verdict().map_err(|err| blocks.locate(err))?;
finite
} else {
exp_half_finite::<M>(col_slice_mut_checked(part.rb_mut(), local)?)
};
if !finite {
return Err(GprError::NonFiniteKernelValue);
}
}
Ok(())
};
if m == 1 {
chunk(0, out)?;
return Ok(true);
}
let n_parts = worker_count();
out.rb_mut()
.par_col_partition_mut(n_parts)
.enumerate()
.try_for_each(|(chunk_idx, part)| chunk(col_chunk(m, chunk_idx, n_parts).0, part))?;
Ok(true)
}
fn cached_rows_offset(row_start: usize, col: usize) -> Result<usize, GprError> {
row_start
.checked_sub(col)
.ok_or_else(|| GprError::UnsupportedKernelOperation {
reason: "the ARD cache holds the lower triangle only".to_owned(),
})
}
fn scale_add(src: &[f64], scale: f64, acc: &mut [f64]) -> Result<(), GprError> {
debug_assert_eq!(src.len(), acc.len());
let sv = f64x4::splat(scale);
let mut i = 0;
while i + LANES <= src.len() {
let s = load4(src, i);
if !all_finite(s) {
return Err(GprError::NonFiniteInput);
}
store4(acc, i, load4(acc, i) + s * sv);
i += LANES;
}
while i < src.len() {
acc[i] += finite_dist(src[i])? * scale;
i += 1;
}
Ok(())
}
fn not_dense() -> GprError {
GprError::UnsupportedKernelOperation {
reason: "the ARD blocks are not dense f64 columns".to_owned(),
}
}
fn weighted_sum<S: BlockState>(gate: &mut Gate<'_, S>, w: &[f64], dest: &mut [f64]) {
let mut i = 0;
while i + LANES <= dest.len() {
store4(dest, i, gate.weighted4(w, i));
i += LANES;
}
while i < dest.len() {
dest[i] = gate.weighted1(w, i);
i += 1;
}
}
fn exp_half_then_sum<M: KernelMath, S: BlockState>(
cur: &mut [f64],
gate: &mut Gate<'_, S>,
w: &[f64],
next: &mut [f64],
) -> bool {
let half = f64x4::splat(-0.5);
let mut overflow = f64x4::ZERO;
let mut i = 0;
while i + LANES <= cur.len() {
store4(next, i, gate.weighted4(w, i));
let c = load4(cur, i);
overflow += c * f64x4::ZERO;
store4(cur, i, M::exp_f64x4(c * half));
i += LANES;
}
let mut tail = 0.0;
while i < cur.len() {
next[i] = gate.weighted1(w, i);
tail += cur[i] * 0.0;
cur[i] = M::exp(-cur[i] * 0.5);
i += 1;
}
overflow.reduce_add() + tail == 0.0
}
fn exp_half_finite<M: KernelMath>(cur: &mut [f64]) -> bool {
let half = f64x4::splat(-0.5);
let mut overflow = f64x4::ZERO;
let mut i = 0;
while i + LANES <= cur.len() {
let c = load4(cur, i);
overflow += c * f64x4::ZERO;
store4(cur, i, M::exp_f64x4(c * half));
i += LANES;
}
let mut tail = 0.0;
while i < cur.len() {
tail += cur[i] * 0.0;
cur[i] = M::exp(-cur[i] * 0.5);
i += 1;
}
overflow.reduce_add() + tail == 0.0
}
fn scale_add_checked(src: &[f64], scale: f64, acc: &mut [f64]) {
debug_assert_eq!(src.len(), acc.len());
let sv = f64x4::splat(scale);
let mut i = 0;
while i + LANES <= src.len() {
store4(acc, i, load4(acc, i) + load4(src, i) * sv);
i += LANES;
}
while i < src.len() {
acc[i] += src[i] * scale;
i += 1;
}
}
fn exp_half_in_place<M: KernelMath>(buf: &mut [f64]) -> Result<(), GprError> {
let scale = f64x4::splat(-0.5);
let mut i = 0;
while i + LANES <= buf.len() {
let d = load4(buf, i);
if !all_finite(d) {
return Err(GprError::NonFiniteInput);
}
store4(buf, i, M::exp_f64x4(d * scale));
i += LANES;
}
while i < buf.len() {
buf[i] = M::exp(-finite_dist(buf[i])? * 0.5);
i += 1;
}
Ok(())
}
fn grad_from_points_in_place<M: KernelMath>(
r2: &mut [f64],
xdim: &[f64],
x0: f64,
inv_dim: f64,
) -> Result<(), GprError> {
debug_assert_eq!(r2.len(), xdim.len());
let half = f64x4::splat(-0.5);
let inv = f64x4::splat(inv_dim);
let x0v = f64x4::splat(x0);
let mut i = 0;
while i + LANES <= r2.len() {
let d = load4(r2, i);
if !all_finite(d) {
return Err(GprError::NonFiniteInput);
}
let delta = load4(xdim, i) - x0v;
store4(r2, i, M::d1_f64x4(d * half) * delta * delta * inv);
i += LANES;
}
while i < r2.len() {
let d = finite_dist(r2[i])?;
let delta = xdim[i] - x0;
r2[i] = M::jet(-0.5 * d).d1 * delta * delta * inv_dim;
i += 1;
}
Ok(())
}
fn grad_from_cache_in_place<M: KernelMath>(
r2: &mut [f64],
dim_sq: &[f64],
inv_dim: f64,
) -> Result<(), GprError> {
debug_assert_eq!(r2.len(), dim_sq.len());
let half = f64x4::splat(-0.5);
let inv = f64x4::splat(inv_dim);
let mut i = 0;
while i + LANES <= r2.len() {
let d = load4(r2, i);
let sq = load4(dim_sq, i);
if !all_finite(d) || !all_finite(sq) {
return Err(GprError::NonFiniteInput);
}
store4(r2, i, M::d1_f64x4(d * half) * sq * inv);
i += LANES;
}
while i < r2.len() {
let d = finite_dist(r2[i])?;
let sq = finite_dist(dim_sq[i])?;
r2[i] = M::jet(-0.5 * d).d1 * sq * inv_dim;
i += 1;
}
Ok(())
}
#[derive(Clone, Copy)]
enum Square<'a> {
Cache(LowerRuns<'a, f64>),
Points(MatRef<'a, f64>),
}
fn accumulate_r2(
source: Square<'_>,
inv_ell_sq: &[f64],
col: usize,
row_start: usize,
dest: &mut [f64],
) -> Result<(), GprError> {
dest.fill(0.0);
let n_rows = dest.len();
match source {
Square::Cache(cache) => {
let offset = cached_rows_offset(row_start, col)?;
for (dim, &w) in inv_ell_sq.iter().enumerate() {
let src = cache.column(dim, col);
scale_add(&src[offset..offset + n_rows], w, dest)?;
}
}
Square::Points(x) => {
for (dim, &w) in inv_ell_sq.iter().enumerate() {
let xdim = col_slice_checked(x, dim)?;
add_squared_diff_scaled(&xdim[row_start..row_start + n_rows], xdim[col], w, dest);
}
}
}
Ok(())
}
fn fill_column<M: KernelMath>(
source: Square<'_>,
inv_ell_sq: &[f64],
param_idx: Option<usize>,
col: usize,
row_start: usize,
dest: &mut [f64],
) -> Result<(), GprError> {
accumulate_r2(source, inv_ell_sq, col, row_start, dest)?;
let Some(dim) = param_idx else {
return exp_half_in_place::<M>(dest);
};
let rows = row_start..row_start + dest.len();
match source {
Square::Cache(cache) => {
let offset = cached_rows_offset(row_start, col)?;
let src = cache.column(dim, col);
grad_from_cache_in_place::<M>(dest, &src[offset..offset + rows.len()], inv_ell_sq[dim])
}
Square::Points(x) => {
let xdim = col_slice_checked(x, dim)?;
grad_from_points_in_place::<M>(dest, &xdim[rows], xdim[col], inv_ell_sq[dim])
}
}
}
fn fill_square<M: KernelMath>(
source: Square<'_>,
mut out: MatMut<'_, f64>,
uplo: Triangle,
inv_ell_sq: &[f64],
param_idx: Option<usize>,
) -> Result<(), GprError> {
let n = out.nrows();
if matches!(uplo, Triangle::Lower) {
return par_lower_cols(out, &|col, rows| {
fill_column::<M>(source, inv_ell_sq, param_idx, col, col, rows_checked(rows)?)
});
}
for col in 0..n {
let end = if matches!(uplo, Triangle::Upper) {
col + 1
} else {
n
};
let dest = &mut col_slice_mut_checked(out.rb_mut(), col)?[..end];
fill_column::<M>(source, inv_ell_sq, param_idx, col, 0, dest)?;
}
Ok(())
}
pub(crate) fn try_apply_cache<M: KernelMath>(
cache: ArdSqDiff<'_, f64>,
out: MatMut<'_, f64>,
uplo: Triangle,
inv_ell_sq: &[f64],
) -> Result<bool, GprError> {
let n = out.nrows();
if out.ncols() != n {
return Ok(false);
}
crate::kernel::dist::require_ard_sq_diff_shape(cache, n, inv_ell_sq.len())?;
if uplo != Triangle::Lower || !unit_row_stride(out.as_ref()) {
return Ok(false);
}
let lower = match cache.runs() {
Runs::Rows(rows) => {
let half = f64x4::splat(-0.5);
return super::rows::try_fill_lower(rows, out, inv_ell_sq, None, &|r2, _| {
M::exp_f64x4(r2 * half)
});
}
Runs::Lower(lower) => lower,
};
fill_square::<M>(Square::Cache(lower), out, uplo, inv_ell_sq, None)?;
Ok(true)
}
pub(crate) fn try_apply_points<M: KernelMath>(
x: MatRef<'_, f64>,
out: MatMut<'_, f64>,
uplo: Triangle,
inv_ell_sq: &[f64],
) -> Result<bool, GprError> {
let n = out.nrows();
if out.ncols() != n || x.nrows() != n || x.ncols() != inv_ell_sq.len() {
return Ok(false);
}
if !unit_row_stride(x) || !unit_row_stride(out.as_ref()) {
return Ok(false);
}
fill_square::<M>(Square::Points(x), out, uplo, inv_ell_sq, None)?;
Ok(true)
}
pub(crate) fn try_grad_cache<M: KernelMath>(
cache: ArdSqDiff<'_, f64>,
d_k: MatMut<'_, f64>,
uplo: Triangle,
inv_ell_sq: &[f64],
param_idx: usize,
) -> Result<bool, GprError> {
let n = d_k.nrows();
if d_k.ncols() != n || param_idx >= inv_ell_sq.len() {
return Ok(false);
}
crate::kernel::dist::require_ard_sq_diff_shape(cache, n, inv_ell_sq.len())?;
if uplo != Triangle::Lower || !unit_row_stride(d_k.as_ref()) {
return Ok(false);
}
let lower = match cache.runs() {
Runs::Rows(rows) => {
let half = f64x4::splat(-0.5);
return super::rows::try_fill_lower(
rows,
d_k,
inv_ell_sq,
Some(param_idx),
&|r2, t| M::d1_f64x4(r2 * half) * t,
);
}
Runs::Lower(lower) => lower,
};
fill_square::<M>(Square::Cache(lower), d_k, uplo, inv_ell_sq, Some(param_idx))?;
Ok(true)
}
pub(crate) fn try_grad_points<M: KernelMath>(
x: MatRef<'_, f64>,
d_k: MatMut<'_, f64>,
uplo: Triangle,
inv_ell_sq: &[f64],
param_idx: usize,
) -> Result<bool, GprError> {
let n = d_k.nrows();
if d_k.ncols() != n
|| x.nrows() != n
|| x.ncols() != inv_ell_sq.len()
|| param_idx >= inv_ell_sq.len()
{
return Ok(false);
}
if !unit_row_stride(x) || !unit_row_stride(d_k.as_ref()) {
return Ok(false);
}
fill_square::<M>(Square::Points(x), d_k, uplo, inv_ell_sq, Some(param_idx))?;
Ok(true)
}
#[derive(Clone, Copy)]
pub(crate) enum Which {
One(usize),
}
impl Which {
fn includes(self, dim: usize) -> bool {
match self {
Self::One(d) => d == dim,
}
}
}
pub(crate) fn cross_inputs(
x1: MatRef<'_, f64>,
x2: MatRef<'_, f64>,
d: usize,
) -> Result<bool, GprError> {
if d == 0 || x1.ncols() != d || x2.ncols() != d {
return Ok(false);
}
if !unit_row_stride(x1) || !unit_row_stride(x2) {
return Ok(false);
}
for dim in 0..d {
finite_slice(col_slice_checked(x1, dim)?)?;
finite_slice(col_slice_checked(x2, dim)?)?;
}
Ok(true)
}
pub(crate) fn try_grad_cross<M: KernelMath>(
x1: MatRef<'_, f64>,
x2: MatRef<'_, f64>,
out: &mut [MatMut<'_, f64>],
inv_ell_sq: &[f64],
which: Which,
) -> Result<bool, GprError> {
let m = x1.nrows();
let n = x2.nrows();
let d = inv_ell_sq.len();
let expected = match which {
Which::One(dim) if dim < d => 1,
Which::One(_) => return Ok(false),
};
if out.len() != expected || !cross_inputs(x1, x2, d)? {
return Ok(false);
}
for dest in out.iter() {
if dest.nrows() != m || dest.ncols() != n || !unit_row_stride(dest.as_ref()) {
return Ok(false);
}
}
if n <= 1024 {
write_cross_rows::<M>(
x1,
x2,
inv_ell_sq,
which,
0,
n,
0,
&mut |_dim, row, col, v| {
out[0][(row, col)] = v;
},
)?;
return Ok(true);
}
let n_parts = worker_count();
let shared = ShareBases(out.iter_mut().map(packed_mut).collect());
let results: Vec<Result<(), GprError>> = (0..n_parts)
.into_par_iter()
.map(|idx| {
let (start, len) = col_chunk(n, idx, n_parts);
write_cross_rows::<M>(
x1,
x2,
inv_ell_sq,
which,
start,
len,
start,
&mut |_dim, row, col, v| {
unsafe { store_packed(&shared.slots()[0], row, col, v) }
},
)
})
.collect();
for result in results {
result?;
}
Ok(true)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn write_cross_rows<M: KernelMath>(
x1: MatRef<'_, f64>,
x2: MatRef<'_, f64>,
inv_ell_sq: &[f64],
which: Which,
x_begin: usize,
len: usize,
dest_col: usize,
store: &mut dyn FnMut(usize, usize, usize, f64),
) -> Result<(), GprError> {
const BLOCK: usize = 256;
let mut r2_buf = [0.0f64; BLOCK];
let mut k_buf = [0.0f64; BLOCK];
let mut term_buf = [0.0f64; BLOCK];
let mut offset = 0;
while offset < len {
let block = BLOCK.min(len - offset);
let begin = x_begin + offset;
let (r2, k, term) = (
&mut r2_buf[..block],
&mut k_buf[..block],
&mut term_buf[..block],
);
for row in 0..x1.nrows() {
r2.fill(0.0);
for (dim, &w) in inv_ell_sq.iter().enumerate() {
let z = col_slice_checked(x1, dim)?[row];
add_squared_diff_scaled(
&col_slice_checked(x2, dim)?[begin..begin + block],
z,
w,
r2,
);
}
d1_half::<M>(r2, k)?;
for (dim, &w) in inv_ell_sq.iter().enumerate() {
if !which.includes(dim) {
continue;
}
let z = col_slice_checked(x1, dim)?[row];
term.fill(0.0);
add_squared_diff_scaled(
&col_slice_checked(x2, dim)?[begin..begin + block],
z,
w,
term,
);
for (i, (&kv, &tv)) in k.iter().zip(term.iter()).enumerate() {
let dk = kv * tv;
if !dk.is_finite() {
return Err(GprError::NonFiniteKernelValue);
}
store(dim, row, dest_col + offset + i, dk);
}
}
}
offset += block;
}
Ok(())
}
fn d1_half<M: KernelMath>(src: &[f64], dest: &mut [f64]) -> Result<(), GprError> {
let scale = f64x4::splat(-0.5);
let mut i = 0;
while i + LANES <= src.len() {
let z = load4(src, i) * scale;
let v = if M::ACCURATE { z.exp() } else { M::d1_f64x4(z) };
if !all_finite(v) {
return Err(GprError::NonFiniteKernelValue);
}
store4(dest, i, v);
i += LANES;
}
while i < src.len() {
let z = -0.5 * src[i];
let v = if M::ACCURATE { z.exp() } else { M::jet(z).d1 };
if !v.is_finite() {
return Err(GprError::NonFiniteKernelValue);
}
dest[i] = v;
i += 1;
}
Ok(())
}
const COLUMN_BATCH: usize = 16;
pub(crate) fn try_contract_cross<M: KernelMath>(
x1: MatRef<'_, f64>,
x2: MatRef<'_, f64>,
weight: MatRef<'_, f64>,
inv_ell_sq: &[f64],
out: &mut [f64],
jobs: &mut Vec<f64>,
want_value: bool,
) -> Result<Option<f64>, GprError> {
let m = x1.nrows();
let n = x2.nrows();
let d = inv_ell_sq.len();
if out.len() != d || weight.nrows() != m || weight.ncols() != n || !cross_inputs(x1, x2, d)? {
return Ok(None);
}
if !unit_row_stride(weight) {
return Ok(None);
}
let batch = COLUMN_BATCH.min(n);
let scratch_len = m
.checked_mul(2)
.and_then(|rows| rows.checked_mul(batch))
.ok_or(GprError::SizeOverflow)?;
let s_len = m.checked_mul(n).ok_or(GprError::SizeOverflow)?;
let prod_len = n.checked_mul(d).ok_or(GprError::SizeOverflow)?;
let need = scratch_len
.checked_add(s_len)
.and_then(|sum| sum.checked_add(m))
.and_then(|sum| sum.checked_add(n))
.and_then(|sum| sum.checked_add(prod_len))
.ok_or(GprError::SizeOverflow)?;
if jobs.len() < need {
jobs.resize(need, 0.0);
}
let (scratch, rest) = jobs.split_at_mut(scratch_len);
let (s, rest) = rest.split_at_mut(s_len);
let (row_sum, rest) = rest.split_at_mut(m);
let (col_sum, prod) = rest.split_at_mut(n);
let prod = &mut prod[..prod_len];
let mut values = [0.0; COLUMN_BATCH];
let mut value = 0.0;
let mut start = 0;
while start < n {
let len = batch.min(n - start);
fill_contract_batch::<M>(
x1,
x2,
weight,
inv_ell_sq,
start,
&mut scratch[..len * 2 * m],
&mut s[start * m..(start + len) * m],
&mut values[..len],
want_value,
)?;
if want_value {
for part in values.iter().take(len) {
value += part;
}
}
start += len;
}
fold_rect_lengthscales(x1, x2, s, inv_ell_sq, row_sum, col_sum, prod, out)?;
if !value.is_finite() {
return Err(GprError::NonFiniteKernelValue);
}
Ok(Some(value))
}
#[allow(clippy::too_many_arguments)]
fn fill_contract_batch<M: KernelMath>(
x1: MatRef<'_, f64>,
x2: MatRef<'_, f64>,
weight: MatRef<'_, f64>,
inv_ell_sq: &[f64],
col0: usize,
scratch: &mut [f64],
s: &mut [f64],
values: &mut [f64],
want_value: bool,
) -> Result<(), GprError> {
let n = values.len();
let m = x1.nrows();
if n == 0 {
return Ok(());
}
if n == 1 {
let (r2, k) = scratch.split_at_mut(m);
values[0] = contract_column::<M>(x1, x2, weight, inv_ell_sq, col0, r2, k, s, want_value)?;
return Ok(());
}
let mid = n / 2;
let (left_scratch, right_scratch) = scratch.split_at_mut(mid * 2 * m);
let (left_s, right_s) = s.split_at_mut(mid * m);
let (left_values, right_values) = values.split_at_mut(mid);
let (left, right) = if rayon::current_num_threads() > 1 {
rayon::join(
|| {
fill_contract_batch::<M>(
x1,
x2,
weight,
inv_ell_sq,
col0,
left_scratch,
left_s,
left_values,
want_value,
)
},
|| {
fill_contract_batch::<M>(
x1,
x2,
weight,
inv_ell_sq,
col0 + mid,
right_scratch,
right_s,
right_values,
want_value,
)
},
)
} else {
let left = fill_contract_batch::<M>(
x1,
x2,
weight,
inv_ell_sq,
col0,
left_scratch,
left_s,
left_values,
want_value,
);
let right = fill_contract_batch::<M>(
x1,
x2,
weight,
inv_ell_sq,
col0 + mid,
right_scratch,
right_s,
right_values,
want_value,
);
(left, right)
};
left.and(right)
}
#[allow(clippy::too_many_arguments)]
fn contract_column<M: KernelMath>(
x1: MatRef<'_, f64>,
x2: MatRef<'_, f64>,
weight: MatRef<'_, f64>,
inv_ell_sq: &[f64],
col: usize,
r2: &mut [f64],
k: &mut [f64],
s_col: &mut [f64],
want_value: bool,
) -> Result<f64, GprError> {
let m = x1.nrows();
r2.fill(0.0);
for (dim, &w) in inv_ell_sq.iter().enumerate() {
let x0 = col_slice_checked(x2, dim)?[col];
add_squared_diff_scaled(col_slice_checked(x1, dim)?, x0, w, r2);
}
d1_half::<M>(r2, k)?;
let wcol = col_slice_checked(weight, col)?;
let mut value = 0.0;
for row in 0..m {
let s = wcol[row] * k[row];
s_col[row] = s;
if want_value {
value += s;
}
}
Ok(value)
}
#[allow(clippy::too_many_arguments)]
fn fold_rect_lengthscales(
x1: MatRef<'_, f64>,
x2: MatRef<'_, f64>,
s: &[f64],
inv_ell_sq: &[f64],
row_sum: &mut [f64],
col_sum: &mut [f64],
prod: &mut [f64],
out: &mut [f64],
) -> Result<(), GprError> {
let m = x1.nrows();
let n = x2.nrows();
let d = inv_ell_sq.len();
row_sum.fill(0.0);
for col in 0..n {
let column = &s[col * m..(col + 1) * m];
let mut sum = 0.0;
for (row, &value) in column.iter().enumerate() {
row_sum[row] += value;
sum += value;
}
col_sum[col] = sum;
}
let s_ref = MatRef::from_column_major_slice(s, m, n);
let mut prod_mat = MatMut::from_column_major_slice_mut(prod, n, d);
matmul(
prod_mat.rb_mut(),
Accum::Replace,
s_ref.transpose(),
x1,
1.0,
Par::Seq,
);
for (dim, &wd) in inv_ell_sq.iter().enumerate() {
let x1_col = col_slice_checked(x1, dim)?;
let x2_col = col_slice_checked(x2, dim)?;
let prod_col = &prod[dim * n..(dim + 1) * n];
let mut left = 0.0;
for row in 0..m {
left += x1_col[row] * x1_col[row] * row_sum[row];
}
let mut right = 0.0;
let mut cross = 0.0;
for col in 0..n {
right += x2_col[col] * x2_col[col] * col_sum[col];
cross += prod_col[col] * x2_col[col];
}
let total = wd * (left + right - 2.0 * cross);
if !total.is_finite() {
return Err(GprError::NonFiniteKernelValue);
}
out[dim] = total;
}
Ok(())
}
pub(crate) fn fold_square_lengthscales(
x: &[f64],
s: &[f64],
inv_ell_sq: &[f64],
row_sum: &mut [f64],
prod: &mut [f64],
out: &mut [f64],
) -> Result<(), GprError> {
let n = row_sum.len();
let d = inv_ell_sq.len();
row_sum.fill(0.0);
for col in 0..n {
let column = &s[col * n..(col + 1) * n];
for (row, &value) in column.iter().enumerate() {
row_sum[row] += value;
}
}
let s_ref = MatRef::from_column_major_slice(s, n, n);
let x_ref = MatRef::from_column_major_slice(x, n, d);
let mut prod_mat = MatMut::from_column_major_slice_mut(prod, n, d);
matmul(
prod_mat.rb_mut(),
Accum::Replace,
s_ref,
x_ref,
1.0,
Par::Seq,
);
for (dim, &wd) in inv_ell_sq.iter().enumerate() {
let x_col = &x[dim * n..(dim + 1) * n];
let prod_col = &prod[dim * n..(dim + 1) * n];
let mut xx = 0.0;
let mut quad = 0.0;
for row in 0..n {
xx += x_col[row] * x_col[row] * row_sum[row];
quad += prod_col[row] * x_col[row];
}
let total = 2.0 * wd * (xx - quad);
if !total.is_finite() {
return Err(GprError::NonFiniteKernelValue);
}
out[dim] = total;
}
Ok(())
}
struct PackedMut {
ptr: *mut f64,
stride: isize,
}
struct ShareBases(Vec<PackedMut>);
impl ShareBases {
fn slots(&self) -> &[PackedMut] {
&self.0
}
}
unsafe impl Send for ShareBases {}
unsafe impl Sync for ShareBases {}
fn packed_mut(mat: &mut MatMut<'_, f64>) -> PackedMut {
let view = mat.rb_mut();
debug_assert!(view.ncols() == 0 || view.row_stride() == 1);
PackedMut {
ptr: view.as_ptr_mut(),
stride: view.col_stride(),
}
}
#[inline(always)]
unsafe fn store_packed(slot: &PackedMut, row: usize, col: usize, value: f64) {
unsafe {
*slot.ptr.offset(row as isize + col as isize * slot.stride) = value;
}
}