use super::{
CompiledKernel, CrossViews, LeafRef, MixedKernelViews, Nested, Term, add_triangle,
ard_needs_coords, mul_triangle, require_scratch_shape, split_terms, term_scratch,
};
use crate::error::GprError;
use crate::kernel::KernelScalar;
use crate::kernel::dist::ArdSqDiff;
use crate::kernel::tree::Supply;
use crate::kernel::{CustomKernel, Triangle, write_rect_from_coords, write_square_from_coords};
use faer::{MatMut, MatRef};
impl<T: KernelScalar> CompiledKernel<T> {
pub fn apply<M: crate::math::KernelMath>(
&self,
dist: MatRef<'_, T>,
out: MatMut<'_, T>,
uplo: Triangle,
scratch: MatMut<'_, T>,
) -> Result<(), GprError> {
let mut nested = self.nested_buffers(out.nrows(), out.ncols());
self.apply_with::<M>(dist, out, uplo, scratch, &mut nested)
}
pub(crate) fn apply_with<M: crate::math::KernelMath>(
&self,
dist: MatRef<'_, T>,
mut out: MatMut<'_, T>,
uplo: Triangle,
mut scratch: MatMut<'_, T>,
nested: &mut Nested<T>,
) -> Result<(), GprError> {
require_scratch_shape(out.as_ref(), scratch.as_ref())?;
match self.term() {
Term::Leaf(leaf) => leaf.apply_dist::<M>(dist, out, uplo),
Term::Supplied(never) => match *never {},
Term::Sum(terms) => fold_terms::<M, _>(
terms,
dist,
out.as_mut(),
uplo,
scratch.as_mut(),
nested,
add_triangle,
),
Term::Product(terms) => fold_terms::<M, _>(
terms,
dist,
out.as_mut(),
uplo,
scratch.as_mut(),
nested,
mul_triangle,
),
}
}
pub fn apply_cross<M: crate::math::KernelMath>(
&self,
dist: MatRef<'_, T>,
out: MatMut<'_, T>,
scratch: MatMut<'_, T>,
) -> Result<(), GprError> {
let mut nested = self.nested_buffers(out.nrows(), out.ncols());
self.apply_cross_with::<M>(dist, out, scratch, &mut nested)
}
pub(crate) fn apply_cross_with<M: crate::math::KernelMath>(
&self,
dist: MatRef<'_, T>,
mut out: MatMut<'_, T>,
mut scratch: MatMut<'_, T>,
nested: &mut Nested<T>,
) -> Result<(), GprError> {
require_scratch_shape(out.as_ref(), scratch.as_ref())?;
match self.term() {
Term::Leaf(leaf) => leaf.apply_cross_dist::<M>(dist, out),
Term::Supplied(never) => match *never {},
Term::Sum(terms) => fold_rect::<M, _>(
terms,
dist,
out.as_mut(),
scratch.as_mut(),
nested,
add_rect,
),
Term::Product(terms) => fold_rect::<M, _>(
terms,
dist,
out.as_mut(),
scratch.as_mut(),
nested,
mul_rect,
),
}
}
pub fn fill_diag_points(&self, x: MatRef<'_, T>, out: &mut [T]) -> Result<(), GprError> {
self.require_columns(x)?;
self.fill_diag_rows(x, out)
}
pub fn apply_points<M: crate::math::KernelMath>(
&self,
x: MatRef<'_, T>,
out: MatMut<'_, T>,
uplo: Triangle,
scratch: MatMut<'_, T>,
) -> Result<(), GprError> {
self.require_columns(x)?;
let mut nested = self.nested_buffers(out.nrows(), out.ncols());
self.apply_points_with::<M>(x, out, uplo, scratch, &mut nested)
}
pub(crate) fn apply_points_with<M: crate::math::KernelMath>(
&self,
x: MatRef<'_, T>,
mut out: MatMut<'_, T>,
uplo: Triangle,
mut scratch: MatMut<'_, T>,
nested: &mut Nested<T>,
) -> Result<(), GprError> {
require_scratch_shape(out.as_ref(), scratch.as_ref())?;
match self.term() {
Term::Leaf(leaf) => leaf.apply_points::<M>(x, out, uplo, scratch),
Term::Supplied(never) => match *never {},
Term::Sum(terms) => fold_terms_points::<M, _>(
terms,
x,
out.as_mut(),
uplo,
scratch.as_mut(),
nested,
add_triangle,
),
Term::Product(terms) => fold_terms_points::<M, _>(
terms,
x,
out.as_mut(),
uplo,
scratch.as_mut(),
nested,
mul_triangle,
),
}
}
pub fn apply_cross_points<M: crate::math::KernelMath>(
&self,
x: MatRef<'_, T>,
xs: MatRef<'_, T>,
out: MatMut<'_, T>,
scratch: MatMut<'_, T>,
) -> Result<(), GprError> {
self.require_columns(x)?;
self.require_columns(xs)?;
self.apply_cross_rows::<M>(x, xs, out, scratch)
}
pub(crate) fn apply_cross_rows<M: crate::math::KernelMath>(
&self,
x: MatRef<'_, T>,
xs: MatRef<'_, T>,
out: MatMut<'_, T>,
scratch: MatMut<'_, T>,
) -> Result<(), GprError> {
let mut nested = self.nested_buffers(out.nrows(), out.ncols());
self.apply_cross_points_with::<M>(x, xs, out, scratch, &mut nested)
}
pub(crate) fn apply_cross_points_with<M: crate::math::KernelMath>(
&self,
x: MatRef<'_, T>,
xs: MatRef<'_, T>,
mut out: MatMut<'_, T>,
mut scratch: MatMut<'_, T>,
nested: &mut Nested<T>,
) -> Result<(), GprError> {
require_scratch_shape(out.as_ref(), scratch.as_ref())?;
match self.term() {
Term::Leaf(leaf) => leaf.apply_cross_points::<M>(x, xs, out, scratch),
Term::Supplied(never) => match *never {},
Term::Sum(terms) => fold_rect_points::<M, _>(
terms,
x,
xs,
out.as_mut(),
scratch.as_mut(),
nested,
add_rect,
),
Term::Product(terms) => fold_rect_points::<M, _>(
terms,
x,
xs,
out.as_mut(),
scratch.as_mut(),
nested,
mul_rect,
),
}
}
pub(crate) fn apply_from_ard_cache<M: crate::math::KernelMath>(
&self,
cache: ArdSqDiff<'_, T>,
x: MatRef<'_, T>,
mut out: MatMut<'_, T>,
uplo: Triangle,
mut scratch: MatMut<'_, T>,
nested: &mut Nested<T>,
) -> Result<(), GprError> {
require_scratch_shape(out.as_ref(), scratch.as_ref())?;
match self.term() {
Term::Leaf(leaf) => leaf.apply_ard_cache::<M>(cache, x, out, uplo, scratch),
Term::Supplied(never) => match *never {},
Term::Sum(terms) => fold_terms_ard_cache::<M, _>(
terms,
cache,
x,
out.as_mut(),
uplo,
scratch.as_mut(),
nested,
),
Term::Product(_) => self.apply_points_with::<M>(x, out, uplo, scratch, nested),
}
}
}
impl<T: KernelScalar, S: Supply> CompiledKernel<T, S> {
pub fn fill_diag(&self, out: &mut [T]) -> Result<(), GprError> {
match self {
Self::Rbf(leaf) => {
leaf.fill_diag(out);
Ok(())
}
Self::RbfArd(leaf) => {
leaf.fill_diag(out);
Ok(())
}
Self::Matern(leaf) => {
leaf.fill_diag(out);
Ok(())
}
Self::MaternArd(leaf) => {
leaf.fill_diag(out);
Ok(())
}
Self::Periodic(leaf) => {
leaf.fill_diag(out);
Ok(())
}
Self::RationalQuadratic(leaf) => {
leaf.fill_diag(out);
Ok(())
}
Self::RationalQuadraticArd(leaf) => {
leaf.fill_diag(out);
Ok(())
}
Self::Constant(leaf) => {
leaf.fill_diag(out);
Ok(())
}
Self::White(leaf) => {
leaf.fill_diag(out);
Ok(())
}
Self::Custom(leaf) => leaf.fill_diag(out),
Self::Supplied(leaf) => S::compiled(leaf).fill_diag(out),
Self::Linear(_) => Err(GprError::UnsupportedKernelOperation {
reason: "linear kernel diagonal needs coordinates".to_owned(),
}),
Self::Sum(terms) => fold_diag(terms, out, add_assign),
Self::Product(terms) => fold_diag(terms, out, mul_assign),
}
}
pub(crate) fn fill_diag_rows(&self, x: MatRef<'_, T>, out: &mut [T]) -> Result<(), GprError> {
match self {
Self::Rbf(leaf) => {
leaf.fill_diag(out);
Ok(())
}
Self::RbfArd(leaf) => {
leaf.fill_diag(out);
Ok(())
}
Self::Matern(leaf) => {
leaf.fill_diag(out);
Ok(())
}
Self::MaternArd(leaf) => {
leaf.fill_diag(out);
Ok(())
}
Self::Periodic(leaf) => {
leaf.fill_diag(out);
Ok(())
}
Self::RationalQuadratic(leaf) => {
leaf.fill_diag(out);
Ok(())
}
Self::RationalQuadraticArd(leaf) => {
leaf.fill_diag(out);
Ok(())
}
Self::Constant(leaf) => {
leaf.fill_diag(out);
Ok(())
}
Self::White(leaf) => {
leaf.fill_diag(out);
Ok(())
}
Self::Custom(leaf) => leaf.fill_diag(out),
Self::Supplied(leaf) => S::compiled(leaf).fill_diag(out),
Self::Linear(leaf) => leaf.fill_diag_points(x, out),
Self::Sum(terms) => fold_diag_points(terms, x, out, add_assign),
Self::Product(terms) => fold_diag_points(terms, x, out, mul_assign),
}
}
pub(crate) fn apply_mixed<M: crate::math::KernelMath>(
&self,
views: MixedKernelViews<'_, T, S>,
mut out: MatMut<'_, T>,
uplo: Triangle,
mut scratch: MatMut<'_, T>,
nested: &mut Nested<T>,
) -> Result<(), GprError> {
require_scratch_shape(out.as_ref(), scratch.as_ref())?;
match self.term() {
Term::Leaf(leaf) => leaf.apply_mixed::<M, _>(views, out, uplo, scratch),
Term::Supplied(leaf) => {
let (leaf, slots) = S::square_leaf(leaf, views.slots);
leaf.apply::<M>(slots, out, uplo)
}
Term::Sum(terms) => fold_terms_mixed::<M, _, _>(
terms,
views,
out.as_mut(),
uplo,
scratch.as_mut(),
nested,
add_triangle,
),
Term::Product(terms) => fold_terms_mixed::<M, _, _>(
terms,
views,
out.as_mut(),
uplo,
scratch.as_mut(),
nested,
mul_triangle,
),
}
}
pub(crate) fn apply_cross_mixed<M: crate::math::KernelMath>(
&self,
views: CrossViews<'_, T, S>,
mut out: MatMut<'_, T>,
mut scratch: MatMut<'_, T>,
nested: &mut Nested<T>,
) -> Result<(), GprError> {
require_scratch_shape(out.as_ref(), scratch.as_ref())?;
match self.term() {
Term::Leaf(leaf) => leaf.apply_cross_mixed::<M, _>(views, out, scratch),
Term::Supplied(leaf) => {
let (leaf, slots) = S::rect_leaf(leaf, views.slots);
leaf.apply_cross::<M>(slots, out)
}
Term::Sum(terms) => fold_rect_mixed::<M, _, _>(
terms,
views,
out.as_mut(),
scratch.as_mut(),
nested,
add_rect,
),
Term::Product(terms) => fold_rect_mixed::<M, _, _>(
terms,
views,
out.as_mut(),
scratch.as_mut(),
nested,
mul_rect,
),
}
}
}
impl<T: KernelScalar> LeafRef<'_, T> {
pub(super) fn needs_ard_sq_diff(self) -> bool {
matches!(
self,
Self::RbfArd(_) | Self::MaternArd(_) | Self::RationalQuadraticArd(_)
)
}
pub(super) fn reads_dist(self) -> bool {
match self {
Self::Rbf(_)
| Self::Matern(_)
| Self::Periodic(_)
| Self::RationalQuadratic(_)
| Self::Custom(_)
| Self::Constant(_)
| Self::White(_) => true,
Self::RbfArd(_)
| Self::Linear(_)
| Self::MaternArd(_)
| Self::RationalQuadraticArd(_) => false,
}
}
pub(super) fn apply_dist<M: crate::math::KernelMath>(
self,
dist: MatRef<'_, T>,
out: MatMut<'_, T>,
uplo: Triangle,
) -> Result<(), GprError> {
match self {
Self::Rbf(leaf) => leaf.apply_math::<M, _>(dist, out, uplo),
Self::RbfArd(_)
| Self::Linear(_)
| Self::MaternArd(_)
| Self::RationalQuadraticArd(_) => Err(ard_needs_coords()),
Self::Matern(leaf) => leaf.apply_math::<M, _>(dist, out, uplo),
Self::Periodic(leaf) => leaf.apply_math::<M, _>(dist, out, uplo),
Self::RationalQuadratic(leaf) => leaf.apply(dist, out, uplo),
Self::Constant(leaf) => leaf.apply(dist, out, uplo),
Self::White(leaf) => leaf.apply(dist, out, uplo),
Self::Custom(leaf) => leaf.apply(dist, out, uplo),
}
}
pub(super) fn apply_cross_dist<M: crate::math::KernelMath>(
self,
dist: MatRef<'_, T>,
out: MatMut<'_, T>,
) -> Result<(), GprError> {
match self {
Self::Rbf(leaf) => leaf.apply_cross_math::<M, _>(dist, out),
Self::RbfArd(_)
| Self::Linear(_)
| Self::MaternArd(_)
| Self::RationalQuadraticArd(_) => Err(ard_needs_coords()),
Self::Matern(leaf) => leaf.apply_cross_math::<M, _>(dist, out),
Self::Periodic(leaf) => leaf.apply_cross_math::<M, _>(dist, out),
Self::RationalQuadratic(leaf) => leaf.apply_cross(dist, out),
Self::Constant(leaf) => leaf.apply_cross(dist, out),
Self::White(leaf) => leaf.apply_cross(dist, out),
Self::Custom(leaf) => leaf.apply_cross(dist, out),
}
}
pub(super) fn apply_points<M: crate::math::KernelMath>(
self,
x: MatRef<'_, T>,
out: MatMut<'_, T>,
uplo: Triangle,
scratch: MatMut<'_, T>,
) -> Result<(), GprError> {
match self {
Self::Rbf(leaf) => leaf.apply_from_coords::<M, _>(x, out, uplo),
Self::Matern(leaf) => leaf.apply_from_coords::<M, _>(x, out, uplo),
Self::Periodic(leaf) => leaf.apply_from_coords::<M, _>(x, out, uplo),
Self::RationalQuadratic(leaf) => leaf.apply_from_coords(x, out, uplo),
Self::Custom(leaf) => apply_custom_from_coords(leaf, x, out, uplo, scratch),
Self::RbfArd(leaf) => leaf.apply_math::<M, _>(x, out, uplo),
Self::Linear(leaf) => leaf.apply(x, out, uplo),
Self::MaternArd(leaf) => leaf.apply_math::<M, _>(x, out, uplo),
Self::RationalQuadraticArd(leaf) => leaf.apply(x, out, uplo),
Self::Constant(leaf) => leaf.apply_rows(x, out, uplo),
Self::White(leaf) => leaf.apply_rows(x, out, uplo),
}
}
pub(super) fn apply_cross_points<M: crate::math::KernelMath>(
self,
x: MatRef<'_, T>,
xs: MatRef<'_, T>,
out: MatMut<'_, T>,
mut scratch: MatMut<'_, T>,
) -> Result<(), GprError> {
match self {
Self::Rbf(leaf) => leaf.apply_cross_from_coords::<M, _>(x, xs, out),
Self::Matern(leaf) => leaf.apply_cross_from_coords::<M, _>(x, xs, out),
Self::Periodic(leaf) => leaf.apply_cross_from_coords::<M, _>(x, xs, out),
Self::RationalQuadratic(leaf) => leaf.apply_cross_from_coords(x, xs, out),
Self::Custom(leaf) => {
write_rect_from_coords(x, xs, scratch.as_mut(), Ok)?;
leaf.apply_cross(scratch.as_ref(), out)
}
Self::RbfArd(leaf) => leaf.apply_cross_math::<M, _>(x, xs, out),
Self::Linear(leaf) => leaf.apply_cross(x, xs, out),
Self::MaternArd(leaf) => leaf.apply_cross_math::<M, _>(x, xs, out),
Self::RationalQuadraticArd(leaf) => leaf.apply_cross(x, xs, out),
Self::Constant(leaf) => leaf.apply_cross_rows(x, xs, out),
Self::White(leaf) => leaf.apply_cross_rows(x, xs, out),
}
}
pub(super) fn apply_ard_cache<M: crate::math::KernelMath>(
self,
cache: ArdSqDiff<'_, T>,
x: MatRef<'_, T>,
out: MatMut<'_, T>,
uplo: Triangle,
scratch: MatMut<'_, T>,
) -> Result<(), GprError> {
match self {
Self::RbfArd(leaf) => leaf.apply_from_sq_diff::<M, _>(cache, out, uplo),
Self::MaternArd(leaf) => leaf.apply_from_sq_diff::<M, _>(cache, out, uplo),
Self::RationalQuadraticArd(leaf) => leaf.apply_from_sq_diff::<M, _>(cache, out, uplo),
Self::Constant(leaf) => leaf.apply_rows(x, out, uplo),
Self::White(leaf) => leaf.apply_rows(x, out, uplo),
_ => self.apply_points::<M>(x, out, uplo, scratch),
}
}
fn apply_mixed<M: crate::math::KernelMath, S: Supply>(
self,
views: MixedKernelViews<'_, T, S>,
out: MatMut<'_, T>,
uplo: Triangle,
scratch: MatMut<'_, T>,
) -> Result<(), GprError> {
if self.reads_dist() {
return match views.dist {
Some(dist) => self.apply_dist::<M>(dist, out, uplo),
None => self.apply_points::<M>(views.x, out, uplo, scratch),
};
}
match views.ard_cache.filter(|_| self.needs_ard_sq_diff()) {
Some(cache) => self.apply_ard_cache::<M>(cache, views.x, out, uplo, scratch),
None => self.apply_points::<M>(views.x, out, uplo, scratch),
}
}
fn apply_cross_mixed<M: crate::math::KernelMath, S: Supply>(
self,
views: CrossViews<'_, T, S>,
out: MatMut<'_, T>,
scratch: MatMut<'_, T>,
) -> Result<(), GprError> {
match views.dist.filter(|_| self.reads_dist()) {
Some(dist) => self.apply_cross_dist::<M>(dist, out),
None => self.apply_cross_points::<M>(views.x1, views.x2, out, scratch),
}
}
}
const DIAG_BLOCK: usize = 64;
pub(super) fn combine_diag<T: KernelScalar>(
out: &mut [T],
combine: fn(&mut T, T),
mut eval: impl FnMut(usize, &mut [T]) -> Result<(), GprError>,
) -> Result<(), GprError> {
let mut buffer = [T::from_f64(0.0); DIAG_BLOCK];
for (index, chunk) in out.chunks_mut(DIAG_BLOCK).enumerate() {
let block = &mut buffer[..chunk.len()];
eval(index * DIAG_BLOCK, block)?;
for (dst, src) in chunk.iter_mut().zip(block.iter()) {
combine(dst, *src);
}
}
Ok(())
}
fn fold_diag<T: KernelScalar, S: Supply>(
terms: &[CompiledKernel<T, S>],
out: &mut [T],
combine: fn(&mut T, T),
) -> Result<(), GprError> {
let (first, rest) = split_terms(terms)?;
first.fill_diag(out)?;
for term in rest {
combine_diag(out, combine, |_, block| term.fill_diag(block))?;
}
Ok(())
}
fn fold_diag_points<T: KernelScalar, S: Supply>(
terms: &[CompiledKernel<T, S>],
x: MatRef<'_, T>,
out: &mut [T],
combine: fn(&mut T, T),
) -> Result<(), GprError> {
if out.len() != x.nrows() {
return Err(GprError::LengthMismatch {
reason: format!("expected {} diagonal entries, got {}", x.nrows(), out.len()),
});
}
let (first, rest) = split_terms(terms)?;
first.fill_diag_rows(x, out)?;
for term in rest {
combine_diag(out, combine, |start, block| {
term.fill_diag_rows(x.subrows(start, block.len()), block)
})?;
}
Ok(())
}
pub(super) fn add_assign<T: KernelScalar>(dst: &mut T, src: T) {
*dst += src;
}
pub(super) fn mul_assign<T: KernelScalar>(dst: &mut T, src: T) {
*dst *= src;
}
fn apply_custom_from_coords<T: KernelScalar>(
leaf: &CustomKernel<T>,
x: MatRef<'_, T>,
mut out: MatMut<'_, T>,
uplo: Triangle,
mut scratch: MatMut<'_, T>,
) -> Result<(), GprError> {
write_square_from_coords(x, scratch.as_mut(), uplo, Ok)?;
leaf.apply(scratch.as_ref(), out.as_mut(), uplo)
}
fn fold_terms<M: crate::math::KernelMath, T: KernelScalar>(
terms: &[CompiledKernel<T>],
dist: MatRef<'_, T>,
mut out: MatMut<'_, T>,
uplo: Triangle,
mut scratch: MatMut<'_, T>,
nested: &mut Nested<T>,
combine: fn(MatMut<'_, T>, MatRef<'_, T>, Triangle),
) -> Result<(), GprError> {
let (first, rest) = split_terms(terms)?;
first.apply_with::<M>(dist, out.as_mut(), uplo, scratch.as_mut(), nested)?;
let (rows, cols) = (out.nrows(), out.ncols());
for term in rest {
let (own, deeper) = term_scratch(term, rows, cols, out.as_mut(), &mut *nested)?;
term.apply_with::<M>(dist, scratch.as_mut(), uplo, own, deeper)?;
combine(out.as_mut(), scratch.as_ref(), uplo);
}
Ok(())
}
fn fold_terms_points<M: crate::math::KernelMath, T: KernelScalar>(
terms: &[CompiledKernel<T>],
x: MatRef<'_, T>,
mut out: MatMut<'_, T>,
uplo: Triangle,
mut scratch: MatMut<'_, T>,
nested: &mut Nested<T>,
combine: fn(MatMut<'_, T>, MatRef<'_, T>, Triangle),
) -> Result<(), GprError> {
let (first, rest) = split_terms(terms)?;
first.apply_points_with::<M>(x, out.as_mut(), uplo, scratch.as_mut(), nested)?;
let (rows, cols) = (out.nrows(), out.ncols());
for term in rest {
let (own, deeper) = term_scratch(term, rows, cols, out.as_mut(), &mut *nested)?;
term.apply_points_with::<M>(x, scratch.as_mut(), uplo, own, deeper)?;
combine(out.as_mut(), scratch.as_ref(), uplo);
}
Ok(())
}
fn fold_terms_ard_cache<M: crate::math::KernelMath, T: KernelScalar>(
terms: &[CompiledKernel<T>],
cache: ArdSqDiff<'_, T>,
x: MatRef<'_, T>,
mut out: MatMut<'_, T>,
uplo: Triangle,
mut scratch: MatMut<'_, T>,
nested: &mut Nested<T>,
) -> Result<(), GprError> {
let (first, rest) = split_terms(terms)?;
first.apply_from_ard_cache::<M>(cache, x, out.as_mut(), uplo, scratch.as_mut(), nested)?;
let (rows, cols) = (out.nrows(), out.ncols());
for term in rest {
let (own, deeper) = term_scratch(term, rows, cols, out.as_mut(), &mut *nested)?;
term.apply_from_ard_cache::<M>(cache, x, scratch.as_mut(), uplo, own, deeper)?;
add_triangle(out.as_mut(), scratch.as_ref(), uplo);
}
Ok(())
}
fn fold_terms_mixed<M: crate::math::KernelMath, T: KernelScalar, S: Supply>(
terms: &[CompiledKernel<T, S>],
views: MixedKernelViews<'_, T, S>,
mut out: MatMut<'_, T>,
uplo: Triangle,
mut scratch: MatMut<'_, T>,
nested: &mut Nested<T>,
combine: fn(MatMut<'_, T>, MatRef<'_, T>, Triangle),
) -> Result<(), GprError> {
let (first, rest) = split_terms(terms)?;
first.apply_mixed::<M>(views, out.as_mut(), uplo, scratch.as_mut(), nested)?;
let (rows, cols) = (out.nrows(), out.ncols());
for term in rest {
let (own, deeper) = term_scratch(term, rows, cols, out.as_mut(), &mut *nested)?;
term.apply_mixed::<M>(views, scratch.as_mut(), uplo, own, deeper)?;
combine(out.as_mut(), scratch.as_ref(), uplo);
}
Ok(())
}
pub(super) fn add_rect<T: KernelScalar>(mut acc: MatMut<'_, T>, src: MatRef<'_, T>) {
for col in 0..acc.ncols() {
for row in 0..acc.nrows() {
acc[(row, col)] += src[(row, col)];
}
}
}
pub(super) fn mul_rect<T: KernelScalar>(mut acc: MatMut<'_, T>, src: MatRef<'_, T>) {
for col in 0..acc.ncols() {
for row in 0..acc.nrows() {
acc[(row, col)] *= src[(row, col)];
}
}
}
fn fold_rect<M: crate::math::KernelMath, T: KernelScalar>(
terms: &[CompiledKernel<T>],
dist: MatRef<'_, T>,
mut out: MatMut<'_, T>,
mut scratch: MatMut<'_, T>,
nested: &mut Nested<T>,
combine: fn(MatMut<'_, T>, MatRef<'_, T>),
) -> Result<(), GprError> {
let (first, rest) = split_terms(terms)?;
first.apply_cross_with::<M>(dist, out.as_mut(), scratch.as_mut(), nested)?;
let (rows, cols) = (out.nrows(), out.ncols());
for term in rest {
let (own, deeper) = term_scratch(term, rows, cols, out.as_mut(), &mut *nested)?;
term.apply_cross_with::<M>(dist, scratch.as_mut(), own, deeper)?;
combine(out.as_mut(), scratch.as_ref());
}
Ok(())
}
fn fold_rect_points<M: crate::math::KernelMath, T: KernelScalar>(
terms: &[CompiledKernel<T>],
x: MatRef<'_, T>,
xs: MatRef<'_, T>,
mut out: MatMut<'_, T>,
mut scratch: MatMut<'_, T>,
nested: &mut Nested<T>,
combine: fn(MatMut<'_, T>, MatRef<'_, T>),
) -> Result<(), GprError> {
let (first, rest) = split_terms(terms)?;
first.apply_cross_points_with::<M>(x, xs, out.as_mut(), scratch.as_mut(), nested)?;
let (rows, cols) = (out.nrows(), out.ncols());
for term in rest {
let (own, deeper) = term_scratch(term, rows, cols, out.as_mut(), &mut *nested)?;
term.apply_cross_points_with::<M>(x, xs, scratch.as_mut(), own, deeper)?;
combine(out.as_mut(), scratch.as_ref());
}
Ok(())
}
fn fold_rect_mixed<M: crate::math::KernelMath, T: KernelScalar, S: Supply>(
terms: &[CompiledKernel<T, S>],
views: CrossViews<'_, T, S>,
mut out: MatMut<'_, T>,
mut scratch: MatMut<'_, T>,
nested: &mut Nested<T>,
combine: fn(MatMut<'_, T>, MatRef<'_, T>),
) -> Result<(), GprError> {
let (first, rest) = split_terms(terms)?;
first.apply_cross_mixed::<M>(views, out.as_mut(), scratch.as_mut(), nested)?;
let (rows, cols) = (out.nrows(), out.ncols());
for term in rest {
let (own, deeper) = term_scratch(term, rows, cols, out.as_mut(), &mut *nested)?;
term.apply_cross_mixed::<M>(views, scratch.as_mut(), own, deeper)?;
combine(out.as_mut(), scratch.as_ref());
}
Ok(())
}