#[allow(
unused_imports,
reason = "a split file takes its parent's imports whole; each uses some"
)]
use super::*;
pub(crate) struct KernelScratch<T> {
scratch: Mat<T>,
nested: Vec<Mat<T>>,
dist: Mat<T>,
transposed: Mat<T>,
square: Vec<Mat<T>>,
cross: Vec<Mat<T>>,
partial: Vec<f64>,
jobs: Vec<f64>,
diag: DiagAccum<T>,
}
impl<T> Clone for KernelScratch<T> {
fn clone(&self) -> Self {
Self {
scratch: Mat::new(),
nested: Vec::new(),
dist: Mat::new(),
transposed: Mat::new(),
square: Vec::new(),
cross: Vec::new(),
partial: Vec::new(),
jobs: Vec::new(),
diag: DiagAccum::new(),
}
}
}
impl<T> std::fmt::Debug for KernelScratch<T> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("KernelScratch").finish_non_exhaustive()
}
}
impl<T: KernelScalar> Default for KernelScratch<T> {
fn default() -> Self {
Self::new()
}
}
impl<T: KernelScalar> KernelScratch<T> {
pub(crate) fn new() -> Self {
Self {
scratch: Mat::new(),
nested: Vec::new(),
dist: Mat::new(),
transposed: Mat::new(),
square: Vec::new(),
cross: Vec::new(),
partial: Vec::new(),
jobs: Vec::new(),
diag: DiagAccum::new(),
}
}
pub(crate) fn gram<M: crate::math::KernelMath, U: Supply>(
&mut self,
compiled: &CompiledKernel<T, U>,
inputs: GramInputs<'_, T, U>,
out: MatMut<'_, T>,
uplo: Triangle,
) -> Result<(), GprError> {
let scratch = view(&mut self.scratch, out.nrows(), out.ncols());
compiled.eval_gram::<M>(inputs, out, uplo, scratch, &mut self.nested)
}
pub(crate) fn grad<M: crate::math::KernelMath, U: Supply>(
&mut self,
compiled: &CompiledKernel<T, U>,
inputs: GramInputs<'_, T, U>,
d_k: MatMut<'_, T>,
param_idx: usize,
uplo: Triangle,
) -> Result<(), GprError> {
let scratch = view(&mut self.scratch, d_k.nrows(), d_k.ncols());
compiled.grad_gram::<M>(inputs, d_k, param_idx, uplo, scratch, &mut self.nested)
}
pub(crate) fn hess<M: crate::math::KernelMath, U: Supply>(
&mut self,
compiled: &CompiledKernel<T, U>,
inputs: GramInputs<'_, T, U>,
d2_k: MatMut<'_, T>,
pair: (usize, usize),
uplo: Triangle,
) -> Result<(), GprError> {
let scratch = view(&mut self.scratch, d2_k.nrows(), d2_k.ncols());
compiled.hess_gram::<M>(inputs, d2_k, pair, uplo, scratch, &mut self.nested)
}
pub(crate) fn cross_into<M: crate::math::KernelMath, U: Supply>(
&mut self,
compiled: &CompiledKernel<T, U>,
views: CrossViews<'_, T, U>,
out: MatMut<'_, T>,
) -> Result<(), GprError> {
let (rows, cols) = (out.nrows(), out.ncols());
let dist = view(&mut self.dist, rows, cols);
let scratch = view(&mut self.scratch, rows, cols);
compiled.eval_cross_slots::<M>(
views.x1,
views.x2,
views.slots,
Some(dist),
out,
scratch,
&mut self.nested,
&mut [],
)
}
pub(crate) fn grad_cross_into<M: crate::math::KernelMath, U: Supply>(
&mut self,
compiled: &CompiledKernel<T, U>,
views: CrossViews<'_, T, U>,
out: MatMut<'_, T>,
param_idx: usize,
) -> Result<(), GprError> {
let (rows, cols) = (out.nrows(), out.ncols());
let KernelScratch {
scratch, nested, ..
} = self;
crate::kernel::ensure_nested_levels(nested, compiled, rows, cols);
compiled.grad_cross_views::<M>(views, out, param_idx, view(scratch, rows, cols), nested)
}
pub(crate) fn hess_cross_into<M: crate::math::KernelMath, U: Supply>(
&mut self,
compiled: &CompiledKernel<T, U>,
views: CrossViews<'_, T, U>,
out: MatMut<'_, T>,
pair: (usize, usize),
) -> Result<(), GprError> {
let (rows, cols) = (out.nrows(), out.ncols());
let KernelScratch {
scratch, nested, ..
} = self;
crate::kernel::ensure_nested_levels(nested, compiled, rows, cols);
compiled.hess_cross_views::<M>(views, out, pair, view(scratch, rows, cols), nested)
}
pub(crate) fn cross<M: crate::math::KernelMath, U: Supply>(
&mut self,
compiled: &CompiledKernel<T, U>,
views: CrossViews<'_, T, U>,
) -> Result<Mat<T>, GprError> {
let mut out = Mat::zeros(views.x1.nrows(), views.x2.nrows());
self.cross_into::<M, U>(compiled, views, out.as_mut())?;
Ok(out)
}
pub(crate) fn cross_mn_into<M: crate::math::KernelMath, U: Supply>(
&mut self,
compiled: &CompiledKernel<T, U>,
sets: SparseSets<'_, T, U>,
mut out: MatMut<'_, T>,
) -> Result<(), GprError> {
if let Some(coords) = U::coordinates(compiled) {
return self.cross_into::<M, NoSupply>(coords, CrossViews::points(sets.z, sets.x), out);
}
let (n, m) = (sets.x.nrows(), sets.z.nrows());
let reads = compiled.reads_distances()?;
let KernelScratch {
dist,
transposed,
scratch,
nested,
..
} = self;
let mut k = view(transposed, n, m);
let views = sets.k_nm();
compiled.eval_cross_slots::<M>(
views.x1,
views.x2,
views.slots,
reads.then(|| view(dist, n, m)),
k.as_mut(),
view(scratch, n, m),
nested,
&mut [],
)?;
out.copy_from(k.transpose());
Ok(())
}
pub(crate) fn cross_mn<M: crate::math::KernelMath, U: Supply>(
&mut self,
compiled: &CompiledKernel<T, U>,
sets: SparseSets<'_, T, U>,
) -> Result<Mat<T>, GprError> {
let mut out = Mat::zeros(sets.z.nrows(), sets.x.nrows());
self.cross_mn_into::<M, U>(compiled, sets, out.as_mut())?;
Ok(out)
}
pub(crate) fn grad_cross_mn_into<M: crate::math::KernelMath, U: Supply>(
&mut self,
compiled: &CompiledKernel<T, U>,
sets: SparseSets<'_, T, U>,
mut out: MatMut<'_, T>,
param_idx: usize,
) -> Result<(), GprError> {
if let Some(coords) = U::coordinates(compiled) {
let views = CrossViews::points(sets.z, sets.x);
return self.grad_cross_into::<M, NoSupply>(coords, views, out, param_idx);
}
let (n, m) = (sets.x.nrows(), sets.z.nrows());
let mut k = std::mem::replace(&mut self.transposed, Mat::new());
let result =
self.grad_cross_into::<M, U>(compiled, sets.k_nm(), view(&mut k, n, m), param_idx);
if result.is_ok() {
out.copy_from(k.as_ref().submatrix(0, 0, n, m).transpose());
}
self.transposed = k;
result
}
pub(crate) fn hess_cross_mn_into<M: crate::math::KernelMath, U: Supply>(
&mut self,
compiled: &CompiledKernel<T, U>,
sets: SparseSets<'_, T, U>,
mut out: MatMut<'_, T>,
pair: (usize, usize),
) -> Result<(), GprError> {
if let Some(coords) = U::coordinates(compiled) {
let views = CrossViews::points(sets.z, sets.x);
return self.hess_cross_into::<M, NoSupply>(coords, views, out, pair);
}
let (n, m) = (sets.x.nrows(), sets.z.nrows());
let mut k = std::mem::replace(&mut self.transposed, Mat::new());
let result = self.hess_cross_into::<M, U>(compiled, sets.k_nm(), view(&mut k, n, m), pair);
if result.is_ok() {
out.copy_from(k.as_ref().submatrix(0, 0, n, m).transpose());
}
self.transposed = k;
result
}
pub(crate) fn add_cross_contraction_mn<M: crate::math::KernelMath, U: Supply>(
&mut self,
compiled: &CompiledKernel<T, U>,
sets: SparseSets<'_, T, U>,
weight: MatRef<'_, T>,
coeff: f64,
out: &mut [f64],
) -> Result<(), GprError> {
if let Some(coords) = U::coordinates(compiled) {
let views = CrossViews::points(sets.z, sets.x);
return self.add_cross_contraction::<M, NoSupply>(coords, views, weight, coeff, out);
}
let (n, m) = (sets.x.nrows(), sets.z.nrows());
let mut w = std::mem::replace(&mut self.transposed, Mat::new());
let mut wt = view(&mut w, n, m);
wt.copy_from(weight.transpose());
let result = self.add_cross_contraction::<M, U>(
compiled,
sets.k_nm(),
w.as_ref().submatrix(0, 0, n, m),
coeff,
out,
);
self.transposed = w;
result
}
pub(crate) fn write_square_contraction<M: crate::math::KernelMath, U: Supply>(
&mut self,
compiled: &CompiledKernel<T, U>,
inputs: GramInputs<'_, T, U>,
weight: MatRef<'_, T>,
out: &mut [f64],
) -> Result<(), GprError> {
let x = inputs.x;
let m = x.nrows();
let reads = compiled.reads_distances()?;
let KernelScratch {
scratch,
nested,
dist,
square,
jobs,
..
} = self;
if reads {
let mut filled = view(dist, m, m);
let mut none: [Mat<T>; 0] = [];
T::write_squared(x, filled.as_mut(), &mut none);
}
let dist_view = reads.then(|| dist.as_ref().submatrix(0, 0, m, m));
let nbuf = compiled.contraction_buffers();
fit_exact(square, nbuf, m, m);
crate::kernel::ensure_nested_levels(nested, compiled, m, m);
let mut walk = WeightedWalk {
inputs: GramInputs {
x,
dist: dist_view,
ard: None,
slots: U::shorter_squares(inputs.slots),
},
scratch: view(scratch, m, m),
nested,
kept: &[],
kept_products: 0,
fold: jobs,
};
compiled.weighted_grads::<M>(&mut walk, weight, out, &mut square[..nbuf])
}
pub(crate) fn add_cross_contraction<M: crate::math::KernelMath, U: Supply>(
&mut self,
compiled: &CompiledKernel<T, U>,
views: CrossViews<'_, T, U>,
weight: MatRef<'_, T>,
coeff: f64,
out: &mut [f64],
) -> Result<(), GprError> {
let n_params = compiled.num_params();
let (rows, cols) = (weight.nrows(), weight.ncols());
let nbuf = compiled.contraction_buffers();
let KernelScratch {
scratch,
nested,
cross,
partial,
jobs,
..
} = self;
if partial.len() < n_params {
partial.resize(n_params, 0.0);
}
fit_exact(cross, nbuf, rows, cols);
crate::kernel::ensure_nested_levels(nested, compiled, rows, cols);
compiled.weighted_cross_grads_views::<M>(
views,
weight,
&mut partial[..n_params],
&mut cross[..nbuf],
view(scratch, rows, cols),
nested,
jobs,
)?;
for (slot, part) in out.iter_mut().zip(partial.iter()) {
*slot += coeff * part;
}
Ok(())
}
pub(crate) fn add_diag_contraction<M: crate::math::KernelMath, U: Supply>(
&mut self,
compiled: &CompiledKernel<T, U>,
x: MatRef<'_, T>,
coeff: f64,
out: &mut [f64],
) -> Result<(), GprError> {
let n_params = compiled.num_params();
let KernelScratch { partial, diag, .. } = self;
if partial.len() < n_params {
partial.resize(n_params, 0.0);
}
compiled.weighted_diag_sums::<M>(x, &mut partial[..n_params], diag)?;
for (slot, part) in out.iter_mut().zip(partial.iter()) {
*slot += coeff * part;
}
Ok(())
}
pub(crate) fn scratch(&mut self, rows: usize, cols: usize) -> MatMut<'_, T> {
view(&mut self.scratch, rows, cols)
}
}
pub(super) fn fit_exact<T: KernelScalar>(
pool: &mut Vec<Mat<T>>,
count: usize,
rows: usize,
cols: usize,
) {
if pool.len() < count {
pool.resize_with(count, Mat::new);
}
for mat in &mut pool[..count] {
if mat.nrows() != rows || mat.ncols() != cols {
*mat = Mat::zeros(rows, cols);
}
}
}
pub(crate) fn view<T: KernelScalar>(buf: &mut Mat<T>, rows: usize, cols: usize) -> MatMut<'_, T> {
if buf.nrows() < rows || buf.ncols() < cols {
*buf = Mat::zeros(rows.max(buf.nrows()), cols.max(buf.ncols()));
}
buf.as_mut().submatrix_mut(0, 0, rows, cols)
}
pub(crate) struct KernelPlan<S: KernelScalar, U: Supply = NoSupply>(
Option<(KernelSpec<U>, CompiledKernel<S, U>)>,
);
impl<S: KernelScalar, U: Supply> KernelPlan<S, U> {
pub(crate) fn get(&mut self, kernel: &KernelSpec<U>) -> &CompiledKernel<S, U> {
if !matches!(&self.0, Some((spec, _)) if spec == kernel) {
self.0 = None;
}
&self
.0
.get_or_insert_with(|| (kernel.clone(), kernel.compile_as::<S>()))
.1
}
}
pub(crate) struct PredictBuffers<S: KernelScalar> {
pub(crate) kernel: KernelScratch<S>,
pub(crate) z: Mat<S>,
pub(crate) query: Mat<S>,
pub(crate) k_sz: Mat<S>,
pub(crate) solved: Mat<S>,
pub(crate) kss: Vec<S>,
pub(crate) column: Vec<S>,
}
impl<S: KernelScalar> Default for PredictBuffers<S> {
fn default() -> Self {
Self {
kernel: KernelScratch::new(),
z: Mat::new(),
query: Mat::new(),
k_sz: Mat::new(),
solved: Mat::new(),
kss: Vec::new(),
column: Vec::new(),
}
}
}
pub(crate) fn pack_into<'a, S: KernelScalar>(
buf: &'a mut Mat<S>,
x: &[f64],
rows: usize,
d: usize,
) -> MatMut<'a, S> {
let mut out = view(buf, rows, d);
crate::data::pack_storage(x, rows, d, out.as_mut());
out
}
pub(crate) struct PredictScratch<S: KernelScalar, U: Supply = NoSupply> {
pub(crate) xs: Vec<f64>,
pub(crate) inverse: InverseBuffers,
pub(crate) plan: KernelPlan<S, U>,
pub(crate) storage: PredictBuffers<S>,
pub(crate) plan64: KernelPlan<f64, U>,
pub(crate) f64: PredictBuffers<f64>,
pub(crate) k_mm64: Mat<f64>,
pub(crate) k_mm64_backup: Mat<f64>,
pub(crate) llt64: Option<(usize, MemBuffer)>,
pub(crate) b_l64: Mat<f64>,
pub(crate) w64: Vec<f64>,
pub(crate) query64: crate::kernel::QueryScratch<f64>,
pub(crate) query_storage: crate::kernel::QueryScratch<S>,
}
impl<S: KernelScalar, U: Supply> Default for PredictScratch<S, U> {
fn default() -> Self {
Self {
xs: Vec::new(),
inverse: InverseBuffers::default(),
plan: KernelPlan(None),
storage: PredictBuffers::default(),
plan64: KernelPlan(None),
f64: PredictBuffers::default(),
k_mm64: Mat::new(),
k_mm64_backup: Mat::new(),
llt64: None,
b_l64: Mat::new(),
w64: Vec::new(),
query64: crate::kernel::QueryScratch::new(),
query_storage: crate::kernel::QueryScratch::new(),
}
}
}
impl<S: KernelScalar, U: Supply> Clone for PredictScratch<S, U> {
fn clone(&self) -> Self {
Self::default()
}
}
impl<S: KernelScalar, U: Supply> fmt::Debug for PredictScratch<S, U> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("PredictScratch").finish_non_exhaustive()
}
}
impl<S: KernelScalar, U: Supply> PredictScratch<S, U> {
pub(crate) fn llt64(llt: &mut Option<(usize, MemBuffer)>, m: usize) -> &mut MemBuffer {
if matches!(llt, Some((size, _)) if *size != m) {
*llt = None;
}
&mut llt
.get_or_insert_with(|| (m, crate::linalg::llt_scratch::<f64>(m)))
.1
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn f64_system<'a, M: crate::math::KernelMath>(
&'a mut self,
kernel: &KernelSpec<U>,
z: &[f64],
m: usize,
d: usize,
zz: <U as SupplyViews>::Held<&BlockStore<f64>>,
jitter: JitterPolicy,
) -> Result<F64System<'a, U>, GprError> {
let Self {
plan64,
f64: bufs,
k_mm64,
k_mm64_backup,
llt64,
b_l64,
w64,
..
} = self;
let compiled = plan64.get(kernel);
if k_mm64.nrows() != m || k_mm64.ncols() != m {
*k_mm64 = Mat::zeros(m, m);
}
let z64 = pack_into(&mut bufs.z, z, m, d);
bufs.kernel.gram::<M, U>(
compiled,
GramInputs::supplied(z64.as_ref(), U::shorter_squares(squares_of::<f64, U>(zz))),
k_mm64.as_mut(),
Triangle::Lower,
)?;
crate::linalg::cholesky_lower_with_backup(
k_mm64,
k_mm64_backup,
Self::llt64(llt64, m),
jitter.retry_jitters(),
crate::error::CholeskyStage::Predict,
)?;
Ok(F64System {
compiled,
bufs,
k_mm_l: k_mm64.as_ref(),
b_l64,
w64,
})
}
}
pub(crate) struct F64System<'a, U: Supply = NoSupply> {
pub(crate) compiled: &'a CompiledKernel<f64, U>,
pub(crate) bufs: &'a mut PredictBuffers<f64>,
pub(crate) k_mm_l: MatRef<'a, f64>,
pub(crate) b_l64: &'a mut Mat<f64>,
pub(crate) w64: &'a mut Vec<f64>,
}
pub(crate) fn reset_prediction<R: KernelScalar>(
out: &mut Prediction<R>,
n_rows: usize,
kind: crate::VarianceKind,
) {
let zero = R::from_f64(0.0);
out.mean.clear();
out.mean.resize(n_rows, zero);
out.variance.clear();
out.variance.resize(n_rows, zero);
out.variance_kind = kind;
}
pub(crate) fn predictive_variance(latent: f64, noise: f64, kind: crate::VarianceKind) -> f64 {
match kind {
crate::VarianceKind::Latent => latent,
crate::VarianceKind::Observation => latent + noise,
}
}
#[derive(Clone, Debug)]
pub(crate) struct SparseScratch<S: KernelScalar, U: Supply = NoSupply> {
pub(crate) storage: KernelScratch<S>,
pub(crate) f64: KernelScratch<f64>,
pub(crate) point: Vec<f64>,
pub(crate) row: <U as SupplyViews>::Held<SupplyAt<f64>>,
pub(crate) predict: PredictScratch<S, U>,
}
impl<S: KernelScalar, U: Supply> Default for SparseScratch<S, U> {
fn default() -> Self {
Self {
storage: KernelScratch::new(),
f64: KernelScratch::new(),
point: Vec::new(),
row: U::hold(SupplyAt::default),
predict: PredictScratch::default(),
}
}
}