use std::fmt;
use faer::{Mat, MatRef};
use crate::math::{self, ExpJet};
pub(crate) mod sealed {
use dyn_stack::MemBuffer;
use faer::{Mat, MatMut, MatRef};
use crate::error::{CholeskyStage, GprError};
use crate::kernel::KernelTerm;
use crate::math::ExpJet;
pub trait ScalarOps: Sized {
type RowCast: Clone + Send + Sync + 'static;
type ColCast: Clone + Send + Sync + 'static;
type F64Factor: Clone + std::fmt::Debug + Send + Sync + 'static;
type F64Held<V: Clone + std::fmt::Debug + Send + Sync + 'static>: Clone
+ std::fmt::Debug
+ Send
+ Sync
+ 'static;
const ROUNDS_FROM_F64: bool;
const DTYPE: safetensors::Dtype;
const READS_ARD_CACHE: bool;
fn write_squared(x: MatRef<'_, Self>, dist: MatMut<'_, Self>, scratch: &mut [Mat<Self>]);
fn write_cross(
x_train: MatRef<'_, Self>,
x_test: MatRef<'_, Self>,
dist: MatMut<'_, Self>,
scratch: &mut [Mat<Self>],
);
fn write_ard(x: MatRef<'_, Self>, cache: &mut [Self]);
fn as_f64_slice(values: &[Self]) -> Option<&[f64]>;
fn as_f64_slice_mut(values: &mut [Self]) -> Option<&mut [f64]>;
fn from_f64_slice(values: &[f64]) -> Option<&[Self]>;
fn from_f64_slices<'a>(values: &'a [&'a [f64]]) -> Option<&'a [&'a [Self]]>;
fn from_f64_vecs(values: &[Vec<f64>]) -> Option<&[Vec<Self>]>;
fn vecs_from_f64(values: Vec<Vec<f64>>) -> Result<Vec<Vec<Self>>, Vec<Vec<f64>>>;
fn as_f64_vecs(values: &[Vec<Self>]) -> Option<&[Vec<f64>]>;
fn empty_rows() -> Self::RowCast;
fn empty_cols() -> Self::ColCast;
fn vec_from_f64(values: Vec<f64>) -> Vec<Self>;
fn storage_rows<'a>(y: &'a [f64], cast: &'a mut Self::RowCast) -> &'a [Self];
fn storage_cols<'a>(x: MatRef<'a, f64>, cast: &'a mut Self::ColCast) -> MatRef<'a, Self>;
fn hold_f64_factor(l: MatRef<'_, Self>) -> Self::F64Factor;
fn f64_factor_from(l: Mat<f64>) -> Self::F64Factor;
fn f64_factor<'a>(l: MatRef<'a, Self>, held: &'a Self::F64Factor) -> MatRef<'a, f64>;
fn f64_factor_mut<'a>(
l: &'a mut Mat<Self>,
held: &'a mut Self::F64Factor,
) -> MatMut<'a, f64>;
fn store_f64_factor(l: &mut Mat<Self>, held: &Self::F64Factor);
fn f64_factor_trial(l: MatRef<'_, Self>, held: &Self::F64Factor) -> Mat<f64>;
fn commit_f64_factor(l: &mut Mat<Self>, held: &mut Self::F64Factor, trial: Mat<f64>);
fn hold_f64<V: Clone + std::fmt::Debug + Send + Sync + 'static>(
make: impl FnOnce() -> V,
) -> Self::F64Held<V>;
fn held_f64<V: Clone + std::fmt::Debug + Send + Sync + 'static>(
held: Self::F64Held<V>,
) -> Option<V>;
fn held_f64_ref<V: Clone + std::fmt::Debug + Send + Sync + 'static>(
held: &Self::F64Held<V>,
) -> Option<&V>;
fn as_f64_ref(m: MatRef<'_, Self>) -> Option<MatRef<'_, f64>>;
fn as_f64_mut(m: MatMut<'_, Self>) -> Option<MatMut<'_, f64>>;
fn pick_term<'a>(
f64_term: &'a dyn KernelTerm<f64>,
f32_term: &'a dyn KernelTerm<f32>,
) -> &'a dyn KernelTerm<Self>
where
Self: crate::kernel::KernelScalar;
fn fast_exp(self) -> Self;
fn fast_jet(self) -> ExpJet<Self>;
fn cholesky_lower(
a: &mut Mat<Self>,
scratch: &mut MemBuffer,
jitter: f64,
stage: CholeskyStage,
) -> Result<(), GprError>;
fn solve_llt_in_place(l: MatRef<'_, Self>, rhs: MatMut<'_, Self>, scratch: &mut MemBuffer);
fn solve_llt_owned_scratch(l: MatRef<'_, Self>, rhs: MatMut<'_, Self>);
fn inv_diag_from_chol_l(l: MatRef<'_, Self>, q_diag: &mut [Self]);
fn matvec_columns(a: MatRef<'_, Self>, y: &[Self], ay: MatMut<'_, Self>);
fn gram_aat(a: MatRef<'_, Self>, b: MatMut<'_, Self>);
fn solve_unit_lower_rows(lt: &faer::Mat<Self>, w: &mut [Self]);
fn solve_ldlt_in_place(ld: MatRef<'_, Self>, rhs: MatMut<'_, Self>, n: usize);
fn ldlt_delete_row_col(
ld: MatMut<'_, Self>,
index: usize,
n: usize,
scratch: &mut MemBuffer,
);
}
}
pub trait KernelScalar:
sealed::ScalarOps + faer_traits::RealField + Copy + fmt::Debug + Default + Send + Sync + 'static
{
fn from_f64(value: f64) -> Self;
fn to_f64(self) -> f64;
fn exp(self) -> Self;
fn ln(self) -> Self;
fn sqrt(self) -> Self;
fn abs(self) -> Self;
fn is_finite(self) -> bool;
fn powf(self, e: Self) -> Self;
fn sin(self) -> Self;
fn cos(self) -> Self;
fn max(self, other: Self) -> Self;
fn min(self, other: Self) -> Self;
}
#[inline(always)]
pub(crate) fn f64_pair<'a, T: KernelScalar>(
src: MatRef<'a, T>,
dest: faer::MatMut<'a, T>,
) -> Option<(MatRef<'a, f64>, faer::MatMut<'a, f64>)> {
Some((T::as_f64_ref(src)?, T::as_f64_mut(dest)?))
}
macro_rules! impl_kernel_scalar {
($t:ty) => {
impl KernelScalar for $t {
#[inline(always)]
fn from_f64(value: f64) -> Self {
value as $t
}
#[inline(always)]
fn to_f64(self) -> f64 {
f64::from(self)
}
#[inline(always)]
fn exp(self) -> Self {
<$t>::exp(self)
}
#[inline(always)]
fn ln(self) -> Self {
<$t>::ln(self)
}
#[inline(always)]
fn sqrt(self) -> Self {
<$t>::sqrt(self)
}
#[inline(always)]
fn abs(self) -> Self {
<$t>::abs(self)
}
#[inline(always)]
fn is_finite(self) -> bool {
<$t>::is_finite(self)
}
#[inline(always)]
fn powf(self, e: Self) -> Self {
<$t>::powf(self, e)
}
#[inline(always)]
fn sin(self) -> Self {
<$t>::sin(self)
}
#[inline(always)]
fn cos(self) -> Self {
<$t>::cos(self)
}
#[inline(always)]
fn max(self, other: Self) -> Self {
<$t>::max(self, other)
}
#[inline(always)]
fn min(self, other: Self) -> Self {
<$t>::min(self, other)
}
}
};
}
impl_kernel_scalar!(f32);
impl_kernel_scalar!(f64);
impl sealed::ScalarOps for f64 {
type RowCast = ();
type ColCast = ();
type F64Factor = ();
type F64Held<V: Clone + std::fmt::Debug + Send + Sync + 'static> = ();
const ROUNDS_FROM_F64: bool = false;
const DTYPE: safetensors::Dtype = safetensors::Dtype::F64;
const READS_ARD_CACHE: bool = true;
fn write_squared(x: MatRef<'_, Self>, dist: faer::MatMut<'_, Self>, scratch: &mut [Mat<Self>]) {
super::dist::fill_squared_euclidean(x, dist, scratch);
}
fn write_cross(
x_train: MatRef<'_, Self>,
x_test: MatRef<'_, Self>,
dist: faer::MatMut<'_, Self>,
scratch: &mut [Mat<Self>],
) {
super::dist::fill_squared_euclidean_cross(x_train, x_test, dist, scratch);
}
fn write_ard(x: MatRef<'_, Self>, cache: &mut [Self]) {
super::dist::fill_ard_squared_diff(x, cache);
}
fn as_f64_slice(values: &[Self]) -> Option<&[f64]> {
Some(values)
}
fn from_f64_slice(values: &[f64]) -> Option<&[Self]> {
Some(values)
}
fn from_f64_slices<'a>(values: &'a [&'a [f64]]) -> Option<&'a [&'a [Self]]> {
Some(values)
}
fn from_f64_vecs(values: &[Vec<f64>]) -> Option<&[Vec<Self>]> {
Some(values)
}
fn vecs_from_f64(values: Vec<Vec<f64>>) -> Result<Vec<Vec<Self>>, Vec<Vec<f64>>> {
Ok(values)
}
fn as_f64_vecs(values: &[Vec<Self>]) -> Option<&[Vec<f64>]> {
Some(values)
}
fn as_f64_slice_mut(values: &mut [Self]) -> Option<&mut [f64]> {
Some(values)
}
fn empty_rows() -> Self::RowCast {}
fn empty_cols() -> Self::ColCast {}
fn vec_from_f64(values: Vec<f64>) -> Vec<Self> {
values
}
fn storage_rows<'a>(y: &'a [f64], _cast: &'a mut Self::RowCast) -> &'a [Self] {
y
}
fn storage_cols<'a>(x: MatRef<'a, f64>, _cast: &'a mut Self::ColCast) -> MatRef<'a, Self> {
x
}
fn pick_term<'a>(
f64_term: &'a dyn crate::kernel::KernelTerm<f64>,
_f32_term: &'a dyn crate::kernel::KernelTerm<f32>,
) -> &'a dyn crate::kernel::KernelTerm<Self> {
f64_term
}
fn hold_f64_factor(_l: MatRef<'_, Self>) -> Self::F64Factor {}
fn f64_factor_from(_l: Mat<f64>) -> Self::F64Factor {}
fn f64_factor<'a>(l: MatRef<'a, Self>, _held: &'a Self::F64Factor) -> MatRef<'a, f64> {
l
}
fn f64_factor_mut<'a>(
l: &'a mut Mat<Self>,
_held: &'a mut Self::F64Factor,
) -> faer::MatMut<'a, f64> {
l.as_mut()
}
fn store_f64_factor(_l: &mut Mat<Self>, _held: &Self::F64Factor) {}
fn f64_factor_trial(l: MatRef<'_, Self>, _held: &Self::F64Factor) -> Mat<f64> {
l.to_owned()
}
fn commit_f64_factor(l: &mut Mat<Self>, _held: &mut Self::F64Factor, trial: Mat<f64>) {
*l = trial;
}
fn hold_f64<V: Clone + std::fmt::Debug + Send + Sync + 'static>(_make: impl FnOnce() -> V) {}
fn held_f64<V: Clone + std::fmt::Debug + Send + Sync + 'static>((): ()) -> Option<V> {
None
}
fn held_f64_ref<V: Clone + std::fmt::Debug + Send + Sync + 'static>((): &()) -> Option<&V> {
None
}
#[inline(always)]
fn as_f64_ref(m: MatRef<'_, Self>) -> Option<MatRef<'_, f64>> {
Some(m)
}
#[inline(always)]
fn as_f64_mut(m: faer::MatMut<'_, Self>) -> Option<faer::MatMut<'_, f64>> {
Some(m)
}
#[inline(always)]
fn fast_exp(self) -> Self {
math::fast_exp_f64(self)
}
#[inline(always)]
fn fast_jet(self) -> ExpJet<Self> {
math::fast_jet_f64(self)
}
fn cholesky_lower(
a: &mut Mat<Self>,
scratch: &mut dyn_stack::MemBuffer,
jitter: f64,
stage: crate::error::CholeskyStage,
) -> Result<(), crate::error::GprError> {
crate::linalg::cholesky_lower_faer(a, scratch, jitter, stage)
}
fn solve_llt_in_place(
l: MatRef<'_, Self>,
rhs: faer::MatMut<'_, Self>,
scratch: &mut dyn_stack::MemBuffer,
) {
crate::linalg::solve_llt_faer(l, rhs, scratch);
}
fn solve_llt_owned_scratch(l: MatRef<'_, Self>, rhs: faer::MatMut<'_, Self>) {
crate::linalg::solve_llt_faer_owned(l, rhs);
}
fn inv_diag_from_chol_l(l: MatRef<'_, Self>, q_diag: &mut [Self]) {
crate::linalg::inv_diag_from_chol_l_faer(l, q_diag);
}
fn matvec_columns(a: MatRef<'_, Self>, y: &[Self], ay: faer::MatMut<'_, Self>) {
crate::linalg::matvec_columns_native(a, y, ay);
}
fn gram_aat(a: MatRef<'_, Self>, b: faer::MatMut<'_, Self>) {
crate::linalg::gram_aat_faer(a, b);
}
fn solve_unit_lower_rows(lt: &faer::Mat<Self>, w: &mut [Self]) {
crate::linalg::solve_unit_lower_rows_f64(lt, w);
}
fn solve_ldlt_in_place(ld: MatRef<'_, Self>, rhs: faer::MatMut<'_, Self>, n: usize) {
crate::linalg::solve_ldlt_faer(ld, rhs, n);
}
fn ldlt_delete_row_col(
ld: faer::MatMut<'_, Self>,
index: usize,
n: usize,
scratch: &mut dyn_stack::MemBuffer,
) {
crate::linalg::ldlt_delete_faer(ld, index, n, scratch);
}
}
impl sealed::ScalarOps for f32 {
type RowCast = Vec<f32>;
type ColCast = Mat<f32>;
type F64Factor = Mat<f64>;
type F64Held<V: Clone + std::fmt::Debug + Send + Sync + 'static> = V;
const ROUNDS_FROM_F64: bool = true;
const DTYPE: safetensors::Dtype = safetensors::Dtype::F32;
const READS_ARD_CACHE: bool = false;
fn write_squared(
x: MatRef<'_, Self>,
dist: faer::MatMut<'_, Self>,
_scratch: &mut [Mat<Self>],
) {
super::dist::fill_squared_scalar(x, dist);
}
fn write_cross(
x_train: MatRef<'_, Self>,
x_test: MatRef<'_, Self>,
dist: faer::MatMut<'_, Self>,
_scratch: &mut [Mat<Self>],
) {
super::dist::fill_cross_scalar(x_train, x_test, dist);
}
fn write_ard(x: MatRef<'_, Self>, cache: &mut [Self]) {
super::dist::fill_ard_scalar(x, cache);
}
fn as_f64_slice(_values: &[Self]) -> Option<&[f64]> {
None
}
fn from_f64_slice(_values: &[f64]) -> Option<&[Self]> {
None
}
fn from_f64_slices<'a>(_values: &'a [&'a [f64]]) -> Option<&'a [&'a [Self]]> {
None
}
fn from_f64_vecs(_values: &[Vec<f64>]) -> Option<&[Vec<Self>]> {
None
}
fn vecs_from_f64(values: Vec<Vec<f64>>) -> Result<Vec<Vec<Self>>, Vec<Vec<f64>>> {
Err(values)
}
fn as_f64_vecs(_values: &[Vec<Self>]) -> Option<&[Vec<f64>]> {
None
}
fn as_f64_slice_mut(_values: &mut [Self]) -> Option<&mut [f64]> {
None
}
fn empty_rows() -> Self::RowCast {
Vec::new()
}
fn empty_cols() -> Self::ColCast {
Mat::zeros(0, 0)
}
fn vec_from_f64(values: Vec<f64>) -> Vec<Self> {
values.into_iter().map(|v| v as f32).collect()
}
fn storage_rows<'a>(y: &'a [f64], cast: &'a mut Self::RowCast) -> &'a [Self] {
if cast.len() != y.len() {
cast.resize(y.len(), 0.0);
}
for (slot, &value) in cast.iter_mut().zip(y.iter()) {
*slot = value as f32;
}
cast.as_slice()
}
fn storage_cols<'a>(x: MatRef<'a, f64>, cast: &'a mut Self::ColCast) -> MatRef<'a, Self> {
if cast.nrows() != x.nrows() || cast.ncols() != x.ncols() {
*cast = Mat::zeros(x.nrows(), x.ncols());
}
for col in 0..x.ncols() {
for row in 0..x.nrows() {
cast[(row, col)] = x[(row, col)] as f32;
}
}
cast.as_ref()
}
fn pick_term<'a>(
_f64_term: &'a dyn crate::kernel::KernelTerm<f64>,
f32_term: &'a dyn crate::kernel::KernelTerm<f32>,
) -> &'a dyn crate::kernel::KernelTerm<Self> {
f32_term
}
fn hold_f64_factor(l: MatRef<'_, Self>) -> Self::F64Factor {
Mat::from_fn(l.nrows(), l.ncols(), |row, col| f64::from(l[(row, col)]))
}
fn f64_factor_from(l: Mat<f64>) -> Self::F64Factor {
l
}
fn f64_factor<'a>(_l: MatRef<'a, Self>, held: &'a Self::F64Factor) -> MatRef<'a, f64> {
held.as_ref()
}
fn f64_factor_mut<'a>(
_l: &'a mut Mat<Self>,
held: &'a mut Self::F64Factor,
) -> faer::MatMut<'a, f64> {
held.as_mut()
}
fn store_f64_factor(l: &mut Mat<Self>, held: &Self::F64Factor) {
for col in 0..l.ncols() {
let from = &held.col_as_slice(col)[col..];
for (slot, &value) in l.col_as_slice_mut(col)[col..].iter_mut().zip(from) {
*slot = value as f32;
}
}
}
fn f64_factor_trial(_l: MatRef<'_, Self>, held: &Self::F64Factor) -> Mat<f64> {
held.clone()
}
fn commit_f64_factor(l: &mut Mat<Self>, held: &mut Self::F64Factor, trial: Mat<f64>) {
*held = trial;
Self::store_f64_factor(l, held);
}
fn hold_f64<V: Clone + std::fmt::Debug + Send + Sync + 'static>(make: impl FnOnce() -> V) -> V {
make()
}
fn held_f64<V: Clone + std::fmt::Debug + Send + Sync + 'static>(held: V) -> Option<V> {
Some(held)
}
fn held_f64_ref<V: Clone + std::fmt::Debug + Send + Sync + 'static>(held: &V) -> Option<&V> {
Some(held)
}
#[inline(always)]
fn as_f64_ref(_m: MatRef<'_, Self>) -> Option<MatRef<'_, f64>> {
None
}
#[inline(always)]
fn as_f64_mut(_m: faer::MatMut<'_, Self>) -> Option<faer::MatMut<'_, f64>> {
None
}
#[inline(always)]
fn fast_exp(self) -> Self {
math::fast_exp_f32(self)
}
#[inline(always)]
fn fast_jet(self) -> ExpJet<Self> {
math::fast_jet_f32(self)
}
fn cholesky_lower(
a: &mut Mat<Self>,
_scratch: &mut dyn_stack::MemBuffer,
jitter: f64,
stage: crate::error::CholeskyStage,
) -> Result<(), crate::error::GprError> {
crate::linalg::cholesky_lower_f64_accum(a, jitter, stage)
}
fn solve_llt_in_place(
l: MatRef<'_, Self>,
rhs: faer::MatMut<'_, Self>,
_scratch: &mut dyn_stack::MemBuffer,
) {
crate::linalg::solve_llt_f64_accum(l, rhs);
}
fn solve_llt_owned_scratch(l: MatRef<'_, Self>, rhs: faer::MatMut<'_, Self>) {
crate::linalg::solve_llt_f64_accum(l, rhs);
}
fn inv_diag_from_chol_l(l: MatRef<'_, Self>, q_diag: &mut [Self]) {
crate::linalg::inv_diag_from_chol_l_f64_accum(l, q_diag);
}
fn matvec_columns(a: MatRef<'_, Self>, y: &[Self], ay: faer::MatMut<'_, Self>) {
crate::linalg::matvec_columns_f64_accum(a, y, ay);
}
fn gram_aat(a: MatRef<'_, Self>, b: faer::MatMut<'_, Self>) {
crate::linalg::gram_aat_f64_accum(a, b);
}
fn solve_unit_lower_rows(lt: &faer::Mat<Self>, w: &mut [Self]) {
crate::linalg::solve_unit_lower_rows_f32(lt, w);
}
fn solve_ldlt_in_place(ld: MatRef<'_, Self>, rhs: faer::MatMut<'_, Self>, n: usize) {
crate::linalg::solve_ldlt_f64_accum(ld, rhs, n);
}
fn ldlt_delete_row_col(
ld: faer::MatMut<'_, Self>,
index: usize,
n: usize,
_scratch: &mut dyn_stack::MemBuffer,
) {
crate::linalg::ldlt_delete_via_f64(ld, index, n);
}
}