use faer::{Mat, MatMut, MatRef};
use crate::error::GprError;
use crate::kernel::{KernelScalar, KernelSpec, RefinedSources, SourceStore, Supply, TrainSources};
use crate::transform::TargetTransform;
pub trait PrecisionPolicy {
type Storage: KernelScalar;
type Refine: KernelScalar;
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct DoublePrecision;
impl PrecisionPolicy for DoublePrecision {
type Storage = f64;
type Refine = f64;
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct SinglePrecision;
impl PrecisionPolicy for SinglePrecision {
type Storage = f32;
type Refine = f32;
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct MixedPrecision<R: ResidualFormula = PromoteStorage>(core::marker::PhantomData<R>);
impl<R: ResidualFormula> PrecisionPolicy for MixedPrecision<R> {
type Storage = f32;
type Refine = f64;
}
mod refine;
mod sparse;
pub(crate) use refine::{COLUMN_BLOCK, RefineSystem, refine};
pub use refine::{StoredFactor, TrainSystem};
pub use sparse::F64Vfe;
mod residual_seal {
pub trait Sealed {
const READS_STORAGE: bool;
}
}
pub trait ResidualFormula: residual_seal::Sealed + Copy + Send + Sync + 'static {}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct PromoteStorage;
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct ReevaluateKernel;
impl residual_seal::Sealed for PromoteStorage {
const READS_STORAGE: bool = true;
}
impl residual_seal::Sealed for ReevaluateKernel {
const READS_STORAGE: bool = false;
}
impl ResidualFormula for PromoteStorage {}
impl ResidualFormula for ReevaluateKernel {}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum PersistKind {
Double,
Single,
MixedPromote,
MixedReevaluate,
}
pub trait ModelPrecision: PrecisionPolicy + Copy + Send + Sync + 'static {
const REFINES_IN_F64: bool;
type Sources: SourceStore<Self::Storage>;
fn persist_kind() -> PersistKind;
fn publish_predict_alpha<M: crate::math::KernelMath, S: Supply>(
sys: &TrainSystem<'_, Self::Storage, S>,
alpha: &mut Vec<Self::Refine>,
) -> Result<(), GprError>;
#[allow(clippy::too_many_arguments)]
fn predict_means<M: crate::math::KernelMath, S: Supply>(
kernel: &KernelSpec<S>,
k_storage: MatRef<'_, Self::Storage>,
x_train: MatRef<'_, f64>,
x_query: &[f64],
n_cols: usize,
cross64: S::Rects<'_, f64>,
alpha: &[Self::Refine],
out: &mut [Self::Refine],
) -> Result<(), GprError>;
fn inverse_mean_variance(
transform: &dyn TargetTransform,
mean: &mut [Self::Refine],
variance: &mut [Self::Refine],
buffers: &mut InverseBuffers,
) -> Result<(), GprError>;
fn inverse_covariance(
transform: &dyn TargetTransform,
covariance: &mut [Self::Refine],
) -> Result<(), GprError>;
fn view_factor<'a>(
mapped_l: Option<MatRef<'a, f64>>,
workspace_l: MatRef<'a, Self::Storage>,
) -> MatRef<'a, Self::Storage>;
fn copy_mapped_l(src: MatRef<'_, f64>, dest: MatMut<'_, Self::Storage>);
fn mean_dot(column: &[Self::Storage], weights: &[Self::Refine]) -> Self::Refine;
fn publish_weights(
a: MatRef<'_, Self::Storage>,
b_l: MatRef<'_, Self::Storage>,
w: &[Self::Storage],
y: &[f64],
noise: f64,
reference: &dyn Fn() -> Result<F64Vfe, GprError>,
) -> Result<Vec<Self::Refine>, GprError>;
}
fn copy_to_refine<P: ModelPrecision>(values: &[P::Storage], out: &mut Vec<P::Refine>) {
if out.len() != values.len() {
out.resize(values.len(), P::Refine::from_f64(0.0));
}
for (slot, &value) in out.iter_mut().zip(values.iter()) {
*slot = P::Refine::from_f64(value.to_f64());
}
}
fn storage_means<P: ModelPrecision>(
k_storage: MatRef<'_, P::Storage>,
alpha: &[P::Refine],
out: &mut [P::Refine],
) {
for (col, slot) in out.iter_mut().enumerate() {
let mut sum = 0.0f64;
for (row, &weight) in alpha.iter().enumerate() {
sum += k_storage[(row, col)].to_f64() * weight.to_f64();
}
*slot = P::Refine::from_f64(sum);
}
}
#[derive(Debug, Default)]
pub struct InverseBuffers {
mean64: Vec<f64>,
variance64: Vec<f64>,
}
fn inverse_f64_mean_variance(
transform: &dyn TargetTransform,
mean: &mut [f64],
variance: &mut [f64],
) -> Result<(), GprError> {
transform.inverse_transform_mean(mean)?;
transform.inverse_transform_variance(variance)
}
impl ModelPrecision for DoublePrecision {
const REFINES_IN_F64: bool = false;
type Sources = TrainSources<f64>;
fn persist_kind() -> PersistKind {
PersistKind::Double
}
fn publish_predict_alpha<M: crate::math::KernelMath, S: Supply>(
sys: &TrainSystem<'_, Self::Storage, S>,
alpha: &mut Vec<Self::Refine>,
) -> Result<(), GprError> {
copy_to_refine::<Self>(sys.factor_alpha, alpha);
Ok(())
}
fn predict_means<M: crate::math::KernelMath, S: Supply>(
_kernel: &KernelSpec<S>,
k_storage: MatRef<'_, Self::Storage>,
_x_train: MatRef<'_, f64>,
_x_query: &[f64],
_n_cols: usize,
_cross64: S::Rects<'_, f64>,
alpha: &[Self::Refine],
out: &mut [Self::Refine],
) -> Result<(), GprError> {
storage_means::<Self>(k_storage, alpha, out);
Ok(())
}
fn inverse_mean_variance(
transform: &dyn TargetTransform,
mean: &mut [Self::Refine],
variance: &mut [Self::Refine],
_buffers: &mut InverseBuffers,
) -> Result<(), GprError> {
inverse_f64_mean_variance(transform, mean, variance)
}
fn inverse_covariance(
transform: &dyn TargetTransform,
covariance: &mut [Self::Refine],
) -> Result<(), GprError> {
transform.inverse_transform_covariance(covariance)
}
fn view_factor<'a>(
mapped_l: Option<MatRef<'a, f64>>,
workspace_l: MatRef<'a, Self::Storage>,
) -> MatRef<'a, Self::Storage> {
match mapped_l {
Some(mapped) => mapped,
None => workspace_l,
}
}
fn copy_mapped_l(src: MatRef<'_, f64>, mut dest: MatMut<'_, Self::Storage>) {
let n = src.nrows().min(dest.nrows());
let cols = src.ncols().min(dest.ncols());
for col in 0..cols {
for row in 0..n {
dest[(row, col)] = src[(row, col)];
}
}
}
fn mean_dot(column: &[Self::Storage], weights: &[Self::Refine]) -> Self::Refine {
sparse::storage_dot(column, weights)
}
fn publish_weights(
_a: MatRef<'_, Self::Storage>,
_b_l: MatRef<'_, Self::Storage>,
w: &[Self::Storage],
_y: &[f64],
_noise: f64,
_reference: &dyn Fn() -> Result<F64Vfe, GprError>,
) -> Result<Vec<Self::Refine>, GprError> {
let mut out = Vec::new();
copy_to_refine::<Self>(w, &mut out);
Ok(out)
}
}
impl ModelPrecision for SinglePrecision {
const REFINES_IN_F64: bool = false;
type Sources = TrainSources<f32>;
fn persist_kind() -> PersistKind {
PersistKind::Single
}
fn publish_predict_alpha<M: crate::math::KernelMath, S: Supply>(
sys: &TrainSystem<'_, Self::Storage, S>,
alpha: &mut Vec<Self::Refine>,
) -> Result<(), GprError> {
copy_to_refine::<Self>(sys.factor_alpha, alpha);
Ok(())
}
fn predict_means<M: crate::math::KernelMath, S: Supply>(
_kernel: &KernelSpec<S>,
k_storage: MatRef<'_, Self::Storage>,
_x_train: MatRef<'_, f64>,
_x_query: &[f64],
_n_cols: usize,
_cross64: S::Rects<'_, f64>,
alpha: &[Self::Refine],
out: &mut [Self::Refine],
) -> Result<(), GprError> {
storage_means::<Self>(k_storage, alpha, out);
Ok(())
}
fn inverse_mean_variance(
transform: &dyn TargetTransform,
mean: &mut [Self::Refine],
variance: &mut [Self::Refine],
buffers: &mut InverseBuffers,
) -> Result<(), GprError> {
let InverseBuffers { mean64, variance64 } = buffers;
mean64.clear();
mean64.extend(mean.iter().copied().map(f32::to_f64));
variance64.clear();
variance64.extend(variance.iter().copied().map(f32::to_f64));
inverse_f64_mean_variance(transform, mean64, variance64)?;
for (slot, value) in mean.iter_mut().zip(mean64.iter()) {
*slot = f32::from_f64(*value);
}
for (slot, value) in variance.iter_mut().zip(variance64.iter()) {
*slot = f32::from_f64(*value);
}
Ok(())
}
fn inverse_covariance(
transform: &dyn TargetTransform,
covariance: &mut [Self::Refine],
) -> Result<(), GprError> {
let mut buf: Vec<f64> = covariance.iter().copied().map(f32::to_f64).collect();
transform.inverse_transform_covariance(&mut buf)?;
for (slot, value) in covariance.iter_mut().zip(buf) {
*slot = f32::from_f64(value);
}
Ok(())
}
fn view_factor<'a>(
_mapped_l: Option<MatRef<'a, f64>>,
workspace_l: MatRef<'a, Self::Storage>,
) -> MatRef<'a, Self::Storage> {
workspace_l
}
fn copy_mapped_l(_src: MatRef<'_, f64>, _dest: MatMut<'_, Self::Storage>) {}
fn mean_dot(column: &[Self::Storage], weights: &[Self::Refine]) -> Self::Refine {
let mut sum = 0.0f64;
for (kernel, weight) in column.iter().zip(weights.iter()) {
sum += kernel.to_f64() * weight.to_f64();
}
Self::Refine::from_f64(sum)
}
fn publish_weights(
_a: MatRef<'_, Self::Storage>,
_b_l: MatRef<'_, Self::Storage>,
w: &[Self::Storage],
_y: &[f64],
_noise: f64,
_reference: &dyn Fn() -> Result<F64Vfe, GprError>,
) -> Result<Vec<Self::Refine>, GprError> {
let mut out = Vec::new();
copy_to_refine::<Self>(w, &mut out);
Ok(out)
}
}
impl<R: ResidualFormula> ModelPrecision for MixedPrecision<R> {
const REFINES_IN_F64: bool = true;
type Sources = RefinedSources;
fn persist_kind() -> PersistKind {
if R::READS_STORAGE {
PersistKind::MixedPromote
} else {
PersistKind::MixedReevaluate
}
}
fn publish_predict_alpha<M: crate::math::KernelMath, S: Supply>(
sys: &TrainSystem<'_, Self::Storage, S>,
alpha: &mut Vec<Self::Refine>,
) -> Result<(), GprError> {
*alpha = refine::refine_alpha::<M, R, _>(sys)?;
Ok(())
}
fn predict_means<M: crate::math::KernelMath, S: Supply>(
kernel: &KernelSpec<S>,
_k_storage: MatRef<'_, Self::Storage>,
x_train: MatRef<'_, f64>,
x_query: &[f64],
n_cols: usize,
cross64: S::Rects<'_, f64>,
alpha: &[Self::Refine],
out: &mut [Self::Refine],
) -> Result<(), GprError> {
f64_cross_means::<M, _>(kernel, x_train, x_query, n_cols, cross64, alpha, out)
}
fn inverse_mean_variance(
transform: &dyn TargetTransform,
mean: &mut [Self::Refine],
variance: &mut [Self::Refine],
_buffers: &mut InverseBuffers,
) -> Result<(), GprError> {
inverse_f64_mean_variance(transform, mean, variance)
}
fn inverse_covariance(
transform: &dyn TargetTransform,
covariance: &mut [Self::Refine],
) -> Result<(), GprError> {
transform.inverse_transform_covariance(covariance)
}
fn view_factor<'a>(
_mapped_l: Option<MatRef<'a, f64>>,
workspace_l: MatRef<'a, Self::Storage>,
) -> MatRef<'a, Self::Storage> {
workspace_l
}
fn copy_mapped_l(_src: MatRef<'_, f64>, _dest: MatMut<'_, Self::Storage>) {}
fn mean_dot(column: &[Self::Storage], weights: &[Self::Refine]) -> Self::Refine {
sparse::promoted_dot(column, weights)
}
fn publish_weights(
a: MatRef<'_, Self::Storage>,
b_l: MatRef<'_, Self::Storage>,
w: &[Self::Storage],
y: &[f64],
noise: f64,
reference: &dyn Fn() -> Result<F64Vfe, GprError>,
) -> Result<Vec<Self::Refine>, GprError> {
sparse::refine_vfe_weights::<R>(a, b_l, w, y, noise, reference)
}
}
fn f64_cross_means<M: crate::math::KernelMath, S: Supply>(
kernel: &KernelSpec<S>,
x_train: MatRef<'_, f64>,
x_query: &[f64],
n_cols: usize,
cross64: S::Rects<'_, f64>,
alpha: &[f64],
out: &mut [f64],
) -> Result<(), GprError> {
let kernel_f64 = kernel.compile();
let n = x_train.nrows();
let m = out.len();
let block = COLUMN_BLOCK.min(m.max(1));
let mut rows = Mat::<f64>::zeros(block, n_cols);
let mut k_block = Mat::<f64>::zeros(n, block);
let mut scratch = Mat::<f64>::zeros(n, block);
let mut dist = Mat::<f64>::zeros(n, block);
let mut nested = Vec::new();
let mut start = 0;
while start < m {
let len = block.min(m - start);
for dim in 0..n_cols {
for j in 0..len {
rows[(j, dim)] = x_query[dim * m + start + j];
}
}
S::with_cols(cross64, start, len, |cols| {
kernel_f64.eval_cross_slots::<M>(
x_train,
rows.as_ref().submatrix(0, 0, len, n_cols),
cols,
Some(dist.as_mut().submatrix_mut(0, 0, n, len)),
k_block.as_mut().submatrix_mut(0, 0, n, len),
scratch.as_mut().submatrix_mut(0, 0, n, len),
&mut nested,
&mut [],
)
})?;
for j in 0..len {
let mut sum = 0.0;
for i in 0..n {
sum += k_block[(i, j)] * alpha[i];
}
out[start + j] = sum;
}
start += len;
}
Ok(())
}
pub trait GpScalar: ModelPrecision {}
impl GpScalar for DoublePrecision {}
impl GpScalar for SinglePrecision {}
impl<R: ResidualFormula> GpScalar for MixedPrecision<R> {}
#[cfg(test)]
mod tests;