use std::path::Path;
use faer::Mat;
use crate::error::GprError;
use crate::error::PersistErrorKind;
use crate::kernel::{BlockAt, DistanceSlot, ModelKernel, SlotShape, SupplyViews};
use crate::optimizer::Fixed;
use crate::points::{IdRegistry, PointRegistry};
use crate::precision::GpScalar;
use crate::sgpr::{FittedSgpr, FixedInducing, InducingRegistry, OnlineSgpr};
use crate::sparse::{PersistedSparse, SparseCore, SparseSpec, SparseSupplied, SparseSupply};
use crate::svgp::FittedSvgp;
use crate::{PredictOptions, Prediction};
use super::config::{
DistanceJson, JitterJson, LikelihoodJson, MathJson, ModelJson, PointsJson, PrecisionJson,
ResidualJson, SparseConfig, SparseDistanceJson, d2_tensor, parse_sparse_config,
};
use super::kernel::KernelJson;
use super::tensors::{TensorFile, f64_tensor, read_f64, write_f64_tensors};
use super::transform::{
encode_fitted_input, encode_fitted_target, encode_unfitted_input, encode_unfitted_target,
};
use super::{FORMAT_VERSION, PersistRegistry, decode_kernel, persist_err, widen};
use safetensors::SafeTensors;
const TENSOR_X: &str = "x";
const TENSOR_Y: &str = "y";
const TENSOR_Z: &str = "z";
const TENSOR_Z_TRAIN: &str = "z_train";
const TENSOR_Q_MEAN: &str = "q_mean";
const TENSOR_Q_L: &str = "q_l";
struct OnlineIds {
points: Vec<u64>,
next_point: u64,
inducing: Vec<u64>,
next_inducing: u64,
}
fn supply_tensors(
supply: &SparseSupply,
slots: &[DistanceSlot],
(n, m): (usize, usize),
) -> Vec<(String, Vec<usize>, Vec<f64>)> {
let xz = &supply.exact().xz;
slots
.iter()
.enumerate()
.map(|(k, slot)| {
let blocks = slot.shape().blocks();
let mut values = Vec::with_capacity(n.saturating_mul(m).saturating_mul(blocks));
let at = crate::kernel::place_in_shape(slots, k);
let shape = match slot.shape() {
SlotShape::Scalar => {
xz.block_into(BlockAt::Scalar(at), &mut values);
vec![n, m]
}
SlotShape::Ard(dims) => {
for dim in 0..dims {
xz.block_into(BlockAt::Ard(at, dim), &mut values);
}
vec![dims, n, m]
}
};
(d2_tensor(k), shape, values)
})
.collect()
}
fn write_sparse<P: GpScalar, K: ModelKernel>(
dir: &Path,
core: &SparseCore<K::Supply>,
model: ModelJson,
ids: Option<OnlineIds>,
q: Option<(&[f64], faer::MatRef<'_, f64>)>,
) -> Result<(), GprError> {
std::fs::create_dir_all(dir)
.map_err(|err| persist_err(PersistErrorKind::Io, format!("create {dir:?}: {err}")))?;
let kind = P::persist_kind();
let (point_ids, next_point_id, inducing_ids, next_inducing_id) = match ids {
Some(ids) => (
Some(ids.points),
Some(ids.next_point),
Some(ids.inducing),
Some(ids.next_inducing),
),
None => (None, None, None, None),
};
let slots = core.slots();
let config = SparseConfig {
format_version: FORMAT_VERSION,
model,
n: core.n,
m: core.m,
d: core.d,
precision: PrecisionJson::from_persist(kind),
residual: ResidualJson::from_persist(kind),
math: MathJson::encode(core.math),
kernel: KernelJson::encode(&core.kernel, slots)?,
likelihood: LikelihoodJson::encode(&core.likelihood),
jitter: JitterJson::encode(core.jitter),
x_unfitted: encode_unfitted_input(core.x_unfitted.as_ref())?,
y_unfitted: encode_unfitted_target(core.y_unfitted.as_ref())?,
x_transform: encode_fitted_input(core.x_transform.as_ref())?,
y_transform: encode_fitted_target(core.y_transform.as_ref())?,
point_ids,
next_point_id,
inducing_ids,
next_inducing_id,
distance: core.supply().map(|supply| SparseDistanceJson {
table: DistanceJson::encode(PointsJson::of::<K>(), slots),
inducing: supply.inducing.clone(),
}),
};
let json = serde_json::to_vec_pretty(&config).map_err(|err| {
persist_err(
PersistErrorKind::Config,
format!("serialize config.json: {err}"),
)
})?;
let (n, m, d) = (core.n, core.m, core.d);
let mut tensors: Vec<(&str, Vec<usize>, &[f64])> = vec![
(TENSOR_X, vec![n, d], &core.x_obs),
(TENSOR_Y, vec![n], &core.y_obs),
(TENSOR_Z, vec![m, d], &core.z_obs),
(TENSOR_Z_TRAIN, vec![m, d], &core.z_train),
];
let q_l_values: Vec<f64>;
if let Some((q_mean, q_l)) = q {
q_l_values = (0..m)
.flat_map(|col| (0..m).map(move |row| (row, col)))
.map(|(row, col)| if row < col { 0.0 } else { q_l[(row, col)] })
.collect();
tensors.push((TENSOR_Q_MEAN, vec![m], q_mean));
tensors.push((TENSOR_Q_L, vec![m, m], &q_l_values));
}
let d2 = core
.supply()
.map_or_else(Vec::new, |supply| supply_tensors(supply, slots, (n, m)));
for (name, shape, values) in &d2 {
tensors.push((name.as_str(), shape.clone(), values));
}
write_f64_tensors(dir, &tensors, Vec::new())?;
super::write_config(dir, &json)
}
pub(crate) fn save_sgpr<O, I, P, K>(
model: &FittedSgpr<O, I, P, K>,
dir: &Path,
) -> Result<(), GprError>
where
I: crate::sgpr::InducingLayout<K::Supply>,
P: GpScalar,
K: ModelKernel,
{
write_sparse::<P, K>(dir, model.core(), ModelJson::Sgpr, None, None)
}
pub(crate) fn save_online_sgpr<O, P: GpScalar, K: ModelKernel>(
model: &OnlineSgpr<O, P, K>,
dir: &Path,
) -> Result<(), GprError> {
let ids = OnlineIds {
points: model.point_registry().raw_ids(),
next_point: model.point_registry().next_id(),
inducing: model.inducing_registry().raw_ids(),
next_inducing: model.inducing_registry().next_id(),
};
write_sparse::<P, K>(dir, model.core(), ModelJson::OnlineSgpr, Some(ids), None)
}
pub(crate) fn save_svgp<P: GpScalar, K: ModelKernel>(
model: &FittedSvgp<P, K>,
dir: &Path,
) -> Result<(), GprError> {
write_sparse::<P, K>(dir, model.core(), ModelJson::Svgp, None, Some(model.q()))
}
pub(super) fn read_config(
dir: &Path,
expected: &[ModelJson],
points: Option<PointsJson>,
) -> Result<SparseConfig, GprError> {
let bytes = super::read_model_bytes(dir, expected, points)?;
parse_sparse_config(&bytes)
}
fn read_core<S: crate::kernel::KernelScalar, K: ModelKernel>(
tensors: &SafeTensors<'_>,
config: &SparseConfig,
registry: &PersistRegistry,
) -> Result<SparseCore<K::Supply>, GprError> {
let (n, m, d) = (config.n, config.m, config.d);
let table = config.distance.as_ref().map(|distance| &distance.table);
let (kernel, slots) = decode_kernel::<K>(&config.kernel, table, registry)?;
let spec = SparseSpec {
kernel,
likelihood: config.likelihood.decode()?,
math: config.math.decode(),
jitter: config.jitter.decode()?,
x_transform: config.x_unfitted.clone().decode(registry)?,
y_transform: config.y_unfitted.clone().decode(registry)?,
};
let supplied = <K::Supply as SupplyViews>::try_hold(|| {
let distance = config.distance.as_ref().ok_or_else(|| {
persist_err(
PersistErrorKind::Config,
"the config of a model on supplied distances has no `distance`",
)
})?;
let supply = read_supply::<S>(tensors, config, &slots, &distance.inducing)?;
Ok::<_, GprError>(SparseSupplied { supply, slots })
})?;
let core = SparseCore::from_persisted(PersistedSparse {
spec,
x_transform: config.x_transform.clone().decode(registry)?,
y_transform: config.y_transform.clone().decode(registry)?,
x_obs: read_f64(tensors, TENSOR_X, &[n, d])?,
y_obs: read_f64(tensors, TENSOR_Y, &[n])?,
z_obs: read_f64(tensors, TENSOR_Z, &[m, d])?,
z_train: read_f64(tensors, TENSOR_Z_TRAIN, &[m, d])?,
n,
m,
d,
supplied,
})?;
if let Some(supply) = core.supply() {
let inducing = &supply.inducing;
same_rows(&core.z_obs, &core.x_obs, (n, d), inducing, TENSOR_Z)?;
same_rows(
&core.z_train,
&core.x_train,
(n, d),
inducing,
TENSOR_Z_TRAIN,
)?;
}
Ok(core)
}
fn same_rows(
z: &[f64],
x: &[f64],
(n, d): (usize, usize),
inducing: &[usize],
name: &str,
) -> Result<(), GprError> {
let m = inducing.len();
for j in 0..d {
for (a, &i) in inducing.iter().enumerate() {
if z[j * m + a].to_bits() != x[j * n + i].to_bits() {
return Err(persist_err(
PersistErrorKind::Config,
format!(
"{name} row {a} is not training point {i}, the inducing point the blocks name"
),
));
}
}
}
Ok(())
}
fn open_tensors(dir: &Path, config: &SparseConfig) -> Result<TensorFile, GprError> {
if config.distance.is_some() {
TensorFile::map(dir)
} else {
TensorFile::read(dir)
}
}
enum SavedBlocks<'a> {
Scalar(crate::kernel::ScalarDistance, &'a [f64]),
Ard(crate::kernel::ArdDistance, Vec<&'a [f64]>),
}
fn read_supply<S: crate::kernel::KernelScalar>(
tensors: &SafeTensors<'_>,
config: &SparseConfig,
slots: &[DistanceSlot],
inducing: &[usize],
) -> Result<SparseSupply, GprError> {
let (n, m) = (config.n, config.m);
if inducing.len() != m {
return Err(persist_err(
PersistErrorKind::Config,
format!(
"config has {} inducing indices, expected m = {m}",
inducing.len()
),
));
}
let len = n.checked_mul(m).ok_or(GprError::SizeOverflow)?;
let tables = slots
.iter()
.enumerate()
.map(|(k, slot)| {
let name = d2_tensor(k);
Ok(match slot {
DistanceSlot::Ard(ard) => SavedBlocks::Ard(
*ard,
f64_tensor(tensors, &name, &[ard.dims(), n, m])?
.chunks_exact(len.max(1))
.collect(),
),
DistanceSlot::Scalar(scalar) => {
SavedBlocks::Scalar(*scalar, f64_tensor(tensors, &name, &[n, m])?)
}
})
})
.collect::<Result<Vec<_>, GprError>>()?;
let sources = tables.iter().map(|table| match table {
SavedBlocks::Ard(ard, blocks) => ard.borrow(blocks),
SavedBlocks::Scalar(scalar, block) => scalar.borrow(block),
});
SparseSupply::bind::<S>(slots, sources, n, inducing)
}
fn read_ids(config: &SparseConfig) -> Result<(PointRegistry, InducingRegistry), GprError> {
let missing = |key: &str| {
persist_err(
PersistErrorKind::Config,
format!("online_sgpr config missing {key}"),
)
};
let points = IdRegistry::from_persisted(
config
.point_ids
.as_deref()
.ok_or_else(|| missing("point_ids"))?,
config
.next_point_id
.ok_or_else(|| missing("next_point_id"))?,
)?;
let inducing = IdRegistry::from_persisted(
config
.inducing_ids
.as_deref()
.ok_or_else(|| missing("inducing_ids"))?,
config
.next_inducing_id
.ok_or_else(|| missing("next_inducing_id"))?,
)?;
Ok((points, inducing))
}
fn read_q(tensors: &SafeTensors<'_>, m: usize) -> Result<(Vec<f64>, Mat<f64>), GprError> {
let q_mean = read_f64(tensors, TENSOR_Q_MEAN, &[m])?;
let values = read_f64(tensors, TENSOR_Q_L, &[m, m])?;
let q_l = Mat::from_fn(m, m, |row, col| values[col * m + row]);
let finite = q_mean.iter().chain(&values).all(|value| value.is_finite());
let lower = (0..m).all(|col| (0..col).all(|row| q_l[(row, col)] == 0.0));
let positive = (0..m).all(|i| q_l[(i, i)] > 0.0);
if !(finite && lower && positive) {
return Err(persist_err(
PersistErrorKind::Tensor,
"q_l must be lower triangular with a positive diagonal, and q finite",
));
}
Ok((q_mean, q_l))
}
#[derive(Clone, Debug)]
#[non_exhaustive]
pub enum LoadedSgpr {
Double(FittedSgpr<Fixed>),
Single(FittedSgpr<Fixed, FixedInducing, crate::SinglePrecision>),
Mixed(FittedSgpr<Fixed, FixedInducing, crate::MixedPrecision>),
Reevaluate(FittedSgpr<Fixed, FixedInducing, crate::MixedPrecision<crate::ReevaluateKernel>>),
OnlineDouble(OnlineSgpr<Fixed>),
OnlineSingle(OnlineSgpr<Fixed, crate::SinglePrecision>),
OnlineMixed(OnlineSgpr<Fixed, crate::MixedPrecision>),
OnlineReevaluate(OnlineSgpr<Fixed, crate::MixedPrecision<crate::ReevaluateKernel>>),
}
pub(super) struct SgprVariants<P: GpScalar, K: ModelKernel, L> {
pub(super) fitted: fn(FittedSgpr<Fixed, FixedInducing, P, K>) -> L,
pub(super) online: fn(OnlineSgpr<Fixed, P, K>) -> L,
}
pub(super) fn load_sgpr_as<P: GpScalar, K: ModelKernel, L>(
dir: &Path,
config: &SparseConfig,
registry: &PersistRegistry,
variants: SgprVariants<P, K, L>,
) -> Result<L, GprError> {
let fitted =
FittedSgpr::<Fixed, FixedInducing, P, K>::from_persisted(read_core::<P::Storage, K>(
&open_tensors(dir, config)?.tensors()?,
config,
registry,
)?)?;
if config.model == ModelJson::OnlineSgpr {
let (points, inducing) = read_ids(config)?;
Ok((variants.online)(OnlineSgpr::from_persisted(
fitted, points, inducing,
)?))
} else {
Ok((variants.fitted)(fitted))
}
}
impl LoadedSgpr {
pub fn load(dir: impl AsRef<Path>, registry: &PersistRegistry) -> Result<Self, GprError> {
let dir = dir.as_ref();
let config = read_config(dir, &[ModelJson::Sgpr, ModelJson::OnlineSgpr], None)?;
by_precision!(config.persist_kind(), Self, |fitted, online| {
load_sgpr_as(dir, &config, registry, SgprVariants { fitted, online })
})
}
pub fn n(&self) -> usize {
self.view().core().n
}
pub fn m(&self) -> usize {
self.view().core().m
}
pub fn d(&self) -> usize {
self.view().core().d
}
pub fn is_online(&self) -> bool {
matches!(
self,
Self::OnlineDouble(_)
| Self::OnlineSingle(_)
| Self::OnlineMixed(_)
| Self::OnlineReevaluate(_)
)
}
pub fn predict(
&self,
xs: &[f64],
n_rows: usize,
n_cols: usize,
) -> Result<Prediction<f64>, GprError> {
self.predict_with(xs, n_rows, n_cols, PredictOptions::default())
}
pub fn predict_with(
&self,
xs: &[f64],
n_rows: usize,
n_cols: usize,
options: PredictOptions,
) -> Result<Prediction<f64>, GprError> {
self.view().predict_f64(xs, n_rows, n_cols, options)
}
fn view(&self) -> &dyn SparseView {
match self {
Self::Double(model) => model,
Self::Single(model) => model,
Self::Mixed(model) => model,
Self::Reevaluate(model) => model,
Self::OnlineDouble(model) => model,
Self::OnlineSingle(model) => model,
Self::OnlineMixed(model) => model,
Self::OnlineReevaluate(model) => model,
}
}
}
#[derive(Clone, Debug)]
#[non_exhaustive]
pub enum LoadedSvgp {
Double(FittedSvgp),
Single(FittedSvgp<crate::SinglePrecision>),
Mixed(FittedSvgp<crate::MixedPrecision>),
Reevaluate(FittedSvgp<crate::MixedPrecision<crate::ReevaluateKernel>>),
}
pub(super) fn load_svgp_as<P: GpScalar, K: ModelKernel, L>(
dir: &Path,
config: &SparseConfig,
registry: &PersistRegistry,
variant: fn(FittedSvgp<P, K>) -> L,
) -> Result<L, GprError> {
let file = open_tensors(dir, config)?;
let tensors = file.tensors()?;
let core = read_core::<P::Storage, K>(&tensors, config, registry)?;
let (q_mean, q_l) = read_q(&tensors, config.m)?;
Ok(variant(FittedSvgp::from_persisted(core, q_mean, q_l)?))
}
impl LoadedSvgp {
pub fn load(dir: impl AsRef<Path>, registry: &PersistRegistry) -> Result<Self, GprError> {
let dir = dir.as_ref();
let config = read_config(dir, &[ModelJson::Svgp], None)?;
by_precision!(config.persist_kind(), Self, |fitted| {
load_svgp_as(dir, &config, registry, fitted)
})
}
pub fn n(&self) -> usize {
self.view().core().n
}
pub fn m(&self) -> usize {
self.view().core().m
}
pub fn d(&self) -> usize {
self.view().core().d
}
pub fn predict(
&self,
xs: &[f64],
n_rows: usize,
n_cols: usize,
) -> Result<Prediction<f64>, GprError> {
self.predict_with(xs, n_rows, n_cols, PredictOptions::default())
}
pub fn predict_with(
&self,
xs: &[f64],
n_rows: usize,
n_cols: usize,
options: PredictOptions,
) -> Result<Prediction<f64>, GprError> {
self.view().predict_f64(xs, n_rows, n_cols, options)
}
fn view(&self) -> &dyn SparseView {
match self {
Self::Double(model) => model,
Self::Single(model) => model,
Self::Mixed(model) => model,
Self::Reevaluate(model) => model,
}
}
}
trait SparseView {
fn core(&self) -> &SparseCore;
fn predict_f64(
&self,
xs: &[f64],
n_rows: usize,
n_cols: usize,
options: PredictOptions,
) -> Result<Prediction<f64>, GprError>;
}
impl<P: GpScalar> SparseView for FittedSgpr<Fixed, FixedInducing, P> {
fn core(&self) -> &SparseCore {
FittedSgpr::core(self)
}
fn predict_f64(
&self,
xs: &[f64],
n_rows: usize,
n_cols: usize,
options: PredictOptions,
) -> Result<Prediction<f64>, GprError> {
self.predict_with(xs, n_rows, n_cols, options).map(widen)
}
}
impl<P: GpScalar> SparseView for OnlineSgpr<Fixed, P> {
fn core(&self) -> &SparseCore {
OnlineSgpr::core(self)
}
fn predict_f64(
&self,
xs: &[f64],
n_rows: usize,
n_cols: usize,
options: PredictOptions,
) -> Result<Prediction<f64>, GprError> {
self.predict_with(xs, n_rows, n_cols, options).map(widen)
}
}
impl<P: GpScalar> SparseView for FittedSvgp<P> {
fn core(&self) -> &SparseCore {
FittedSvgp::core(self)
}
fn predict_f64(
&self,
xs: &[f64],
n_rows: usize,
n_cols: usize,
options: PredictOptions,
) -> Result<Prediction<f64>, GprError> {
self.predict_with(xs, n_rows, n_cols, options).map(widen)
}
}