#[allow(
unused_imports,
reason = "a split file takes its parent's imports whole; each uses some"
)]
use super::*;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum BlockKind {
Square,
Rect,
}
pub(crate) fn unlocated() -> GprError {
GprError::InvalidDistance {
slot: None,
dim: None,
pair: None,
reason: "the table failed its check, but no value could be located".to_owned(),
}
}
pub(super) fn invalid(row: usize, col: usize, reason: impl Into<String>) -> GprError {
GprError::InvalidDistance {
slot: None,
dim: None,
pair: Some((row, col)),
reason: reason.into(),
}
}
#[inline]
#[allow(clippy::manual_range_contains)]
pub(crate) fn valid(v: f64) -> bool {
(v >= 0.0) & (v <= f64::MAX)
}
#[inline]
pub(crate) fn same(a: f64, b: f64) -> bool {
a - b == 0.0
}
pub(super) fn first_invalid(block: &[f64], rows: usize) -> GprError {
first_invalid_from(block, rows, 0)
}
pub(crate) fn first_invalid_from(block: &[f64], rows: usize, col0: usize) -> GprError {
let rows = rows.max(1);
let at = block.iter().position(|&v| !valid(v)).unwrap_or(0);
let v = block.get(at).copied().unwrap_or(0.0);
invalid_value(v, at % rows, col0 + at / rows)
}
pub(super) fn out_of_range(row: usize, col: usize) -> GprError {
invalid(row, col, "is past the range of the model's storage scalar")
}
pub(super) fn negative_past(v: f64, tol: f64, row: usize, col: usize) -> GprError {
invalid(
row,
col,
format!("{v} is negative past the tolerance {tol}"),
)
}
pub(super) fn nonzero_diagonal(v: f64, j: usize) -> GprError {
invalid(j, j, format!("the diagonal is {v}, not zero"))
}
pub(super) fn check_lower_run<T: KernelScalar>(run: &[T], col: usize) -> Result<(), GprError> {
if let Some(diag) = run.first().map(|v| v.to_f64())
&& valid(diag)
&& diag != 0.0
{
return Err(nonzero_diagonal(diag, col));
}
if !run.iter().fold(true, |ok, v| ok & valid(v.to_f64())) {
let at = run.iter().position(|v| !valid(v.to_f64())).unwrap_or(0);
return Err(invalid_value(run[at].to_f64(), col + at, col));
}
Ok(())
}
pub(crate) fn invalid_value(v: f64, row: usize, col: usize) -> GprError {
let reason = if v.is_finite() {
format!("{v} is negative")
} else {
format!("{v} is not finite")
};
invalid(row, col, reason)
}
pub(super) fn check_block(
block: &[f64],
rows: usize,
cols: usize,
kind: BlockKind,
tidy: Tidy,
) -> Result<bool, GprError> {
match tidy {
Tidy::Exact => exact_block(block, rows, cols, kind).map(|()| false),
Tidy::Within(rel) => within_block(block, rows, cols, kind, rel),
}
}
pub(super) fn exact_block(
block: &[f64],
rows: usize,
cols: usize,
kind: BlockKind,
) -> Result<(), GprError> {
if kind == BlockKind::Rect {
return if crate::kernel::simd::all_valid_distances(block) {
Ok(())
} else {
Err(first_invalid(block, rows))
};
}
let bands = rows.div_ceil(BAND);
let band = |band: usize| {
let j0 = band * BAND;
crate::kernel::simd::square_band(block, rows, (j0, (j0 + BAND).min(rows)))
};
let ok = if rows < PAR_ROWS || rayon::current_num_threads() == 1 {
(0..bands).all(band)
} else {
(0..bands).into_par_iter().all(band)
};
if ok {
return Ok(());
}
if !crate::kernel::simd::all_valid_distances(block) {
return Err(first_invalid(block, rows));
}
for j in 0..cols {
let diag = block[j + j * rows];
if diag != 0.0 {
return Err(nonzero_diagonal(diag, j));
}
}
let mut found = Ok(());
let _ = for_each_lower_pair(rows, |i, j| {
let (a, b) = (block[i + j * rows], block[j + i * rows]);
if !same(a, b) {
found = Err(invalid(
i,
j,
format!("{a} differs from its mirror ({j}, {i}), {b}"),
));
return Err(());
}
Ok(())
});
found?;
Err(unlocated())
}
pub(super) const PAR_ROWS: usize = 256;
pub(super) const BAND: usize = crate::kernel::simd::SQUARE_BAND;
pub(super) fn pack_exact_ard<'b, T: KernelScalar>(
n: usize,
d: usize,
block: impl Fn(usize) -> &'b [f64] + Sync,
) -> Result<ArdSqDiffBuf<T>, GprError> {
let per_dim = packed_len(n)?;
let len = per_dim.checked_mul(d).ok_or(GprError::SizeOverflow)?;
let (data, ok) = if rayon::current_num_threads() > 1 {
let mut data = vec![T::from_f64(0.0); len];
let ok = data
.par_chunks_mut(per_dim.max(1))
.enumerate()
.all(|(k, dest)| pack_bands(block(k), n, 0..n.div_ceil(BAND), dest));
(data, ok)
} else {
let mut data = Vec::with_capacity(len);
let ok = (0..d).fold(true, |ok, k| {
ok & crate::kernel::simd::pack_columns(block(k), n, (0, n), SquareOut::Push(&mut data))
});
(data, ok)
};
if !ok {
for k in 0..d {
exact_block(block(k), n, n, BlockKind::Square).map_err(|err| err.in_dim(k))?;
}
return Err(unlocated());
}
Ok(ArdSqDiffBuf::from_packed(data, n, d))
}
pub(super) fn keep_exact_ard<T: KernelScalar>(
tables: Vec<Vec<f64>>,
n: usize,
d: usize,
) -> Result<ArdSqDiffBuf<T>, GprError> {
if !reads_in_place::<T>() {
return pack_exact_ard(n, d, |k| &tables[k]);
}
for (k, table) in tables.iter().enumerate() {
exact_block(table, n, n, BlockKind::Square).map_err(|err| err.in_dim(k))?;
}
match T::vecs_from_f64(tables) {
Ok(tables) => Ok(ArdSqDiffBuf::from_tables(tables, n)),
Err(tables) => pack_exact_ard(n, d, |k| &tables[k]),
}
}
pub(super) fn pack_bands<T: KernelScalar>(
block: &[f64],
n: usize,
bands: std::ops::Range<usize>,
dest: &mut [T],
) -> bool {
let (j0, j1) = (bands.start * BAND, (bands.end * BAND).min(n));
if bands.len() <= 1 {
return j0 >= j1
|| crate::kernel::simd::pack_columns(block, n, (j0, j1), SquareOut::Over(dest));
}
let mid = bands.start + bands.len() / 2;
let split = (j0..mid * BAND).map(|j| n - j).sum();
let (head, tail) = dest.split_at_mut(split);
let (a, b) = rayon::join(
|| pack_bands(block, n, bands.start..mid, head),
|| pack_bands(block, n, mid..bands.end, tail),
);
a & b
}
pub(super) fn within_block(
block: &[f64],
rows: usize,
cols: usize,
kind: BlockKind,
rel: f64,
) -> Result<bool, GprError> {
if !block.iter().fold(true, |ok, &v| ok & v.is_finite()) {
let at = block.iter().position(|v| !v.is_finite()).unwrap_or(0);
let rows = rows.max(1);
return Err(invalid_value(block[at], at % rows, at / rows));
}
let tol = rel * block.iter().fold(0.0f64, |acc, v| acc.max(v.abs()));
let mut repair = false;
for (at, &v) in block.iter().enumerate() {
if v < -tol {
let rows = rows.max(1);
return Err(negative_past(v, tol, at % rows, at / rows));
}
repair |= v < 0.0;
}
if kind == BlockKind::Square {
for j in 0..cols {
let diag = block[j + j * rows];
if diag.abs() > tol {
return Err(invalid(
j,
j,
format!("the diagonal is {diag}, past the tolerance {tol}"),
));
}
repair |= diag != 0.0;
}
for_each_lower_pair(rows, |i, j| {
let (a, b) = (block[i + j * rows], block[j + i * rows]);
let gap = (a - b).abs();
if gap > tol {
return Err(invalid(
i,
j,
format!(
"{a} differs from its mirror ({j}, {i}), {b}, past the tolerance {tol}"
),
));
}
repair |= gap > 0.0;
Ok(())
})?;
}
Ok(repair)
}
pub(super) const PAIR_TILE: usize = 64;
pub(super) fn for_each_lower_pair<E>(
n: usize,
mut visit: impl FnMut(usize, usize) -> Result<(), E>,
) -> Result<(), E> {
for j0 in (0..n).step_by(PAIR_TILE) {
let j1 = (j0 + PAIR_TILE).min(n);
for i0 in (j0..n).step_by(PAIR_TILE) {
let i1 = (i0 + PAIR_TILE).min(n);
for j in j0..j1 {
for i in i0.max(j + 1)..i1 {
visit(i, j)?;
}
}
}
}
Ok(())
}
pub(super) fn repair_block(block: &mut [f64], rows: usize, cols: usize, kind: BlockKind) {
for v in block.iter_mut() {
*v = v.max(0.0);
}
if kind == BlockKind::Square {
for j in 0..cols {
block[j + j * rows] = 0.0;
}
let walked = for_each_lower_pair::<std::convert::Infallible>(rows, |i, j| {
let (a, b) = (block[i + j * rows], block[j + i * rows]);
let diff = b - a;
if diff != 0.0 {
let mean = a + 0.5 * diff;
block[i + j * rows] = mean;
block[j + i * rows] = mean;
}
Ok(())
});
if let Err(never) = walked {
match never {}
}
}
}
#[derive(Default)]
pub(super) struct FillRounding {
max: f64,
negative: Option<(f64, usize, usize)>,
diagonal: Option<(f64, usize, usize)>,
}
impl FillRounding {
pub(super) fn note(&mut self, v: f64, row: usize, col: usize) -> f64 {
self.max = self.max.max(v.abs());
if row == col {
if self.diagonal.is_none_or(|(worst, _, _)| v.abs() > worst) && v != 0.0 {
self.diagonal = Some((v.abs(), row, col));
}
return 0.0;
}
if v < 0.0 {
if self.negative.is_none_or(|(worst, _, _)| -v > worst) {
self.negative = Some((-v, row, col));
}
return 0.0;
}
v
}
pub(super) fn judge(&self, rel: f64) -> Result<(), GprError> {
let tol = rel * self.max;
if let Some((v, row, col)) = self.negative.filter(|(v, _, _)| *v > tol) {
return Err(negative_past(-v, tol, row, col));
}
if let Some((v, row, col)) = self.diagonal.filter(|(v, _, _)| *v > tol) {
return Err(invalid(
row,
col,
format!("the diagonal is {v} in size, past the tolerance {tol}"),
));
}
Ok(())
}
}
pub(super) fn require_blocks(
lens: impl ExactSizeIterator<Item = usize>,
blocks: usize,
len: usize,
) -> Result<(), GprError> {
if lens.len() != blocks {
return Err(GprError::LengthMismatch {
reason: format!(
"expected {blocks} tables of squared distances, got {}",
lens.len()
),
});
}
for got in lens {
crate::data::require_count(got, len, "squared distances")?;
}
Ok(())
}
pub(super) fn require_ard_data(data: &ArdData<'_>, d: usize, len: usize) -> Result<(), GprError> {
match data {
ArdData::Blocks(tables) => require_blocks(tables.iter().map(Vec::len), d, len),
ArdData::Slices(tables) => require_blocks(tables.iter().map(|table| table.len()), d, len),
ArdData::Fill(_) => Ok(()),
}
}