#[allow(
unused_imports,
reason = "a split file takes its parent's imports whole; each uses some"
)]
use super::*;
pub(super) struct RawSlot<'a> {
place: usize,
pub(crate) shape: SlotShape,
data: RawData<'a>,
rows: usize,
cols: usize,
cast_at: usize,
unchecked: bool,
tidy: Tidy,
}
pub(super) enum RawData<'a> {
Values(Cow<'a, [f64]>),
Blocks(Vec<Vec<f64>>),
Slices(&'a [&'a [f64]]),
Written(usize),
}
impl RawSlot<'_> {
pub(crate) fn block<'b>(&'b self, k: usize, written: &'b [f64]) -> &'b [f64] {
let len = self.rows * self.cols;
match &self.data {
RawData::Values(values) => values,
RawData::Blocks(blocks) => blocks.get(k).map_or(&[], Vec::as_slice),
RawData::Slices(blocks) => blocks.get(k).copied().unwrap_or(&[]),
RawData::Written(at) => written.get(at + k * len..at + (k + 1) * len).unwrap_or(&[]),
}
}
}
pub(crate) struct QueryScratch<T> {
raw: Vec<RawSlot<'static>>,
written: Vec<f64>,
column: Vec<f64>,
pub(super) cast: Vec<T>,
}
impl<T> QueryScratch<T> {
pub(crate) const fn new() -> Self {
Self {
raw: Vec::new(),
written: Vec::new(),
column: Vec::new(),
cast: Vec::new(),
}
}
}
impl<T> Default for QueryScratch<T> {
fn default() -> Self {
Self::new()
}
}
impl<T> Clone for QueryScratch<T> {
fn clone(&self) -> Self {
Self::new()
}
}
impl<T> fmt::Debug for QueryScratch<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("QueryScratch")
.field("written", &self.written.capacity())
.field("cast", &self.cast.capacity())
.finish_non_exhaustive()
}
}
#[allow(clippy::unnecessary_filter_map)]
pub(super) fn recycle<'x, 'y>(mut raw: Vec<RawSlot<'x>>) -> Vec<RawSlot<'y>> {
raw.clear();
raw.into_iter().filter_map(|_| None).collect()
}
pub(crate) struct QuerySources<'a, T: KernelScalar> {
raw: Vec<RawSlot<'a>>,
scalars: usize,
scratch: &'a mut QueryScratch<T>,
}
pub(crate) struct QuerySquares<'a, T: KernelScalar>(pub(super) QuerySources<'a, T>);
impl<T: KernelScalar> Drop for QuerySources<'_, T> {
fn drop(&mut self) {
self.scratch.raw = recycle(std::mem::take(&mut self.raw));
}
}
impl<'a, T: KernelScalar> QuerySources<'a, T> {
pub(crate) fn bind_rect<'s: 'a>(
slots: &[DistanceSlot],
sources: impl IntoIterator<Item = DistanceSource<'s>>,
(rows, cols): (usize, usize),
scratch: &'a mut QueryScratch<T>,
) -> Result<Self, GprError> {
Self::bind(slots, sources, rows, cols, BlockKind::Rect, true, scratch)
}
pub(crate) fn bind_column<'s: 'a>(
slots: &[DistanceSlot],
sources: impl IntoIterator<Item = DistanceSource<'s>>,
n: usize,
scratch: &'a mut QueryScratch<T>,
) -> Result<Self, GprError> {
Self::bind(slots, sources, n, 1, BlockKind::Rect, false, scratch)
}
pub(crate) fn bind_square<'s: 'a>(
slots: &[DistanceSlot],
sources: impl IntoIterator<Item = DistanceSource<'s>>,
n: usize,
scratch: &'a mut QueryScratch<T>,
) -> Result<QuerySquares<'a, T>, GprError> {
Self::bind(slots, sources, n, n, BlockKind::Square, false, scratch).map(QuerySquares)
}
pub(super) fn bind<'s: 'a>(
slots: &[DistanceSlot],
sources: impl IntoIterator<Item = DistanceSource<'s>>,
rows: usize,
cols: usize,
kind: BlockKind,
defer_check: bool,
scratch: &'a mut QueryScratch<T>,
) -> Result<Self, GprError> {
crate::data::require_nonempty(rows)?;
crate::data::require_nonempty(cols)?;
let len = rows.checked_mul(cols).ok_or(GprError::SizeOverflow)?;
if kind == BlockKind::Square && rows != cols {
return Err(GprError::ShapeMismatch {
reason: format!("a square of squared distances is {rows}x{cols}"),
});
}
let raw = recycle(std::mem::take(&mut scratch.raw));
let mut this = Self {
raw,
scalars: 0,
scratch,
};
this.scratch.written.clear();
for source in sources {
let SlotPlace { slot, place, .. } = slot_of(slots, &source)?;
if this.raw.iter().any(|raw| raw.place == place) {
return Err(duplicate(place));
}
let shape = slot.shape();
let blocks = shape.blocks();
let tidy = source.tidy;
let QueryScratch {
written, column, ..
} = &mut *this.scratch;
let data = match source.data {
SourceData::Scalar(ScalarData::Values(values)) => RawData::Values(values),
SourceData::Ard(_, ArdData::Blocks(tables)) => RawData::Blocks(tables),
SourceData::Ard(_, ArdData::Slices(tables)) => RawData::Slices(tables),
SourceData::Scalar(ScalarData::Fill(filler))
| SourceData::Ard(_, ArdData::Fill(filler)) => {
let at = written.len();
fill_dense(filler, (rows, cols), blocks, kind, written, column)?;
RawData::Written(at)
}
};
let mut raw_slot = RawSlot {
place,
shape,
data,
rows,
cols,
cast_at: 0,
unchecked: false,
tidy,
};
let read_checked = defer_check
&& kind == BlockKind::Rect
&& tidy == Tidy::Exact
&& matches!(shape, SlotShape::Ard(_));
if read_checked {
check_counts(&raw_slot, blocks, len, written)?;
raw_slot.unchecked = true;
} else {
check_slot(&mut raw_slot, blocks, len, kind, tidy, written)
.map_err(|err| err.in_slot(place))?;
}
this.raw.push(raw_slot);
}
if this.raw.len() != slots.len() {
let raw = &this.raw;
let bound = |place: &usize| raw.iter().any(|raw| raw.place == *place);
return Err(missing((0..slots.len()).find(|place| !bound(place))));
}
this.raw
.sort_unstable_by_key(|raw| (matches!(raw.shape, SlotShape::Ard(_)), raw.place));
this.scalars = this
.raw
.iter()
.filter(|raw| raw.shape == SlotShape::Scalar)
.count();
if !reads_in_place::<T>() {
let Self { raw, scratch, .. } = &mut this;
let QueryScratch { written, cast, .. } = &mut **scratch;
cast.clear();
cast.reserve(raw.iter().map(|slot| slot.shape.blocks() * len).sum());
for slot in raw.iter_mut() {
slot.cast_at = cast.len();
for k in 0..slot.shape.blocks() {
let block = slot.block(k, written);
let place = |err| in_block(err, slot.shape, k).in_slot(slot.place);
for tile in block.chunks(CAST_TILE) {
if slot.unchecked && !crate::kernel::simd::all_valid_distances(tile) {
return Err(place(first_invalid_from(block, slot.rows, 0)));
}
let at = cast.len();
cast.extend(tile.iter().map(|&v| T::from_f64(v)));
if let Some(i) = cast[at..].iter().position(|v| !v.is_finite()) {
let pos = (at - slot.cast_at) % len + i;
return Err(place(out_of_range(pos % slot.rows, pos / slot.rows)));
}
}
}
slot.unchecked = false;
}
}
Ok(this)
}
fn blocks(&self) -> BoundBlocks<'_> {
BoundBlocks {
raw: &self.raw,
scalars: self.scalars,
written: &self.scratch.written,
}
}
pub(crate) fn f64_view(&self) -> F64Blocks<'_> {
F64Blocks(self.blocks())
}
pub(crate) fn tidy(&self, block: BlockAt) -> Result<Tidy, GprError> {
self.raw_of(block).map(|raw| raw.tidy)
}
pub(crate) fn locate(&self, block: BlockAt, err: GprError) -> GprError {
let err = match block {
BlockAt::Scalar(_) => err,
BlockAt::Ard(_, k) => err.in_dim(k),
};
match self.raw_of(block) {
Ok(raw) => err.in_slot(raw.place),
Err(_) => err,
}
}
fn raw_of(&self, block: BlockAt) -> Result<&RawSlot<'a>, GprError> {
let at = match block {
BlockAt::Scalar(at) => at,
BlockAt::Ard(at, _) => self.scalars + at,
};
self.raw.get(at).ok_or_else(unbound)
}
}
impl<T: KernelScalar> RectSlots<T> for QuerySources<'_, T> {
fn scalar(&self, at: usize) -> Result<MatRef<'_, T>, GprError> {
self.blocks().scalar(&self.scratch.cast, at)
}
fn ard(&self, at: usize) -> Result<ArdRect<'_, T>, GprError> {
self.blocks().ard(&self.scratch.cast, at)
}
}
impl<T: KernelScalar> SquareSlots<T> for QuerySquares<'_, T> {
fn scalar(&self, at: usize) -> Result<MatRef<'_, T>, GprError> {
self.0.blocks().scalar(&self.0.scratch.cast, at)
}
fn ard(&self, at: usize) -> Result<ArdSquare<'_, T>, GprError> {
let blocks = self.0.blocks();
let raw = blocks.ard_slot(at)?;
if raw.unchecked {
return Err(GprError::UnsupportedKernelOperation {
reason: "a square of squared distances was bound unchecked".to_owned(),
});
}
let (list, _) = block_list(blocks, &self.0.scratch.cast, raw);
Ok(ArdSquare::Dense(ArdBlocks::new(
list, raw.rows, raw.cols, 0,
)))
}
}
#[derive(Clone, Copy)]
pub(super) struct BoundBlocks<'v> {
raw: &'v [RawSlot<'v>],
scalars: usize,
written: &'v [f64],
}
impl<'v> BoundBlocks<'v> {
fn scalar<U: KernelScalar>(self, cast: &'v [U], at: usize) -> Result<MatRef<'v, U>, GprError> {
let raw = self
.raw
.get(at)
.filter(|_| at < self.scalars)
.ok_or_else(unbound)?;
let (list, _) = block_list(self, cast, raw);
let checked = ArdBlocks::<U, Checked>::new(list, raw.rows, raw.cols, 0);
Ok(MatRef::from_column_major_slice(
&checked.block(0)[..raw.rows * raw.cols],
raw.rows,
raw.cols,
))
}
fn ard_slot(self, at: usize) -> Result<&'v RawSlot<'v>, GprError> {
self.raw
.get(self.scalars.saturating_add(at))
.ok_or_else(unbound)
}
fn ard<U: KernelScalar>(self, cast: &'v [U], at: usize) -> Result<ArdRect<'v, U>, GprError> {
let raw = self.ard_slot(at)?;
let (list, in_place) = block_list(self, cast, raw);
Ok(if raw.unchecked && in_place {
ArdRect::Unchecked(ArdBlocks::new(list, raw.rows, raw.cols, 0).of_slot(raw.place))
} else {
ArdRect::Checked(ArdBlocks::new(list, raw.rows, raw.cols, 0))
})
}
}
pub(crate) struct F64Blocks<'v>(BoundBlocks<'v>);
impl RectSlots<f64> for F64Blocks<'_> {
fn scalar(&self, at: usize) -> Result<MatRef<'_, f64>, GprError> {
self.0.scalar(&[], at)
}
fn ard(&self, at: usize) -> Result<ArdRect<'_, f64>, GprError> {
self.0.ard(&[], at)
}
}
pub(super) fn block_list<'v, U: KernelScalar>(
blocks: BoundBlocks<'v>,
cast: &'v [U],
raw: &'v RawSlot<'v>,
) -> (BlockList<'v, U>, bool) {
let len = raw.rows * raw.cols;
let dims = raw.shape.blocks();
let in_place = match &raw.data {
RawData::Values(values) => U::from_f64_slice(values).map(|v| BlockList::Packed(v, len, 1)),
RawData::Blocks(tables) => U::from_f64_vecs(tables).map(BlockList::Vecs),
RawData::Slices(tables) => U::from_f64_slices(tables).map(BlockList::Slices),
RawData::Written(at) => {
U::from_f64_slice(&blocks.written[*at..]).map(|v| BlockList::Packed(v, len, dims))
}
};
match in_place {
Some(list) => (list, true),
None => (BlockList::Packed(&cast[raw.cast_at..], len, dims), false),
}
}
pub(super) fn fill_dense(
filler: &dyn DistanceFill,
(rows, cols): (usize, usize),
blocks: usize,
kind: BlockKind,
all: &mut Vec<f64>,
column: &mut Vec<f64>,
) -> Result<(), GprError> {
let len = rows.checked_mul(cols).ok_or(GprError::SizeOverflow)?;
let total = len.checked_mul(blocks).ok_or(GprError::SizeOverflow)?;
let at = all.len();
all.resize(at.checked_add(total).ok_or(GprError::SizeOverflow)?, 0.0);
let all = &mut all[at..];
column.clear();
column.resize(rows.checked_mul(blocks).ok_or(GprError::SizeOverflow)?, 0.0);
for col in 0..cols {
let first = if kind == BlockKind::Square { col } else { 0 };
let run = rows - first;
let column = &mut column[..run * blocks];
filler.fill_column(col, first..rows, column);
for k in 0..blocks {
let src = &column[k * run..(k + 1) * run];
let block = &mut all[k * len..(k + 1) * len];
block[col * rows + first..(col + 1) * rows].copy_from_slice(src);
}
}
if kind == BlockKind::Square {
for block in all.chunks_exact_mut(len.max(1)) {
mirror_lower(block, rows);
}
}
Ok(())
}
pub(super) fn check_counts(
slot: &RawSlot<'_>,
blocks: usize,
len: usize,
written: &[f64],
) -> Result<(), GprError> {
let count = match &slot.data {
RawData::Values(_) => 1,
RawData::Blocks(tables) => tables.len(),
RawData::Slices(tables) => tables.len(),
RawData::Written(_) => blocks,
};
require_blocks(
(0..count).map(|k| slot.block(k, written).len()),
blocks,
len,
)
}
pub(super) const CAST_TILE: usize = 4096;
pub(super) fn check_slot(
slot: &mut RawSlot<'_>,
blocks: usize,
len: usize,
kind: BlockKind,
tidy: Tidy,
written: &mut Vec<f64>,
) -> Result<(), GprError> {
let (rows, cols) = (slot.rows, slot.cols);
check_counts(slot, blocks, len, written)?;
for k in 0..blocks {
let checked = check_block(slot.block(k, written), rows, cols, kind, tidy);
if !checked.map_err(|err| in_block(err, slot.shape, k))? {
continue;
}
let at = match &mut slot.data {
RawData::Values(Cow::Owned(values)) => {
repair_block(values, rows, cols, kind);
continue;
}
RawData::Blocks(tables) => {
repair_block(&mut tables[k], rows, cols, kind);
continue;
}
RawData::Written(at) => *at,
RawData::Values(Cow::Borrowed(values)) => {
let at = written.len();
written.extend_from_slice(values);
at
}
RawData::Slices(tables) => {
let at = written.len();
for table in tables.iter() {
written.extend_from_slice(table);
}
at
}
};
slot.data = RawData::Written(at);
repair_block(
&mut written[at + k * len..at + (k + 1) * len],
rows,
cols,
kind,
);
}
Ok(())
}