use crate::application::index::validate_mutable_output;
use crate::domain::scalar::Scalar;
use leto::{Array, ArrayView, ArrayViewMut, Layout, LetoError, Result, VecStorage};
pub fn embedding_gather_into<T: Scalar>(
table: &ArrayView<'_, T, 2>,
indices: &ArrayView<'_, u32, 1>,
output: &mut ArrayViewMut<'_, T, 2>,
) -> Result<()> {
let [num_embeddings, embedding_dim] = table.shape();
let [rows] = indices.shape();
if output.shape() != [rows, embedding_dim] {
return Err(LetoError::ShapeMismatch {
lhs: [rows, embedding_dim].to_vec(),
rhs: output.shape().to_vec(),
});
}
table.layout().validate_storage_len(table.data().len())?;
indices
.layout()
.validate_storage_len(indices.data().len())?;
validate_mutable_output(output, "embedding gather output")?;
if let (Some(table_slice), Some(index_slice), Some(out_slice)) =
(table.as_slice(), indices.as_slice(), output.as_mut_slice())
{
for (row, &index) in index_slice.iter().enumerate() {
let selected = usize::try_from(index).map_err(|_| LetoError::Overflow {
reason: "embedding index exceeds the addressable row count",
})?;
if selected >= num_embeddings {
return Err(LetoError::OutOfBounds {
index: vec![selected],
shape: vec![num_embeddings],
});
}
let table_base = selected * embedding_dim;
let out_base = row * embedding_dim;
out_slice[out_base..out_base + embedding_dim]
.copy_from_slice(&table_slice[table_base..table_base + embedding_dim]);
}
return Ok(());
}
let table_layout = table.layout();
let index_layout = indices.layout();
let output_layout = output.layout();
let table_data = table.data();
let index_data = indices.data();
let output_data = output.data_mut();
let table_base = table_layout.offset() as isize;
let index_base = index_layout.offset() as isize;
let output_base = output_layout.offset() as isize;
let (tab_row, tab_col) = (table_layout.strides()[0], table_layout.strides()[1]);
let index_stride = index_layout.strides()[0];
let (out_row, out_col) = (output_layout.strides()[0], output_layout.strides()[1]);
for row in 0..rows {
let at = (index_base + row as isize * index_stride) as usize;
let selected = usize::try_from(index_data[at]).map_err(|_| LetoError::Overflow {
reason: "embedding index exceeds the addressable row count",
})?;
if selected >= num_embeddings {
return Err(LetoError::OutOfBounds {
index: vec![selected],
shape: vec![num_embeddings],
});
}
for col in 0..embedding_dim {
let from = (table_base + selected as isize * tab_row + col as isize * tab_col) as usize;
let to = (output_base + row as isize * out_row + col as isize * out_col) as usize;
output_data[to] = table_data[from];
}
}
Ok(())
}
pub fn embedding_gather<T: Scalar>(
table: &ArrayView<'_, T, 2>,
indices: &ArrayView<'_, u32, 1>,
) -> Result<Array<T, VecStorage<T>, 2>> {
let [_, embedding_dim] = table.shape();
let [rows] = indices.shape();
let layout = Layout::c_contiguous([rows, embedding_dim])?;
let size = layout.checked_size()?;
let mut output = Array::new(layout, VecStorage::uninit(size))?;
embedding_gather_into(table, indices, &mut output.view_mut())?;
Ok(output)
}