use cpueinsum::Exec;
use tenferro_tensor::backend::Engine;
use tenferro_tensor::{
CompareDir, DType, DotGeneralConfig, ElementwiseReadOp, GatherConfig, NativeSessionRef,
PadConfig, ScatterConfig, SliceConfig, TensorRead, TensorWrite,
};
use tenferro_tensor::{
SessionCachedDot, TensorAnalytic, TensorBuffer, TensorDeviceTransfer, TensorDot,
TensorElementwise, TensorFusion, TensorIndexing, TensorReduction, TensorStructural,
};
use crate::buffer_pool::BufferPool;
use crate::CpuEngine;
struct BorrowedCpuMarker;
pub(crate) struct BorrowedCpuEngine<'a, 'e> {
exec: &'a mut Exec<'e>,
pool: &'a mut BufferPool,
}
impl<'a, 'e> BorrowedCpuEngine<'a, 'e> {
pub(crate) fn new(exec: &'a mut Exec<'e>, pool: &'a mut BufferPool) -> Self {
Self { exec, pool }
}
}
impl std::fmt::Debug for BorrowedCpuEngine<'_, '_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("BorrowedCpuEngine")
.field("budget", &self.exec.budget())
.field("pool", &self.pool.stats())
.finish()
}
}
#[doc(hidden)]
pub fn with_borrowed_cpu_engine<B, R>(
engine: &mut B,
f: impl FnOnce(&mut Exec<'_>, &mut BufferPool) -> R,
) -> Option<R>
where
B: Engine + ?Sized,
{
let data = engine
.native_engine()?
.into_marked_ptr::<BorrowedCpuMarker>()?;
let this = unsafe { data.cast::<BorrowedCpuEngine<'_, '_>>().as_mut() };
Some(f(&mut *this.exec, &mut *this.pool))
}
fn refused<T>(op: &'static str) -> tenferro_tensor::Result<T> {
Err(tenferro_tensor::Error::unsupported(
op,
"a lane of CpuEngine::with_lanes runs only the linalg factorizations \
svd/svdvals/qr/eigh/eigvalsh/eigvals; this operation needs a CpuEngine",
))
}
impl SessionCachedDot for BorrowedCpuEngine<'_, '_> {}
impl TensorFusion for BorrowedCpuEngine<'_, '_> {}
impl TensorBuffer for BorrowedCpuEngine<'_, '_> {}
impl Engine for BorrowedCpuEngine<'_, '_> {
fn native_engine(&mut self) -> Option<NativeSessionRef<'_>> {
Some(unsafe { NativeSessionRef::new::<BorrowedCpuMarker, _>(self) })
}
}
impl TensorStructural for BorrowedCpuEngine<'_, '_> {
fn transpose_read(
&mut self,
_a0: TensorRead<'_>,
_a1: &[usize],
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("transpose_read")
}
fn reshape_read(
&mut self,
_a0: TensorRead<'_>,
_a1: &[usize],
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("reshape_read")
}
fn broadcast_in_dim_read(
&mut self,
_a0: TensorRead<'_>,
_a1: &[usize],
_a2: &[usize],
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("broadcast_in_dim_read")
}
fn cast(
&mut self,
_a0: &tenferro_tensor::Tensor,
_a1: DType,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("cast")
}
fn extract_diagonal(
&mut self,
_a0: &tenferro_tensor::Tensor,
_a1: usize,
_a2: usize,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("extract_diagonal")
}
fn embed_diagonal(
&mut self,
_a0: &tenferro_tensor::Tensor,
_a1: usize,
_a2: usize,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("embed_diagonal")
}
fn tril(
&mut self,
_a0: &tenferro_tensor::Tensor,
_a1: i64,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("tril")
}
fn triu(
&mut self,
_a0: &tenferro_tensor::Tensor,
_a1: i64,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("triu")
}
}
impl TensorElementwise for BorrowedCpuEngine<'_, '_> {
fn elementwise_read_into(
&mut self,
_a0: ElementwiseReadOp,
_a1: &[TensorRead<'_>],
_a2: TensorWrite<'_>,
) -> Result<(), tenferro_tensor::Error> {
refused("elementwise_read_into")
}
fn add_read(
&mut self,
_a0: TensorRead<'_>,
_a1: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("add_read")
}
fn sub_read(
&mut self,
_a0: TensorRead<'_>,
_a1: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("sub_read")
}
fn mul_read(
&mut self,
_a0: TensorRead<'_>,
_a1: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("mul_read")
}
fn neg_read(
&mut self,
_a0: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("neg_read")
}
fn conj_read(
&mut self,
_a0: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("conj_read")
}
fn div_read(
&mut self,
_a0: TensorRead<'_>,
_a1: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("div_read")
}
fn abs_read(
&mut self,
_a0: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("abs_read")
}
fn sign_read(
&mut self,
_a0: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("sign_read")
}
fn maximum_read(
&mut self,
_a0: TensorRead<'_>,
_a1: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("maximum_read")
}
fn minimum_read(
&mut self,
_a0: TensorRead<'_>,
_a1: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("minimum_read")
}
fn compare_read(
&mut self,
_a0: TensorRead<'_>,
_a1: TensorRead<'_>,
_a2: &CompareDir,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("compare_read")
}
fn select_read(
&mut self,
_a0: TensorRead<'_>,
_a1: TensorRead<'_>,
_a2: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("select_read")
}
fn clamp_read(
&mut self,
_a0: TensorRead<'_>,
_a1: TensorRead<'_>,
_a2: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("clamp_read")
}
}
impl TensorAnalytic for BorrowedCpuEngine<'_, '_> {
fn exp_read(
&mut self,
_a0: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("exp_read")
}
fn log_read(
&mut self,
_a0: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("log_read")
}
fn sin_read(
&mut self,
_a0: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("sin_read")
}
fn cos_read(
&mut self,
_a0: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("cos_read")
}
fn tanh_read(
&mut self,
_a0: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("tanh_read")
}
fn sqrt_read(
&mut self,
_a0: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("sqrt_read")
}
fn rsqrt_read(
&mut self,
_a0: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("rsqrt_read")
}
fn pow_read(
&mut self,
_a0: TensorRead<'_>,
_a1: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("pow_read")
}
fn expm1_read(
&mut self,
_a0: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("expm1_read")
}
fn log1p_read(
&mut self,
_a0: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("log1p_read")
}
fn erf_read(
&mut self,
_a0: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("erf_read")
}
}
impl TensorReduction for BorrowedCpuEngine<'_, '_> {
fn reduce_sum_read(
&mut self,
_a0: TensorRead<'_>,
_a1: &[usize],
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("reduce_sum_read")
}
fn reduce_prod_read(
&mut self,
_a0: TensorRead<'_>,
_a1: &[usize],
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("reduce_prod_read")
}
fn reduce_max_read(
&mut self,
_a0: TensorRead<'_>,
_a1: &[usize],
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("reduce_max_read")
}
fn reduce_min_read(
&mut self,
_a0: TensorRead<'_>,
_a1: &[usize],
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("reduce_min_read")
}
}
impl TensorIndexing for BorrowedCpuEngine<'_, '_> {
fn gather(
&mut self,
_a0: &tenferro_tensor::Tensor,
_a1: &tenferro_tensor::Tensor,
_a2: &GatherConfig,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("gather")
}
fn scatter(
&mut self,
_a0: &tenferro_tensor::Tensor,
_a1: &tenferro_tensor::Tensor,
_a2: &tenferro_tensor::Tensor,
_a3: &ScatterConfig,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("scatter")
}
fn slice(
&mut self,
_a0: &tenferro_tensor::Tensor,
_a1: &SliceConfig,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("slice")
}
fn dynamic_slice(
&mut self,
_a0: &tenferro_tensor::Tensor,
_a1: &tenferro_tensor::Tensor,
_a2: &[usize],
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("dynamic_slice")
}
fn dynamic_update_slice(
&mut self,
_a0: &tenferro_tensor::Tensor,
_a1: &tenferro_tensor::Tensor,
_a2: &tenferro_tensor::Tensor,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("dynamic_update_slice")
}
fn pad(
&mut self,
_a0: &tenferro_tensor::Tensor,
_a1: &PadConfig,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("pad")
}
fn concatenate(
&mut self,
_a0: &[&tenferro_tensor::Tensor],
_a1: usize,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("concatenate")
}
fn reverse(
&mut self,
_a0: &tenferro_tensor::Tensor,
_a1: &[usize],
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("reverse")
}
}
impl TensorDot for BorrowedCpuEngine<'_, '_> {
fn dot_general_read(
&mut self,
_a0: TensorRead<'_>,
_a1: TensorRead<'_>,
_a2: &DotGeneralConfig,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("dot_general_read")
}
}
impl TensorDeviceTransfer for BorrowedCpuEngine<'_, '_> {
fn download_to_host(
&mut self,
_a0: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("download_to_host")
}
fn upload_host_tensor(
&mut self,
_a0: TensorRead<'_>,
) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
refused("upload_host_tensor")
}
}
struct Lane<'p, R> {
pool: &'p mut BufferPool,
out: Option<R>,
}
impl CpuEngine {
pub fn with_lanes<R: Send>(
&mut self,
shares: &[usize],
pools: &mut [BufferPool],
f: impl Fn(usize, &mut dyn Engine) -> R + Sync,
) -> crate::Result<Vec<R>> {
const OP: &str = "CpuEngine::with_lanes";
if shares.len() != pools.len() {
return Err(tenferro_tensor::Error::invalid_argument(
OP,
"pools",
format!(
"need one pool per lane: {} shares but {} pools",
shares.len(),
pools.len()
),
));
}
if shares.contains(&0) {
return Err(tenferro_tensor::Error::invalid_argument(
OP,
"shares",
format!("every lane needs at least one thread, got {shares:?}"),
));
}
let budget = self.num_threads();
let total = shares
.iter()
.try_fold(0_usize, |sum, &share| sum.checked_add(share));
if total.is_none_or(|total| total > budget) {
return Err(tenferro_tensor::Error::invalid_argument(
OP,
"shares",
format!("the shares {shares:?} exceed the engine's thread budget of {budget}"),
));
}
self.with_exec(|exec| {
let workspace = exec.workspace();
let policy = *exec.batch_policy();
let mut lanes: Vec<Lane<'_, R>> = pools
.iter_mut()
.map(|pool| Lane { pool, out: None })
.collect();
exec.token()
.fork_sized_slots(shares, &mut lanes, |index, lane, child| {
let mut bundle = Exec::from_token(child, workspace, policy);
let mut engine = BorrowedCpuEngine::new(&mut bundle, &mut *lane.pool);
lane.out = Some(f(index, &mut engine));
})
.map_err(|error| {
tenferro_tensor::Error::invalid_argument(OP, "shares", error.to_string())
})?;
lanes
.into_iter()
.map(|lane| {
lane.out.ok_or_else(|| {
tenferro_tensor::Error::Internal("a lane produced no result".into())
})
})
.collect()
})?
}
}
#[cfg(test)]
mod tests;