use super::Tensor;
use crate::backend_authoring::{Backend, SupportsDType};
use crate::dist::{Placement, PlacementKind};
use crate::shapes::{Dyn, Shape, ShapeValue};
use crate::tensor::device::{Device, DeviceId};
use crate::tensor::dtype::{DType, DTypeDescriptor};
use crate::tensor::grad::RequiresGrad;
use alloc::string::ToString;
#[non_exhaustive]
#[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)]
pub enum PlacedTensorError {
#[error("invalid tensor shape at placement boundary: {0}")]
Shape(alloc::string::String),
#[error("tensor global shape {tensor:?} does not match distributed proof {proof:?}")]
GlobalShape {
tensor: alloc::vec::Vec<usize>,
proof: alloc::vec::Vec<usize>,
},
#[error("tensor placement {tensor:?} does not match distributed proof {proof:?}")]
OutputPlacement {
tensor: PlacementKind,
proof: PlacementKind,
},
#[error("rank {rank} is outside a distributed result with {ranks} local values")]
RankOutOfRange {
rank: usize,
ranks: usize,
},
#[error("rank {rank} storage shape {storage:?} does not match proof {proof:?}")]
LocalShape {
rank: usize,
storage: alloc::vec::Vec<usize>,
proof: alloc::vec::Vec<usize>,
},
#[error("tensor dtype {expected:?} does not match rank-local storage {got:?}")]
DType {
expected: DTypeDescriptor,
got: DTypeDescriptor,
},
#[error("tensor device {expected:?} does not match rank-local storage {got:?}")]
Device {
expected: DeviceId,
got: DeviceId,
},
#[error("cannot resolve placed tensor metadata: {message}")]
MetadataResolution {
message: alloc::string::String,
},
#[error(
"gradient tracking requires a floating dtype, got {dtype:?} on backend {backend} for {op}"
)]
GradientDType {
dtype: DTypeDescriptor,
backend: &'static str,
op: &'static str,
},
#[error("distributed proof expects input {proof:?}, tensor is {tensor:?}")]
InputPlacement {
tensor: PlacementKind,
proof: PlacementKind,
},
#[error(transparent)]
Distributed(#[from] crate::dist::DistributedError),
#[error("static transition {expected:?} does not match distributed proof {proof:?}")]
Transition {
expected: crate::dist::PlacementTransition,
proof: crate::dist::PlacementTransition,
},
}
impl<S: Shape, B: Backend, K: DType, G: RequiresGrad, P: Placement> Tensor<S, B, K, G, P> {
pub fn try_from_distributed_storage<O>(
inner: B::Storage<K>,
global_shape: crate::shapes::ShapeBuf,
dtype: K::Field,
device: <B::Device as Device>::Field,
grad: G::Field,
rank: usize,
proof: &crate::dist::ValidatedDistributed<O>,
) -> core::result::Result<Self, PlacedTensorError>
where
O: crate::exec::ExecutionDescriptor,
B: SupportsDType<K>,
{
if G::requires_grad(&grad) && !K::descriptor(&dtype).is_float() {
return Err(PlacedTensorError::GradientDType {
dtype: K::descriptor(&dtype),
backend: B::BACKEND_NAME,
op: "gradient tracking",
});
}
let shape = ShapeValue::<S>::try_new(global_shape.clone())
.map_err(|error| PlacedTensorError::Shape(error.to_string()))?;
let tensor_global = global_shape.as_ref().to_vec();
let proof_global = proof.global_shape().dims().to_vec();
if tensor_global != proof_global {
return Err(PlacedTensorError::GlobalShape {
tensor: tensor_global,
proof: proof_global,
});
}
let Some(placement) = P::try_from_incin(proof.output_placement(), rank) else {
return Err(PlacedTensorError::OutputPlacement {
tensor: P::to_incin(&P::Field::default()),
proof: proof.output_placement(),
});
};
let Some(expected_local) = proof.local_shapes().get(rank) else {
return Err(PlacedTensorError::RankOutOfRange {
rank,
ranks: proof.local_shapes().len(),
});
};
let storage_local = B::shape(&inner);
if storage_local.as_ref() != expected_local.dims() {
return Err(PlacedTensorError::LocalShape {
rank,
storage: storage_local.as_ref().to_vec(),
proof: expected_local.dims().to_vec(),
});
}
let expected_device = B::Device::to_incin(&device).map_err(|error| {
PlacedTensorError::MetadataResolution {
message: error.to_string(),
}
})?;
let expected_dtype = B::resolve_dtype(&dtype, &expected_device).map_err(|error| {
PlacedTensorError::MetadataResolution {
message: error.to_string(),
}
})?;
if let Some(got) = B::storage_dtype(&inner)
&& got != expected_dtype
{
return Err(PlacedTensorError::DType {
expected: expected_dtype,
got,
});
}
if let Some(got) = B::storage_device(&inner)
&& got != expected_device
{
return Err(PlacedTensorError::Device {
expected: expected_device,
got,
});
}
Ok(Self {
inner,
_shape: shape,
_dtype: dtype,
_device: device,
_grad: grad,
_placement: placement,
})
}
pub fn try_reshard<To, O>(
self,
inner: B::Storage<K>,
rank: usize,
proof: &crate::dist::ValidatedDistributed<O>,
) -> core::result::Result<Tensor<S, B, K, G, To>, PlacedTensorError>
where
O: crate::exec::ExecutionDescriptor,
P: crate::dist::ConstPlacement + crate::dist::LegalTransition<To>,
To: crate::dist::ConstPlacement,
B: SupportsDType<K>,
{
validate_reshard_proof(P::PLACEMENT, To::PLACEMENT, proof)?;
if proof.transition() != P::TRANSITION {
return Err(PlacedTensorError::Transition {
expected: P::TRANSITION,
proof: proof.transition(),
});
}
let shape = self.shape_buf_value();
Tensor::<S, B, K, G, To>::try_from_distributed_storage(
inner,
shape,
self._dtype,
self._device,
self._grad,
rank,
proof,
)
}
}
impl<S: Shape, B: Backend, K: DType, G: RequiresGrad> Tensor<S, B, K, G, Dyn> {
pub fn try_reshard_dyn<O>(
self,
inner: B::Storage<K>,
to: PlacementKind,
rank: usize,
proof: &crate::dist::ValidatedDistributed<O>,
) -> core::result::Result<Self, PlacedTensorError>
where
O: crate::exec::ExecutionDescriptor,
B: SupportsDType<K>,
{
let from = self.placement();
validate_reshard_proof(from, to, proof)?;
let expected = crate::dist::validate_transition(from, to)?;
if proof.transition() != expected {
return Err(PlacedTensorError::Transition {
expected,
proof: proof.transition(),
});
}
let shape = self.shape_buf_value();
Self::try_from_distributed_storage(
inner,
shape,
self._dtype,
self._device,
self._grad,
rank,
proof,
)
}
}
fn validate_reshard_proof<O>(
from: PlacementKind,
to: PlacementKind,
proof: &crate::dist::ValidatedDistributed<O>,
) -> core::result::Result<(), PlacedTensorError> {
let Some(&proof_input) = proof.input_placements().as_slice().first() else {
return Err(crate::dist::DistributedError::NoInputPlacements.into());
};
if proof_input != from {
return Err(PlacedTensorError::InputPlacement {
tensor: from,
proof: proof_input,
});
}
if proof
.input_placements()
.as_slice()
.iter()
.any(|&placement| placement != from)
{
let proof_input = proof
.input_placements()
.as_slice()
.iter()
.copied()
.find(|&placement| placement != from)
.unwrap_or(proof_input);
return Err(PlacedTensorError::InputPlacement {
tensor: from,
proof: proof_input,
});
}
if proof.output_placement() != to {
return Err(PlacedTensorError::OutputPlacement {
tensor: to,
proof: proof.output_placement(),
});
}
Ok(())
}