use std::collections::BTreeSet;
use std::num::NonZeroU32;
use serde::{Deserialize, Deserializer, Serialize};
use super::super::{
CanonicalRational, ContractVersion, QuantizationFormatId, VNextError, WeightFormatId, WeightId,
WeightLayoutId,
};
use super::ElementType;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum QuantizationPacking {
Linear,
Interleaved,
Tiled,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum QuantizationGrouping {
Fixed { size: u32 },
WholeAxis,
Block2d { block_shape: [NonZeroU32; 2] },
}
impl QuantizationGrouping {
pub const fn fixed(size: u32) -> Self {
Self::Fixed { size }
}
pub const fn fixed_size(self) -> Option<u32> {
match self {
Self::Fixed { size } => Some(size),
Self::WholeAxis | Self::Block2d { .. } => None,
}
}
pub const fn block_2d(block_shape: [NonZeroU32; 2]) -> Self {
Self::Block2d { block_shape }
}
pub const fn block_shape_2d(self) -> Option<[NonZeroU32; 2]> {
match self {
Self::Block2d { block_shape } => Some(block_shape),
Self::Fixed { .. } | Self::WholeAxis => None,
}
}
pub const fn resolved_size(self, axis_extent: u64) -> u64 {
match self {
Self::Fixed { size } => size as u64,
Self::WholeAxis => axis_extent,
Self::Block2d { .. } => 0,
}
}
const fn is_valid(self) -> bool {
match self {
Self::Fixed { size } => size != 0 && size.is_power_of_two(),
Self::WholeAxis => true,
Self::Block2d { block_shape } => {
block_shape[0].get().is_power_of_two() && block_shape[1].get().is_power_of_two()
}
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct QuantizationSpec {
pub format_id: QuantizationFormatId,
pub bits_per_weight: u8,
pub grouping: QuantizationGrouping,
pub packing: QuantizationPacking,
pub scale_type: ElementType,
pub zero_point_type: Option<ElementType>,
}
impl QuantizationSpec {
pub fn validate(&self) -> Result<(), VNextError> {
if !(1..=8).contains(&self.bits_per_weight)
|| !self.grouping.is_valid()
|| !matches!(
self.scale_type,
ElementType::U8 | ElementType::F16 | ElementType::Bf16 | ElementType::F32
)
|| self.zero_point_type.is_some_and(|element_type| {
!matches!(
element_type,
ElementType::U8 | ElementType::U32 | ElementType::I8 | ElementType::I32
)
})
{
return Err(VNextError::InvalidExecutionPlan {
reason: format!("invalid quantization format `{}`", self.format_id),
});
}
Ok(())
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct BlockQuantizationSpec {
pub format_id: QuantizationFormatId,
pub logical_values_per_block: u32,
pub bytes_per_block: u32,
}
impl BlockQuantizationSpec {
pub fn validate(&self) -> Result<(), VNextError> {
if self.logical_values_per_block == 0 || self.bytes_per_block == 0 {
return Err(VNextError::InvalidExecutionPlan {
reason: format!("invalid block quantization format `{}`", self.format_id),
});
}
Ok(())
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum WeightEncoding {
Dense {
element_type: ElementType,
},
DenseAffine {
element_type: ElementType,
scale: CanonicalRational,
bias: CanonicalRational,
},
Quantized(QuantizationSpec),
BlockQuantized(BlockQuantizationSpec),
}
impl WeightEncoding {
pub const fn dense_element_type(&self) -> Option<ElementType> {
match self {
Self::Dense { element_type } | Self::DenseAffine { element_type, .. } => {
Some(*element_type)
}
Self::Quantized(_) | Self::BlockQuantized(_) => None,
}
}
pub(crate) fn physical_bytes(
&self,
dimensions: &[u64],
component_id: &WeightId,
) -> Result<u64, VNextError> {
let elements =
checked_elements(dimensions).ok_or_else(|| VNextError::InvalidExecutionPlan {
reason: format!("physical component `{component_id}` size overflows u64"),
})?;
match self {
Self::Dense { element_type } | Self::DenseAffine { element_type, .. } => elements
.checked_mul(element_type.size_bytes())
.ok_or_else(|| VNextError::InvalidExecutionPlan {
reason: format!("physical component `{component_id}` byte size overflows u64"),
}),
Self::Quantized(_) => Ok(elements),
Self::BlockQuantized(spec) => {
spec.validate()?;
elements
.checked_mul(u64::from(spec.bytes_per_block))
.ok_or_else(|| VNextError::InvalidExecutionPlan {
reason: format!(
"physical block component `{component_id}` byte size overflows u64"
),
})
}
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum WeightComponentRole {
Values,
PackedValues,
Scales,
ZeroPoints,
Indices,
Permutation,
Codebook,
Metadata,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum PhysicalWeightPadding {
Exact,
ZeroFill { padded_dimensions: Vec<u64> },
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum PhysicalStorageLayout {
Contiguous {
padding: PhysicalWeightPadding,
},
Strided {
strides_in_elements: Vec<u64>,
padding: PhysicalWeightPadding,
},
Tiled {
tile_shape: Vec<u64>,
axis_order: Vec<u32>,
tile_strides_in_elements: Vec<u64>,
padding: PhysicalWeightPadding,
},
}
impl PhysicalStorageLayout {
pub fn exact_contiguous() -> Self {
Self::Contiguous {
padding: PhysicalWeightPadding::Exact,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PhysicalWeightComponentBinding {
pub component_id: WeightId,
pub storage: PhysicalStorageLayout,
}
impl PhysicalWeightComponentBinding {
pub fn exact_contiguous(component_id: WeightId) -> Self {
Self {
component_id,
storage: PhysicalStorageLayout::exact_contiguous(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct AxisWeightComponent {
pub component: PhysicalWeightComponentBinding,
pub axis: u32,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CompositeWeightPart {
pub layout: Box<PhysicalWeightLayout>,
pub logical_offsets: Vec<u64>,
pub extents: Vec<u64>,
}
pub const MAX_PHYSICAL_WEIGHT_LAYOUT_DEPTH: usize = 16;
pub const MAX_PHYSICAL_WEIGHT_LAYOUT_NODES: usize = 4096;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum PhysicalWeightLayout {
Dense {
component_id: WeightId,
},
Stored {
component: PhysicalWeightComponentBinding,
},
Composite {
parts: Vec<CompositeWeightPart>,
},
Quantized {
packed_values: PhysicalWeightComponentBinding,
packed_dimensions: Vec<u64>,
scales: PhysicalWeightComponentBinding,
zero_points: Option<PhysicalWeightComponentBinding>,
zero_point_packed_dimensions: Option<Vec<u64>>,
axis_indices: Option<AxisWeightComponent>,
permutation: Option<AxisWeightComponent>,
codebook: Option<PhysicalWeightComponentBinding>,
group_axis: u32,
group_padding: PhysicalWeightPadding,
},
QuantizedBlockGrid {
packed_values: PhysicalWeightComponentBinding,
packed_dimensions: Vec<u64>,
scales: PhysicalWeightComponentBinding,
block_axes: [u32; 2],
},
BlockQuantized {
blocks: PhysicalWeightComponentBinding,
block_axis: u32,
block_padding: PhysicalWeightPadding,
},
AxisReshapePermutation {
values: Box<PhysicalWeightLayout>,
axis: u32,
logical_offset: u64,
extent: u64,
reshape: Vec<u64>,
stored_axis_order: Vec<u32>,
},
Indexed {
indices: AxisWeightComponent,
values: Box<PhysicalWeightLayout>,
source_axis_extent: u64,
},
ExpertStack {
experts: Vec<PhysicalWeightLayout>,
expert_axis: u32,
},
}
impl PhysicalWeightLayout {
pub(crate) fn normalize(&mut self) {
match self {
Self::Composite { parts } => {
for part in parts.iter_mut() {
part.layout.normalize();
}
parts.sort_by(|left, right| {
left.logical_offsets
.cmp(&right.logical_offsets)
.then_with(|| left.extents.cmp(&right.extents))
});
}
Self::AxisReshapePermutation { values, .. } | Self::Indexed { values, .. } => {
values.normalize()
}
Self::ExpertStack { experts, .. } => {
for expert in experts {
expert.normalize();
}
}
Self::Dense { .. }
| Self::Stored { .. }
| Self::Quantized { .. }
| Self::QuantizedBlockGrid { .. }
| Self::BlockQuantized { .. } => {}
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ResolvedWeightComponentLayout {
component_id: WeightId,
role: WeightComponentRole,
physical_dimensions: Vec<u64>,
encoding: WeightEncoding,
}
impl ResolvedWeightComponentLayout {
pub(crate) fn from_parts(
component_id: WeightId,
role: WeightComponentRole,
physical_dimensions: Vec<u64>,
encoding: WeightEncoding,
) -> Self {
Self {
component_id,
role,
physical_dimensions,
encoding,
}
}
pub fn component_id(&self) -> &WeightId {
&self.component_id
}
pub const fn role(&self) -> WeightComponentRole {
self.role
}
pub fn physical_dimensions(&self) -> &[u64] {
&self.physical_dimensions
}
pub fn encoding(&self) -> &WeightEncoding {
&self.encoding
}
pub fn physical_bytes(&self) -> Result<u64, VNextError> {
self.encoding
.physical_bytes(&self.physical_dimensions, &self.component_id)
}
pub fn physical_element_type(&self) -> ElementType {
self.encoding
.dense_element_type()
.unwrap_or(ElementType::U8)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct ResolvedWeightBinding {
weight_id: WeightId,
#[serde(rename = "format_id")]
schema_format_id: WeightFormatId,
layout_id: WeightLayoutId,
schema_version: ContractVersion,
physical_layout: PhysicalWeightLayout,
components: Vec<ResolvedWeightComponentLayout>,
}
pub(crate) trait ResolvedWeightLogicalValidation {
fn validate_logical_contract(
&self,
logical_dimensions: &[u64],
logical_element_type: ElementType,
) -> Result<(), VNextError>;
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct ResolvedWeightBindingWire {
weight_id: WeightId,
format_id: WeightFormatId,
layout_id: WeightLayoutId,
schema_version: ContractVersion,
physical_layout: PhysicalWeightLayout,
components: Vec<ResolvedWeightComponentLayout>,
}
impl<'de> Deserialize<'de> for ResolvedWeightBinding {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let wire = ResolvedWeightBindingWire::deserialize(deserializer)?;
Self::from_parts(
wire.weight_id,
wire.format_id,
wire.layout_id,
wire.schema_version,
wire.physical_layout,
wire.components,
)
.map_err(serde::de::Error::custom)
}
}
impl ResolvedWeightBinding {
pub(crate) fn from_parts(
weight_id: WeightId,
schema_format_id: WeightFormatId,
layout_id: WeightLayoutId,
schema_version: ContractVersion,
physical_layout: PhysicalWeightLayout,
components: Vec<ResolvedWeightComponentLayout>,
) -> Result<Self, VNextError> {
let binding = Self {
weight_id,
schema_format_id,
layout_id,
schema_version,
physical_layout,
components,
};
binding.validate_structure()?;
Ok(binding)
}
pub(crate) fn validate_structure(&self) -> Result<(), VNextError> {
validate_physical_layout_budget(&self.physical_layout).map_err(|reason| {
VNextError::InvalidExecutionPlan {
reason: format!("resolved weight `{}` layout: {reason}", self.weight_id),
}
})?;
let referenced = physical_component_ids(&self.physical_layout).map_err(|reason| {
VNextError::InvalidExecutionPlan {
reason: format!("resolved weight `{}` layout: {reason}", self.weight_id),
}
})?;
let component_ids = self
.components
.iter()
.map(|component| component.component_id.clone())
.collect::<BTreeSet<_>>();
let canonical_components = self
.components
.windows(2)
.all(|pair| pair[0].component_id < pair[1].component_id);
if self.schema_version.major == 0
|| self.components.is_empty()
|| !canonical_components
|| component_ids.len() != self.components.len()
|| component_ids != referenced
|| self.components.iter().any(|component| {
component.physical_dimensions.is_empty()
|| component
.physical_dimensions
.iter()
.any(|extent| *extent == 0)
|| component.physical_bytes().is_err()
})
{
return Err(VNextError::InvalidExecutionPlan {
reason: format!(
"resolved weight `{}` physical identity is invalid or non-canonical",
self.weight_id
),
});
}
Ok(())
}
pub fn weight_id(&self) -> &WeightId {
&self.weight_id
}
pub(crate) fn schema_format_id(&self) -> &WeightFormatId {
&self.schema_format_id
}
pub fn layout_id(&self) -> &WeightLayoutId {
&self.layout_id
}
pub const fn schema_version(&self) -> ContractVersion {
self.schema_version
}
pub fn physical_layout(&self) -> &PhysicalWeightLayout {
&self.physical_layout
}
pub fn components(&self) -> &[ResolvedWeightComponentLayout] {
&self.components
}
pub fn quantization_formats(&self) -> BTreeSet<QuantizationFormatId> {
self.components
.iter()
.filter_map(|component| match &component.encoding {
WeightEncoding::Quantized(spec) => Some(spec.format_id.clone()),
WeightEncoding::BlockQuantized(spec) => Some(spec.format_id.clone()),
WeightEncoding::Dense { .. } | WeightEncoding::DenseAffine { .. } => None,
})
.collect()
}
}
fn push_physical_layout_child<'a>(
stack: &mut Vec<(&'a PhysicalWeightLayout, usize)>,
child: &'a PhysicalWeightLayout,
child_depth: usize,
visited: usize,
) -> Result<(), String> {
if visited
.checked_add(stack.len())
.is_none_or(|pending| pending >= MAX_PHYSICAL_WEIGHT_LAYOUT_NODES)
{
return Err(format!(
"physical layout node count exceeds {MAX_PHYSICAL_WEIGHT_LAYOUT_NODES}"
));
}
stack.push((child, child_depth));
Ok(())
}
pub(crate) fn validate_physical_layout_budget(layout: &PhysicalWeightLayout) -> Result<(), String> {
let mut stack = vec![(layout, 1_usize)];
let mut visited = 0_usize;
while let Some((node, depth)) = stack.pop() {
if depth > MAX_PHYSICAL_WEIGHT_LAYOUT_DEPTH {
return Err(format!(
"physical layout depth exceeds {MAX_PHYSICAL_WEIGHT_LAYOUT_DEPTH}"
));
}
let direct_bindings = match node {
PhysicalWeightLayout::Dense { .. } | PhysicalWeightLayout::Stored { .. } => 1,
PhysicalWeightLayout::Quantized {
zero_points,
axis_indices,
permutation,
codebook,
..
} => {
2 + usize::from(zero_points.is_some())
+ usize::from(axis_indices.is_some())
+ usize::from(permutation.is_some())
+ usize::from(codebook.is_some())
}
PhysicalWeightLayout::QuantizedBlockGrid { .. } => 2,
PhysicalWeightLayout::BlockQuantized { .. } => 1,
PhysicalWeightLayout::AxisReshapePermutation { .. } => 0,
PhysicalWeightLayout::Indexed { .. } => 1,
PhysicalWeightLayout::Composite { .. } | PhysicalWeightLayout::ExpertStack { .. } => 0,
};
visited = visited
.checked_add(1 + direct_bindings)
.ok_or_else(|| "physical layout node count overflows usize".to_owned())?;
if visited > MAX_PHYSICAL_WEIGHT_LAYOUT_NODES {
return Err(format!(
"physical layout node count exceeds {MAX_PHYSICAL_WEIGHT_LAYOUT_NODES}"
));
}
let child_depth = depth
.checked_add(1)
.ok_or_else(|| "physical layout depth overflows usize".to_owned())?;
match node {
PhysicalWeightLayout::Composite { parts } => {
for part in parts {
push_physical_layout_child(&mut stack, &part.layout, child_depth, visited)?;
}
}
PhysicalWeightLayout::AxisReshapePermutation { values, .. }
| PhysicalWeightLayout::Indexed { values, .. } => {
push_physical_layout_child(&mut stack, values, child_depth, visited)?;
}
PhysicalWeightLayout::ExpertStack { experts, .. } => {
for expert in experts {
push_physical_layout_child(&mut stack, expert, child_depth, visited)?;
}
}
PhysicalWeightLayout::Dense { .. }
| PhysicalWeightLayout::Stored { .. }
| PhysicalWeightLayout::Quantized { .. }
| PhysicalWeightLayout::QuantizedBlockGrid { .. }
| PhysicalWeightLayout::BlockQuantized { .. } => {}
}
}
Ok(())
}
pub(crate) fn physical_component_ids(
layout: &PhysicalWeightLayout,
) -> Result<BTreeSet<WeightId>, String> {
validate_physical_layout_budget(layout)?;
let mut ids = BTreeSet::new();
let mut stack = vec![layout];
while let Some(node) = stack.pop() {
let mut insert_binding = |binding: &PhysicalWeightComponentBinding| {
ids.insert(binding.component_id.clone());
};
match node {
PhysicalWeightLayout::Dense { component_id } => {
ids.insert(component_id.clone());
}
PhysicalWeightLayout::Stored { component } => insert_binding(component),
PhysicalWeightLayout::Composite { parts } => {
stack.extend(parts.iter().map(|part| part.layout.as_ref()));
}
PhysicalWeightLayout::Quantized {
packed_values,
scales,
zero_points,
axis_indices,
permutation,
codebook,
..
} => {
insert_binding(packed_values);
insert_binding(scales);
if let Some(binding) = zero_points {
insert_binding(binding);
}
if let Some(axis_component) = axis_indices {
insert_binding(&axis_component.component);
}
if let Some(axis_component) = permutation {
insert_binding(&axis_component.component);
}
if let Some(binding) = codebook {
insert_binding(binding);
}
}
PhysicalWeightLayout::QuantizedBlockGrid {
packed_values,
scales,
..
} => {
insert_binding(packed_values);
insert_binding(scales);
}
PhysicalWeightLayout::BlockQuantized { blocks, .. } => insert_binding(blocks),
PhysicalWeightLayout::AxisReshapePermutation { values, .. } => stack.push(values),
PhysicalWeightLayout::Indexed {
indices, values, ..
} => {
insert_binding(&indices.component);
stack.push(values);
}
PhysicalWeightLayout::ExpertStack { experts, .. } => {
stack.extend(experts);
}
}
}
Ok(ids)
}
pub(crate) fn checked_elements(dimensions: &[u64]) -> Option<u64> {
dimensions
.iter()
.try_fold(1_u64, |elements, extent| elements.checked_mul(*extent))
}