use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use crate::checkpoint::TensorDtype;
#[derive(Debug, Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum InputModality {
Text,
Image,
Video,
Audio,
}
impl InputModality {
pub const fn as_str(self) -> &'static str {
match self {
Self::Text => "text",
Self::Image => "image",
Self::Video => "video",
Self::Audio => "audio",
}
}
const fn wire_tag(self) -> u32 {
match self {
Self::Text => 0,
Self::Image => 1,
Self::Video => 2,
Self::Audio => 3,
}
}
fn from_wire_tag(tag: u32) -> Result<Self, PreparedInputError> {
match tag {
0 => Ok(Self::Text),
1 => Ok(Self::Image),
2 => Ok(Self::Video),
3 => Ok(Self::Audio),
_ => Err(PreparedInputError::InvalidWireValue {
field: "modality",
value: tag,
}),
}
}
}
#[derive(Debug, Clone, Copy, Eq, Hash, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum InputPayloadKind {
TokenIds,
Tensor,
Embeddings,
}
impl InputPayloadKind {
pub const fn accepts(self, modality: InputModality) -> bool {
match self {
Self::TokenIds => matches!(modality, InputModality::Text),
Self::Tensor => !matches!(modality, InputModality::Text),
Self::Embeddings => true,
}
}
const fn wire_tag(self) -> u32 {
match self {
Self::TokenIds => 0,
Self::Tensor => 1,
Self::Embeddings => 2,
}
}
fn from_wire_tag(tag: u32) -> Result<Self, PreparedInputError> {
match tag {
0 => Ok(Self::TokenIds),
1 => Ok(Self::Tensor),
2 => Ok(Self::Embeddings),
_ => Err(PreparedInputError::InvalidWireValue {
field: "payload kind",
value: tag,
}),
}
}
}
#[derive(Debug, Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum InputMetadataKey {
PatchGrid,
PatchPositions,
AudioMask,
}
#[derive(Debug, Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum InputExtent {
PatchGrid {
time: usize,
height: usize,
width: usize,
},
AudioValidFrames(usize),
}
impl InputExtent {
pub const fn accepts(self, modality: InputModality) -> bool {
match self {
Self::PatchGrid { .. } => {
matches!(modality, InputModality::Image | InputModality::Video)
}
Self::AudioValidFrames(_) => matches!(modality, InputModality::Audio),
}
}
const fn wire_tag(self) -> u32 {
match self {
Self::PatchGrid { .. } => 0,
Self::AudioValidFrames(_) => 1,
}
}
const fn key(self) -> u32 {
self.wire_tag()
}
fn encode_words(self, output: &mut Vec<u32>) -> Result<(), PreparedInputError> {
output.push(self.wire_tag());
let values: &[usize] = match &self {
Self::PatchGrid {
time,
height,
width,
} => &[*time, *height, *width],
Self::AudioValidFrames(frames) => &[*frames],
};
for value in values {
output.push(
u32::try_from(*value)
.map_err(|_| PreparedInputError::WireValueOverflow("input extent"))?,
);
}
Ok(())
}
fn decode_words(cursor: &mut WordCursor<'_>) -> Result<Self, PreparedInputError> {
match cursor.next("input extent")? {
0 => Ok(Self::PatchGrid {
time: cursor.usize("patch grid time")?,
height: cursor.usize("patch grid height")?,
width: cursor.usize("patch grid width")?,
}),
1 => Ok(Self::AudioValidFrames(cursor.usize("valid audio frames")?)),
value => Err(PreparedInputError::InvalidWireValue {
field: "input extent",
value,
}),
}
}
}
impl InputMetadataKey {
pub const fn accepts(self, modality: InputModality) -> bool {
match self {
Self::PatchGrid | Self::PatchPositions => {
matches!(modality, InputModality::Image | InputModality::Video)
}
Self::AudioMask => matches!(modality, InputModality::Audio),
}
}
const fn wire_tag(self) -> u32 {
match self {
Self::PatchGrid => 0,
Self::PatchPositions => 1,
Self::AudioMask => 2,
}
}
fn from_wire_tag(tag: u32) -> Result<Self, PreparedInputError> {
match tag {
0 => Ok(Self::PatchGrid),
1 => Ok(Self::PatchPositions),
2 => Ok(Self::AudioMask),
_ => Err(PreparedInputError::InvalidWireValue {
field: "metadata key",
value: tag,
}),
}
}
}
#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
pub struct InputTensorIdentity {
dtype: TensorDtype,
shape: Vec<usize>,
}
impl InputTensorIdentity {
pub fn new(dtype: TensorDtype, shape: Vec<usize>) -> Result<Self, PreparedInputError> {
if shape.is_empty() || shape.contains(&0) {
return Err(PreparedInputError::InvalidTensorShape { shape });
}
Ok(Self { dtype, shape })
}
pub const fn dtype(&self) -> &TensorDtype {
&self.dtype
}
pub fn shape(&self) -> &[usize] {
&self.shape
}
fn encode_words(&self, output: &mut Vec<u32>) -> Result<(), PreparedInputError> {
encode_dtype(&self.dtype, output)?;
output.push(
u32::try_from(self.shape.len())
.map_err(|_| PreparedInputError::WireValueOverflow("tensor rank"))?,
);
for dimension in &self.shape {
output.push(
u32::try_from(*dimension)
.map_err(|_| PreparedInputError::WireValueOverflow("tensor dimension"))?,
);
}
Ok(())
}
fn decode_words(cursor: &mut WordCursor<'_>) -> Result<Self, PreparedInputError> {
let dtype = decode_dtype(cursor)?;
let rank = cursor.usize("tensor rank")?;
if rank == 0 || rank > 8 {
return Err(PreparedInputError::InvalidWireRank(rank));
}
let shape = (0..rank)
.map(|_| cursor.usize("tensor dimension"))
.collect::<Result<Vec<_>, _>>()?;
Self::new(dtype, shape)
}
}
#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
pub struct InputPartDescriptor {
modality: InputModality,
payload_kind: InputPayloadKind,
payload: InputTensorIdentity,
metadata: BTreeMap<InputMetadataKey, InputTensorIdentity>,
extents: BTreeMap<u32, InputExtent>,
}
impl InputPartDescriptor {
pub fn new(
modality: InputModality,
payload_kind: InputPayloadKind,
payload: InputTensorIdentity,
metadata: impl IntoIterator<Item = (InputMetadataKey, InputTensorIdentity)>,
) -> Result<Self, PreparedInputError> {
Self::new_with_extents(modality, payload_kind, payload, metadata, [])
}
pub fn new_with_extents(
modality: InputModality,
payload_kind: InputPayloadKind,
payload: InputTensorIdentity,
metadata: impl IntoIterator<Item = (InputMetadataKey, InputTensorIdentity)>,
extents: impl IntoIterator<Item = InputExtent>,
) -> Result<Self, PreparedInputError> {
if !payload_kind.accepts(modality) {
return Err(PreparedInputError::IncompatiblePayload {
modality,
payload: payload_kind,
});
}
let mut typed_metadata = BTreeMap::new();
for (key, identity) in metadata {
if !key.accepts(modality) {
return Err(PreparedInputError::IncompatibleMetadata { modality, key });
}
if typed_metadata.insert(key, identity).is_some() {
return Err(PreparedInputError::DuplicateMetadata { key });
}
}
let mut typed_extents = BTreeMap::new();
for extent in extents {
if !extent.accepts(modality) {
return Err(PreparedInputError::IncompatibleExtent { modality, extent });
}
if typed_extents.insert(extent.key(), extent).is_some() {
return Err(PreparedInputError::DuplicateExtent { extent });
}
}
Ok(Self {
modality,
payload_kind,
payload,
metadata: typed_metadata,
extents: typed_extents,
})
}
pub const fn modality(&self) -> InputModality {
self.modality
}
pub const fn payload_kind(&self) -> InputPayloadKind {
self.payload_kind
}
pub const fn payload(&self) -> &InputTensorIdentity {
&self.payload
}
pub const fn metadata(&self) -> &BTreeMap<InputMetadataKey, InputTensorIdentity> {
&self.metadata
}
pub fn extents(&self) -> impl ExactSizeIterator<Item = InputExtent> + '_ {
self.extents.values().copied()
}
pub fn metadata_value(&self, key: InputMetadataKey) -> Option<&InputTensorIdentity> {
self.metadata.get(&key)
}
pub fn require_metadata(
&self,
part: usize,
key: InputMetadataKey,
) -> Result<&InputTensorIdentity, PreparedInputError> {
self.metadata
.get(&key)
.ok_or(PreparedInputError::MissingMetadata {
part,
modality: self.modality,
key,
})
}
}
#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
pub struct PreparedInputIdentity {
parts: Vec<InputPartDescriptor>,
}
impl PreparedInputIdentity {
pub fn new(parts: Vec<InputPartDescriptor>) -> Result<Self, PreparedInputError> {
if parts.is_empty() {
return Err(PreparedInputError::EmptyInput);
}
Ok(Self { parts })
}
pub fn parts(&self) -> &[InputPartDescriptor] {
&self.parts
}
pub fn len(&self) -> usize {
self.parts.len()
}
pub fn is_empty(&self) -> bool {
self.parts.is_empty()
}
pub fn encode_words(&self) -> Result<Vec<u32>, PreparedInputError> {
let mut output = Vec::new();
output.push(
u32::try_from(self.parts.len())
.map_err(|_| PreparedInputError::WireValueOverflow("part count"))?,
);
for part in &self.parts {
output.extend_from_slice(&[part.modality.wire_tag(), part.payload_kind.wire_tag()]);
part.payload.encode_words(&mut output)?;
output.push(
u32::try_from(part.metadata.len())
.map_err(|_| PreparedInputError::WireValueOverflow("metadata count"))?,
);
for (key, identity) in &part.metadata {
output.push(key.wire_tag());
identity.encode_words(&mut output)?;
}
output.push(
u32::try_from(part.extents.len())
.map_err(|_| PreparedInputError::WireValueOverflow("extent count"))?,
);
for extent in part.extents.values().copied() {
extent.encode_words(&mut output)?;
}
}
Ok(output)
}
pub fn decode_words(words: &[u32]) -> Result<Self, PreparedInputError> {
let mut cursor = WordCursor { words, offset: 0 };
let part_count = cursor.usize("part count")?;
if part_count == 0 {
return Err(PreparedInputError::EmptyInput);
}
let mut parts = Vec::with_capacity(part_count);
for _ in 0..part_count {
let modality = InputModality::from_wire_tag(cursor.next("modality")?)?;
let payload_kind = InputPayloadKind::from_wire_tag(cursor.next("payload kind")?)?;
let payload = InputTensorIdentity::decode_words(&mut cursor)?;
let metadata_count = cursor.usize("metadata count")?;
if metadata_count > 3 {
return Err(PreparedInputError::InvalidMetadataCount(metadata_count));
}
let metadata = (0..metadata_count)
.map(|_| {
let key = InputMetadataKey::from_wire_tag(cursor.next("metadata key")?)?;
Ok((key, InputTensorIdentity::decode_words(&mut cursor)?))
})
.collect::<Result<Vec<_>, PreparedInputError>>()?;
let extent_count = cursor.usize("extent count")?;
if extent_count > 2 {
return Err(PreparedInputError::InvalidExtentCount(extent_count));
}
let extents = (0..extent_count)
.map(|_| InputExtent::decode_words(&mut cursor))
.collect::<Result<Vec<_>, _>>()?;
parts.push(InputPartDescriptor::new_with_extents(
modality,
payload_kind,
payload,
metadata,
extents,
)?);
}
if cursor.offset != words.len() {
return Err(PreparedInputError::TrailingWireValues);
}
Self::new(parts)
}
}
#[derive(Debug, Clone, Eq, PartialEq, thiserror::Error)]
#[non_exhaustive]
pub enum PreparedInputError {
#[error("prepared model input must contain at least one part")]
EmptyInput,
#[error("prepared input tensor has invalid shape {shape:?}")]
InvalidTensorShape {
shape: Vec<usize>,
},
#[error("{modality:?} input is incompatible with {payload:?} payload")]
IncompatiblePayload {
modality: InputModality,
payload: InputPayloadKind,
},
#[error("{key:?} metadata is incompatible with {modality:?} input")]
IncompatibleMetadata {
modality: InputModality,
key: InputMetadataKey,
},
#[error("prepared input contains duplicate {key:?} metadata")]
DuplicateMetadata {
key: InputMetadataKey,
},
#[error("{extent:?} extent is incompatible with {modality:?} input")]
IncompatibleExtent {
modality: InputModality,
extent: InputExtent,
},
#[error("prepared input contains duplicate {extent:?} extent")]
DuplicateExtent {
extent: InputExtent,
},
#[error("prepared input part {part} ({modality:?}) is missing {key:?} metadata")]
MissingMetadata {
part: usize,
modality: InputModality,
key: InputMetadataKey,
},
#[error("prepared-input descriptor has invalid {field} value {value}")]
InvalidWireValue {
field: &'static str,
value: u32,
},
#[error("prepared-input descriptor ended while reading {0}")]
TruncatedWireDescriptor(&'static str),
#[error("prepared-input descriptor tensor rank {0} is outside 1..=8")]
InvalidWireRank(usize),
#[error("prepared-input descriptor metadata count {0} exceeds 3")]
InvalidMetadataCount(usize),
#[error("prepared-input descriptor extent count {0} exceeds 2")]
InvalidExtentCount(usize),
#[error("prepared-input descriptor has trailing values")]
TrailingWireValues,
#[error("prepared-input {0} exceeds descriptor range")]
WireValueOverflow(&'static str),
#[error("prepared-input wire payload has {actual} values; expected {expected}")]
WireValueCount {
expected: usize,
actual: usize,
},
#[error("prepared-input wire payload does not match its identity")]
WireIdentityMismatch,
#[error("encoded dtype {0:?} cannot identify a prepared runtime tensor")]
EncodedRuntimeDtype(String),
#[error("backend prepared-tensor identity failed: {0}")]
BackendTensorIdentity(String),
}
struct WordCursor<'a> {
words: &'a [u32],
offset: usize,
}
impl WordCursor<'_> {
fn next(&mut self, field: &'static str) -> Result<u32, PreparedInputError> {
let value = self
.words
.get(self.offset)
.copied()
.ok_or(PreparedInputError::TruncatedWireDescriptor(field))?;
self.offset += 1;
Ok(value)
}
fn usize(&mut self, field: &'static str) -> Result<usize, PreparedInputError> {
usize::try_from(self.next(field)?).map_err(|_| PreparedInputError::WireValueOverflow(field))
}
}
fn encode_dtype(dtype: &TensorDtype, output: &mut Vec<u32>) -> Result<(), PreparedInputError> {
let tag = match dtype {
TensorDtype::Bool => 0,
TensorDtype::U8 => 1,
TensorDtype::U16 => 2,
TensorDtype::U32 => 3,
TensorDtype::U64 => 4,
TensorDtype::I8 => 5,
TensorDtype::I16 => 6,
TensorDtype::I32 => 7,
TensorDtype::I64 => 8,
TensorDtype::F16 => 9,
TensorDtype::F32 => 10,
TensorDtype::F64 => 11,
TensorDtype::Bf16 => 12,
TensorDtype::Complex64 => 13,
TensorDtype::Encoded(name) => {
return Err(PreparedInputError::EncodedRuntimeDtype(name.clone()))
}
};
output.push(tag);
Ok(())
}
fn decode_dtype(cursor: &mut WordCursor<'_>) -> Result<TensorDtype, PreparedInputError> {
let tag = cursor.next("dtype")?;
match tag {
0 => Ok(TensorDtype::Bool),
1 => Ok(TensorDtype::U8),
2 => Ok(TensorDtype::U16),
3 => Ok(TensorDtype::U32),
4 => Ok(TensorDtype::U64),
5 => Ok(TensorDtype::I8),
6 => Ok(TensorDtype::I16),
7 => Ok(TensorDtype::I32),
8 => Ok(TensorDtype::I64),
9 => Ok(TensorDtype::F16),
10 => Ok(TensorDtype::F32),
11 => Ok(TensorDtype::F64),
12 => Ok(TensorDtype::Bf16),
13 => Ok(TensorDtype::Complex64),
value => Err(PreparedInputError::InvalidWireValue {
field: "dtype",
value,
}),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn tensor(dtype: TensorDtype, shape: &[usize]) -> InputTensorIdentity {
InputTensorIdentity::new(dtype, shape.to_vec()).unwrap()
}
#[test]
fn identity_round_trip_preserves_order_geometry_and_typed_metadata() {
let identity = PreparedInputIdentity::new(vec![
InputPartDescriptor::new(
InputModality::Text,
InputPayloadKind::TokenIds,
tensor(TensorDtype::U32, &[1, 2]),
[],
)
.unwrap(),
InputPartDescriptor::new_with_extents(
InputModality::Image,
InputPayloadKind::Tensor,
tensor(TensorDtype::F32, &[4, 12]),
[(
InputMetadataKey::PatchGrid,
tensor(TensorDtype::I32, &[1, 3]),
)],
[InputExtent::PatchGrid {
time: 1,
height: 2,
width: 2,
}],
)
.unwrap(),
])
.unwrap();
let words = identity.encode_words().unwrap();
assert_eq!(
PreparedInputIdentity::decode_words(&words).unwrap(),
identity
);
assert_eq!(
identity.parts()[1].extents().collect::<Vec<_>>(),
[InputExtent::PatchGrid {
time: 1,
height: 2,
width: 2,
}]
);
}
#[test]
fn rejects_duplicate_missing_and_modality_incompatible_metadata() {
let grid = tensor(TensorDtype::I32, &[1, 3]);
let duplicate = InputPartDescriptor::new(
InputModality::Image,
InputPayloadKind::Tensor,
tensor(TensorDtype::F32, &[2, 4]),
[
(InputMetadataKey::PatchGrid, grid.clone()),
(InputMetadataKey::PatchGrid, grid.clone()),
],
);
assert!(matches!(
duplicate,
Err(PreparedInputError::DuplicateMetadata { .. })
));
let image = InputPartDescriptor::new(
InputModality::Image,
InputPayloadKind::Tensor,
tensor(TensorDtype::F32, &[2, 4]),
[],
)
.unwrap();
assert!(matches!(
image.require_metadata(0, InputMetadataKey::PatchGrid),
Err(PreparedInputError::MissingMetadata { .. })
));
assert!(matches!(
InputPartDescriptor::new(
InputModality::Audio,
InputPayloadKind::Tensor,
tensor(TensorDtype::F32, &[2, 4]),
[(InputMetadataKey::PatchGrid, grid)]
),
Err(PreparedInputError::IncompatibleMetadata { .. })
));
}
#[test]
fn malformed_wire_descriptors_fail_closed() {
assert!(matches!(
PreparedInputIdentity::decode_words(&[]),
Err(PreparedInputError::TruncatedWireDescriptor(_))
));
assert!(matches!(
PreparedInputIdentity::decode_words(&[1, 99]),
Err(PreparedInputError::InvalidWireValue {
field: "modality",
..
})
));
}
}