use super::types::{
encoder_capacity, fusion_supports, layer_modality, FusionOp, Modality, ModalityEncoder,
MultimodalLayer,
};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fmt;
pub const DEFAULT_MAX_CAPACITY_RATIO: f64 = 8.0;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub enum MultimodalValidationError {
NoModalities,
DuplicateModality {
modality: Modality,
},
MissingEncoder {
modality: Modality,
},
EmptyEncoder {
modality: Modality,
},
ForeignLayerInEncoder {
modality: Modality,
layer: MultimodalLayer,
},
UndeclaredEncoder {
modality: Modality,
},
EmptyHead,
FusionModalityMismatch {
fusion: FusionOp,
modalities: usize,
},
CapacityImbalance {
ratio: f64,
max_ratio: f64,
},
}
impl fmt::Display for MultimodalValidationError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
MultimodalValidationError::NoModalities => {
write!(f, "architecture declares no modalities")
}
MultimodalValidationError::DuplicateModality { modality } => {
write!(f, "modality {} is declared more than once", modality)
}
MultimodalValidationError::MissingEncoder { modality } => {
write!(f, "modality {} has no encoder before fusion", modality)
}
MultimodalValidationError::EmptyEncoder { modality } => {
write!(f, "modality {} has an empty encoder", modality)
}
MultimodalValidationError::ForeignLayerInEncoder { modality, layer } => {
write!(
f,
"encoder for {} contains foreign layer {:?}",
modality, layer
)
}
MultimodalValidationError::UndeclaredEncoder { modality } => {
write!(f, "encoder present for undeclared modality {}", modality)
}
MultimodalValidationError::EmptyHead => write!(f, "post-fusion head is empty"),
MultimodalValidationError::FusionModalityMismatch { fusion, modalities } => write!(
f,
"fusion {:?} is incompatible with {} modalities",
fusion, modalities
),
MultimodalValidationError::CapacityImbalance { ratio, max_ratio } => write!(
f,
"modality capacity ratio {:.3} exceeds maximum {:.3}",
ratio, max_ratio
),
}
}
}
impl std::error::Error for MultimodalValidationError {}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct MultimodalArchitecture {
pub modalities: Vec<Modality>,
pub encoders: Vec<ModalityEncoder>,
pub fusion: FusionOp,
pub head: Vec<MultimodalLayer>,
pub parameters: HashMap<String, f64>,
pub model_id: String,
}
impl MultimodalArchitecture {
pub fn validate(&self) -> std::result::Result<(), MultimodalValidationError> {
self.validate_with(DEFAULT_MAX_CAPACITY_RATIO)
}
pub fn validate_with(
&self,
max_capacity_ratio: f64,
) -> std::result::Result<(), MultimodalValidationError> {
if self.modalities.is_empty() {
return Err(MultimodalValidationError::NoModalities);
}
for (i, &m) in self.modalities.iter().enumerate() {
if self.modalities[i + 1..].contains(&m) {
return Err(MultimodalValidationError::DuplicateModality { modality: m });
}
}
for &m in &self.modalities {
let matching: Vec<&ModalityEncoder> =
self.encoders.iter().filter(|e| e.modality == m).collect();
match matching.as_slice() {
[] => return Err(MultimodalValidationError::MissingEncoder { modality: m }),
[enc] => {
if enc.layers.is_empty() {
return Err(MultimodalValidationError::EmptyEncoder { modality: m });
}
for &layer in &enc.layers {
if let Some(owner) = layer_modality(&layer) {
if owner != m {
return Err(MultimodalValidationError::ForeignLayerInEncoder {
modality: m,
layer,
});
}
}
}
}
_ => return Err(MultimodalValidationError::DuplicateModality { modality: m }),
}
}
for enc in &self.encoders {
if !self.modalities.contains(&enc.modality) {
return Err(MultimodalValidationError::UndeclaredEncoder {
modality: enc.modality,
});
}
}
if self.head.is_empty() {
return Err(MultimodalValidationError::EmptyHead);
}
let num_modalities = self.modalities.len();
if !fusion_supports(&self.fusion, num_modalities) {
return Err(MultimodalValidationError::FusionModalityMismatch {
fusion: self.fusion,
modalities: num_modalities,
});
}
if num_modalities >= 2 {
let mut min_c = f64::INFINITY;
let mut max_c = f64::NEG_INFINITY;
for &m in &self.modalities {
if let Some(enc) = self.encoders.iter().find(|e| e.modality == m) {
let c = encoder_capacity(enc);
if c < min_c {
min_c = c;
}
if c > max_c {
max_c = c;
}
}
}
if min_c > 0.0 && max_c.is_finite() {
let ratio = max_c / min_c;
if ratio > max_capacity_ratio {
return Err(MultimodalValidationError::CapacityImbalance {
ratio,
max_ratio: max_capacity_ratio,
});
}
}
}
Ok(())
}
pub fn encoder_for(&self, modality: Modality) -> Option<&ModalityEncoder> {
self.encoders.iter().find(|e| e.modality == modality)
}
}