use super::engine::*;
use crate::core::ModelType;
use crate::utils::{OcrError, Result};
use serde::{Deserialize, Serialize};
use std::path::Path;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HybridConfig {
pub architecture: HybridArchitecture,
pub model_path: String,
pub supported_languages: Vec<LanguageVariant>,
pub input_shape: (u32, u32, u32),
pub confidence_threshold: f32,
pub device: DeviceType,
pub quantization: Option<QuantizationType>,
pub fusion_strategy: FusionStrategy,
pub ensemble_weights: Vec<f32>,
pub use_attention_fusion: bool,
pub use_late_fusion: bool,
pub use_early_fusion: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub enum HybridArchitecture {
CNNLSTM,
CNNTransformer,
ViTLSTM,
CNNViTLSTM,
MultiScaleCNNTransformer,
Custom(String),
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub enum FusionStrategy {
Concatenation,
WeightedAverage,
Attention,
Gating,
MixtureOfExperts,
Custom(String),
}
pub struct HybridModel {
config: HybridConfig,
model_loaded: bool,
components: Vec<ModelComponent>,
fusion_layer: FusionLayer,
output_head: OutputHead,
}
pub struct ModelComponent {
component_type: ComponentType,
model: Box<dyn OcrModel>,
weight: f32,
is_active: bool,
}
#[derive(Debug, Clone)]
pub enum ComponentType {
CNN,
LSTM,
Transformer,
VisionTransformer,
Custom(String),
}
pub struct FusionLayer {
strategy: FusionStrategy,
attention_weights: Option<AttentionWeights>,
gating_weights: Option<GatingWeights>,
mixture_weights: Option<MixtureWeights>,
}
pub struct AttentionWeights {
query_weights: Vec<Vec<f32>>,
key_weights: Vec<Vec<f32>>,
value_weights: Vec<Vec<f32>>,
output_weights: Vec<Vec<f32>>,
}
pub struct GatingWeights {
gate_weights: Vec<Vec<f32>>,
gate_bias: Vec<f32>,
}
pub struct MixtureWeights {
expert_weights: Vec<Vec<f32>>,
gating_network: Vec<Vec<f32>>,
}
pub struct OutputHead {
layers: Vec<OutputLayer>,
num_classes: usize,
}
pub struct OutputLayer {
layer_type: OutputLayerType,
input_size: usize,
output_size: usize,
weights: Vec<Vec<f32>>,
bias: Option<Vec<f32>>,
}
#[derive(Debug, Clone)]
pub enum OutputLayerType {
Dense,
Attention,
Convolution,
LSTM,
}
impl HybridModel {
pub fn new(config: HybridConfig) -> Self {
let components = Self::create_components(&config);
let fusion_layer = FusionLayer::new(&config);
let output_head = OutputHead::new(config.supported_languages.len());
Self {
config,
model_loaded: false,
components,
fusion_layer,
output_head,
}
}
fn create_components(config: &HybridConfig) -> Vec<ModelComponent> {
let mut components = Vec::new();
match config.architecture {
HybridArchitecture::CNNLSTM => {
components.push(ModelComponent {
component_type: ComponentType::CNN,
model: Box::new(create_dummy_model(ModelType::CNN)),
weight: 0.5,
is_active: true,
});
components.push(ModelComponent {
component_type: ComponentType::LSTM,
model: Box::new(create_dummy_model(ModelType::LSTM)),
weight: 0.5,
is_active: true,
});
}
HybridArchitecture::CNNTransformer => {
components.push(ModelComponent {
component_type: ComponentType::CNN,
model: Box::new(create_dummy_model(ModelType::CNN)),
weight: 0.4,
is_active: true,
});
components.push(ModelComponent {
component_type: ComponentType::Transformer,
model: Box::new(create_dummy_model(ModelType::Transformer)),
weight: 0.6,
is_active: true,
});
}
HybridArchitecture::ViTLSTM => {
components.push(ModelComponent {
component_type: ComponentType::VisionTransformer,
model: Box::new(create_dummy_model(ModelType::VisionTransformer)),
weight: 0.6,
is_active: true,
});
components.push(ModelComponent {
component_type: ComponentType::LSTM,
model: Box::new(create_dummy_model(ModelType::LSTM)),
weight: 0.4,
is_active: true,
});
}
HybridArchitecture::CNNViTLSTM => {
components.push(ModelComponent {
component_type: ComponentType::CNN,
model: Box::new(create_dummy_model(ModelType::CNN)),
weight: 0.3,
is_active: true,
});
components.push(ModelComponent {
component_type: ComponentType::VisionTransformer,
model: Box::new(create_dummy_model(ModelType::VisionTransformer)),
weight: 0.4,
is_active: true,
});
components.push(ModelComponent {
component_type: ComponentType::LSTM,
model: Box::new(create_dummy_model(ModelType::LSTM)),
weight: 0.3,
is_active: true,
});
}
_ => {
components.push(ModelComponent {
component_type: ComponentType::CNN,
model: Box::new(create_dummy_model(ModelType::CNN)),
weight: 0.5,
is_active: true,
});
components.push(ModelComponent {
component_type: ComponentType::LSTM,
model: Box::new(create_dummy_model(ModelType::LSTM)),
weight: 0.5,
is_active: true,
});
}
}
components
}
pub fn load_from_file<P: AsRef<Path>>(&mut self, path: P) -> Result<()> {
let path = path.as_ref();
if !path.exists() {
return Err(OcrError::ModelNotFound(format!(
"Model file not found: {}",
path.display()
))
.into());
}
self.model_loaded = true;
Ok(())
}
pub fn config(&self) -> &HybridConfig {
&self.config
}
pub fn is_loaded(&self) -> bool {
self.model_loaded
}
pub fn architecture(&self) -> &HybridArchitecture {
&self.config.architecture
}
pub fn supported_languages(&self) -> &[LanguageVariant] {
&self.config.supported_languages
}
pub fn supports_language(&self, language: &LanguageVariant) -> bool {
self.config.supported_languages.contains(language)
}
fn forward_components(&self, input: &[u8]) -> Result<Vec<RecognitionResult>> {
let mut results = Vec::new();
for component in &self.components {
if component.is_active {
let result = component.model.predict(input)?;
results.push(result);
}
}
Ok(results)
}
fn fuse_outputs(&self, outputs: &[RecognitionResult]) -> Result<RecognitionResult> {
self.fusion_layer.fuse(outputs)
}
}
impl OcrModel for HybridModel {
fn model_type(&self) -> ModelType {
ModelType::Hybrid
}
fn supported_languages(&self) -> Vec<LanguageVariant> {
self.config.supported_languages.clone()
}
fn supports_language(&self, language: &LanguageVariant) -> bool {
self.supports_language(language)
}
fn input_shape(&self) -> (usize, usize, usize) {
let (h, w, c) = self.config.input_shape;
(h as usize, w as usize, c as usize)
}
fn config(&self) -> &ModelConfig {
todo!("Implement proper config storage")
}
fn predict(&self, input: &[u8]) -> Result<RecognitionResult> {
if !self.model_loaded {
return Err(OcrError::ModelNotFound("Model not loaded".to_string()).into());
}
let component_outputs = self.forward_components(input)?;
let fused_output = self.fuse_outputs(&component_outputs)?;
let final_output = self.output_head.process(&fused_output)?;
Ok(final_output)
}
}
impl FusionLayer {
fn new(config: &HybridConfig) -> Self {
let attention_weights = if config.use_attention_fusion {
Some(AttentionWeights::new())
} else {
None
};
let gating_weights = if config.fusion_strategy == FusionStrategy::Gating {
Some(GatingWeights::new())
} else {
None
};
let mixture_weights = if config.fusion_strategy == FusionStrategy::MixtureOfExperts {
Some(MixtureWeights::new())
} else {
None
};
Self {
strategy: config.fusion_strategy.clone(),
attention_weights,
gating_weights,
mixture_weights,
}
}
fn fuse(&self, outputs: &[RecognitionResult]) -> Result<RecognitionResult> {
match self.strategy {
FusionStrategy::Concatenation => self.concatenate_outputs(outputs),
FusionStrategy::WeightedAverage => self.weighted_average_outputs(outputs),
FusionStrategy::Attention => self.attention_fuse_outputs(outputs),
FusionStrategy::Gating => self.gating_fuse_outputs(outputs),
FusionStrategy::MixtureOfExperts => self.mixture_fuse_outputs(outputs),
FusionStrategy::Custom(_) => self.custom_fuse_outputs(outputs),
}
}
fn concatenate_outputs(&self, outputs: &[RecognitionResult]) -> Result<RecognitionResult> {
if outputs.is_empty() {
return Err(OcrError::ModelNotFound("No outputs to fuse".to_string()).into());
}
let mut fused_text = String::new();
let mut total_confidence = 0.0;
let mut total_processing_time = 0;
for (i, output) in outputs.iter().enumerate() {
if i > 0 {
fused_text.push(' ');
}
fused_text.push_str(&output.text);
total_confidence += output.confidence;
total_processing_time += output.processing_time_ms;
}
let avg_confidence = total_confidence / outputs.len() as f32;
Ok(RecognitionResult {
text: fused_text,
confidence: avg_confidence,
bounding_boxes: outputs[0].bounding_boxes.clone(),
character_results: outputs[0].character_results.clone(),
word_results: outputs[0].word_results.clone(),
line_results: outputs[0].line_results.clone(),
language: outputs[0].language.clone(),
model_type: ModelType::Hybrid,
processing_time_ms: total_processing_time,
})
}
fn weighted_average_outputs(&self, outputs: &[RecognitionResult]) -> Result<RecognitionResult> {
if outputs.is_empty() {
return Err(OcrError::ModelNotFound("No outputs to fuse".to_string()).into());
}
let total_weight: f32 = outputs.iter().map(|o| o.confidence).sum();
let mut weighted_text = String::new();
let mut total_processing_time = 0;
for output in outputs {
let weight = output.confidence / total_weight;
if !weighted_text.is_empty() {
weighted_text.push(' ');
}
weighted_text.push_str(&output.text);
total_processing_time += output.processing_time_ms;
}
Ok(RecognitionResult {
text: weighted_text,
confidence: total_weight / outputs.len() as f32,
bounding_boxes: outputs[0].bounding_boxes.clone(),
character_results: outputs[0].character_results.clone(),
word_results: outputs[0].word_results.clone(),
line_results: outputs[0].line_results.clone(),
language: outputs[0].language.clone(),
model_type: ModelType::Hybrid,
processing_time_ms: total_processing_time,
})
}
fn attention_fuse_outputs(&self, outputs: &[RecognitionResult]) -> Result<RecognitionResult> {
self.weighted_average_outputs(outputs)
}
fn gating_fuse_outputs(&self, outputs: &[RecognitionResult]) -> Result<RecognitionResult> {
self.weighted_average_outputs(outputs)
}
fn mixture_fuse_outputs(&self, outputs: &[RecognitionResult]) -> Result<RecognitionResult> {
self.weighted_average_outputs(outputs)
}
fn custom_fuse_outputs(&self, outputs: &[RecognitionResult]) -> Result<RecognitionResult> {
self.weighted_average_outputs(outputs)
}
}
impl AttentionWeights {
fn new() -> Self {
Self {
query_weights: vec![vec![0.0; 128]; 128],
key_weights: vec![vec![0.0; 128]; 128],
value_weights: vec![vec![0.0; 128]; 128],
output_weights: vec![vec![0.0; 128]; 128],
}
}
}
impl GatingWeights {
fn new() -> Self {
Self {
gate_weights: vec![vec![0.0; 128]; 128],
gate_bias: vec![0.0; 128],
}
}
}
impl MixtureWeights {
fn new() -> Self {
Self {
expert_weights: vec![vec![0.0; 128]; 128],
gating_network: vec![vec![0.0; 128]; 128],
}
}
}
impl OutputHead {
fn new(num_classes: usize) -> Self {
let mut layers = Vec::new();
layers.push(OutputLayer {
layer_type: OutputLayerType::Dense,
input_size: 128,
output_size: 64,
weights: vec![vec![0.0; 128]; 64],
bias: Some(vec![0.0; 64]),
});
layers.push(OutputLayer {
layer_type: OutputLayerType::Dense,
input_size: 64,
output_size: num_classes,
weights: vec![vec![0.0; 64]; num_classes],
bias: Some(vec![0.0; num_classes]),
});
Self {
layers,
num_classes,
}
}
fn process(&self, input: &RecognitionResult) -> Result<RecognitionResult> {
let mut result = input.clone();
result.model_type = ModelType::Hybrid;
Ok(result)
}
}
impl OutputLayer {
fn forward(&self, input: &[f32]) -> Result<Vec<f32>> {
let mut output = vec![0.0; self.output_size];
for (i, row) in self.weights.iter().enumerate() {
for (j, &weight) in row.iter().enumerate() {
if j < input.len() {
output[i] += weight * input[j];
}
}
if let Some(ref bias) = self.bias {
if i < bias.len() {
output[i] += bias[i];
}
}
}
Ok(output)
}
}
fn create_dummy_model(model_type: ModelType) -> impl OcrModel {
struct DummyModel {
model_type: ModelType,
}
impl OcrModel for DummyModel {
fn model_type(&self) -> ModelType {
self.model_type.clone()
}
fn supported_languages(&self) -> Vec<LanguageVariant> {
vec![LanguageVariant::English]
}
fn supports_language(&self, _language: &LanguageVariant) -> bool {
true
}
fn input_shape(&self) -> (usize, usize, usize) {
(224, 224, 3)
}
fn config(&self) -> &ModelConfig {
todo!("Implement proper config storage")
}
fn predict(&self, _input: &[u8]) -> Result<RecognitionResult> {
Ok(RecognitionResult {
text: format!("Dummy {:?} Result", self.model_type),
confidence: 0.8,
bounding_boxes: vec![],
character_results: vec![],
word_results: vec![],
line_results: vec![],
language: Some("en".to_string()),
model_type: self.model_type.clone(),
processing_time_ms: 100,
})
}
}
DummyModel { model_type }
}
pub struct HybridModelBuilder {
config: Option<HybridConfig>,
}
impl HybridModelBuilder {
pub fn new() -> Self {
Self { config: None }
}
pub fn with_config(mut self, config: HybridConfig) -> Self {
self.config = Some(config);
self
}
pub fn build(self) -> Result<HybridModel> {
let config = self
.config
.ok_or_else(|| OcrError::ModelNotFound("Configuration not provided".to_string()))?;
Ok(HybridModel::new(config))
}
}
impl Default for HybridModelBuilder {
fn default() -> Self {
Self::new()
}
}