Struct MultiModalEmbedding 

pub struct MultiModalEmbedding {

    pub config: CrossModalConfig,
    pub model_id: Uuid,
    pub text_embeddings: HashMap<String, Array1<f32>>,
    pub kg_embeddings: HashMap<String, Array1<f32>>,
    pub unified_embeddings: HashMap<String, Array1<f32>>,
    pub text_kg_alignments: HashMap<String, String>,
    pub entity_descriptions: HashMap<String, String>,
    pub property_texts: HashMap<String, String>,
    pub multilingual_mappings: HashMap<String, Vec<String>>,
    pub cross_domain_mappings: HashMap<String, String>,
    pub text_encoder: TextEncoder,
    pub kg_encoder: KGEncoder,
    pub alignment_network: AlignmentNetwork,
    pub training_stats: TrainingStats,
    pub model_stats: ModelStats,
    pub is_trained: bool,
}

Multi-modal embedding model for unified representation learning

Fields

config: CrossModalConfig

model_id: Uuid

text_embeddings: HashMap<String, Array1<f32>>

Text embeddings cache

kg_embeddings: HashMap<String, Array1<f32>>

Knowledge graph embeddings cache

unified_embeddings: HashMap<String, Array1<f32>>

Unified cross-modal embeddings

text_kg_alignments: HashMap<String, String>

Cross-modal alignment mappings

entity_descriptions: HashMap<String, String>

Entity descriptions for alignment

property_texts: HashMap<String, String>

Property-text mappings

multilingual_mappings: HashMap<String, Vec<String>>

Multi-language mappings

cross_domain_mappings: HashMap<String, String>

Cross-domain mappings

text_encoder: TextEncoder

Training components

kg_encoder: KGEncoder

alignment_network: AlignmentNetwork

training_stats: TrainingStats

Training statistics

model_stats: ModelStats

is_trained: bool

Implementations

impl MultiModalEmbedding

pub fn new(config: CrossModalConfig) -> Self

Create new multi-modal embedding model

pub fn add_text_kg_alignment(&mut self, text: &str, entity: &str)

Add text-KG alignment pair

pub fn add_entity_description(&mut self, entity: &str, description: &str)

Add entity description

pub fn add_property_text(&mut self, property: &str, text_description: &str)

Add property-text mapping

pub fn add_multilingual_mapping( &mut self, concept: &str, translations: Vec<String>, )

Add multilingual mapping

pub fn add_cross_domain_mapping( &mut self, source_concept: &str, target_concept: &str, )

Add cross-domain mapping

pub async fn generate_unified_embedding( &mut self, text: &str, entity: &str, ) -> Result<Array1<f32>>

Generate unified embedding from text and KG

pub fn get_or_create_kg_embedding(&self, entity: &str) -> Result<Array1<f32>>

Get or create KG embedding for entity

pub fn contrastive_loss( &self, positive_pairs: &[(String, String)], negative_pairs: &[(String, String)], ) -> Result<f32>

Perform contrastive learning

pub async fn zero_shot_prediction( &self, text: &str, candidate_entities: &[String], ) -> Result<Vec<(String, f32)>>

Perform zero-shot learning

pub async fn cross_domain_transfer( &mut self, source_domain: &str, target_domain: &str, ) -> Result<f32>

Cross-domain transfer

pub async fn multilingual_alignment( &self, concept: &str, ) -> Result<Vec<(String, f32)>>

Multi-language alignment

pub fn get_multimodal_stats(&self) -> MultiModalStats

Get multi-modal statistics

pub fn with_few_shot_learning(self, _few_shot_config: FewShotLearning) -> Self

Add few-shot learning capability

pub async fn few_shot_learn( &self, support_examples: &[(String, String, String)], query_examples: &[(String, String)], ) -> Result<Vec<(String, f32)>>

Perform few-shot learning task

pub fn with_real_time_finetuning(self, _rt_config: RealTimeFinetuning) -> Self

Add real-time fine-tuning capability

pub async fn real_time_update( &mut self, text: &str, entity: &str, label: &str, ) -> Result<f32>

Update model with new example in real-time

Trait Implementations

impl Clone for MultiModalEmbedding

fn clone(&self) -> MultiModalEmbedding

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for MultiModalEmbedding

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for MultiModalEmbedding

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl EmbeddingModel for MultiModalEmbedding

fn config(&self) -> &ModelConfig

fn model_id(&self) -> &Uuid

fn model_type(&self) -> &'static str

fn add_triple(&mut self, triple: Triple) -> Result<()>

fn train<'life0, 'async_trait>( &'life0 mut self, epochs: Option<usize>, ) -> Pin<Box<dyn Future<Output = Result<TrainingStats>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

fn get_entity_embedding(&self, entity: &str) -> Result<Vector>

fn get_relation_embedding(&self, relation: &str) -> Result<Vector>

fn score_triple( &self, subject: &str, predicate: &str, object: &str, ) -> Result<f64>

fn predict_objects( &self, subject: &str, predicate: &str, k: usize, ) -> Result<Vec<(String, f64)>>

fn predict_subjects( &self, predicate: &str, object: &str, k: usize, ) -> Result<Vec<(String, f64)>>

fn predict_relations( &self, subject: &str, object: &str, k: usize, ) -> Result<Vec<(String, f64)>>

fn get_entities(&self) -> Vec<String>

fn get_relations(&self) -> Vec<String>

fn get_stats(&self) -> ModelStats

fn save(&self, _path: &str) -> Result<()>

fn load(&mut self, _path: &str) -> Result<()>

fn clear(&mut self)

fn is_trained(&self) -> bool

fn encode<'life0, 'life1, 'async_trait>( &'life0 self, texts: &'life1 [String], ) -> Pin<Box<dyn Future<Output = Result<Vec<Vec<f32>>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Encode text strings into embeddings

impl Serialize for MultiModalEmbedding

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.