1use std::collections::HashMap;
8
9#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
15pub enum Modality {
16 Text,
18 Image,
20 Audio,
22 Code,
24 Structured,
26}
27
28#[derive(Clone, Debug)]
34pub struct ModalityResult {
35 pub result_id: u64,
37 pub modality: Modality,
39 pub score: f64,
41 pub cid: String,
43}
44
45#[derive(Clone, Copy, Debug, PartialEq)]
51pub enum FusionStrategy {
52 ScoreSum,
54 ScoreMax,
56 WeightedSum {
58 text_w: f64,
60 image_w: f64,
62 audio_w: f64,
64 code_w: f64,
66 struct_w: f64,
68 },
69 RankFusion,
73}
74
75#[derive(Clone, Debug)]
81pub struct SearchQuery {
82 pub query_id: u64,
84 pub modalities: Vec<Modality>,
86 pub k: usize,
88}
89
90#[derive(Clone, Debug)]
96pub struct FusedResult {
97 pub result_id: u64,
99 pub fused_score: f64,
101 pub contributing_modalities: Vec<Modality>,
103 pub cid: String,
105}
106
107#[derive(Clone, Debug, Default)]
113pub struct CoordinatorStats {
114 pub total_queries: u64,
116 pub modality_counts: HashMap<Modality, u64>,
118}
119
120impl CoordinatorStats {
121 pub fn most_used_modality(&self) -> Option<Modality> {
124 self.modality_counts
125 .iter()
126 .max_by_key(|(_m, &count)| count)
127 .map(|(&m, _)| m)
128 }
129}
130
131pub struct MultiModalSearchCoordinator {
156 pub stats: CoordinatorStats,
158}
159
160impl MultiModalSearchCoordinator {
161 pub fn new() -> Self {
163 Self {
164 stats: CoordinatorStats::default(),
165 }
166 }
167
168 pub fn fuse_results(
177 &mut self,
178 query: &SearchQuery,
179 per_modality_results: HashMap<Modality, Vec<ModalityResult>>,
180 strategy: FusionStrategy,
181 ) -> Vec<FusedResult> {
182 self.stats.total_queries += 1;
184 for modality in &query.modalities {
185 *self.stats.modality_counts.entry(*modality).or_insert(0) += 1;
186 }
187
188 let mut accumulator: HashMap<u64, Vec<(Modality, f64, usize, String)>> = HashMap::new();
191
192 for (modality, results) in &per_modality_results {
193 for (rank, r) in results.iter().enumerate() {
194 accumulator.entry(r.result_id).or_default().push((
195 *modality,
196 r.score,
197 rank,
198 r.cid.clone(),
199 ));
200 }
201 }
202
203 let mut fused: Vec<FusedResult> = accumulator
205 .into_iter()
206 .map(|(result_id, contributions)| {
207 let fused_score = match strategy {
208 FusionStrategy::ScoreSum => {
209 contributions.iter().map(|(_, score, _, _)| *score).sum()
210 }
211 FusionStrategy::ScoreMax => contributions
212 .iter()
213 .map(|(_, score, _, _)| *score)
214 .fold(f64::NEG_INFINITY, f64::max),
215 FusionStrategy::WeightedSum {
216 text_w,
217 image_w,
218 audio_w,
219 code_w,
220 struct_w,
221 } => contributions
222 .iter()
223 .map(|(modality, score, _, _)| {
224 let weight = match modality {
225 Modality::Text => text_w,
226 Modality::Image => image_w,
227 Modality::Audio => audio_w,
228 Modality::Code => code_w,
229 Modality::Structured => struct_w,
230 };
231 weight * score
232 })
233 .sum(),
234 FusionStrategy::RankFusion => {
235 contributions
237 .iter()
238 .map(|(_, _, rank, _)| 1.0 / (*rank as f64 + 60.0))
239 .sum()
240 }
241 };
242
243 let mut seen_modalities: Vec<Modality> = Vec::new();
245 for (modality, _, _, _) in &contributions {
246 if !seen_modalities.contains(modality) {
247 seen_modalities.push(*modality);
248 }
249 }
250
251 let cid = contributions[0].3.clone();
254
255 FusedResult {
256 result_id,
257 fused_score,
258 contributing_modalities: seen_modalities,
259 cid,
260 }
261 })
262 .collect();
263
264 fused.sort_by(|a, b| {
266 b.fused_score
267 .partial_cmp(&a.fused_score)
268 .unwrap_or(std::cmp::Ordering::Equal)
269 });
270 fused.truncate(query.k);
271
272 fused
273 }
274
275 pub fn stats(&self) -> &CoordinatorStats {
277 &self.stats
278 }
279}
280
281impl Default for MultiModalSearchCoordinator {
282 fn default() -> Self {
283 Self::new()
284 }
285}
286
287#[cfg(test)]
292mod tests {
293 use super::*;
294
295 fn mr(result_id: u64, modality: Modality, score: f64, cid: &str) -> ModalityResult {
297 ModalityResult {
298 result_id,
299 modality,
300 score,
301 cid: cid.to_string(),
302 }
303 }
304
305 fn query(modalities: Vec<Modality>, k: usize) -> SearchQuery {
307 SearchQuery {
308 query_id: 1,
309 modalities,
310 k,
311 }
312 }
313
314 #[test]
318 fn test_score_sum_fuses_two_modalities() {
319 let mut c = MultiModalSearchCoordinator::new();
320 let q = query(vec![Modality::Text, Modality::Image], 10);
321
322 let mut pm: HashMap<Modality, Vec<ModalityResult>> = HashMap::new();
323 pm.insert(Modality::Text, vec![mr(1, Modality::Text, 0.8, "cid1")]);
324 pm.insert(Modality::Image, vec![mr(1, Modality::Image, 0.5, "cid1")]);
325
326 let results = c.fuse_results(&q, pm, FusionStrategy::ScoreSum);
327
328 assert_eq!(results.len(), 1);
329 assert_eq!(results[0].result_id, 1);
330 let expected = 0.8 + 0.5;
331 assert!((results[0].fused_score - expected).abs() < 1e-10);
332 }
333
334 #[test]
338 fn test_score_max_picks_max() {
339 let mut c = MultiModalSearchCoordinator::new();
340 let q = query(vec![Modality::Text, Modality::Audio], 10);
341
342 let mut pm: HashMap<Modality, Vec<ModalityResult>> = HashMap::new();
343 pm.insert(Modality::Text, vec![mr(7, Modality::Text, 0.3, "cid7")]);
344 pm.insert(Modality::Audio, vec![mr(7, Modality::Audio, 0.9, "cid7")]);
345
346 let results = c.fuse_results(&q, pm, FusionStrategy::ScoreMax);
347
348 assert_eq!(results.len(), 1);
349 assert!((results[0].fused_score - 0.9).abs() < 1e-10);
350 }
351
352 #[test]
356 fn test_score_max_single_modality() {
357 let mut c = MultiModalSearchCoordinator::new();
358 let q = query(vec![Modality::Code], 5);
359
360 let mut pm: HashMap<Modality, Vec<ModalityResult>> = HashMap::new();
361 pm.insert(
362 Modality::Code,
363 vec![
364 mr(10, Modality::Code, 0.6, "cidA"),
365 mr(20, Modality::Code, 0.2, "cidB"),
366 ],
367 );
368
369 let results = c.fuse_results(&q, pm, FusionStrategy::ScoreMax);
370 assert_eq!(results[0].result_id, 10);
371 assert!((results[0].fused_score - 0.6).abs() < 1e-10);
372 }
373
374 #[test]
378 fn test_weighted_sum_applies_weights() {
379 let mut c = MultiModalSearchCoordinator::new();
380 let q = query(vec![Modality::Text, Modality::Image], 10);
381
382 let strategy = FusionStrategy::WeightedSum {
383 text_w: 2.0,
384 image_w: 0.5,
385 audio_w: 1.0,
386 code_w: 1.0,
387 struct_w: 1.0,
388 };
389
390 let mut pm: HashMap<Modality, Vec<ModalityResult>> = HashMap::new();
391 pm.insert(Modality::Text, vec![mr(1, Modality::Text, 1.0, "cid1")]);
392 pm.insert(Modality::Image, vec![mr(1, Modality::Image, 1.0, "cid1")]);
393
394 let results = c.fuse_results(&q, pm, strategy);
395
396 assert_eq!(results.len(), 1);
397 assert!((results[0].fused_score - 2.5).abs() < 1e-10);
399 }
400
401 #[test]
405 fn test_weighted_sum_ordering() {
406 let mut c = MultiModalSearchCoordinator::new();
407 let q = query(vec![Modality::Text, Modality::Audio], 10);
408
409 let strategy = FusionStrategy::WeightedSum {
410 text_w: 1.0,
411 image_w: 1.0,
412 audio_w: 10.0,
413 code_w: 1.0,
414 struct_w: 1.0,
415 };
416
417 let mut pm: HashMap<Modality, Vec<ModalityResult>> = HashMap::new();
418 pm.insert(
421 Modality::Text,
422 vec![
423 mr(1, Modality::Text, 0.9, "cid1"),
424 mr(2, Modality::Text, 0.1, "cid2"),
425 ],
426 );
427 pm.insert(
428 Modality::Audio,
429 vec![
430 mr(2, Modality::Audio, 0.9, "cid2"),
431 mr(1, Modality::Audio, 0.0, "cid1"),
432 ],
433 );
434
435 let results = c.fuse_results(&q, pm, strategy);
436
437 assert_eq!(results[0].result_id, 2);
440 }
441
442 #[test]
446 fn test_rank_fusion_reciprocal_rank() {
447 let mut c = MultiModalSearchCoordinator::new();
448 let q = query(vec![Modality::Text], 10);
449
450 let mut pm: HashMap<Modality, Vec<ModalityResult>> = HashMap::new();
452 pm.insert(
453 Modality::Text,
454 vec![
455 mr(1, Modality::Text, 0.9, "cid1"), mr(2, Modality::Text, 0.4, "cid2"), ],
458 );
459
460 let results = c.fuse_results(&q, pm, FusionStrategy::RankFusion);
461
462 assert_eq!(results[0].result_id, 1);
465 assert!((results[0].fused_score - 1.0 / 60.0).abs() < 1e-10);
466 assert!((results[1].fused_score - 1.0 / 61.0).abs() < 1e-10);
467 }
468
469 #[test]
473 fn test_rank_fusion_two_modalities() {
474 let mut c = MultiModalSearchCoordinator::new();
475 let q = query(vec![Modality::Text, Modality::Image], 10);
476
477 let mut pm: HashMap<Modality, Vec<ModalityResult>> = HashMap::new();
478 pm.insert(Modality::Text, vec![mr(42, Modality::Text, 0.5, "cidX")]);
480 pm.insert(Modality::Image, vec![mr(42, Modality::Image, 0.5, "cidX")]);
481
482 let results = c.fuse_results(&q, pm, FusionStrategy::RankFusion);
483
484 assert_eq!(results.len(), 1);
485 let expected = 2.0 / 60.0;
487 assert!((results[0].fused_score - expected).abs() < 1e-10);
488 }
489
490 #[test]
494 fn test_deduplication_merges_same_result_id() {
495 let mut c = MultiModalSearchCoordinator::new();
496 let q = query(vec![Modality::Text, Modality::Image], 10);
497
498 let mut pm: HashMap<Modality, Vec<ModalityResult>> = HashMap::new();
499 pm.insert(Modality::Text, vec![mr(99, Modality::Text, 0.7, "cidZ")]);
500 pm.insert(Modality::Image, vec![mr(99, Modality::Image, 0.3, "cidZ")]);
501
502 let results = c.fuse_results(&q, pm, FusionStrategy::ScoreSum);
503
504 assert_eq!(results.len(), 1, "same result_id must be deduplicated");
505 assert_eq!(results[0].result_id, 99);
506 }
507
508 #[test]
512 fn test_top_k_truncation() {
513 let mut c = MultiModalSearchCoordinator::new();
514 let q = query(vec![Modality::Text], 2);
515
516 let mut pm: HashMap<Modality, Vec<ModalityResult>> = HashMap::new();
517 pm.insert(
518 Modality::Text,
519 vec![
520 mr(1, Modality::Text, 0.9, "c1"),
521 mr(2, Modality::Text, 0.8, "c2"),
522 mr(3, Modality::Text, 0.7, "c3"),
523 mr(4, Modality::Text, 0.6, "c4"),
524 ],
525 );
526
527 let results = c.fuse_results(&q, pm, FusionStrategy::ScoreSum);
528
529 assert_eq!(results.len(), 2);
530 assert_eq!(results[0].result_id, 1);
531 assert_eq!(results[1].result_id, 2);
532 }
533
534 #[test]
538 fn test_contributing_modalities_list() {
539 let mut c = MultiModalSearchCoordinator::new();
540 let q = query(vec![Modality::Text, Modality::Image, Modality::Audio], 10);
541
542 let mut pm: HashMap<Modality, Vec<ModalityResult>> = HashMap::new();
543 pm.insert(Modality::Text, vec![mr(5, Modality::Text, 0.5, "c5")]);
544 pm.insert(Modality::Image, vec![mr(5, Modality::Image, 0.4, "c5")]);
545 pm.insert(Modality::Audio, vec![mr(5, Modality::Audio, 0.3, "c5")]);
546
547 let results = c.fuse_results(&q, pm, FusionStrategy::ScoreSum);
548
549 assert_eq!(results.len(), 1);
550 let contribs = &results[0].contributing_modalities;
551 assert_eq!(contribs.len(), 3);
552 assert!(contribs.contains(&Modality::Text));
553 assert!(contribs.contains(&Modality::Image));
554 assert!(contribs.contains(&Modality::Audio));
555 }
556
557 #[test]
561 fn test_contributing_modalities_single() {
562 let mut c = MultiModalSearchCoordinator::new();
563 let q = query(vec![Modality::Code], 10);
564
565 let mut pm: HashMap<Modality, Vec<ModalityResult>> = HashMap::new();
566 pm.insert(Modality::Code, vec![mr(3, Modality::Code, 0.7, "c3")]);
567
568 let results = c.fuse_results(&q, pm, FusionStrategy::ScoreSum);
569
570 assert_eq!(results[0].contributing_modalities.len(), 1);
571 assert_eq!(results[0].contributing_modalities[0], Modality::Code);
572 }
573
574 #[test]
578 fn test_empty_per_modality_returns_empty() {
579 let mut c = MultiModalSearchCoordinator::new();
580 let q = query(vec![Modality::Text], 10);
581
582 let pm: HashMap<Modality, Vec<ModalityResult>> = HashMap::new();
583 let results = c.fuse_results(&q, pm, FusionStrategy::ScoreSum);
584
585 assert!(results.is_empty());
586 }
587
588 #[test]
592 fn test_single_modality_passthrough() {
593 let mut c = MultiModalSearchCoordinator::new();
594 let q = query(vec![Modality::Image], 10);
595
596 let mut pm: HashMap<Modality, Vec<ModalityResult>> = HashMap::new();
597 pm.insert(
598 Modality::Image,
599 vec![
600 mr(10, Modality::Image, 0.2, "a"),
601 mr(20, Modality::Image, 0.8, "b"),
602 mr(30, Modality::Image, 0.5, "c"),
603 ],
604 );
605
606 let results = c.fuse_results(&q, pm, FusionStrategy::ScoreSum);
607
608 assert_eq!(results.len(), 3);
609 assert_eq!(results[0].result_id, 20);
610 assert_eq!(results[1].result_id, 30);
611 assert_eq!(results[2].result_id, 10);
612 }
613
614 #[test]
618 fn test_stats_total_queries_increments() {
619 let mut c = MultiModalSearchCoordinator::new();
620 assert_eq!(c.stats().total_queries, 0);
621
622 let q = query(vec![Modality::Text], 5);
623 c.fuse_results(&q, HashMap::new(), FusionStrategy::ScoreSum);
624 assert_eq!(c.stats().total_queries, 1);
625 c.fuse_results(&q, HashMap::new(), FusionStrategy::ScoreSum);
626 assert_eq!(c.stats().total_queries, 2);
627 }
628
629 #[test]
633 fn test_stats_modality_counts() {
634 let mut c = MultiModalSearchCoordinator::new();
635
636 let q1 = query(vec![Modality::Text, Modality::Image], 5);
637 c.fuse_results(&q1, HashMap::new(), FusionStrategy::ScoreSum);
638
639 let q2 = query(vec![Modality::Text], 5);
640 c.fuse_results(&q2, HashMap::new(), FusionStrategy::ScoreSum);
641
642 let counts = &c.stats().modality_counts;
643 assert_eq!(*counts.get(&Modality::Text).unwrap_or(&0), 2);
644 assert_eq!(*counts.get(&Modality::Image).unwrap_or(&0), 1);
645 assert_eq!(*counts.get(&Modality::Audio).unwrap_or(&0), 0);
646 }
647
648 #[test]
652 fn test_most_used_modality() {
653 let mut c = MultiModalSearchCoordinator::new();
654
655 for _ in 0..3 {
657 let q = query(vec![Modality::Audio], 1);
658 c.fuse_results(&q, HashMap::new(), FusionStrategy::ScoreSum);
659 }
660 for _ in 0..2 {
661 let q = query(vec![Modality::Text], 1);
662 c.fuse_results(&q, HashMap::new(), FusionStrategy::ScoreSum);
663 }
664 let q = query(vec![Modality::Code], 1);
665 c.fuse_results(&q, HashMap::new(), FusionStrategy::ScoreSum);
666
667 assert_eq!(c.stats().most_used_modality(), Some(Modality::Audio));
668 }
669
670 #[test]
674 fn test_most_used_modality_none_when_empty() {
675 let c = MultiModalSearchCoordinator::new();
676 assert_eq!(c.stats().most_used_modality(), None);
677 }
678
679 #[test]
683 fn test_cid_preserved_in_fused_result() {
684 let mut c = MultiModalSearchCoordinator::new();
685 let q = query(vec![Modality::Structured], 5);
686
687 let mut pm: HashMap<Modality, Vec<ModalityResult>> = HashMap::new();
688 pm.insert(
689 Modality::Structured,
690 vec![mr(11, Modality::Structured, 0.5, "bafybeiabc123")],
691 );
692
693 let results = c.fuse_results(&q, pm, FusionStrategy::ScoreSum);
694 assert_eq!(results[0].cid, "bafybeiabc123");
695 }
696
697 #[test]
701 fn test_results_sorted_descending() {
702 let mut c = MultiModalSearchCoordinator::new();
703 let q = query(vec![Modality::Text], 10);
704
705 let scores = [0.3, 0.9, 0.1, 0.7, 0.5];
706 let results_in: Vec<ModalityResult> = scores
707 .iter()
708 .enumerate()
709 .map(|(i, &s)| mr(i as u64, Modality::Text, s, "c"))
710 .collect();
711
712 let mut pm: HashMap<Modality, Vec<ModalityResult>> = HashMap::new();
713 pm.insert(Modality::Text, results_in);
714
715 let results = c.fuse_results(&q, pm, FusionStrategy::ScoreSum);
716
717 assert_eq!(results.len(), 5);
718 for i in 1..results.len() {
719 assert!(
720 results[i - 1].fused_score >= results[i].fused_score,
721 "results not sorted descending at index {}",
722 i
723 );
724 }
725 }
726}