microscope_memory/
multimodal.rs

1//! Multi-Modal Memory for Microscope Memory.
2//!
3//! Extends beyond text to store and recall images (perceptual hashes),
4//! audio (spectral fingerprints), and structured data (typed key-value pairs)
5//! within the same spatial coordinate framework.
6//!
7//! Binary format: modalities.bin (MOD1)
8//!
9//! Modalities are stored as a sidecar index — the core BlockHeader (32B mmap'd)
10//! is unchanged. Each entry maps a block_idx to its modality metadata.
11
12use std::collections::HashMap;
13use std::fs;
14use std::path::Path;
15
16// ─── Constants ──────────────────────────────────────
17
18const IMAGE_PAYLOAD_BYTES: usize = 32; // 2+2+8+12+4+4
19const AUDIO_PAYLOAD_BYTES: usize = 32; // 4+2+16+4+4+2
20
21// ─── Modality types ─────────────────────────────────
22
23#[derive(Clone, Debug, PartialEq)]
24pub enum Modality {
25    Text,
26    Image(ImageMeta),
27    Audio(AudioMeta),
28    Structured(StructuredMeta),
29}
30
31impl Modality {
32    pub fn tag(&self) -> u8 {
33        match self {
34            Modality::Text => 0,
35            Modality::Image(_) => 1,
36            Modality::Audio(_) => 2,
37            Modality::Structured(_) => 3,
38        }
39    }
40
41    pub fn name(&self) -> &'static str {
42        match self {
43            Modality::Text => "text",
44            Modality::Image(_) => "image",
45            Modality::Audio(_) => "audio",
46            Modality::Structured(_) => "structured",
47        }
48    }
49}
50
51#[derive(Clone, Debug, PartialEq)]
52pub struct ImageMeta {
53    pub width: u16,
54    pub height: u16,
55    pub phash: [u8; 8],
56    pub color_histogram: [u8; 12],
57    pub content_hash: u32,
58}
59
60#[derive(Clone, Debug, PartialEq)]
61pub struct AudioMeta {
62    pub duration_ms: u32,
63    pub sample_rate: u16,
64    pub spectral_fingerprint: [u8; 16],
65    pub peak_freq: f32,
66    pub bpm_estimate: f32,
67}
68
69#[derive(Clone, Debug, PartialEq)]
70pub struct StructuredMeta {
71    pub fields: Vec<(String, FieldValue)>,
72}
73
74#[derive(Clone, Debug, PartialEq)]
75pub enum FieldValue {
76    Str(String),
77    Int(i64),
78    Float(f64),
79    Bool(bool),
80}
81
82impl FieldValue {
83    pub fn type_tag(&self) -> u8 {
84        match self {
85            FieldValue::Str(_) => 0,
86            FieldValue::Int(_) => 1,
87            FieldValue::Float(_) => 2,
88            FieldValue::Bool(_) => 3,
89        }
90    }
91}
92
93// ─── ModalityIndex ──────────────────────────────────
94
95/// Sidecar index mapping block indices to modality metadata.
96pub struct ModalityIndex {
97    pub entries: HashMap<u32, Modality>,
98}
99
100pub struct ModalityStats {
101    pub total_entries: usize,
102    pub text_count: usize,
103    pub image_count: usize,
104    pub audio_count: usize,
105    pub structured_count: usize,
106}
107
108impl ModalityIndex {
109    pub fn load_or_init(output_dir: &Path) -> Self {
110        let path = output_dir.join("modalities.bin");
111        if let Ok(data) = fs::read(&path) {
112            if data.len() >= 8 && &data[0..4] == b"MOD1" {
113                let entry_count = read_u32(&data, 4) as usize;
114                let mut entries = HashMap::new();
115                let mut pos = 8;
116
117                for _ in 0..entry_count {
118                    if pos + 7 > data.len() {
119                        break;
120                    }
121                    let block_idx = read_u32(&data, pos);
122                    let modality_tag = data[pos + 4];
123                    let payload_len = read_u16(&data, pos + 5) as usize;
124                    pos += 7;
125
126                    if pos + payload_len > data.len() {
127                        break;
128                    }
129
130                    let modality = match modality_tag {
131                        0 => Modality::Text,
132                        1 if payload_len >= IMAGE_PAYLOAD_BYTES => {
133                            let m = decode_image_meta(&data[pos..pos + payload_len]);
134                            Modality::Image(m)
135                        }
136                        2 if payload_len >= AUDIO_PAYLOAD_BYTES => {
137                            let m = decode_audio_meta(&data[pos..pos + payload_len]);
138                            Modality::Audio(m)
139                        }
140                        3 => {
141                            if let Some(m) = decode_structured_meta(&data[pos..pos + payload_len]) {
142                                Modality::Structured(m)
143                            } else {
144                                pos += payload_len;
145                                continue;
146                            }
147                        }
148                        _ => {
149                            pos += payload_len;
150                            continue;
151                        }
152                    };
153
154                    entries.insert(block_idx, modality);
155                    pos += payload_len;
156                }
157
158                return Self { entries };
159            }
160        }
161        Self {
162            entries: HashMap::new(),
163        }
164    }
165
166    pub fn save(&self, output_dir: &Path) -> Result<(), String> {
167        let path = output_dir.join("modalities.bin");
168        let mut buf = Vec::new();
169        buf.extend_from_slice(b"MOD1");
170        buf.extend_from_slice(&(self.entries.len() as u32).to_le_bytes());
171
172        for (&block_idx, modality) in &self.entries {
173            buf.extend_from_slice(&block_idx.to_le_bytes());
174            buf.push(modality.tag());
175
176            match modality {
177                Modality::Text => {
178                    buf.extend_from_slice(&0u16.to_le_bytes());
179                }
180                Modality::Image(m) => {
181                    buf.extend_from_slice(&(IMAGE_PAYLOAD_BYTES as u16).to_le_bytes());
182                    encode_image_meta(m, &mut buf);
183                }
184                Modality::Audio(m) => {
185                    buf.extend_from_slice(&(AUDIO_PAYLOAD_BYTES as u16).to_le_bytes());
186                    encode_audio_meta(m, &mut buf);
187                }
188                Modality::Structured(m) => {
189                    let payload = encode_structured_meta(m);
190                    buf.extend_from_slice(&(payload.len() as u16).to_le_bytes());
191                    buf.extend_from_slice(&payload);
192                }
193            }
194        }
195
196        fs::write(&path, &buf).map_err(|e| format!("write modalities.bin: {}", e))
197    }
198
199    /// Register a block's modality.
200    pub fn register(&mut self, block_idx: u32, modality: Modality) {
201        self.entries.insert(block_idx, modality);
202    }
203
204    /// Get modality for a block.
205    pub fn get(&self, block_idx: u32) -> Option<&Modality> {
206        self.entries.get(&block_idx)
207    }
208
209    /// Search by perceptual hash similarity (hamming distance).
210    /// Returns Vec of (block_idx, hamming_distance), sorted by distance.
211    pub fn search_image_similar(&self, phash: &[u8; 8], max_distance: u32) -> Vec<(u32, u32)> {
212        let mut results: Vec<(u32, u32)> = self
213            .entries
214            .iter()
215            .filter_map(|(&idx, m)| {
216                if let Modality::Image(img) = m {
217                    let dist = hamming_distance(&img.phash, phash);
218                    if dist <= max_distance {
219                        Some((idx, dist))
220                    } else {
221                        None
222                    }
223                } else {
224                    None
225                }
226            })
227            .collect();
228        results.sort_by_key(|&(_, d)| d);
229        results
230    }
231
232    /// Search by spectral fingerprint similarity.
233    /// Returns Vec of (block_idx, similarity_score), sorted by score descending.
234    pub fn search_audio_similar(&self, fingerprint: &[u8; 16], threshold: f32) -> Vec<(u32, f32)> {
235        let mut results: Vec<(u32, f32)> = self
236            .entries
237            .iter()
238            .filter_map(|(&idx, m)| {
239                if let Modality::Audio(aud) = m {
240                    let sim = spectral_similarity(&aud.spectral_fingerprint, fingerprint);
241                    if sim >= threshold {
242                        Some((idx, sim))
243                    } else {
244                        None
245                    }
246                } else {
247                    None
248                }
249            })
250            .collect();
251        results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
252        results
253    }
254
255    /// Search structured data by field name and value.
256    pub fn search_structured(&self, field: &str, value: &FieldValue) -> Vec<u32> {
257        self.entries
258            .iter()
259            .filter_map(|(&idx, m)| {
260                if let Modality::Structured(s) = m {
261                    if s.fields.iter().any(|(k, v)| k == field && v == value) {
262                        Some(idx)
263                    } else {
264                        None
265                    }
266                } else {
267                    None
268                }
269            })
270            .collect()
271    }
272
273    /// Compute spatial coordinates for a modality.
274    /// Images: from phash, Audio: from spectral features, Structured: from field hashing.
275    pub fn modality_coords(modality: &Modality) -> (f32, f32, f32) {
276        match modality {
277            Modality::Text => (0.0, 0.0, 0.0), // handled by normal content_coords
278            Modality::Image(m) => {
279                // Derive from phash bytes
280                let x = (m.phash[0] as f32 + m.phash[1] as f32) / 510.0 * 0.25;
281                let y = (m.phash[2] as f32 + m.phash[3] as f32) / 510.0 * 0.25;
282                let z = (m.phash[4] as f32 + m.phash[5] as f32) / 510.0 * 0.25;
283                // Offset into associative region
284                (0.3 + x, 0.0 + y, 0.0 + z)
285            }
286            Modality::Audio(m) => {
287                // Derive from spectral features
288                let x = (m.peak_freq / 20000.0).clamp(0.0, 1.0) * 0.25;
289                let y = (m.bpm_estimate / 300.0).clamp(0.0, 1.0) * 0.25;
290                let z = (m.duration_ms as f32 / 600_000.0).clamp(0.0, 1.0) * 0.25;
291                // Offset into echo_cache region
292                (0.0 + x, 0.3 + y, 0.3 + z)
293            }
294            Modality::Structured(m) => {
295                // Hash field names for coordinates
296                let mut h: u64 = 0xcbf29ce484222325;
297                for (k, _) in &m.fields {
298                    for &b in k.as_bytes() {
299                        h = h.wrapping_mul(0x100000001b3) ^ b as u64;
300                    }
301                }
302                let x = ((h & 0xFFFF) as f32 / 65535.0) * 0.25;
303                let y = (((h >> 16) & 0xFFFF) as f32 / 65535.0) * 0.25;
304                let z = (((h >> 32) & 0xFFFF) as f32 / 65535.0) * 0.25;
305                // Offset into rust_state region
306                (0.15 + x, 0.0 + y, 0.15 + z)
307            }
308        }
309    }
310
311    pub fn stats(&self) -> ModalityStats {
312        let mut text = 0;
313        let mut image = 0;
314        let mut audio = 0;
315        let mut structured = 0;
316        for m in self.entries.values() {
317            match m {
318                Modality::Text => text += 1,
319                Modality::Image(_) => image += 1,
320                Modality::Audio(_) => audio += 1,
321                Modality::Structured(_) => structured += 1,
322            }
323        }
324        ModalityStats {
325            total_entries: self.entries.len(),
326            text_count: text,
327            image_count: image,
328            audio_count: audio,
329            structured_count: structured,
330        }
331    }
332}
333
334// ─── Perceptual hashing ─────────────────────────────
335
336/// Compute perceptual hash (dHash-like) from grayscale pixel data.
337/// Expects row-major grayscale pixels.
338pub fn compute_phash(pixels: &[u8], width: u32, height: u32) -> [u8; 8] {
339    // Downsample to 9x8 grid and compute differences
340    let mut hash = [0u8; 8];
341    if width < 2 || height < 2 || pixels.len() < (width * height) as usize {
342        return hash;
343    }
344
345    let mut bit_idx = 0;
346    for row in 0..8u32 {
347        let src_y = (row * height / 8).min(height - 1);
348        for col in 0..8u32 {
349            let src_x1 = (col * width / 8).min(width - 1);
350            let src_x2 = ((col + 1) * width / 8).min(width - 1);
351            let p1 = pixels[(src_y * width + src_x1) as usize];
352            let p2 = pixels[(src_y * width + src_x2) as usize];
353            if p1 > p2 {
354                hash[bit_idx / 8] |= 1 << (bit_idx % 8);
355            }
356            bit_idx += 1;
357        }
358    }
359    hash
360}
361
362/// Simple spectral fingerprint from audio samples.
363/// Divides into 16 frequency bands and computes energy per band.
364pub fn compute_spectral_fingerprint(samples: &[f32], _sample_rate: u32) -> [u8; 16] {
365    let mut fingerprint = [0u8; 16];
366    if samples.is_empty() {
367        return fingerprint;
368    }
369
370    let band_size = (samples.len() / 16).max(1);
371    for (i, fp_byte) in fingerprint.iter_mut().enumerate() {
372        let start = i * band_size;
373        let end = ((i + 1) * band_size).min(samples.len());
374        let energy: f32 =
375            samples[start..end].iter().map(|s| s * s).sum::<f32>() / (end - start) as f32;
376        *fp_byte = (energy.sqrt() * 255.0).clamp(0.0, 255.0) as u8;
377    }
378    fingerprint
379}
380
381/// Hamming distance between two byte arrays.
382pub fn hamming_distance(a: &[u8], b: &[u8]) -> u32 {
383    a.iter()
384        .zip(b.iter())
385        .map(|(x, y)| (x ^ y).count_ones())
386        .sum()
387}
388
389/// Spectral similarity: normalized dot product of fingerprints.
390fn spectral_similarity(a: &[u8; 16], b: &[u8; 16]) -> f32 {
391    let dot: u32 = a
392        .iter()
393        .zip(b.iter())
394        .map(|(&x, &y)| x as u32 * y as u32)
395        .sum();
396    let mag_a: f32 = a
397        .iter()
398        .map(|&x| (x as f32) * (x as f32))
399        .sum::<f32>()
400        .sqrt();
401    let mag_b: f32 = b
402        .iter()
403        .map(|&x| (x as f32) * (x as f32))
404        .sum::<f32>()
405        .sqrt();
406    if mag_a < 0.001 || mag_b < 0.001 {
407        return 0.0;
408    }
409    dot as f32 / (mag_a * mag_b)
410}
411
412// ─── Binary encoding helpers ────────────────────────
413
414fn encode_image_meta(m: &ImageMeta, buf: &mut Vec<u8>) {
415    buf.extend_from_slice(&m.width.to_le_bytes());
416    buf.extend_from_slice(&m.height.to_le_bytes());
417    buf.extend_from_slice(&m.phash);
418    buf.extend_from_slice(&m.color_histogram);
419    buf.extend_from_slice(&m.content_hash.to_le_bytes());
420    // pad to IMAGE_PAYLOAD_BYTES
421    let written = 2 + 2 + 8 + 12 + 4; // 28
422    for _ in written..IMAGE_PAYLOAD_BYTES {
423        buf.push(0);
424    }
425}
426
427fn decode_image_meta(data: &[u8]) -> ImageMeta {
428    let width = u16::from_le_bytes(data[0..2].try_into().unwrap());
429    let height = u16::from_le_bytes(data[2..4].try_into().unwrap());
430    let mut phash = [0u8; 8];
431    phash.copy_from_slice(&data[4..12]);
432    let mut color_histogram = [0u8; 12];
433    color_histogram.copy_from_slice(&data[12..24]);
434    let content_hash = u32::from_le_bytes(data[24..28].try_into().unwrap());
435    ImageMeta {
436        width,
437        height,
438        phash,
439        color_histogram,
440        content_hash,
441    }
442}
443
444fn encode_audio_meta(m: &AudioMeta, buf: &mut Vec<u8>) {
445    buf.extend_from_slice(&m.duration_ms.to_le_bytes());
446    buf.extend_from_slice(&m.sample_rate.to_le_bytes());
447    buf.extend_from_slice(&m.spectral_fingerprint);
448    buf.extend_from_slice(&m.peak_freq.to_le_bytes());
449    buf.extend_from_slice(&m.bpm_estimate.to_le_bytes());
450    // pad to AUDIO_PAYLOAD_BYTES
451    let written = 4 + 2 + 16 + 4 + 4; // 30
452    for _ in written..AUDIO_PAYLOAD_BYTES {
453        buf.push(0);
454    }
455}
456
457fn decode_audio_meta(data: &[u8]) -> AudioMeta {
458    let duration_ms = u32::from_le_bytes(data[0..4].try_into().unwrap());
459    let sample_rate = u16::from_le_bytes(data[4..6].try_into().unwrap());
460    let mut spectral_fingerprint = [0u8; 16];
461    spectral_fingerprint.copy_from_slice(&data[6..22]);
462    let peak_freq = f32::from_le_bytes(data[22..26].try_into().unwrap());
463    let bpm_estimate = f32::from_le_bytes(data[26..30].try_into().unwrap());
464    AudioMeta {
465        duration_ms,
466        sample_rate,
467        spectral_fingerprint,
468        peak_freq,
469        bpm_estimate,
470    }
471}
472
473fn encode_structured_meta(m: &StructuredMeta) -> Vec<u8> {
474    let mut buf = Vec::new();
475    buf.push(m.fields.len() as u8);
476    for (key, val) in &m.fields {
477        let key_bytes = key.as_bytes();
478        buf.push(key_bytes.len().min(255) as u8);
479        buf.extend_from_slice(&key_bytes[..key_bytes.len().min(255)]);
480        buf.push(val.type_tag());
481        match val {
482            FieldValue::Str(s) => {
483                let vb = s.as_bytes();
484                buf.push(vb.len().min(255) as u8);
485                buf.extend_from_slice(&vb[..vb.len().min(255)]);
486            }
487            FieldValue::Int(i) => {
488                buf.push(8);
489                buf.extend_from_slice(&i.to_le_bytes());
490            }
491            FieldValue::Float(f) => {
492                buf.push(8);
493                buf.extend_from_slice(&f.to_le_bytes());
494            }
495            FieldValue::Bool(b) => {
496                buf.push(1);
497                buf.push(if *b { 1 } else { 0 });
498            }
499        }
500    }
501    buf
502}
503
504fn decode_structured_meta(data: &[u8]) -> Option<StructuredMeta> {
505    if data.is_empty() {
506        return None;
507    }
508    let field_count = data[0] as usize;
509    let mut pos = 1;
510    let mut fields = Vec::new();
511
512    for _ in 0..field_count {
513        if pos >= data.len() {
514            break;
515        }
516        let key_len = data[pos] as usize;
517        pos += 1;
518        if pos + key_len >= data.len() {
519            break;
520        }
521        let key = String::from_utf8_lossy(&data[pos..pos + key_len]).to_string();
522        pos += key_len;
523
524        if pos + 2 > data.len() {
525            break;
526        }
527        let type_tag = data[pos];
528        let val_len = data[pos + 1] as usize;
529        pos += 2;
530
531        if pos + val_len > data.len() {
532            break;
533        }
534
535        let value = match type_tag {
536            0 => FieldValue::Str(String::from_utf8_lossy(&data[pos..pos + val_len]).to_string()),
537            1 if val_len == 8 => {
538                FieldValue::Int(i64::from_le_bytes(data[pos..pos + 8].try_into().unwrap()))
539            }
540            2 if val_len == 8 => {
541                FieldValue::Float(f64::from_le_bytes(data[pos..pos + 8].try_into().unwrap()))
542            }
543            3 if val_len >= 1 => FieldValue::Bool(data[pos] != 0),
544            _ => {
545                pos += val_len;
546                continue;
547            }
548        };
549
550        fields.push((key, value));
551        pos += val_len;
552    }
553
554    Some(StructuredMeta { fields })
555}
556
557fn read_u32(b: &[u8], off: usize) -> u32 {
558    u32::from_le_bytes(b[off..off + 4].try_into().unwrap())
559}
560fn read_u16(b: &[u8], off: usize) -> u16 {
561    u16::from_le_bytes(b[off..off + 2].try_into().unwrap())
562}
563
564// ─── Tests ──────────────────────────────────────────
565
566#[cfg(test)]
567mod tests {
568    use super::*;
569
570    #[test]
571    fn test_phash_deterministic() {
572        let pixels = vec![128u8; 64 * 64];
573        let h1 = compute_phash(&pixels, 64, 64);
574        let h2 = compute_phash(&pixels, 64, 64);
575        assert_eq!(h1, h2);
576    }
577
578    #[test]
579    fn test_phash_different_images() {
580        // Ascending gradient — many left<right transitions
581        let mut pixels1 = vec![0u8; 64 * 64];
582        for (i, p) in pixels1.iter_mut().enumerate() {
583            *p = ((i % 64) * 4) as u8; // 0..252 across each row
584        }
585        // Descending gradient — many left>right transitions
586        let mut pixels2 = vec![0u8; 64 * 64];
587        for (i, p) in pixels2.iter_mut().enumerate() {
588            *p = (255 - (i % 64) * 4) as u8;
589        }
590        let h1 = compute_phash(&pixels1, 64, 64);
591        let h2 = compute_phash(&pixels2, 64, 64);
592        assert_ne!(h1, h2);
593    }
594
595    #[test]
596    fn test_hamming_distance() {
597        let a = [0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00];
598        let b = [0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00];
599        assert_eq!(hamming_distance(&a, &b), 0);
600
601        let c = [0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF];
602        assert_eq!(hamming_distance(&a, &c), 64); // all bits differ
603    }
604
605    #[test]
606    fn test_spectral_fingerprint_deterministic() {
607        let samples: Vec<f32> = (0..1000).map(|i| (i as f32 * 0.01).sin()).collect();
608        let fp1 = compute_spectral_fingerprint(&samples, 44100);
609        let fp2 = compute_spectral_fingerprint(&samples, 44100);
610        assert_eq!(fp1, fp2);
611    }
612
613    #[test]
614    fn test_modality_coords() {
615        let text_coords = ModalityIndex::modality_coords(&Modality::Text);
616        assert_eq!(text_coords, (0.0, 0.0, 0.0));
617
618        let img = Modality::Image(ImageMeta {
619            width: 100,
620            height: 100,
621            phash: [128, 64, 32, 16, 8, 4, 2, 1],
622            color_histogram: [0; 12],
623            content_hash: 42,
624        });
625        let img_coords = ModalityIndex::modality_coords(&img);
626        // Should be in associative region (0.3+, ...)
627        assert!(img_coords.0 >= 0.3);
628
629        let aud = Modality::Audio(AudioMeta {
630            duration_ms: 30000,
631            sample_rate: 44100,
632            spectral_fingerprint: [0; 16],
633            peak_freq: 440.0,
634            bpm_estimate: 120.0,
635        });
636        let aud_coords = ModalityIndex::modality_coords(&aud);
637        // Should be in echo_cache region
638        assert!(aud_coords.1 >= 0.3);
639    }
640
641    #[test]
642    fn test_image_meta_roundtrip() {
643        let meta = ImageMeta {
644            width: 1920,
645            height: 1080,
646            phash: [1, 2, 3, 4, 5, 6, 7, 8],
647            color_histogram: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120],
648            content_hash: 0xDEADBEEF,
649        };
650        let mut buf = Vec::new();
651        encode_image_meta(&meta, &mut buf);
652        let decoded = decode_image_meta(&buf);
653        assert_eq!(meta, decoded);
654    }
655
656    #[test]
657    fn test_audio_meta_roundtrip() {
658        let meta = AudioMeta {
659            duration_ms: 180000,
660            sample_rate: 44100,
661            spectral_fingerprint: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
662            peak_freq: 440.0,
663            bpm_estimate: 120.5,
664        };
665        let mut buf = Vec::new();
666        encode_audio_meta(&meta, &mut buf);
667        let decoded = decode_audio_meta(&buf);
668        assert_eq!(meta, decoded);
669    }
670
671    #[test]
672    fn test_structured_meta_roundtrip() {
673        let meta = StructuredMeta {
674            fields: vec![
675                ("name".to_string(), FieldValue::Str("test".to_string())),
676                ("count".to_string(), FieldValue::Int(42)),
677                ("ratio".to_string(), FieldValue::Float(3.125)),
678                ("active".to_string(), FieldValue::Bool(true)),
679            ],
680        };
681        let encoded = encode_structured_meta(&meta);
682        let decoded = decode_structured_meta(&encoded).unwrap();
683        assert_eq!(meta, decoded);
684    }
685
686    #[test]
687    fn test_search_image_similar() {
688        let mut index = ModalityIndex {
689            entries: HashMap::new(),
690        };
691        index.register(
692            0,
693            Modality::Image(ImageMeta {
694                width: 100,
695                height: 100,
696                phash: [0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF],
697                color_histogram: [0; 12],
698                content_hash: 1,
699            }),
700        );
701        index.register(
702            1,
703            Modality::Image(ImageMeta {
704                width: 100,
705                height: 100,
706                phash: [0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE], // 1 bit diff
707                color_histogram: [0; 12],
708                content_hash: 2,
709            }),
710        );
711        index.register(
712            2,
713            Modality::Image(ImageMeta {
714                width: 100,
715                height: 100,
716                phash: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], // max diff
717                color_histogram: [0; 12],
718                content_hash: 3,
719            }),
720        );
721
722        let target = [0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF];
723        let results = index.search_image_similar(&target, 5);
724        assert_eq!(results.len(), 2); // blocks 0 and 1
725        assert_eq!(results[0].1, 0); // exact match first
726        assert_eq!(results[1].1, 1); // 1 bit diff second
727    }
728
729    #[test]
730    fn test_search_structured_by_field() {
731        let mut index = ModalityIndex {
732            entries: HashMap::new(),
733        };
734        index.register(
735            0,
736            Modality::Structured(StructuredMeta {
737                fields: vec![("type".to_string(), FieldValue::Str("report".to_string()))],
738            }),
739        );
740        index.register(
741            1,
742            Modality::Structured(StructuredMeta {
743                fields: vec![("type".to_string(), FieldValue::Str("note".to_string()))],
744            }),
745        );
746
747        let results = index.search_structured("type", &FieldValue::Str("report".to_string()));
748        assert_eq!(results.len(), 1);
749        assert_eq!(results[0], 0);
750    }
751
752    #[test]
753    fn test_modality_index_save_load() {
754        let tmp = tempfile::tempdir().expect("tempdir");
755        let mut index = ModalityIndex {
756            entries: HashMap::new(),
757        };
758
759        index.register(
760            0,
761            Modality::Image(ImageMeta {
762                width: 640,
763                height: 480,
764                phash: [1, 2, 3, 4, 5, 6, 7, 8],
765                color_histogram: [10; 12],
766                content_hash: 42,
767            }),
768        );
769        index.register(
770            1,
771            Modality::Audio(AudioMeta {
772                duration_ms: 60000,
773                sample_rate: 44100,
774                spectral_fingerprint: [5; 16],
775                peak_freq: 440.0,
776                bpm_estimate: 120.0,
777            }),
778        );
779        index.register(
780            2,
781            Modality::Structured(StructuredMeta {
782                fields: vec![("key".to_string(), FieldValue::Int(99))],
783            }),
784        );
785
786        index.save(tmp.path()).unwrap();
787        let loaded = ModalityIndex::load_or_init(tmp.path());
788        assert_eq!(loaded.entries.len(), 3);
789        assert!(matches!(loaded.get(0), Some(Modality::Image(_))));
790        assert!(matches!(loaded.get(1), Some(Modality::Audio(_))));
791        assert!(matches!(loaded.get(2), Some(Modality::Structured(_))));
792    }
793}
microscope_memory/multimodal.rs

microscope_memory/
multimodal.rs
