microscope_memory/

lib.rs

#![recursion_limit = "512"]
//! Microscope Memory library interface.
//! Re-exports core types and functions for integration tests and external use.

pub mod archetype;
pub mod attention;
#[cfg(not(target_arch = "wasm32"))]
pub mod build;
#[cfg(not(target_arch = "wasm32"))]
pub mod cache;
pub mod config;
#[cfg(not(target_arch = "wasm32"))]
pub mod doctor;
#[cfg(not(target_arch = "wasm32"))]
pub mod embedding_index;
pub mod embeddings;
#[cfg(not(target_arch = "wasm32"))]
pub mod emotional;
#[cfg(not(target_arch = "wasm32"))]
pub mod emotional_state;
#[cfg(not(target_arch = "wasm32"))]
pub mod federation;
pub mod fingerprint;
pub mod hebbian;
pub mod no_learn;
pub mod parallel;
pub mod sync_guard;
pub mod types;

#[cfg(not(target_arch = "wasm32"))]
pub mod ai_adapter;
#[cfg(not(target_arch = "wasm32"))]
pub mod bridge;
#[cfg(not(target_arch = "wasm32"))]
pub mod mcp;
#[cfg(all(not(target_arch = "wasm32"), feature = "mcp_http"))]
pub mod mcp_http;
pub mod merkle;
pub mod mirror;
#[cfg(not(target_arch = "wasm32"))]
pub mod query;
#[cfg(not(target_arch = "wasm32"))]
pub mod reader;
#[cfg(not(target_arch = "wasm32"))]
pub mod recall_ctx;
pub mod resonance;
#[cfg(not(target_arch = "wasm32"))]
pub mod sequential_thinking;
pub mod snapshot;
#[cfg(not(target_arch = "wasm32"))]
pub mod text_index;

#[cfg(not(target_arch = "wasm32"))]
pub mod consciousness_seqlock;
#[cfg(not(target_arch = "wasm32"))]
pub mod consciousness_stream;
#[cfg(not(target_arch = "wasm32"))]
pub mod daydream;
#[cfg(not(target_arch = "wasm32"))]
pub mod dream;
#[cfg(not(target_arch = "wasm32"))]
pub mod emotion_extraction;
pub mod emotional_contagion;
#[cfg(not(target_arch = "wasm32"))]
pub mod emotional_episode;
#[cfg(not(target_arch = "wasm32"))]
pub mod emotional_gate;
#[cfg(not(target_arch = "wasm32"))]
pub mod eureka;
#[cfg(not(target_arch = "wasm32"))]
#[cfg(not(target_arch = "wasm32"))]
pub mod narrative;
#[cfg(not(target_arch = "wasm32"))]
pub mod pipeline;
#[cfg(not(target_arch = "wasm32"))]
pub mod reconsolidation;
#[cfg(not(target_arch = "wasm32"))]
pub mod salience;
#[cfg(not(target_arch = "wasm32"))]
pub mod spaced_repetition;
#[cfg(not(target_arch = "wasm32"))]
pub mod working_memory;

// New cognitive enhancement modules
#[cfg(not(target_arch = "wasm32"))]
pub mod absentia;
#[cfg(not(target_arch = "wasm32"))]
pub mod advanced_cognition;
#[cfg(not(target_arch = "wasm32"))]
pub mod architecture_generator;
#[cfg(not(target_arch = "wasm32"))]
pub mod architecture_simulator;
#[cfg(not(target_arch = "wasm32"))]
pub mod autopoiesis;
#[cfg(not(target_arch = "wasm32"))]
pub mod chatgpt;
#[cfg(not(target_arch = "wasm32"))]
pub mod code_memory;
#[cfg(not(target_arch = "wasm32"))]
pub mod cognitive_morphogenesis;
#[cfg(not(target_arch = "wasm32"))]
pub mod enforcement;
#[cfg(not(target_arch = "wasm32"))]
pub mod executive;
#[cfg(not(target_arch = "wasm32"))]
pub mod explicit_memory;
#[cfg(not(target_arch = "wasm32"))]
pub mod functional_plasticity;
#[cfg(not(target_arch = "wasm32"))]
pub mod heuristic_decision;
#[cfg(not(target_arch = "wasm32"))]
pub mod hippocampus;
#[cfg(not(target_arch = "wasm32"))]
pub mod hyperfocus;
#[cfg(not(target_arch = "wasm32"))]
pub mod implicit_memory;
#[cfg(not(target_arch = "wasm32"))]
pub mod impulse_control;
#[cfg(not(target_arch = "wasm32"))]
pub mod intent;
pub mod keystore;
#[cfg(not(target_arch = "wasm32"))]
pub mod knowledge_base;
#[cfg(not(target_arch = "wasm32"))]
pub mod mental_sandbox;
#[cfg(not(target_arch = "wasm32"))]
pub mod mental_stimulation;
#[cfg(not(target_arch = "wasm32"))]
pub mod meta_supervision;
pub mod morphogenesis;
pub mod multimodal;
#[cfg(not(target_arch = "wasm32"))]
pub mod neuroplasticity;
#[cfg(not(target_arch = "wasm32"))]
pub mod pattern_recognition;
#[cfg(not(target_arch = "wasm32"))]
pub mod planning;
pub mod predictive_cache;
pub mod relevance;
#[cfg(not(target_arch = "wasm32"))]
pub mod spine;
#[cfg(not(target_arch = "wasm32"))]
pub mod structural_plasticity;
#[cfg(not(target_arch = "wasm32"))]
pub mod synaptic_plasticity;
pub mod temporal_archetype;
pub mod thought_graph;
#[cfg(not(target_arch = "wasm32"))]
pub mod vagus;
#[cfg(not(target_arch = "wasm32"))]
pub mod viz;
pub mod zen_keystore;

#[cfg(target_arch = "wasm32")]
pub mod wasm;

#[cfg(feature = "python")]
#[allow(non_local_definitions)]
pub mod python;

#[cfg(feature = "gpu")]
pub mod gpu;

#[cfg(not(target_arch = "wasm32"))]
pub mod cli;
pub mod open_loops;
pub mod timeline;

// Re-export commonly used items
#[cfg(not(target_arch = "wasm32"))]
pub use emotional_state::{emotional_prime_weight, EmotionalStateRing};
#[cfg(not(target_arch = "wasm32"))]
pub use reader::{
    append_emotion_log, build_emotions_from_log, emotional_similarity, format_emotion,
    load_emotion_lookup, read_append_log, store_memory, store_memory_temporary,
    store_memory_with_embedding, store_memory_with_emotion, DataStore, MicroscopeReader,
    RadialResult, ResultSet, EMOTION_DIMS, EMOTION_VECTOR_SIZE,
};
#[cfg(not(target_arch = "wasm32"))]
pub use types::{AppendEntry, BlockHeader, MemoryBlockHeader, MemoryQueryOptions, ProjectId};

#[cfg(target_arch = "wasm32")]
pub use types::{AppendEntry, BlockHeader, MemoryBlockHeader, MemoryQueryOptions, ProjectId};

// Re-export CLI
#[cfg(not(target_arch = "wasm32"))]
pub use cli::{Cli, Cmd};

// ─── Shared constants ────────────────────────────────
pub const DEFAULT_CONFIG_PATH: &str = "config.toml";

/// Maximum bytes of text stored in one block.
///
/// Raised from 1024 on the evidence of the BEIR SciFact run, and the reason is
/// worth recording because the fix looked like a storage-format change and was
/// not. At 1024, `to_block` truncated 4,300 of the 5,183 abstracts at byte 1,021,
/// so most of that corpus was silently cut. Splitting the tail into further
/// blocks stopped the data loss but cost 9.4 points of R@1 (51.0% -> 41.6%),
/// because a 1,400-character abstract is a better retrieval unit than two
/// ~700-character fragments: the sentence that answers a query gets separated
/// from the title that the query matches against.
///
/// 16 KiB is large enough to hold the longest SciFact abstract (10,127 bytes)
/// whole, so the index stores one block per document and the embedded set
/// matches what the FAISS baseline embeds -- one vector per abstract. It does
/// not mean long inputs are embedded whole: the MiniLM provider truncates at
/// 512 tokens regardless, exactly as the reference encoder does, so the two
/// systems see the same text.
///
/// This was always safe to raise. data.bin is a packed, variable-length file
/// whose real span lives in BlockHeader's data_offset (u32) and data_len (u16),
/// not a fixed-stride grid, so no reader assumes a particular block size. The
/// u16 is the real ceiling: 65,535 bytes. Entries longer than BLOCK_DATA_SIZE
/// are still split rather than truncated, by `build::split_oversized_entry`.
pub const BLOCK_DATA_SIZE: usize = 16384;
pub const HEADER_SIZE: usize = 50;
pub const LEGACY_HEADER_SIZE: usize = 32;
pub const APV3_MAGIC: &[u8] = b"APv3";
pub const META_HEADER_SIZE: usize = 16;
pub const DEPTH_ENTRY_SIZE: usize = 8;
pub const LAYER_NAMES: &[&str] = &[
    "identity",
    "long_term",
    "short_term",
    "associative",
    "emotional",
    "relational",
    "reflections",
    "crypto_chain",
    "echo_cache",
    "rust_state",
    "code",
    "session",
];

// ─── Shared utility functions ────────────────────────

pub fn layer_to_id(name: &str) -> u8 {
    LAYER_NAMES.iter().position(|&n| n == name).unwrap_or(0) as u8
}

/// CRC16-CCITT (poly=0x1021, init=0xFFFF) over arbitrary data.
pub fn crc16_ccitt(data: &[u8]) -> u16 {
    let mut crc: u16 = 0xFFFF;
    for &byte in data {
        crc ^= (byte as u16) << 8;
        for _ in 0..8 {
            if crc & 0x8000 != 0 {
                crc = (crc << 1) ^ 0x1021;
            } else {
                crc <<= 1;
            }
        }
    }
    crc
}

pub fn content_coords(text: &str, layer: &str) -> (f32, f32, f32) {
    let mut h: [u64; 3] = [0xcbf29ce484222325, 0x100000001b3, 0xa5a5a5a5a5a5a5a5];
    for &b in text.as_bytes().iter().take(128) {
        h[0] = h[0].wrapping_mul(0x100000001b3) ^ b as u64;
        h[1] = h[1].wrapping_mul(0x100000001b3) ^ b as u64;
        h[2] = h[2].wrapping_mul(0x1000193) ^ b as u64;
    }
    let bx = (h[0] & 0xFFFF) as f32 / 65535.0;
    let by = (h[1] & 0xFFFF) as f32 / 65535.0;
    let bz = (h[2] & 0xFFFF) as f32 / 65535.0;

    let (ox, oy, oz) = match layer {
        "long_term" => (0.0, 0.0, 0.0),
        "associative" => (0.3, 0.0, 0.0),
        "emotional" => (0.1, 0.3, 0.2),
        "relational" => (0.3, 0.3, 0.0),
        "reflections" => (0.0, 0.0, 0.3),
        "crypto_chain" => (0.3, 0.0, 0.3),
        "echo_cache" => (0.0, 0.3, 0.3),
        "short_term" => (0.15, 0.15, 0.15),
        "rust_state" => (0.15, 0.0, 0.15),
        "session" => (0.0, 0.0, 0.0),
        _ => (0.25, 0.25, 0.25),
    };

    (ox + bx * 0.25, oy + by * 0.25, oz + bz * 0.25)
}

fn semantic_coords(text: &str, weight: f32) -> Option<(f32, f32, f32)> {
    if weight <= 0.0 {
        return None;
    }
    use embeddings::{EmbeddingProvider, MockEmbeddingProvider};
    let provider = MockEmbeddingProvider::new(128);
    if let Ok(emb) = provider.embed(text) {
        if emb.len() >= 3 {
            let sx = (emb[0] + 1.0) / 2.0;
            let sy = (emb[1] + 1.0) / 2.0;
            let sz = (emb[2] + 1.0) / 2.0;
            return Some((sx, sy, sz));
        }
    }
    None
}

pub fn content_coords_blended(text: &str, layer: &str, weight: f32) -> (f32, f32, f32) {
    let (hx, hy, hz) = content_coords(text, layer);
    if weight <= 0.0 {
        return (hx, hy, hz);
    }
    match semantic_coords(text, weight) {
        Some((sx, sy, sz)) => {
            let w = weight.clamp(0.0, 1.0);
            (
                (1.0 - w) * hx + w * sx,
                (1.0 - w) * hy + w * sy,
                (1.0 - w) * hz + w * sz,
            )
        }
        None => (hx, hy, hz),
    }
}

pub fn hex_str(bytes: &[u8]) -> String {
    bytes
        .iter()
        .map(|b| format!("{:02x}", b))
        .collect::<Vec<_>>()
        .join("")
}

pub fn safe_truncate(s: &str, max_bytes: usize) -> String {
    if s.len() <= max_bytes {
        return s.to_string();
    }
    let mut end = max_bytes;
    while end > 0 && !s.is_char_boundary(end) {
        end -= 1;
    }
    s[..end].to_string()
}

pub fn to_block(text: &str) -> Vec<u8> {
    let bytes = text.as_bytes();
    if bytes.len() <= BLOCK_DATA_SIZE {
        bytes.to_vec()
    } else {
        let mut v = bytes[..BLOCK_DATA_SIZE - 3].to_vec();
        v.extend_from_slice(b"...");
        v
    }
}

// ─── AUTO ZOOM / AUTO DEPTH ──────────────────────────

pub fn auto_zoom(query: &str) -> (u8, u8) {
    let stopwords = ["a", "the", "is", "of", "and", "to", "in", "it", "on", "for"];
    let unique_content_words = query
        .to_lowercase()
        .split_whitespace()
        .filter(|w| !stopwords.contains(w) && w.len() > 2)
        .count();

    if unique_content_words <= 1 {
        return (1, 1);
    }
    if unique_content_words <= 3 {
        return (2, 1);
    }
    if unique_content_words <= 6 {
        return (3, 1);
    }
    if unique_content_words <= 10 {
        return (4, 1);
    }
    (5, 1)
}

pub fn auto_depth(text: &str) -> u8 {
    let len = text.len();
    if len >= 100 {
        3
    } else if len >= 40 {
        4
    } else if len >= 15 {
        5
    } else {
        6
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_crc16_ccitt_known_vector() {
        let data = b"123456789";
        assert_eq!(crc16_ccitt(data), 0x29B1);
    }

    #[test]
    fn test_crc16_empty() {
        assert_eq!(crc16_ccitt(b""), 0xFFFF);
    }

    #[test]
    fn test_crc16_deterministic() {
        let a = crc16_ccitt(b"hello world");
        let b = crc16_ccitt(b"hello world");
        assert_eq!(a, b);
        assert_ne!(a, crc16_ccitt(b"hello worl!"));
    }
}

// Additional modules for main binary
#[cfg(not(target_arch = "wasm32"))]
pub mod auto_context;
#[cfg(not(target_arch = "wasm32"))]
pub mod autonomous;
#[cfg(not(target_arch = "wasm32"))]
pub mod curiosity;
pub mod emotional_21d;
pub mod epistemic;
#[cfg(not(target_arch = "wasm32"))]
pub mod inner_monologue;
#[cfg(not(target_arch = "wasm32"))]
pub mod mermaid;
#[cfg(not(target_arch = "wasm32"))]
pub mod narrative_memory;
#[cfg(not(target_arch = "wasm32"))]
pub mod self_model;
#[cfg(not(target_arch = "wasm32"))]
pub mod self_reflect;