st/mem8/

format.rs

//! .m8 file format implementation with Markqant compression
//! Achieves 100:1 semantic-preserving compression for AI-native storage

use crate::mem8::wave::MemoryWave;
use anyhow::{anyhow, Result};
use std::collections::HashMap;
use std::io::Write;

/// Magic bytes for .m8 format
const M8_MAGIC: &[u8] = b"M8\x02\x09";

/// Section types in .m8 files
#[derive(Debug, Clone, Copy, PartialEq)]
#[repr(u8)]
pub enum SectionType {
    MarkqantText = 0x09,
    QuantumDirectory = 0x0A,
    WaveMemory = 0x0B,
    Metadata = 0x0C,
    Index = 0x0D,
}

/// .m8 file header
#[derive(Debug)]
pub struct M8Header {
    /// File format version
    pub version: u16,
    /// Number of sections
    pub section_count: u16,
    /// Total file size (excluding CRC)
    pub file_size: u64,
    /// Creation timestamp
    pub timestamp: u64,
}

/// .m8 file section
#[derive(Debug)]
pub struct M8Section {
    /// Section type
    pub section_type: SectionType,
    /// Section size in bytes
    pub size: u32,
    /// Section data
    pub data: Vec<u8>,
}

/// Wave memory compressed to 32 bytes
#[derive(Debug, Clone)]
pub struct CompressedWave {
    pub id: u64,           // 8 bytes - unique identifier
    pub amplitude: u8,     // 1 byte - logarithmically quantized
    pub frequency: u16,    // 2 bytes - frequency in Hz
    pub phase: u8,         // 1 byte - phase in radians * 40.74
    pub valence: i8,       // 1 byte - emotional valence * 127
    pub arousal: u8,       // 1 byte - emotional arousal * 255
    pub decay_tau: u16,    // 2 bytes - decay constant in seconds
    pub timestamp: u64,    // 8 bytes - creation time
    pub interference: u64, // 8 bytes - interference pattern hash
}

impl CompressedWave {
    /// Compress a MemoryWave to 32 bytes
    pub fn from_wave(wave: &MemoryWave, id: u64) -> Self {
        Self {
            id,
            amplitude: quantize_amplitude(wave.amplitude),
            frequency: wave.frequency as u16,
            phase: ((wave.phase / std::f32::consts::PI + 1.0) * 127.5) as u8,
            valence: (wave.valence * 127.0) as i8,
            arousal: (wave.arousal * 255.0) as u8,
            decay_tau: wave
                .decay_tau
                .map(|d| d.as_secs() as u16)
                .unwrap_or(u16::MAX),
            timestamp: wave.created_at.elapsed().as_secs(),
            interference: 0, // Calculated separately
        }
    }

    /// Decompress to MemoryWave
    pub fn to_wave(&self) -> MemoryWave {
        let mut wave = MemoryWave::new(self.frequency as f32, dequantize_amplitude(self.amplitude));

        wave.phase = (self.phase as f32 / 127.5 - 1.0) * std::f32::consts::PI;
        wave.valence = self.valence as f32 / 127.0;
        wave.arousal = self.arousal as f32 / 255.0;
        wave.decay_tau = if self.decay_tau == u16::MAX {
            None
        } else {
            Some(std::time::Duration::from_secs(self.decay_tau as u64))
        };

        wave
    }

    /// Serialize to bytes
    pub fn to_bytes(&self) -> [u8; 32] {
        let mut bytes = [0u8; 32];
        bytes[0..8].copy_from_slice(&self.id.to_le_bytes());
        bytes[8] = self.amplitude;
        bytes[9..11].copy_from_slice(&self.frequency.to_le_bytes());
        bytes[11] = self.phase;
        bytes[12] = self.valence as u8;
        bytes[13] = self.arousal;
        bytes[14..16].copy_from_slice(&self.decay_tau.to_le_bytes());
        bytes[16..24].copy_from_slice(&self.timestamp.to_le_bytes());
        bytes[24..32].copy_from_slice(&self.interference.to_le_bytes());
        bytes
    }

    /// Deserialize from bytes
    pub fn from_bytes(bytes: &[u8]) -> Result<Self> {
        if bytes.len() != 32 {
            return Err(anyhow!("CompressedWave must be exactly 32 bytes"));
        }

        Ok(Self {
            id: u64::from_le_bytes(bytes[0..8].try_into()?),
            amplitude: bytes[8],
            frequency: u16::from_le_bytes(bytes[9..11].try_into()?),
            phase: bytes[11],
            valence: bytes[12] as i8,
            arousal: bytes[13],
            decay_tau: u16::from_le_bytes(bytes[14..16].try_into()?),
            timestamp: u64::from_le_bytes(bytes[16..24].try_into()?),
            interference: u64::from_le_bytes(bytes[24..32].try_into()?),
        })
    }
}

/// Logarithmic amplitude quantization
fn quantize_amplitude(amplitude: f32) -> u8 {
    if amplitude <= 0.0 {
        0
    } else {
        (32.0 * amplitude.log2()).clamp(0.0, 255.0) as u8
    }
}

/// Inverse logarithmic quantization
fn dequantize_amplitude(quantized: u8) -> f32 {
    if quantized == 0 {
        0.0
    } else {
        2.0_f32.powf(quantized as f32 / 32.0)
    }
}

/// Markqant v2.0 rotating token system
pub struct MarkqantEncoder {
    /// Token assignments (pattern -> token)
    tokens: HashMap<String, u8>,
    /// Reverse mapping (token -> pattern)
    patterns: HashMap<u8, String>,
    /// Pattern frequencies
    frequencies: HashMap<String, usize>,
    /// Next available token
    next_token: u8,
}

impl Default for MarkqantEncoder {
    fn default() -> Self {
        Self::new()
    }
}

impl MarkqantEncoder {
    pub fn new() -> Self {
        Self {
            tokens: HashMap::new(),
            patterns: HashMap::new(),
            frequencies: HashMap::new(),
            next_token: 0x80, // Start at 128
        }
    }

    /// Analyze text and build token assignments
    pub fn analyze(&mut self, text: &str) {
        // Find all repeated substrings
        let words: Vec<&str> = text.split_whitespace().collect();

        // Count frequencies
        for window_size in 1..=5 {
            for i in 0..words.len().saturating_sub(window_size - 1) {
                let pattern = words[i..i + window_size].join(" ");
                *self.frequencies.entry(pattern).or_insert(0) += 1;
            }
        }

        // Score patterns by (length - 1) * (frequency - 1)
        let mut scored_patterns: Vec<_> = self
            .frequencies
            .iter()
            .filter(|(_, &freq)| freq >= 2)
            .map(|(pattern, &freq)| {
                let score = (pattern.len() - 1) * (freq - 1);
                (pattern.clone(), score)
            })
            .collect();

        scored_patterns.sort_by_key(|(_, score)| std::cmp::Reverse(*score));

        // Assign tokens to top patterns
        for (pattern, _) in scored_patterns.iter().take(128) {
            if self.next_token < 255 {
                self.tokens.insert(pattern.clone(), self.next_token);
                self.patterns.insert(self.next_token, pattern.clone());
                self.next_token = self.next_token.saturating_add(1);
            }
        }
    }

    /// Encode text using assigned tokens
    pub fn encode(&self, text: &str) -> Vec<u8> {
        let mut result = Vec::new();
        let mut remaining = text;

        while !remaining.is_empty() {
            let mut found = false;

            // Try to match longest pattern first
            for len in (1..=remaining.len()).rev() {
                if let Some(&token) = self.tokens.get(&remaining[..len]) {
                    result.push(token);
                    remaining = &remaining[len..];
                    found = true;
                    break;
                }
            }

            if !found {
                // No pattern match, encode as raw byte
                result.extend_from_slice(remaining.chars().next().unwrap().to_string().as_bytes());
                remaining = &remaining[remaining.chars().next().unwrap().len_utf8()..];
            }
        }

        result
    }

    /// Decode tokens back to text
    pub fn decode(&self, data: &[u8]) -> Result<String> {
        let mut result = String::new();
        let mut i = 0;

        while i < data.len() {
            if data[i] >= 0x80 {
                // Token
                if let Some(pattern) = self.patterns.get(&data[i]) {
                    result.push_str(pattern);
                } else {
                    return Err(anyhow!("Unknown token: 0x{:02x}", data[i]));
                }
                i += 1;
            } else {
                // Raw UTF-8
                let ch = data[i] as char;
                result.push(ch);
                i += 1;
            }
        }

        Ok(result)
    }
}

/// .m8 file writer
pub struct M8Writer<W: Write> {
    writer: W,
    sections: Vec<M8Section>,
}

impl<W: Write> M8Writer<W> {
    pub fn new(writer: W) -> Self {
        Self {
            writer,
            sections: Vec::new(),
        }
    }

    /// Add a wave memory section
    pub fn add_wave_memory(&mut self, waves: &[CompressedWave]) -> Result<()> {
        let mut data = Vec::with_capacity(waves.len() * 32);

        for wave in waves {
            data.extend_from_slice(&wave.to_bytes());
        }

        self.sections.push(M8Section {
            section_type: SectionType::WaveMemory,
            size: data.len() as u32,
            data,
        });

        Ok(())
    }

    /// Add a Markqant-compressed text section
    pub fn add_markqant_text(&mut self, text: &str) -> Result<()> {
        let mut encoder = MarkqantEncoder::new();
        encoder.analyze(text);
        let encoded = encoder.encode(text);

        // Store token table followed by encoded data
        let mut data = Vec::new();

        // Token table header
        data.extend_from_slice(&(encoder.patterns.len() as u16).to_le_bytes());

        // Token definitions
        for (token, pattern) in &encoder.patterns {
            data.push(*token);
            data.extend_from_slice(&(pattern.len() as u16).to_le_bytes());
            data.extend_from_slice(pattern.as_bytes());
        }

        // Encoded text
        data.extend_from_slice(&(encoded.len() as u32).to_le_bytes());
        data.extend_from_slice(&encoded);

        self.sections.push(M8Section {
            section_type: SectionType::MarkqantText,
            size: data.len() as u32,
            data,
        });

        Ok(())
    }

    /// Write the complete .m8 file
    pub fn finish(mut self) -> Result<()> {
        // Write magic bytes
        self.writer.write_all(M8_MAGIC)?;

        // Calculate total size
        let header_size = 16; // Magic + header fields
        let section_headers_size = self.sections.len() * 8; // Type + size per section
        let data_size: usize = self.sections.iter().map(|s| s.data.len()).sum();
        let total_size = header_size + section_headers_size + data_size + 4; // +4 for CRC

        // Write header
        let header = M8Header {
            version: 1,
            section_count: self.sections.len() as u16,
            file_size: total_size as u64,
            timestamp: std::time::SystemTime::now()
                .duration_since(std::time::UNIX_EPOCH)?
                .as_secs(),
        };

        self.writer.write_all(&header.version.to_le_bytes())?;
        self.writer.write_all(&header.section_count.to_le_bytes())?;
        self.writer.write_all(&header.file_size.to_le_bytes())?;
        self.writer.write_all(&header.timestamp.to_le_bytes())?;

        // Write sections
        for section in &self.sections {
            self.writer.write_all(&[section.section_type as u8])?;
            self.writer.write_all(&section.size.to_le_bytes())?;
            self.writer.write_all(&section.data)?;
        }

        // Calculate and write CRC32
        let crc = 0u32; // TODO: Implement actual CRC32
        self.writer.write_all(&crc.to_le_bytes())?;

        Ok(())
    }
}

/// Example of .m8 format usage
pub fn create_example_m8() -> Result<Vec<u8>> {
    let mut buffer = Vec::new();
    let mut writer = M8Writer::new(&mut buffer);

    // Add some wave memories
    let waves = vec![
        CompressedWave::from_wave(&MemoryWave::new(440.0, 0.8), 1),
        CompressedWave::from_wave(&MemoryWave::new(880.0, 0.6), 2),
    ];
    writer.add_wave_memory(&waves)?;

    // Add some text
    writer.add_markqant_text("The user is cooking in the kitchen at 6PM")?;

    writer.finish()?;
    Ok(buffer)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_wave_compression() {
        let mut wave = MemoryWave::new(440.0, 0.8);
        wave.valence = 0.7;
        wave.arousal = 0.4;

        let compressed = CompressedWave::from_wave(&wave, 12345);
        assert_eq!(compressed.to_bytes().len(), 32);

        let decompressed = compressed.to_wave();
        assert!((decompressed.frequency - 440.0).abs() < 1.0);
        assert!((decompressed.valence - 0.7).abs() < 0.01);
    }

    #[test]
    fn test_markqant_encoding() {
        let mut encoder = MarkqantEncoder::new();
        let text = "the cat in the hat sat on the mat";
        encoder.analyze(text);

        let encoded = encoder.encode(text);
        let decoded = encoder.decode(&encoded).unwrap();

        assert_eq!(decoded, text);
        assert!(encoded.len() < text.len()); // Should compress
    }

    #[test]
    fn test_m8_creation() {
        let m8_data = create_example_m8().unwrap();
        assert!(m8_data.starts_with(M8_MAGIC));
        assert!(m8_data.len() > 100); // Should have some content
    }
}