token-optimizer 0.1.4

// ================================================================================
// PRODUCT  : sovereign-stack:token-optimizer
// COMPONENT: Sovereign Token Optimization & Enveloping Engine
// FILE     : crates/token-optimizer/src/lib.rs
// ROLE     : Core library entry point with guarded minification.
// AUTHOR   : Chamara Somaratne assisted by Gemini, Ashby, Serge & Ciara 
//            (Sovereign Gestalt)
// LICENSE  : Apache-2.0
// ================================================================================

pub mod llm_envelope;

pub use llm_envelope::{Envelope, Envelopable};

pub type OptimizerResult<T> = Result<T, String>;

// --------------------------------------------------------------------------------
// UTILITY: Guarded Text Minifier
// PURPOSE: Strips structural bloat from unstructured text (emails, prompts).
// SAFETY:  Automatically detects JSON/TOML and skips minification to prevent
//          corrupting structured data that relies on specific formatting.
// --------------------------------------------------------------------------------
pub fn minify_text(input: &str) -> String {
    let trimmed = input.trim();
    
    if trimmed.is_empty() {
        return String::new();
    }

    // 1. Check if input is a valid JSON structure
    if serde_json::from_str::<serde_json::Value>(trimmed).is_ok() {
        return input.to_string();
    }

    // 2. Check if input is a valid TOML structure
    // Note: TOML requires at least one key-value pair to be valid.
    if toml::from_str::<toml::Value>(trimmed).is_ok() {
        return input.to_string();
    }

    // 3. Unstructured Prose detected: Safe to compress.
    // O(N) internal whitespace stripping.
    input
        .split_whitespace()
        .collect::<Vec<&str>>()
        .join(" ")
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_minify_ignores_toml() {
        let toml_data = "key = \"value\"\n[table]\nitem = 1";
        // Should remain unchanged because newlines are semantic in TOML
        assert_eq!(minify_text(toml_data), toml_data);
    }

    #[test]
    fn test_minify_packs_prose() {
        let prose = "  This    is   unstructured    text.  ";
        assert_eq!(minify_text(prose), "This is unstructured text.");
    }
}