token_optimizer/
lib.rs

1// ================================================================================
2// PRODUCT  : sovereign-stack:token-optimizer
3// COMPONENT: Sovereign Token Optimization & Enveloping Engine
4// FILE     : crates/token-optimizer/src/lib.rs
5// ROLE     : Core library entry point with guarded minification.
6// AUTHOR   : Chamara Somaratne assisted by Gemini, Ashby, Serge & Ciara 
7//            (Sovereign Gestalt)
8// LICENSE  : Apache-2.0
9// ================================================================================
10
11pub mod llm_envelope;
12
13pub use llm_envelope::{Envelope, Envelopable};
14
15pub type OptimizerResult<T> = Result<T, String>;
16
17// --------------------------------------------------------------------------------
18// UTILITY: Guarded Text Minifier
19// PURPOSE: Strips structural bloat from unstructured text (emails, prompts).
20// SAFETY:  Automatically detects JSON/TOML and skips minification to prevent
21//          corrupting structured data that relies on specific formatting.
22// --------------------------------------------------------------------------------
23pub fn minify_text(input: &str) -> String {
24    let trimmed = input.trim();
25    
26    if trimmed.is_empty() {
27        return String::new();
28    }
29
30    // 1. Check if input is a valid JSON structure
31    if serde_json::from_str::<serde_json::Value>(trimmed).is_ok() {
32        return input.to_string();
33    }
34
35    // 2. Check if input is a valid TOML structure
36    // Note: TOML requires at least one key-value pair to be valid.
37    if toml::from_str::<toml::Value>(trimmed).is_ok() {
38        return input.to_string();
39    }
40
41    // 3. Unstructured Prose detected: Safe to compress.
42    // O(N) internal whitespace stripping.
43    input
44        .split_whitespace()
45        .collect::<Vec<&str>>()
46        .join(" ")
47}
48
49#[cfg(test)]
50mod tests {
51    use super::*;
52
53    #[test]
54    fn test_minify_ignores_toml() {
55        let toml_data = "key = \"value\"\n[table]\nitem = 1";
56        // Should remain unchanged because newlines are semantic in TOML
57        assert_eq!(minify_text(toml_data), toml_data);
58    }
59
60    #[test]
61    fn test_minify_packs_prose() {
62        let prose = "  This    is   unstructured    text.  ";
63        assert_eq!(minify_text(prose), "This is unstructured text.");
64    }
65}
token_optimizer/lib.rs

token_optimizer/
lib.rs
