marqant 0.1.2

Quantum-compressed markdown format for AI consumption with 90% token reduction
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

use marqant::Marqant;

#[test]
fn tokenize_reduces_length_for_repetitions() {
    // Repeat header enough times so static token "# " yields positive savings
    let content = "# Title\n\n".repeat(10);
    let (tokens, tokenized) = Marqant::tokenize_content(&content);
    assert!(!tokens.is_empty());
    assert!(tokenized.len() < content.len());
}

#[test]
fn tokenize_static_tokens_present() {
    // Ensure "## " and "- " appear enough times to meet savings > 0
    let mut content = String::new();
    for _ in 0..4 {
        content.push_str("## Heading\n\n");
    }
    content.push_str("- a\n- b\n- c\n- d\n- e\n- f\n");

    let (tokens, _tokenized) = Marqant::tokenize_content(&content);
    let has_expected = tokens.values().any(|v| v == "## " || v == "- ");
    assert!(has_expected);
}