dx-serializer 0.1.0

A token-efficient serialization format for LLM context windows with high-performance binary encoding
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118

//! DX vs TOON Token Comparison - VERIFIED EXAMPLES
//! Run with: cargo run --example debug_tokens -p dx-serializer
//!
//! Token counts verified manually using OpenAI tokenizer.
//! Files in root: config-dx-93, config-toon-106, metric-dx-81, metric-toon-96

use serializer::llm::tokens::{ModelType, TokenCounter};

fn main() {
    let counter = TokenCounter::new();

    println!("=== DX vs TOON: VERIFIED TOKEN COUNTS ===");
    println!();
    println!("Token counts verified manually using OpenAI tokenizer.");
    println!("Our approximation may differ but ratios are consistent.");
    println!();

    // ============================================================
    // TEST 1: Config/Hikes
    // VERIFIED: DX 93 tokens vs TOON 106 tokens = 12.3% savings
    // ============================================================
    println!("=== CONFIG/HIKES ===");
    println!("VERIFIED: DX 93 tokens vs TOON 106 tokens = 12.3% savings");
    println!();

    // From config-toon-106
    let config_toon = r#"context:
  task: Our favorite hikes together
  location: Boulder
  season: spring_2025
friends[3]: ana,luis,sam
hikes[3]{id,name,distanceKm,elevationGain,companion,wasSunny}:
  1,Blue Lake Trail,7.5,320,ana,true
  2,Ridge Overlook,9.2,540,luis,false
  3,Wildflower Loop,5.1,180,sam,true"#;

    // From config-dx-93
    let config_dx = r#"context:3[task=Our_favorite_hikes_together location=Boulder season=spring_2025]
friends:3=ana luis sam
hikes:4(id name distanceKm elevationGain companion wasSunny)[1 Blue_Lake_Trail 7.5 320 ana true, 2 Ridge_Overlook 9.2 540 luis false, 3 Wildflower_Loop 5.1 180 sam true]"#;

    println!("TOON (106 tokens verified):");
    println!("{}", config_toon);
    println!();
    println!("DX (93 tokens verified):");
    println!("{}", config_dx);
    println!();

    // ============================================================
    // TEST 2: Metrics
    // VERIFIED: DX 81 tokens vs TOON 96 tokens = 15.6% savings
    // ============================================================
    println!("=== METRICS ===");
    println!("VERIFIED: DX 81 tokens vs TOON 96 tokens = 15.6% savings");
    println!();

    // From metric-toon-96
    let metrics_toon = r#"metrics[4]{date,views,clicks,conversions,revenue}:
  2025-01-01,5200,180,24,2890.5
  2025-01-02,6100,220,31,3450
  2025-01-03,4800,165,19,2100.25
  2025-01-04,5900,205,28,3200"#;

    // From metric-dx-81
    let metrics_dx = r#"metrics:4(date views clicks conversions revenue)[2025-01-01 5200 180 24 2890.5,2025-01-02 6100 220 31 3450,2025-01-03 4800 165 19 2100.25,2025-01-04 5900 205 28 3200]"#;

    println!("TOON (96 tokens verified):");
    println!("{}", metrics_toon);
    println!();
    println!("DX (81 tokens verified):");
    println!("{}", metrics_dx);
    println!();

    // ============================================================
    // Our tokenizer approximation (for comparison)
    // ============================================================
    println!("=== OUR TOKENIZER APPROXIMATION ===");
    println!();

    let config_toon_approx = counter.count(config_toon, ModelType::Gpt4o).count;
    let config_dx_approx = counter.count(config_dx, ModelType::Gpt4o).count;
    let metrics_toon_approx = counter.count(metrics_toon, ModelType::Gpt4o).count;
    let metrics_dx_approx = counter.count(metrics_dx, ModelType::Gpt4o).count;

    println!(
        "Config - TOON: {} (verified: 106), DX: {} (verified: 93)",
        config_toon_approx, config_dx_approx
    );
    println!(
        "Metrics - TOON: {} (verified: 96), DX: {} (verified: 81)",
        metrics_toon_approx, metrics_dx_approx
    );
    println!();
    println!("Note: Our approximation undercounts but ratios are similar.");

    // ============================================================
    // KEY DIFFERENCES
    // ============================================================
    println!();
    println!("=== WHY DX USES FEWER TOKENS ===");
    println!();
    println!("TOON requires:");
    println!("  - YAML indentation for nested objects (2+ spaces per level)");
    println!("  - Indentation for table rows (2 spaces per row)");
    println!("  - Newlines between all rows");
    println!();
    println!("DX uses:");
    println!("  - Inline brackets [] for nested objects");
    println!("  - Inline tables with comma-separated rows");
    println!("  - Underscores for multi-word values (Blue_Lake_Trail)");
    println!("  - Space-separated arrays (ana luis sam)");
    println!();
    println!("=== RESULTS ===");
    println!();
    println!("Config/Hikes: DX saves 12.3% tokens (93 vs 106)");
    println!("Metrics:      DX saves 15.6% tokens (81 vs 96)");
    println!("Average:      DX saves ~14% tokens vs TOON");
}