ralph-coder 0.2.1

/// Token counting tests — verifies tiktoken-rs is used, not char/4.
use ralph::compaction::{count_tokens, estimate_tokens, should_compact};
use ralph::providers::{Message, MessageContent, Role};

#[test]
fn count_tokens_empty_string() {
    assert_eq!(count_tokens(""), 0);
}

#[test]
fn count_tokens_single_word() {
    // "hello" is 1 token in cl100k_base
    let t = count_tokens("hello");
    assert!(t >= 1, "expected at least 1 token, got {}", t);
}

#[test]
fn count_tokens_longer_text() {
    let short = count_tokens("hi");
    let long =
        count_tokens("The quick brown fox jumps over the lazy dog and then continues running.");
    assert!(long > short, "longer text should produce more tokens");
}

#[test]
fn count_tokens_not_char_over_four() {
    // "aaaa" is 1 token in cl100k_base, not 1 (char/4 would give 1, tiktoken gives 1 too)
    // But "aaaaaaaaaaaaaaaa" (16 chars) should be 1 token (all same char merges in BPE)
    // while char/4 would give 4. This distinguishes the two approaches.
    let repeated = "aaaaaaaaaaaaaaaa"; // 16 chars
    let tiktoken_count = count_tokens(repeated);
    let char_estimate = (repeated.len() as u64) / 4;
    // tiktoken should give significantly fewer tokens than char/4 for repeated chars
    assert!(
        tiktoken_count <= char_estimate,
        "tiktoken ({}) should be <= char/4 estimate ({}) for repeated chars",
        tiktoken_count,
        char_estimate
    );
}

#[test]
fn estimate_tokens_sums_messages() {
    let msgs = vec![
        Message {
            role: Role::User,
            content: MessageContent::Text("hello world".to_string()),
            tool_call_id: None,
            name: None,
            reasoning_content: None,
        },
        Message {
            role: Role::Assistant,
            content: MessageContent::Text("hi there friend".to_string()),
            tool_call_id: None,
            name: None,
            reasoning_content: None,
        },
    ];
    let total = estimate_tokens(&msgs);
    assert!(total > 0);
    // Should be more than just the first message
    let single = estimate_tokens(&msgs[..1]);
    assert!(total > single);
}

#[test]
fn should_compact_false_with_tiny_messages() {
    let msgs = vec![Message {
        role: Role::User,
        content: MessageContent::Text("hi".to_string()),
        tool_call_id: None,
        name: None,
        reasoning_content: None,
    }];
    // 200k context window, 80% threshold = 160k tokens. "hi" is nowhere near that.
    assert!(!should_compact(&msgs, 200_000, 80));
}

#[test]
fn should_compact_true_when_over_threshold() {
    // Create enough messages to exceed a tiny context window
    let msgs: Vec<Message> = (0..20)
        .map(|i| Message {
            role: if i % 2 == 0 {
                Role::User
            } else {
                Role::Assistant
            },
            content: MessageContent::Text("word ".repeat(50)),
            tool_call_id: None,
            name: None,
            reasoning_content: None,
        })
        .collect();
    // Very small context window of 50 tokens, 10% threshold = 5 tokens
    assert!(should_compact(&msgs, 50, 10));
}