token-count 0.4.0

Count tokens for LLM models using exact tokenization
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

use assert_cmd::Command;

#[test]
fn test_stdin_pipe_basic() {
    let mut cmd = Command::cargo_bin("token-count").unwrap();
    cmd.write_stdin("Hello world");

    // For now, we're in Phase 3, so we just verify it doesn't crash
    // Full functionality will be tested in Phase 6 after integration
    cmd.assert().success();
}

#[test]
fn test_empty_input() {
    let mut cmd = Command::cargo_bin("token-count").unwrap();
    cmd.write_stdin("");

    cmd.assert().success();
}

#[test]
fn test_unicode_input() {
    let mut cmd = Command::cargo_bin("token-count").unwrap();
    cmd.write_stdin("Hello δΈ–η•Œ 🌍");

    cmd.assert().success();
}

#[test]
fn test_multiline_input() {
    let mut cmd = Command::cargo_bin("token-count").unwrap();
    cmd.write_stdin("Line 1\nLine 2\nLine 3");

    cmd.assert().success();
}