lowfat-core 0.3.0

Core types, config, tokens, SQLite tracking for lowfat
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

/// Estimate token count from text. ~4 chars = 1 token.
/// Matches bash: `(len + 3) / 4`
pub fn estimate_tokens(s: &str) -> usize {
    (s.len() + 3) / 4
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn empty_string() {
        assert_eq!(estimate_tokens(""), 0);
    }

    #[test]
    fn one_char() {
        assert_eq!(estimate_tokens("a"), 1);
    }

    #[test]
    fn four_chars() {
        assert_eq!(estimate_tokens("abcd"), 1);
    }

    #[test]
    fn five_chars() {
        assert_eq!(estimate_tokens("abcde"), 2);
    }

    #[test]
    fn eight_chars() {
        assert_eq!(estimate_tokens("abcdefgh"), 2);
    }

    #[test]
    fn nine_chars() {
        assert_eq!(estimate_tokens("abcdefghi"), 3);
    }

    #[test]
    fn hundred_chars() {
        let s = "a".repeat(100);
        assert_eq!(estimate_tokens(&s), 25);
    }
}