tinyjuice 0.2.1

Pluggable token compression for OpenHuman.
Documentation
1
2
3
4
5
6
7
8
9
10
pub(crate) fn utf8_safe_prefix_at_byte_boundary(s: &str, max_bytes: usize) -> &str {
    if s.len() <= max_bytes {
        return s;
    }
    let mut end = max_bytes;
    while end > 0 && !s.is_char_boundary(end) {
        end -= 1;
    }
    &s[..end]
}