use crate::tokens::{Count, Method};
#[must_use]
pub(crate) fn of_text(text: &str, method: Method) -> Count {
let tokens = match method {
Method::Estimate => itok::estimate::dummy(width(text)),
Method::Bpe => itok::bpe::count(text),
};
Count { tokens, method }
}
fn width(text: &str) -> u64 {
u64::try_from(text.len()).unwrap_or(u64::MAX)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_count_carries_its_method() {
assert_eq!(of_text("hi", Method::Bpe).method, Method::Bpe);
let cheap = of_text("hi", Method::Estimate);
assert_eq!(cheap.method, Method::Estimate);
}
}