use std::path::Path;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Count {
pub tokens: u64,
pub method: &'static str,
}
impl std::fmt::Display for Count {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{} tok ({})", self.tokens, self.method)
}
}
#[must_use]
pub fn count(text: &str) -> Count {
Count {
tokens: itok::bpe::count(text),
method: "o200k",
}
}
pub fn count_file(path: &Path) -> std::io::Result<Count> {
Ok(count(&std::fs::read_to_string(path)?))
}
pub const ENTRY_COST: u64 = 28_543;
#[must_use]
pub const fn working(window: u64) -> u64 {
window.saturating_sub(ENTRY_COST)
}
#[derive(Debug, Default, Clone)]
pub struct Ceilings {
rows: Vec<(String, u64)>,
default: u64,
cold: bool,
}
pub const DEFAULT_NODE: u64 = 2_000;
impl Ceilings {
pub fn parse(text: &str) -> Result<Self, String> {
let mut rows = Vec::new();
for (n, line) in text.lines().enumerate() {
let l = line.trim();
if l.is_empty() || l.starts_with('#') {
continue;
}
let mut it = l.split_whitespace();
match (it.next(), it.next()) {
(Some(path), Some(limit)) => {
let v: u64 = limit.parse()
.map_err(|_| format!(".context-limits:{}: `{limit}` is not a token count", n + 1))?;
rows.push((path.to_string(), v));
}
_ => {
return Err(format!(
".context-limits:{}: expected `<path> <limit>`",
n + 1
));
}
}
}
Ok(Self {
rows,
default: DEFAULT_NODE,
cold: false,
})
}
pub fn load(root: &std::path::Path) -> Result<Self, String> {
match std::fs::read_to_string(root.join(".context-limits")) {
Ok(t) => Self::parse(&t),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Self {
cold: true,
..Self::default_only()
}),
Err(e) => Err(format!(".context-limits: {e}")),
}
}
#[must_use]
pub const fn is_cold(&self) -> bool {
self.cold
}
#[must_use]
pub fn default_only() -> Self {
Self {
rows: Vec::new(),
default: DEFAULT_NODE,
cold: false,
}
}
#[must_use]
pub fn for_path(&self, path: &str) -> u64 {
self.rows
.iter()
.filter(|(p, _)| path.starts_with(p.as_str()))
.max_by_key(|(p, _)| p.len())
.map_or(self.default, |(_, v)| *v)
}
}
#[cfg(test)]
#[path = "tests/tokens.rs"]
mod tests;