use crate::args::Opts;
use crate::walk::{bytes, tracked};
use std::path::Path;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Estimate {
pub path: String,
pub tokens: u64,
}
#[must_use]
pub fn dummy(bytes: u64) -> u64 {
bytes.checked_div(4).unwrap_or(0)
}
pub(crate) fn measure(
opts: &Opts,
root: &Path,
) -> Result<Vec<Estimate>, String> {
let ests: Vec<Estimate> = select(opts, root)?
.iter()
.filter_map(|f| {
count(&root.join(f), opts.is_bpe()).map(|tokens| Estimate {
path: f.clone(),
tokens,
})
})
.collect();
Ok(cap(ests, opts.top))
}
pub(crate) fn count(path: &Path, bpe: bool) -> Option<u64> {
if bpe {
bpe_count(path)
} else {
bytes(path).map(dummy)
}
}
#[cfg(feature = "bpe")]
fn bpe_count(path: &Path) -> Option<u64> {
std::fs::read_to_string(path)
.ok()
.map(|t| crate::bpe::count(&t))
}
#[cfg(not(feature = "bpe"))]
fn bpe_count(_path: &Path) -> Option<u64> {
None
}
pub(crate) fn cap(
mut ests: Vec<Estimate>,
top: Option<usize>,
) -> Vec<Estimate> {
ests.sort_by_key(|e| std::cmp::Reverse(e.tokens));
if let Some(n) = top {
ests.truncate(n);
}
ests
}
pub(crate) fn select(opts: &Opts, root: &Path) -> Result<Vec<String>, String> {
select_paths(&opts.paths, root)
}
pub fn select_paths(
paths: &[String],
root: &Path,
) -> Result<Vec<String>, String> {
if paths.is_empty() {
return Ok(tracked(root));
}
for p in paths {
usable(&root.join(p), p)?;
}
Ok(paths.to_vec())
}
fn usable(full: &Path, shown: &str) -> Result<(), String> {
let md = std::fs::metadata(full).map_err(|e| format!("{shown}: {e}"))?;
if md.is_dir() {
return Err(format!(
"{shown} is a directory -- itok costs a flat fileset, so name \
files, or omit paths for the git-tracked set"
));
}
Ok(())
}
#[must_use]
pub(crate) fn over_budget(
ests: &[Estimate],
budget: Option<u64>,
) -> Vec<&Estimate> {
match budget {
None => Vec::new(),
Some(n) => ests.iter().filter(|e| e.tokens > n).collect(),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_directory_argument_is_refused_with_its_reason() {
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
let err = select_paths(&["src".to_owned()], root)
.err()
.unwrap_or_default();
assert!(err.contains("src"), "names the path: {err}");
assert!(err.contains("directory"), "names the reason: {err}");
assert!(err.contains("flat fileset"), "names the rule: {err}");
}
#[test]
fn a_missing_path_is_named_not_dropped() {
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
let err = select_paths(&["no/such/file".to_owned()], root)
.err()
.unwrap_or_default();
assert!(err.contains("no/such/file"), "{err}");
}
#[test]
fn files_and_the_tracked_default_still_select() {
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
assert_eq!(
select_paths(&["Cargo.toml".to_owned()], root),
Ok(vec!["Cargo.toml".to_owned()])
);
assert!(select_paths(&[], root).is_ok());
}
use proptest::prelude::*;
#[test]
fn four_bytes_is_one_token() {
assert_eq!(dummy(4), 1);
}
#[test]
fn an_empty_file_is_zero() {
assert_eq!(dummy(0), 0);
}
#[test]
fn it_is_a_quarter_of_the_bytes() {
assert_eq!(dummy(1000), 250);
}
#[test]
fn a_missing_named_file_is_an_error_not_an_omission() {
let opts = Opts {
paths: vec!["no-such-itok-file-xyz".to_owned()],
..Default::default()
};
let err = measure(&opts, Path::new(".")).err().unwrap_or_default();
assert!(err.contains("no-such-itok-file-xyz"), "{err}");
}
#[test]
fn no_budget_flags_nothing() {
let all = ests(&[10, 20, 30]);
assert!(over_budget(&all, None).is_empty());
}
fn ests(toks: &[u64]) -> Vec<Estimate> {
toks.iter()
.enumerate()
.map(|(i, t)| Estimate {
path: format!("f{i}"),
tokens: *t,
})
.collect()
}
proptest! {
#![proptest_config(ProptestConfig::with_cases(64))]
#[test]
fn cap_never_exceeds_top(
toks in prop::collection::vec(0u64..1000, 0..20),
n in 0usize..25,
) {
prop_assert!(cap(ests(&toks), Some(n)).len() <= n);
}
#[test]
fn cap_sorts_descending(toks in prop::collection::vec(0u64..1000, 1..20)) {
let out = cap(ests(&toks), None);
let sorted = out
.windows(2)
.all(|w| matches!(w, [a, b] if a.tokens >= b.tokens));
prop_assert!(sorted);
}
#[test]
fn over_budget_only_flags_files_above(
toks in prop::collection::vec(0u64..1000, 0..20),
n in 0u64..1000,
) {
let all = ests(&toks);
let flagged = over_budget(&all, Some(n));
prop_assert!(flagged.iter().all(|e| e.tokens > n));
}
}
}