use crate::cli::Output;
use crate::render::DUMMY;
use crate::units;
#[derive(Default, Debug, PartialEq, Eq)]
struct Req {
budget: Option<u64>,
footer: Kind,
asked: Vec<crate::ladder::Rung>,
}
#[derive(Default, Debug, PartialEq, Eq)]
enum Kind {
#[default]
Human,
Json,
}
pub(crate) fn cap(rest: &[String], input: &str) -> Output {
match parse(rest) {
Ok(req) => Output::ok(run(&req, input)),
Err(e) => Output::usage_err(format!("itok: {e}")),
}
}
fn run(req: &Req, input: &str) -> String {
let (reduced, ran) = climb(req, input);
let c = cut(&reduced, input, req.budget);
let mut out = c.body.clone();
if !out.is_empty() && !out.ends_with('\n') {
out.push('\n');
}
match req.footer {
Kind::Human => out.push_str(&human_footer(&c, &ran)),
Kind::Json => out.push_str(&json_footer(&c, req.budget, &ran)),
}
out.push('\n');
out
}
fn climb(req: &Req, input: &str) -> (String, Vec<&'static str>) {
let mut text = input.to_owned();
let mut ran = Vec::new();
let mut asked = req.asked.clone();
asked.sort_unstable();
asked.dedup();
for rung in asked {
if !over(len(&text), req.budget) {
break;
}
text = rung.apply(&text);
ran.push(rung.label());
}
(text, ran)
}
struct Cut {
body: String,
kept_lines: u64,
total_bytes: u64,
total_lines: u64,
}
impl Cut {
fn kept_bytes(&self) -> u64 {
len(&self.body)
}
fn kept_tokens(&self) -> u64 {
crate::estimate::dummy(self.kept_bytes())
}
fn input_tokens(&self) -> u64 {
crate::estimate::dummy(self.total_bytes)
}
fn elided_tokens(&self) -> u64 {
self.input_tokens().saturating_sub(self.kept_tokens())
}
fn elided_bytes(&self) -> u64 {
self.total_bytes.saturating_sub(self.kept_bytes())
}
fn elided_lines(&self) -> u64 {
self.total_lines.saturating_sub(self.kept_lines)
}
fn elided(&self) -> bool {
self.elided_lines() > 0
}
fn advances(&self) -> bool {
self.kept_bytes() > 0
}
fn resume_line(&self) -> u64 {
self.kept_lines.saturating_add(1)
}
}
fn cut(reduced: &str, original: &str, budget: Option<u64>) -> Cut {
let body = kept_prefix(reduced, budget);
Cut {
kept_lines: lines(&body),
total_bytes: len(original),
total_lines: lines(original),
body,
}
}
fn kept_prefix(input: &str, budget: Option<u64>) -> String {
let mut out = String::new();
for l in input.split_inclusive('\n') {
if over(len(&out).saturating_add(len(l)), budget) {
break;
}
out.push_str(l);
}
out
}
fn over(n: u64, budget: Option<u64>) -> bool {
budget.is_some_and(|b| crate::estimate::dummy(n) > b)
}
fn lines(s: &str) -> u64 {
count(s.split_inclusive('\n').count())
}
fn len(s: &str) -> u64 {
count(s.len())
}
fn count(n: usize) -> u64 {
u64::try_from(n).unwrap_or(u64::MAX)
}
fn mark() -> &'static str {
if DUMMY.approximate { "~" } else { "" }
}
fn human_footer(c: &Cut, ran: &[&'static str]) -> String {
let (t, m) = (mark(), DUMMY.label());
if c.elided() || !ran.is_empty() {
return format!(
"[itok cap: kept {t}{} of {t}{} itok ({m}); {}]",
c.kept_tokens(),
c.input_tokens(),
elision(c, ran),
);
}
format!(
"[itok cap: kept {t}{} itok ({m}); nothing elided]",
c.kept_tokens()
)
}
fn elision(c: &Cut, ran: &[&'static str]) -> String {
let mut names: Vec<&str> = ran.to_vec();
if c.elided() {
names.push("cap");
}
format!(
"elided {} of {} lines, {} bytes; rungs: {}; {}",
c.elided_lines(),
c.total_lines,
c.elided_bytes(),
names.join(", "),
resume_note(c, ran),
)
}
fn resume_note(c: &Cut, ran: &[&'static str]) -> String {
if !ran.is_empty() {
return "no resume point: the text was reduced, so offsets do not \
index the input"
.to_owned();
}
if !c.advances() {
return "no resume point: nothing fits this budget, so an offset \
would send the next read round the same bytes"
.to_owned();
}
format!(
"resume: line {} (byte offset {})",
c.resume_line(),
c.kept_bytes()
)
}
fn json_footer(c: &Cut, budget: Option<u64>, ran: &[&'static str]) -> String {
format!(
"{{\"tool\":\"cap\"{},\"unit\":\"{}\",\"method\":\"{}\",\
\"estimated\":{},\"kept\":{},\"input\":{},\"elided\":{},\
\"rungs\":[{}]{}}}",
budget_field(budget),
crate::json::UNIT,
DUMMY.tier,
DUMMY.approximate,
span(c.kept_tokens(), c.kept_lines, c.kept_bytes()),
span(c.input_tokens(), c.total_lines, c.total_bytes),
span(c.elided_tokens(), c.elided_lines(), c.elided_bytes()),
rungs(c, ran),
resume(c, ran),
)
}
fn span(tokens: u64, lines: u64, bytes: u64) -> String {
format!("{{\"tokens\":{tokens},\"lines\":{lines},\"bytes\":{bytes}}}")
}
fn rungs(c: &Cut, ran: &[&'static str]) -> String {
let mut all: Vec<String> = ran.iter().map(|r| format!("\"{r}\"")).collect();
if c.elided() {
all.push("\"cap\"".to_owned());
}
all.join(",")
}
fn resume(c: &Cut, ran: &[&'static str]) -> String {
if !c.elided() || !ran.is_empty() || !c.advances() {
return String::new();
}
format!(
",\"resume\":{{\"line\":{},\"byte\":{}}}",
c.resume_line(),
c.kept_bytes()
)
}
fn budget_field(budget: Option<u64>) -> String {
budget.map_or(String::new(), |n| format!(",\"budget\":{n}"))
}
fn parse(rest: &[String]) -> Result<Req, String> {
let mut r = Req::default();
let mut i = 0usize;
while let Some(a) = rest.get(i) {
apply(&mut r, a, rest, &mut i)?;
i = i.saturating_add(1);
}
Ok(r)
}
fn apply(
r: &mut Req,
a: &str,
rest: &[String],
i: &mut usize,
) -> Result<(), String> {
match a {
"--footer" => r.footer = kind(&val(rest, i)?)?,
p if crate::ladder::Rung::parse(p).is_some() => {
r.asked.extend(crate::ladder::Rung::parse(p));
}
p if p.starts_with('-') => return Err(format!("unknown flag '{p}'")),
n if r.budget.is_none() => r.budget = Some(units::parse(n)?),
extra => {
return Err(format!(
"unexpected argument '{extra}' -- cap filters stdin, so it \
takes one budget and no paths"
));
}
}
Ok(())
}
fn val(rest: &[String], i: &mut usize) -> Result<String, String> {
*i = i.saturating_add(1);
rest.get(*i)
.cloned()
.ok_or_else(|| "flag needs a value".to_owned())
}
fn kind(s: &str) -> Result<Kind, String> {
match s {
"human" => Ok(Kind::Human),
"json" => Ok(Kind::Json),
other => {
Err(format!("unknown footer '{other}' (want 'human' or 'json')"))
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn args(parts: &[&str]) -> Vec<String> {
parts.iter().map(|s| (*s).to_owned()).collect()
}
fn noisy() -> String {
let mut t = String::new();
for _ in 0..6 {
t.push_str("\u{1b}[32mrepeated line with padding\u{1b}[0m \n");
}
for n in 0..6 {
t.push_str(&format!("fn f{n}() {{\n let body = {n};\n}}\n"));
}
t
}
#[test]
fn the_ladder_order_wins_over_the_flag_order() {
let out = cap(&args(&["4", "--outline", "--strip"]), &noisy()).out;
let strip = out.find("strip").unwrap_or(usize::MAX);
let outline = out.find("outline").unwrap_or(0);
assert!(strip < outline, "{out}");
}
#[test]
fn the_ladder_stops_at_the_first_rung_that_fits() {
let text = noisy();
let stripped = crate::ladder::Rung::Strip.apply(&text);
let budget = crate::estimate::dummy(len(&stripped)).to_string();
let asked = args(&[&budget, "--strip", "--dedup", "--outline"]);
let out = cap(&asked, &text).out;
assert!(out.contains("rungs: strip"), "{out}");
assert!(!out.contains("dedup"), "stopped after strip: {out}");
assert!(!out.contains("outline"), "{out}");
}
#[test]
fn no_budget_runs_no_rung_however_many_were_asked_for() {
let text = noisy();
let out = cap(&args(&["--strip", "--dedup", "--outline"]), &text).out;
assert!(out.starts_with(&text), "body untouched: {out}");
assert!(out.contains("nothing elided"), "{out}");
}
#[test]
fn a_reduced_text_publishes_no_resume_point() {
let reduced = cap(&args(&["4", "--outline"]), &noisy()).out;
assert!(reduced.contains("the text was reduced"), "{reduced}");
let truncated = cap(&args(&["20"]), &noisy()).out;
assert!(truncated.contains("resume: line"), "{truncated}");
}
#[test]
fn the_totals_are_the_callers_own_input() {
let text = noisy();
let out =
cap(&args(&["4", "--outline", "--footer", "json"]), &text).out;
let bytes = format!("\"bytes\":{}", text.len());
assert!(out.contains(&bytes), "input span is the original: {out}");
}
#[test]
fn the_same_cut_twice_is_the_same_cut() {
let once = cap(&args(&["4", "--strip", "--dedup"]), &noisy()).out;
let twice = cap(&args(&["4", "--strip", "--dedup"]), &noisy()).out;
assert_eq!(once, twice);
}
#[test]
fn json_lists_every_rung_that_ran() {
let out = cap(
&args(&["4", "--strip", "--outline", "--footer", "json"]),
&noisy(),
)
.out;
assert!(
out.contains("\"rungs\":[\"strip\",\"outline\",\"cap\"]"),
"{out}"
);
}
fn follow(input: &str, budget: &str) -> Option<String> {
let mut at = 0usize;
let mut whole = String::new();
for _ in 0..64 {
let rest = input.get(at..)?;
let Some(n) = step(rest, budget) else {
whole.push_str(rest);
return Some(whole);
};
if n == 0 {
return None;
}
whole.push_str(rest.get(..n)?);
at = at.checked_add(n)?;
}
None
}
fn step(rest: &str, budget: &str) -> Option<usize> {
resume_byte(&cap(&args(&[budget]), rest).out)
}
fn resume_byte(out: &str) -> Option<usize> {
let (_, tail) = out.rsplit_once("byte offset ")?;
tail.split(')').next()?.parse().ok()
}
#[test]
fn the_resume_selector_walks_the_whole_input() {
let text = input();
let walked = follow(&text, "6").unwrap_or_default();
assert_eq!(walked, text);
}
#[test]
fn the_same_input_cuts_the_same_way_twice() {
let a = cap(&args(&["6"]), &input()).out;
let b = cap(&args(&["6"]), &input()).out;
assert_eq!(a, b);
}
#[test]
fn a_budget_nothing_fits_publishes_no_selector() {
let text = "a line far longer than the budget allows\nshort\n";
let out = cap(&args(&["1"]), text).out;
assert!(!out.contains("byte offset"), "would loop: {out}");
assert!(out.contains("nothing fits"), "{out}");
}
mod walks {
use super::{args, cap, follow};
use proptest::prelude::*;
proptest! {
#![proptest_config(ProptestConfig::with_cases(64))]
#[test]
fn the_selector_reconstructs_any_input(
lines in prop::collection::vec("[a-z ]{0,12}", 1..12),
budget in 5u64..40,
) {
let text: String =
lines.iter().map(|l| format!("{l}\n")).collect();
let walked = follow(&text, &budget.to_string());
prop_assert_eq!(walked, Some(text));
}
#[test]
fn the_same_cut_twice_matches(
lines in prop::collection::vec("[a-z ]{0,12}", 1..12),
budget in 1u64..40,
) {
let text: String =
lines.iter().map(|l| format!("{l}\n")).collect();
let n = budget.to_string();
let once = cap(&args(&[&n]), &text).out;
prop_assert_eq!(once, cap(&args(&[&n]), &text).out);
}
}
}
fn input() -> String {
(0..10).map(|n| format!("line {n}x\n")).collect()
}
#[test]
fn the_cut_is_by_tokens() {
let c = cut(&input(), &input(), Some(6));
assert_eq!(c.kept_lines, 3);
assert_eq!(c.kept_tokens(), 6);
assert_eq!(c.body, "line 0x\nline 1x\nline 2x\n");
}
#[test]
fn a_partial_line_is_never_emitted() {
let c = cut(&input(), &input(), Some(7));
assert_eq!(c.kept_lines, 3, "a 4th line would cost 8, over 7");
assert_eq!(c.kept_tokens(), 6, "the odd token is left unspent");
assert!(c.body.ends_with('\n'));
}
#[test]
fn a_cut_that_elides_nothing_still_announces() {
let out = run(&Req::default(), &input());
assert!(out.contains("[itok cap:"), "{out}");
assert!(out.contains("nothing elided"), "{out}");
assert!(out.starts_with("line 0x\n"), "body passes through");
}
#[test]
fn the_footer_carries_the_resume_selector() {
let c = cut(&input(), &input(), Some(6));
assert_eq!(c.resume_line(), 4);
assert_eq!(c.kept_bytes(), 24);
let f = human_footer(&c, &[]);
assert!(f.contains("resume: line 4 (byte offset 24)"), "{f}");
}
#[test]
fn the_elided_counts_complement_the_kept_ones() {
let c = cut(&input(), &input(), Some(6));
assert_eq!(c.elided_lines(), 7);
assert_eq!(c.elided_bytes(), 56);
assert_eq!(c.kept_tokens().saturating_add(c.elided_tokens()), 20);
}
#[test]
fn the_cut_is_idempotent() {
let once = run(&Req::default(), &input());
assert_eq!(once, run(&Req::default(), &input()));
let capped = Req {
budget: Some(6),
..Default::default()
};
assert_eq!(run(&capped, &input()), run(&capped, &input()));
}
#[test]
fn the_applied_rung_is_named() {
assert!(
human_footer(&cut(&input(), &input(), Some(6)), &[])
.contains("rungs: cap")
);
let whole = json_footer(&cut(&input(), &input(), None), None, &[]);
assert!(whole.contains("\"rungs\":[]"), "no rung ran: {whole}");
}
#[test]
fn the_human_footer_names_unit_method_and_estimate() {
let f = human_footer(&cut(&input(), &input(), Some(6)), &[]);
assert!(f.contains("kept ~6 of ~20 itok"), "{f}");
assert!(f.contains("(bytes/4)"), "{f}");
}
#[test]
fn the_json_footer_is_the_machine_contract() {
let j = json_footer(&cut(&input(), &input(), Some(6)), Some(6), &[]);
for key in [
"\"tool\":\"cap\"",
"\"budget\":6",
"\"unit\":\"input_tokens\"",
"\"method\":\"bytes/4\"",
"\"estimated\":true",
"\"resume\":{\"line\":4,\"byte\":24}",
] {
assert!(j.contains(key), "missing {key}: {j}");
}
assert!(!j.contains('~'), "no tilde in json: {j}");
}
#[test]
fn json_omits_what_is_absent() {
let j = json_footer(&cut(&input(), &input(), None), None, &[]);
assert!(!j.contains("\"resume\""), "nothing to resume from: {j}");
assert!(!j.contains("\"budget\""), "no budget was given: {j}");
assert!(
j.contains("\"elided\":{\"tokens\":0,\"lines\":0,\"bytes\":0}")
);
}
#[test]
fn a_budget_below_the_first_line_keeps_nothing() {
let out = run(
&Req {
budget: Some(1),
..Default::default()
},
&input(),
);
assert!(out.starts_with("[itok cap:"), "empty body: {out}");
assert!(!out.contains("byte offset"), "would loop: {out}");
assert!(out.contains("nothing fits"), "{out}");
}
#[test]
fn the_line_selector_survives_a_lossy_decode() {
let raw = [0xffu8, b'\n', 0xfe, b'\n', 0xfd, b'\n'];
let text = String::from_utf8_lossy(&raw).into_owned();
let c = cut(&text, &text, Some(1));
assert_eq!(c.total_lines, 3, "three newlines, three lines");
assert_eq!(c.kept_lines, 1, "4 bytes decoded is 1 itok");
assert_eq!(c.resume_line(), 2);
assert!(c.kept_bytes() > 2, "the decoded byte count is inflated");
}
#[test]
fn empty_input_is_empty_output_plus_a_footer() {
let out = run(&Req::default(), "");
assert!(out.starts_with("[itok cap: kept ~0 itok"), "{out}");
}
#[test]
fn an_unterminated_body_does_not_shift_the_offset() {
let c = cut("abcdef", "abcdef", None);
assert_eq!(c.kept_bytes(), 6);
assert!(run(&Req::default(), "abcdef").starts_with("abcdef\n[itok"));
}
#[test]
fn the_budget_uses_the_shared_unit_parser() {
assert_eq!(parse(&args(&["10k"])), Ok(Req::default().budgeted(10_000)));
assert!(parse(&args(&["xx"])).is_err());
}
#[test]
fn the_footer_kind_is_a_flag_with_two_values() {
let j = parse(&args(&["--footer", "json"])).ok();
assert_eq!(j.map(|r| r.footer), Some(Kind::Json));
assert!(parse(&args(&["--footer", "yaml"])).is_err());
assert!(parse(&args(&["--footer"])).is_err(), "needs a value");
}
#[test]
fn a_second_positional_is_refused_with_its_reason() {
let e = parse(&args(&["10k", "SPEC.md"])).err().unwrap_or_default();
assert!(e.contains("SPEC.md"), "names it: {e}");
assert!(e.contains("stdin"), "says why: {e}");
assert!(parse(&args(&["--nope"])).is_err());
}
#[test]
fn cap_never_gates() {
assert_eq!(cap(&args(&["0"]), &input()).code, 0);
assert_eq!(cap(&args(&["--bogus"]), &input()).code, 2, "usage only");
}
impl Req {
fn budgeted(self, n: u64) -> Self {
Self {
budget: Some(n),
..self
}
}
}
}