use lscolors::{Indicator, LsColors};
use serde::Deserialize;
use std::cmp::Ordering;
use std::collections::{BTreeMap, BinaryHeap, HashMap};
use std::path::Component;
use std::path::Path;
use unicode_width::UnicodeWidthStr;
#[derive(Debug, Clone)]
pub enum TuiColor {
White,
Gray,
Red,
Green,
Blue,
Yellow,
Cyan,
Magenta,
LightRed,
LightGreen,
LightBlue,
LightYellow,
LightCyan,
LightMagenta,
}
#[derive(Debug, Clone)]
pub struct TuiTokenMapLine {
pub tokens_part: String,
pub prefix_part: String,
pub name_part: String,
pub name_color: TuiColor,
pub bar_part: String,
pub percentage_part: String,
}
#[derive(Debug, Clone, Copy, Deserialize)]
pub struct EntryMetadata {
pub is_dir: bool,
}
pub struct TokenMapFile {
pub path: String,
pub tokens: usize,
}
#[derive(Debug, Clone)]
struct TreeNode {
tokens: usize,
children: BTreeMap<String, TreeNode>,
path: String,
metadata: Option<EntryMetadata>,
}
impl TreeNode {
fn with_path(path: String) -> Self {
TreeNode {
tokens: 0,
children: BTreeMap::new(),
path,
metadata: None,
}
}
}
#[derive(Debug, Clone, Eq, PartialEq)]
struct NodePriority {
tokens: usize,
path: String,
depth: usize,
}
impl Ord for NodePriority {
fn cmp(&self, other: &Self) -> Ordering {
self.tokens
.cmp(&other.tokens)
.then_with(|| other.depth.cmp(&self.depth))
.then_with(|| self.path.cmp(&other.path))
}
}
impl PartialOrd for NodePriority {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}
pub fn generate_token_map_with_limit(
files: &[TokenMapFile],
total_tokens: usize,
max_lines: Option<usize>,
min_percent: Option<f64>,
) -> Vec<TokenMapEntry> {
let max_lines = max_lines.unwrap_or(20);
let min_percent = min_percent.unwrap_or(0.1);
let mut root = TreeNode::with_path(String::new());
root.tokens = total_tokens;
for file in files {
let path = Path::new(&file.path);
let components: Vec<&str> = path
.components()
.filter_map(|c| match c {
Component::Normal(s) => s.to_str(),
_ => None,
})
.collect();
insert_path(
&mut root,
&components,
file.tokens,
String::new(),
EntryMetadata { is_dir: false },
);
}
let allowed_nodes = select_nodes_to_display(&root, total_tokens, max_lines, min_percent);
let mut entries = Vec::new();
rebuild_filtered_tree(
&root,
String::new(),
&allowed_nodes,
&mut entries,
0,
total_tokens,
true,
);
let displayed_tokens: usize = entries
.iter()
.enumerate()
.filter_map(|(idx, e)| {
let is_leaf = entries
.get(idx + 1)
.is_none_or(|next| next.depth <= e.depth);
is_leaf.then_some(e.tokens)
})
.sum();
let hidden_tokens = total_tokens.saturating_sub(displayed_tokens);
if hidden_tokens > 0 {
entries.push(TokenMapEntry {
path: "(other files)".to_string(),
name: "(other files)".to_string(),
tokens: hidden_tokens,
percentage: (hidden_tokens as f64 / total_tokens as f64) * 100.0,
depth: 0,
is_last: true,
metadata: EntryMetadata { is_dir: false },
});
}
entries
}
fn insert_path(
node: &mut TreeNode,
components: &[&str],
tokens: usize,
parent_path: String,
file_metadata: EntryMetadata,
) {
if components.is_empty() {
return;
}
if components.len() == 1 {
let file_name = components[0].to_string();
let file_path = if parent_path.is_empty() {
file_name.clone()
} else {
format!("{}/{}", parent_path, file_name)
};
let child = node
.children
.entry(file_name)
.or_insert_with(|| TreeNode::with_path(file_path));
child.tokens = tokens;
child.metadata = Some(file_metadata);
} else {
let dir_name = components[0].to_string();
let dir_path = if parent_path.is_empty() {
dir_name.clone()
} else {
format!("{}/{}", parent_path, dir_name)
};
let child = node
.children
.entry(dir_name)
.or_insert_with(|| TreeNode::with_path(dir_path.clone()));
child.tokens += tokens;
child.metadata = Some(EntryMetadata { is_dir: true });
insert_path(child, &components[1..], tokens, dir_path, file_metadata);
}
}
#[derive(Debug, Clone)]
pub struct TokenMapEntry {
pub path: String,
pub name: String,
pub tokens: usize,
pub percentage: f64,
pub depth: usize,
pub is_last: bool,
pub metadata: EntryMetadata,
}
fn select_nodes_to_display(
root: &TreeNode,
total_tokens: usize,
max_lines: usize,
min_percent: f64,
) -> HashMap<String, usize> {
let mut heap = BinaryHeap::new();
let mut allowed_nodes = HashMap::new();
let min_tokens = (total_tokens as f64 * min_percent / 100.0) as usize;
for child in root.children.values() {
if child.tokens >= min_tokens {
heap.push(NodePriority {
tokens: child.tokens,
path: child.path.clone(),
depth: 0,
});
}
}
while allowed_nodes.len() < max_lines.saturating_sub(1) && !heap.is_empty() {
if let Some(node_priority) = heap.pop() {
allowed_nodes.insert(node_priority.path.clone(), node_priority.depth);
if let Some(node) = find_node_by_path(root, &node_priority.path) {
for child in node.children.values() {
if child.tokens >= min_tokens && !allowed_nodes.contains_key(&child.path) {
heap.push(NodePriority {
tokens: child.tokens,
path: child.path.clone(),
depth: node_priority.depth + 1,
});
}
}
}
}
}
allowed_nodes
}
fn find_node_by_path<'a>(root: &'a TreeNode, path: &str) -> Option<&'a TreeNode> {
if path.is_empty() {
return Some(root);
}
let components: Vec<&str> = path.split('/').collect();
let mut current = root;
for component in components {
let child = current.children.get(component)?;
current = child;
}
Some(current)
}
fn rebuild_filtered_tree(
node: &TreeNode,
path: String,
allowed_nodes: &HashMap<String, usize>,
entries: &mut Vec<TokenMapEntry>,
depth: usize,
total_tokens: usize,
is_last: bool,
) {
if !path.is_empty() && allowed_nodes.contains_key(&path) {
let percentage = (node.tokens as f64 / total_tokens as f64) * 100.0;
let name = path.split('/').next_back().unwrap_or(&path).to_string();
let metadata = node.metadata.unwrap_or(EntryMetadata { is_dir: true });
entries.push(TokenMapEntry {
path: path.clone(),
name,
tokens: node.tokens,
percentage,
depth,
is_last,
metadata,
});
}
let mut filtered_children: Vec<_> = node
.children
.iter()
.filter(|(_, child)| allowed_nodes.contains_key(&child.path))
.collect();
filtered_children.sort_by_key(|b| std::cmp::Reverse(b.1.tokens));
let child_count = filtered_children.len();
for (i, (name, child)) in filtered_children.into_iter().enumerate() {
let child_path = if path.is_empty() {
name.clone()
} else {
format!("{}/{}", path, name)
};
let is_last_child = i == child_count - 1;
rebuild_filtered_tree(
child,
child_path,
allowed_nodes,
entries,
depth + 1,
total_tokens,
is_last_child,
);
}
}
fn should_enable_colors() -> bool {
if std::env::var_os("NO_COLOR").is_some() {
return false;
}
if terminal_size::terminal_size().is_none() {
return false;
}
#[cfg(windows)]
{
use log::error;
match ansi_term::enable_ansi_support() {
Ok(_) => true,
Err(_) => {
error!("This version of Windows does not support ANSI colors");
false
}
}
}
#[cfg(not(windows))]
{
true
}
}
pub fn display_token_map(entries: &[TokenMapEntry], total_tokens: usize) {
if entries.is_empty() {
return;
}
let ls_colors = LsColors::from_env().unwrap_or_default();
let colors_enabled = should_enable_colors();
let terminal_width = terminal_size::terminal_size()
.map(|(terminal_size::Width(w), _)| w as usize)
.unwrap_or(80);
let max_token_width = entries
.iter()
.map(|e| format_tokens(e.tokens).len())
.max()
.unwrap_or(3)
.max(format_tokens(total_tokens).len())
.max(4);
let max_name_length = entries
.iter()
.map(|e| {
let prefix_width = if e.depth == 0 { 3 } else { (e.depth * 2) + 3 };
prefix_width + UnicodeWidthStr::width(e.name.as_str())
})
.max()
.unwrap_or(20)
.min(terminal_width / 2);
let bar_width = terminal_width
.saturating_sub(max_token_width + 3 + max_name_length + 2 + 2 + 5)
.max(20);
let mut parent_bars: Vec<String> = vec![String::new(); 10];
parent_bars[0] = "█".repeat(bar_width);
for (i, entry) in entries.iter().enumerate() {
let prefix = build_tree_prefix(entry, entries, i);
let tokens_str = format_tokens(entry.tokens);
let parent_bar = if entry.depth > 0 {
&parent_bars[(entry.depth - 1).min(parent_bars.len() - 1)]
} else {
&parent_bars[0]
};
let bar = generate_hierarchical_bar(bar_width, parent_bar, entry.percentage, entry.depth);
let slot = entry.depth.min(parent_bars.len() - 1);
parent_bars[slot] = bar.clone();
let percentage_str = format!("{:>4.0}%", entry.percentage);
let prefix_display_width = prefix.chars().count();
let name_padding = max_name_length
.saturating_sub(prefix_display_width + UnicodeWidthStr::width(entry.name.as_str()));
let name_with_padding = format!("{}{}", entry.name, " ".repeat(name_padding));
let colored_name_with_padding = if colors_enabled && entry.name != "(other files)" {
let ansi_style = if entry.metadata.is_dir {
ls_colors
.style_for_indicator(Indicator::Directory)
.map(|s| s.to_ansi_term_style())
.unwrap_or_default()
} else {
ls_colors
.style_for_path(std::path::Path::new(&entry.path))
.map(lscolors::Style::to_ansi_term_style)
.unwrap_or_default()
};
format!("{}", ansi_style.paint(name_with_padding))
} else {
name_with_padding
};
eprintln!(
"{:>width$} {}{} │{}│ {}",
tokens_str,
prefix,
colored_name_with_padding,
bar,
percentage_str,
width = max_token_width
);
}
}
fn build_tree_prefix(entry: &TokenMapEntry, entries: &[TokenMapEntry], index: usize) -> String {
let mut prefix = String::new();
for d in 0..entry.depth {
if d < entry.depth - 1 {
let needs_line = entries
.iter()
.skip(index + 1)
.take_while(|entry| entry.depth > d)
.any(|entry| entry.depth == d + 1);
if needs_line {
prefix.push_str("│ ");
} else {
prefix.push_str(" ");
}
} else if entry.is_last {
prefix.push_str("└─");
} else {
prefix.push_str("├─");
}
}
if entry.depth == 0 && index == 0 && entry.name != "(other files)" {
prefix = "┌─".to_string();
}
let has_children = entries
.get(index + 1)
.map(|next| next.depth > entry.depth)
.unwrap_or(false);
if entry.depth > 0 || entry.name == "(other files)" {
if has_children {
prefix.push('┬');
} else {
prefix.push('─');
}
} else if index == 0 {
prefix.push('┴');
}
prefix.push(' ');
prefix
}
fn determine_tui_color(entry: &TokenMapEntry) -> TuiColor {
if entry.metadata.is_dir {
TuiColor::Cyan
} else {
match entry.name.split('.').next_back().unwrap_or("") {
"rs" => TuiColor::Yellow,
"c" | "h" | "cpp" | "cxx" | "hpp" => TuiColor::Blue,
"go" => TuiColor::LightBlue,
"java" | "kt" | "kts" => TuiColor::Red,
"swift" => TuiColor::LightRed,
"zig" => TuiColor::LightYellow,
"js" | "mjs" | "cjs" => TuiColor::LightGreen,
"ts" | "tsx" | "jsx" => TuiColor::LightCyan,
"html" | "htm" => TuiColor::Magenta,
"css" | "scss" | "less" => TuiColor::LightMagenta,
"py" => TuiColor::LightYellow,
"sh" | "bash" | "zsh" => TuiColor::Gray,
"rb" => TuiColor::LightRed,
"pl" => TuiColor::LightCyan,
"php" => TuiColor::LightMagenta,
"lua" => TuiColor::LightBlue,
"json" | "toml" | "yaml" | "yml" => TuiColor::Magenta,
"xml" => TuiColor::LightGreen,
"csv" => TuiColor::Green,
"ini" => TuiColor::Gray,
"md" | "txt" | "rst" | "adoc" => TuiColor::Green,
"pdf" => TuiColor::Red,
_ => TuiColor::White,
}
}
}
pub fn format_token_map_for_tui(
entries: &[TokenMapEntry],
total_tokens: usize,
terminal_width: usize,
) -> Vec<TuiTokenMapLine> {
if entries.is_empty() {
return Vec::new();
}
let terminal_width = terminal_width.max(80);
let max_token_width = entries
.iter()
.map(|e| format_tokens(e.tokens).len())
.max()
.unwrap_or(3)
.max(format_tokens(total_tokens).len())
.max(4);
let max_name_length = entries
.iter()
.map(|e| {
let prefix_width = if e.depth == 0 { 3 } else { (e.depth * 2) + 3 };
prefix_width + UnicodeWidthStr::width(e.name.as_str())
})
.max()
.unwrap_or(20)
.min(terminal_width / 2);
let bar_width = terminal_width
.saturating_sub(max_token_width + 3 + max_name_length + 2 + 2 + 7) .max(15);
let mut parent_bars: Vec<String> = vec![String::new(); 10];
parent_bars[0] = "█".repeat(bar_width);
let mut lines = Vec::new();
for (i, entry) in entries.iter().enumerate() {
let prefix = build_tree_prefix(entry, entries, i);
let tokens_str = format_tokens(entry.tokens);
let parent_bar = if entry.depth > 0 {
&parent_bars[entry.depth - 1]
} else {
&parent_bars[0]
};
let bar = generate_hierarchical_bar(bar_width, parent_bar, entry.percentage, entry.depth);
if entry.depth < parent_bars.len() {
parent_bars[entry.depth] = bar.clone();
}
let percentage_str = format!("{:>4.0}%", entry.percentage);
let prefix_display_width = prefix.chars().count();
let name_padding = max_name_length
.saturating_sub(prefix_display_width + UnicodeWidthStr::width(entry.name.as_str()));
let name_with_padding = format!("{}{}", entry.name, " ".repeat(name_padding));
let name_color = determine_tui_color(entry);
lines.push(TuiTokenMapLine {
tokens_part: format!("{:>width$}", tokens_str, width = max_token_width),
prefix_part: prefix,
name_part: name_with_padding,
name_color,
bar_part: format!("│{}│", bar),
percentage_part: percentage_str,
});
}
lines
}
fn format_tokens(tokens: usize) -> String {
if tokens >= 1_000_000 {
let millions = (tokens + 500_000) / 1_000_000;
format!("{}M", millions)
} else if tokens >= 1_000 {
let thousands = (tokens + 500) / 1_000;
format!("{}K", thousands)
} else {
format!("{}", tokens)
}
}
fn generate_hierarchical_bar(
bar_width: usize,
parent_bar: &str,
percentage: f64,
depth: usize,
) -> String {
let filled_chars = ((percentage / 100.0) * bar_width as f64).round() as usize;
let mut result = String::new();
let shade_char = match depth.max(1) {
1 => ' ', 2 => '░', 3 => '▒', _ => '▓', };
let parent_chars: Vec<char> = parent_bar.chars().collect();
for i in 0..bar_width {
if i < filled_chars {
result.push('█');
} else if i < parent_chars.len() {
let parent_char = parent_chars[i];
if parent_char == '█' {
result.push(shade_char);
} else {
result.push(parent_char);
}
} else {
result.push(' ');
}
}
result
}
#[cfg(test)]
mod tests {
use super::*;
fn mf(path: &str, tokens: usize) -> TokenMapFile {
TokenMapFile {
path: path.to_string(),
tokens,
}
}
#[test]
fn builds_simple_tree_with_dir_aggregate() {
let files = vec![mf("src/main.rs", 100), mf("src/lib.rs", 200)];
let entries = generate_token_map_with_limit(&files, 300, Some(100), Some(0.0));
let src = entries.iter().find(|e| e.name == "src").expect("src dir");
assert_eq!(src.tokens, 300, "src should aggregate its children");
assert!(src.metadata.is_dir, "src is a synthesized directory node");
let main = entries
.iter()
.find(|e| e.name == "main.rs")
.expect("main.rs");
assert!(!main.metadata.is_dir, "leaf files are not directories");
}
#[test]
fn directories_accumulate_child_tokens() {
let files = vec![
mf("src/a.rs", 100),
mf("src/b.rs", 200),
mf("src/sub/c.rs", 50),
];
let entries = generate_token_map_with_limit(&files, 350, Some(100), Some(0.0));
let src = entries.iter().find(|e| e.name == "src").expect("src");
assert_eq!(src.tokens, 350, "src = 100 + 200 + 50");
let sub = entries.iter().find(|e| e.name == "sub").expect("sub");
assert_eq!(sub.tokens, 50, "sub holds only its own child");
}
#[test]
fn nested_structure_increases_depth() {
let files = vec![
mf("website/public/assets/logo.svg", 100),
mf("website/public/favicon.ico", 50),
];
let entries = generate_token_map_with_limit(&files, 150, Some(100), Some(0.0));
let depth = |name: &str| {
entries
.iter()
.find(|e| e.name == name)
.map(|e| e.depth)
.unwrap_or_else(|| panic!("missing entry: {name}"))
};
let website = depth("website");
let public = depth("public");
let assets = depth("assets");
let logo = depth("logo.svg");
let favicon = depth("favicon.ico");
assert!(
website < public && public < assets && assets < logo,
"depth must strictly increase down the chain: {website} {public} {assets} {logo}"
);
assert_eq!(favicon, assets, "favicon.ico is a sibling level of assets");
}
#[test]
fn max_lines_limits_displayed_files() {
let files = vec![
mf("a.rs", 1000),
mf("b.rs", 500),
mf("c.rs", 200),
mf("d.rs", 100),
];
let entries = generate_token_map_with_limit(&files, 1800, Some(2), Some(0.0));
assert!(
entries.iter().any(|e| e.name == "a.rs"),
"a.rs must survive"
);
let file_rows = entries.iter().filter(|e| !e.metadata.is_dir).count();
assert!(
file_rows <= 3,
"expected at most 3 file rows, got {file_rows}"
);
}
#[test]
fn min_percent_filters_small_files() {
let files = vec![mf("big.rs", 900), mf("small.rs", 50)];
let entries = generate_token_map_with_limit(&files, 950, Some(100), Some(10.0));
assert!(entries.iter().any(|e| e.name == "big.rs"), "big.rs kept");
assert!(
entries.iter().all(|e| e.name != "small.rs"),
"small.rs filtered by min_percent"
);
}
#[test]
fn hidden_tokens_become_other_files() {
let files = vec![mf("big.rs", 900), mf("small.rs", 50)];
let entries = generate_token_map_with_limit(&files, 950, Some(100), Some(10.0));
let other = entries
.iter()
.find(|e| e.name == "(other files)")
.expect("filtered tokens should surface as (other files)");
assert_eq!(other.tokens, 50, "the dropped 50 tokens are aggregated");
}
#[test]
fn no_other_files_when_everything_shown() {
let files = vec![mf("a.rs", 100), mf("b.rs", 200)];
let entries = generate_token_map_with_limit(&files, 300, Some(100), Some(0.0));
assert!(
entries.iter().all(|e| e.name != "(other files)"),
"nothing hidden ⇒ no (other files) row"
);
}
#[test]
fn single_file_is_one_leaf() {
let entries = generate_token_map_with_limit(&[mf("a.rs", 100)], 100, Some(100), Some(0.0));
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].name, "a.rs");
assert_eq!(entries[0].tokens, 100);
assert!(!entries[0].metadata.is_dir);
}
#[test]
fn empty_input_yields_no_entries() {
let entries = generate_token_map_with_limit(&[], 0, Some(100), Some(0.0));
assert!(
entries.is_empty(),
"no files ⇒ no entries (and no divide-by-zero)"
);
}
#[test]
fn realistic_multi_crate_layout() {
let files = vec![
mf("crates/gnaw/src/main.rs", 5000),
mf("crates/gnaw/src/args.rs", 3000),
mf("crates/gnaw-core/src/analysis.rs", 2000),
mf("website/src/index.html", 1000),
mf("website/public/favicon.ico", 500),
mf("README.md", 200),
];
let total: usize = files.iter().map(|f| f.tokens).sum();
let entries = generate_token_map_with_limit(&files, total, Some(50), Some(0.0));
assert!(!entries.is_empty(), "should produce a tree");
assert!(
entries.iter().any(|e| e.name == "crates"),
"top-level 'crates' directory present"
);
assert!(
entries.iter().any(|e| e.depth >= 3),
"deep nesting (crates/<pkg>/src/<file>) should reach depth >= 3"
);
}
#[test]
fn entries_are_valid_preorder() {
let files = vec![mf("a/b/c/d.rs", 100), mf("a/b/e.rs", 200), mf("f.rs", 50)];
let entries = generate_token_map_with_limit(&files, 350, Some(100), Some(0.0));
for w in entries.windows(2) {
assert!(
w[1].depth <= w[0].depth + 1,
"depth jumped from {} to {} — not a valid pre-order walk",
w[0].depth,
w[1].depth
);
}
}
#[test]
fn percentages_match_token_share() {
let files = vec![mf("a.rs", 250), mf("b.rs", 750)];
let total = 1000;
let entries = generate_token_map_with_limit(&files, total, Some(100), Some(0.0));
for e in &entries {
let expected = e.tokens as f64 / total as f64 * 100.0;
assert!(
(e.percentage - expected).abs() < 1e-6,
"{}: percentage {} != {}",
e.name,
e.percentage,
expected
);
}
}
}