tokenburn_core/utils/
files.rs

1//! Filesystem + JSON helpers shared by the collectors.
2
3use std::fs::{self, File};
4use std::io::{BufRead, BufReader};
5use std::path::{Path, PathBuf};
6use std::time::SystemTime;
7
8use chrono::{DateTime, Utc};
9use serde_json::Value;
10
11/// Recursively collect files below `root` whose extension is one of `exts`
12/// (case-sensitive, without the dot). Unreadable directories are skipped.
13pub fn walk_ext(root: &Path, exts: &[&str]) -> Vec<PathBuf> {
14    let mut out = Vec::new();
15    walk(root, exts, &mut out);
16    out.sort();
17    out
18}
19
20fn walk(dir: &Path, exts: &[&str], out: &mut Vec<PathBuf>) {
21    let Ok(rd) = fs::read_dir(dir) else { return };
22    for entry in rd.flatten() {
23        let path = entry.path();
24        if path.is_dir() {
25            walk(&path, exts, out);
26        } else if path
27            .extension()
28            .and_then(|e| e.to_str())
29            .is_some_and(|e| exts.contains(&e))
30        {
31            out.push(path);
32        }
33    }
34}
35
36/// `false` only when the file's mtime is known and older than `start` — cheap
37/// pre-filter so untouched history is never parsed.
38pub fn modified_since(path: &Path, start: DateTime<Utc>) -> bool {
39    let start_sys: SystemTime = start.into();
40    !matches!(
41        fs::metadata(path).and_then(|m| m.modified()),
42        Ok(m) if m < start_sys
43    )
44}
45
46/// Stream the lines of a (possibly huge) file; invalid UTF-8 lines are skipped.
47pub fn read_lines(path: &Path) -> Option<impl Iterator<Item = String>> {
48    let f = File::open(path).ok()?;
49    Some(BufReader::new(f).lines().map_while(Result::ok))
50}
51
52/// A JSON number as `u64`: accepts integers and floats, clamps negatives to 0.
53pub fn num(v: Option<&Value>) -> u64 {
54    match v {
55        Some(Value::Number(n)) => n
56            .as_u64()
57            .or_else(|| n.as_f64().map(|f| f.max(0.0) as u64))
58            .unwrap_or(0),
59        _ => 0,
60    }
61}
62
63/// An optional JSON number (already deserialised as `f64`) as `u64`: clamps
64/// negatives and NaN to 0. Used by the typed per-line parsers.
65pub fn num_f(v: Option<f64>) -> u64 {
66    v.filter(|x| x.is_finite()).map_or(0, |x| x.max(0.0) as u64)
67}
68
69/// First of `keys` present in `obj` as a number (aliases across versions).
70pub fn num_any(obj: &Value, keys: &[&str]) -> u64 {
71    keys.iter()
72        .find_map(|k| obj.get(*k).filter(|v| v.is_number()))
73        .map(|v| num(Some(v)))
74        .unwrap_or(0)
75}
76
77/// A non-empty environment variable as a path.
78pub fn env_path(key: &str) -> Option<PathBuf> {
79    std::env::var_os(key)
80        .filter(|v| !v.is_empty())
81        .map(PathBuf::from)
82}
83
84/// Split a path-list environment value (`:` / `;` / `,` separated).
85pub fn split_paths(value: &str) -> Vec<PathBuf> {
86    value
87        .split([',', ';', if cfg!(windows) { '\u{0}' } else { ':' }])
88        .map(str::trim)
89        .filter(|s| !s.is_empty())
90        .map(PathBuf::from)
91        .collect()
92}
93
94/// `-Users-me-Projects-app-` → `Users-me-Projects-app`.
95pub fn project_from_dirname(name: &str) -> String {
96    name.trim_matches('-').to_string()
97}
98
99/// Last path component of a working directory (`/a/b/app` → `app`), or `None`.
100pub fn dir_basename(cwd: &str) -> Option<String> {
101    cwd.trim_end_matches(['/', '\\'])
102        .rsplit(['/', '\\'])
103        .next()
104        .filter(|s| !s.is_empty())
105        .map(str::to_string)
106}
107
108#[cfg(test)]
109mod tests {
110    use super::*;
111    use serde_json::json;
112
113    #[test]
114    fn num_handles_ints_floats_negatives_and_junk() {
115        assert_eq!(num(Some(&json!(7))), 7);
116        assert_eq!(num(Some(&json!(7.9))), 7);
117        assert_eq!(num(Some(&json!(-3))), 0);
118        assert_eq!(num(Some(&json!("7"))), 0);
119        assert_eq!(num(None), 0);
120    }
121
122    #[test]
123    fn num_f_clamps_and_tolerates_junk() {
124        assert_eq!(num_f(Some(7.9)), 7);
125        assert_eq!(num_f(Some(-1.0)), 0);
126        assert_eq!(num_f(Some(f64::NAN)), 0);
127        assert_eq!(num_f(Some(f64::INFINITY)), 0);
128        assert_eq!(num_f(None), 0);
129    }
130
131    #[test]
132    fn num_any_takes_the_first_alias_present() {
133        let v = json!({"prompt": 4, "input": 9});
134        assert_eq!(num_any(&v, &["input", "prompt"]), 9);
135        assert_eq!(num_any(&v, &["nope", "prompt"]), 4);
136        assert_eq!(num_any(&v, &["nope"]), 0);
137    }
138
139    #[test]
140    fn basename_handles_both_separators_and_trailing_slash() {
141        assert_eq!(dir_basename("/a/b/app/").as_deref(), Some("app"));
142        assert_eq!(dir_basename("C:\\Users\\me\\app").as_deref(), Some("app"));
143        assert_eq!(dir_basename("").as_deref(), None);
144    }
145
146    #[test]
147    fn project_dirname_trims_dashes() {
148        assert_eq!(project_from_dirname("--home-me--"), "home-me");
149    }
150
151    #[test]
152    fn split_paths_accepts_commas_and_semicolons() {
153        assert_eq!(split_paths("/a, /b;/c").len(), 3);
154        assert!(split_paths("").is_empty());
155    }
156
157    #[test]
158    fn walk_ext_finds_nested_files_sorted() {
159        let d = tempfile::tempdir().unwrap();
160        fs::create_dir_all(d.path().join("x/y")).unwrap();
161        fs::write(d.path().join("x/y/a.jsonl"), "").unwrap();
162        fs::write(d.path().join("b.jsonl"), "").unwrap();
163        fs::write(d.path().join("c.txt"), "").unwrap();
164        let files = walk_ext(d.path(), &["jsonl"]);
165        assert_eq!(files.len(), 2);
166        assert!(files.windows(2).all(|w| w[0] <= w[1]));
167    }
168}
tokenburn_core/utils/files.rs

tokenburn_core/utils/
files.rs
