very_normal/
corpus.rs

1//! What you typed: the messages you sent Claude Code, and any other writing
2//! you point at. Nothing here leaves your machine.
3
4use std::io::Read;
5use std::path::{Path, PathBuf};
6
7use anyhow::{Context, Result};
8use serde_json::Value;
9
10/// Messages longer than this were probably pasted rather than typed.
11const LONGEST: usize = 2000;
12
13/// File types `--text` reads when it's given a folder.
14const TEXT_EXTENSIONS: [&str; 8] = ["txt", "md", "markdown", "org", "rst", "text", "tex", "eml"];
15
16/// Where Claude Code keeps its transcripts.
17pub fn claude_projects() -> PathBuf {
18    let home = std::env::var_os("HOME")
19        .map(PathBuf::from)
20        .unwrap_or_default();
21    home.join(".claude/projects")
22}
23
24/// The messages you typed to Claude Code: user turns with plain text, minus
25/// pastes, slash commands, interruptions and anything long enough to be a
26/// paste. Never Claude's replies or tool output.
27pub fn typed_messages(projects: &Path) -> Vec<String> {
28    let mut files = Vec::new();
29    walk(projects, &mut files, &|p| {
30        p.extension().is_some_and(|e| e == "jsonl")
31    });
32    files.retain(|p| !p.to_string_lossy().contains("/subagents/"));
33    files.sort();
34    let mut out = Vec::new();
35    for file in files {
36        let Ok(bytes) = std::fs::read(&file) else {
37            continue;
38        };
39        for line in String::from_utf8_lossy(&bytes).lines() {
40            if let Some(m) = typed_message(line) {
41                out.push(m);
42            }
43        }
44    }
45    out
46}
47
48fn typed_message(line: &str) -> Option<String> {
49    let o: Value = serde_json::from_str(line).ok()?;
50    let truthy = |k: &str| {
51        o.get(k)
52            .is_some_and(|v| v.as_bool().unwrap_or(!v.is_null()))
53    };
54    if o.get("type").and_then(Value::as_str) != Some("user")
55        || truthy("isMeta")
56        || truthy("isSidechain")
57    {
58        return None;
59    }
60    let content = o.pointer("/message/content")?;
61    let text = match content {
62        Value::String(s) => s.clone(),
63        Value::Array(parts) => parts
64            .iter()
65            .filter(|p| p.get("type").and_then(Value::as_str) == Some("text"))
66            .map(|p| p.get("text").and_then(Value::as_str).unwrap_or(""))
67            .collect::<Vec<_>>()
68            .join(" "),
69        _ => return None,
70    };
71    let text = without_pastes(&text);
72    let text = text.trim();
73    let head: String = text.chars().take(20).collect();
74    if text.is_empty()
75        || text.starts_with('<')
76        || text.starts_with("[Request interrupted")
77        || head.contains("Caveat:")
78        || text.chars().count() > LONGEST
79    {
80        return None;
81    }
82    Some(text.to_string())
83}
84
85/// Drops `<pasted_content …>…</pasted_content …>` blocks.
86fn without_pastes(text: &str) -> String {
87    let mut out = String::new();
88    let mut rest = text;
89    while let Some(start) = rest.find("<pasted_content") {
90        let Some(close) = rest[start..].find("</pasted_content") else {
91            break;
92        };
93        let after_close = start + close;
94        let Some(end) = rest[after_close..].find('>') else {
95            break;
96        };
97        out.push_str(&rest[..start]);
98        rest = &rest[after_close + end + 1..];
99    }
100    out.push_str(rest);
101    out
102}
103
104/// Paragraphs from text files, folders of them (their .txt, .md, .org and
105/// similar files), or `-` for stdin.
106pub fn text_messages(paths: &[String]) -> Result<Vec<String>> {
107    let mut out = Vec::new();
108    for path in paths {
109        if path == "-" {
110            let mut body = String::new();
111            std::io::stdin().read_to_string(&mut body)?;
112            out.extend(paragraphs(&body));
113            continue;
114        }
115        let p = Path::new(path);
116        let mut files = Vec::new();
117        if p.is_dir() {
118            walk(p, &mut files, &|f| {
119                f.extension()
120                    .and_then(|e| e.to_str())
121                    .is_some_and(|e| TEXT_EXTENSIONS.contains(&e.to_ascii_lowercase().as_str()))
122            });
123            files.sort();
124        } else {
125            files.push(p.to_path_buf());
126        }
127        for f in files {
128            let bytes = std::fs::read(&f).with_context(|| format!("reading {}", f.display()))?;
129            out.extend(paragraphs(&String::from_utf8_lossy(&bytes)));
130        }
131    }
132    Ok(out)
133}
134
135/// Splits on blank lines.
136fn paragraphs(body: &str) -> Vec<String> {
137    let mut out = Vec::new();
138    let mut current: Vec<&str> = Vec::new();
139    for line in body.lines() {
140        if line.trim().is_empty() {
141            if !current.is_empty() {
142                out.push(current.join("\n").trim().to_string());
143                current.clear();
144            }
145        } else {
146            current.push(line);
147        }
148    }
149    if !current.is_empty() {
150        out.push(current.join("\n").trim().to_string());
151    }
152    out.retain(|p| !p.is_empty());
153    out
154}
155
156/// Every file under `dir` that `keep` wants, skipping hidden folders.
157fn walk(dir: &Path, out: &mut Vec<PathBuf>, keep: &dyn Fn(&Path) -> bool) {
158    let Ok(entries) = std::fs::read_dir(dir) else {
159        return;
160    };
161    for entry in entries.flatten() {
162        let path = entry.path();
163        let hidden = entry.file_name().to_string_lossy().starts_with('.');
164        match entry.file_type() {
165            Ok(t) if t.is_dir() && !hidden => walk(&path, out, keep),
166            Ok(t) if t.is_file() && keep(&path) => out.push(path),
167            _ => {}
168        }
169    }
170}
171
172#[cfg(test)]
173mod tests {
174    use super::*;
175
176    #[test]
177    fn keeps_only_what_you_typed() {
178        let msg = |content: &str| {
179            format!(r#"{{"type":"user","message":{{"role":"user","content":{content}}}}}"#)
180        };
181        assert_eq!(
182            typed_message(&msg(r#""make it faster""#)).as_deref(),
183            Some("make it faster")
184        );
185        assert_eq!(
186            typed_message(&msg(
187                r#"[{"type":"text","text":"hi"},{"type":"tool_result","content":"x"}]"#
188            ))
189            .as_deref(),
190            Some("hi")
191        );
192        assert_eq!(
193            typed_message(&msg(
194                r#""look <pasted_content id=\"1\">secret</pasted_content id=\"1\"> here""#
195            ))
196            .as_deref(),
197            Some("look  here")
198        );
199        assert_eq!(
200            typed_message(&msg(r#""<command-name>/clear</command-name>""#)),
201            None
202        );
203        assert_eq!(
204            typed_message(&msg(r#""[Request interrupted by user]""#)),
205            None
206        );
207        assert_eq!(
208            typed_message(&msg(&format!("\"{}\"", "a".repeat(2001)))),
209            None
210        );
211        assert_eq!(
212            typed_message(r#"{"type":"assistant","message":{"content":"hello"}}"#),
213            None
214        );
215        assert_eq!(
216            typed_message(r#"{"type":"user","isMeta":true,"message":{"content":"hello"}}"#),
217            None
218        );
219    }
220
221    #[test]
222    fn splits_text_into_paragraphs() {
223        assert_eq!(
224            paragraphs("one\ntwo\n\n  \nthree\n"),
225            vec!["one\ntwo".to_string(), "three".to_string()]
226        );
227    }
228}
very_normal/corpus.rs

very_normal/
corpus.rs
