ljos_cli/
extract.rs

1//! Facts a session leaves behind, kept without anyone asking. When a
2//! runner ends a session, the hook reads its transcript and keeps up to
3//! [`MAX_PER_SESSION`] sentences that state something durable: a rule
4//! ("always", "never", "must"), a cause ("root cause", "because", "turns
5//! out"), or a fix. Each is written at once as a `note` whose
6//! `source.basis` is `agent`, carrying `via:session-end`, the session,
7//! and the open sitting's issue. The read side goes by that stored basis:
8//! recall, search and the island label it an unverified agent note, and
9//! no prompt hook hands it on as a fact or a rule. With no sitting,
10//! packset has no place for a note, and the write is skipped. No model
11//! runs: the extraction is a bounded, deterministic read. A sentence
12//! already kept (by its normalized form) is not written again.
13//!
14//! The read runs at session end. `LJOS_EXTRACT=off` turns it off.
15//! A mined sentence is still nobody's claim: the note stays on the
16//! tracker, and no prompt hook injects it.
17
18use serde_json::Value;
19use std::collections::BTreeSet;
20
21/// The most facts one session writes.
22pub const MAX_PER_SESSION: usize = 5;
23/// The longest transcript read, in bytes, from its end.
24pub const MAX_TRANSCRIPT_BYTES: usize = 2 * 1024 * 1024;
25
26const CUES: &[&str] = &[
27    "always ",
28    "never ",
29    " must ",
30    "root cause",
31    "turns out",
32    "turned out",
33    "the fix ",
34    "fixed by",
35    " because ",
36    " requires ",
37    " instead of ",
38    "does not work",
39    "doesn't work",
40    "is broken",
41    "the reason ",
42    "make sure ",
43    "remember that",
44];
45
46/// A sentence's form for deduplication: lowercase words only.
47#[must_use]
48pub fn normal(s: &str) -> String {
49    s.to_lowercase()
50        .split(|c: char| !c.is_alphanumeric())
51        .filter(|w| !w.is_empty())
52        .collect::<Vec<_>>()
53        .join(" ")
54}
55
56fn texts_of(v: &Value, out: &mut Vec<String>) {
57    match v {
58        Value::String(s) => out.push(s.clone()),
59        Value::Array(a) => a.iter().for_each(|x| texts_of(x, out)),
60        Value::Object(o) => {
61            // Tool calls and results are not what the session concluded.
62            if matches!(
63                o.get("type").and_then(Value::as_str),
64                Some("tool_use" | "tool_result" | "thinking" | "image")
65            ) {
66                return;
67            }
68            for key in ["text", "content", "message"] {
69                if let Some(x) = o.get(key) {
70                    texts_of(x, out);
71                }
72            }
73        }
74        _ => {}
75    }
76}
77
78/// The prose of a transcript: JSONL with one message a line (Claude Code,
79/// Codex and most runners), or plain text.
80#[must_use]
81pub fn prose(transcript: &str) -> Vec<String> {
82    let mut out = Vec::new();
83    for line in transcript.lines() {
84        match serde_json::from_str::<Value>(line) {
85            Ok(v) => {
86                let role = v["type"]
87                    .as_str()
88                    .or_else(|| v["role"].as_str())
89                    .or_else(|| v["message"]["role"].as_str())
90                    .unwrap_or("");
91                if matches!(role, "user" | "assistant" | "") {
92                    texts_of(&v, &mut out);
93                }
94            }
95            Err(_) => out.push(line.to_string()),
96        }
97    }
98    out
99}
100
101fn sentences(text: &str) -> Vec<String> {
102    let mut out = Vec::new();
103    let mut in_fence = false;
104    for line in text.lines() {
105        if line.trim_start().starts_with("```") {
106            in_fence = !in_fence;
107            continue;
108        }
109        if in_fence {
110            continue;
111        }
112        let mut cur = String::new();
113        let mut chars = line.chars().peekable();
114        while let Some(c) = chars.next() {
115            cur.push(c);
116            // A stop ends a sentence only before a space or the line's end,
117            // so `build.rs` and `v0.29.1` stay whole.
118            if matches!(c, '.' | '!' | '?') && chars.peek().is_none_or(|n| n.is_whitespace()) {
119                out.push(std::mem::take(&mut cur));
120            }
121        }
122        if !cur.trim().is_empty() {
123            out.push(cur);
124        }
125    }
126    out.into_iter()
127        .map(|s| {
128            s.trim()
129                .trim_start_matches(['-', '*', '>', ' '])
130                .trim()
131                .to_string()
132        })
133        .collect()
134}
135
136/// Whether a sentence states something durable enough to keep.
137#[must_use]
138pub fn is_fact(s: &str) -> bool {
139    let n = s.chars().count();
140    if !(30..=240).contains(&n) || s.ends_with('?') || s.contains('`') || s.contains("://") {
141        return false;
142    }
143    let low = format!(" {} ", s.to_lowercase());
144    // A question, an instruction to this session, or a placeholder.
145    if low.starts_with(" can you") || low.starts_with(" please") || low.contains(" i will ") {
146        return false;
147    }
148    CUES.iter().any(|c| low.contains(c))
149}
150
151/// Up to [`MAX_PER_SESSION`] facts from a transcript, newest last, with no
152/// two of the same normalized form and none in `kept`.
153#[must_use]
154pub fn facts(transcript: &str, kept: &BTreeSet<String>) -> Vec<String> {
155    let mut seen = kept.clone();
156    let mut out: Vec<String> = Vec::new();
157    for text in prose(transcript) {
158        for s in sentences(&text) {
159            if !is_fact(&s) {
160                continue;
161            }
162            let key = normal(&s);
163            if seen.insert(key) {
164                out.push(s);
165            }
166        }
167    }
168    // The end of a session is what it concluded.
169    let skip = out.len().saturating_sub(MAX_PER_SESSION);
170    out.into_iter().skip(skip).collect()
171}
172
173fn ledger() -> Option<std::path::PathBuf> {
174    let base = std::env::var_os("XDG_STATE_HOME")
175        .map(std::path::PathBuf::from)
176        .or_else(|| {
177            std::env::var_os("HOME").map(|h| std::path::PathBuf::from(h).join(".local/state"))
178        })?;
179    Some(base.join("ljos").join("extracted"))
180}
181
182/// The normalized forms already kept on this machine.
183#[must_use]
184pub fn kept() -> BTreeSet<String> {
185    ledger()
186        .and_then(|p| std::fs::read_to_string(p).ok())
187        .map(|t| t.lines().map(str::to_string).collect())
188        .unwrap_or_default()
189}
190
191/// The pack's reason with the HTTP status left off, so a debug line does
192/// not repeat `400:`.
193fn plain_failure(err: &anyhow::Error) -> String {
194    let text = format!("{err:#}");
195    let parts: Vec<&str> = text.split(": ").collect();
196    for (i, part) in parts.iter().enumerate() {
197        let status = part.trim().parse::<u16>();
198        if status.is_ok_and(|code| (400..=599).contains(&code)) {
199            let rest = parts[i + 1..].join(": ");
200            if !rest.is_empty() {
201                return rest;
202            }
203        }
204    }
205    text
206}
207
208/// Whether session-end extraction runs. It does, unless `LJOS_EXTRACT=off`.
209#[must_use]
210pub fn asked_for() -> bool {
211    asked_by(std::env::var("LJOS_EXTRACT").ok().as_deref())
212}
213
214/// [`asked_for`] over the variable's value, so a test names it.
215#[must_use]
216pub fn asked_by(set: Option<&str>) -> bool {
217    !matches!(set.map(str::trim), Some("off"))
218}
219
220/// Read the session's transcript (the hook payload's `transcript_path`),
221/// write each new sentence as an unverified agent note on the open
222/// sitting, and return how many were written. Assistant prose is the
223/// agent's own reading, so it is kept as a note the hook never shows as a
224/// fact, not a lesson. With no sitting the write is skipped: packset keeps
225/// a note only on a ticket. Quiet on every other failure: a hook must not
226/// stop a runner ending.
227pub fn at_session_end(input: &str, session: Option<&str>) -> usize {
228    if !asked_for() {
229        return 0;
230    }
231    let Ok(v) = serde_json::from_str::<Value>(input) else {
232        return 0;
233    };
234    let Some(path) = v["transcript_path"]
235        .as_str()
236        .or_else(|| v["transcriptPath"].as_str())
237    else {
238        return 0;
239    };
240    let Ok(bytes) = std::fs::read(path) else {
241        return 0;
242    };
243    let tail = &bytes[bytes.len().saturating_sub(MAX_TRANSCRIPT_BYTES)..];
244    let text = String::from_utf8_lossy(tail);
245    let found = facts(&text, &kept());
246    let debug = std::env::var_os("LJOS_EXTRACT_DEBUG").is_some();
247    if debug {
248        eprintln!("ljos extract: {} facts from {path}", found.len());
249    }
250    if found.is_empty() {
251        return 0;
252    }
253    // Packset keeps a note only on a ticket, and has no form for one that
254    // names none. Skip before the post, one line, rather than collect a
255    // refusal per fact.
256    if crate::live_held_issue().is_none() {
257        eprintln!("ljos extract: {}", crate::NOTE_WITHOUT_TICKET);
258        return 0;
259    }
260    let mut written = 0;
261    let mut keys = Vec::new();
262    for f in found {
263        let mut tags = vec!["via:session-end".to_string()];
264        if let Some(s) = session {
265            tags.push(format!("session:{s}"));
266        }
267        match crate::packset_write_agent_note_tagged(&f, &tags) {
268            Ok(_) => {
269                written += 1;
270                keys.push(normal(&f));
271            }
272            Err(e) => {
273                if debug {
274                    eprintln!("ljos extract: {f:?} not written: {}", plain_failure(&e));
275                }
276            }
277        }
278    }
279    if debug {
280        eprintln!("ljos extract: wrote {written}");
281    }
282    if let (Some(p), false) = (ledger(), keys.is_empty()) {
283        if let Some(dir) = p.parent() {
284            let _ = std::fs::create_dir_all(dir);
285        }
286        let mut all: Vec<String> = kept().into_iter().collect();
287        all.extend(keys);
288        // Bounded: the newest 5000 forms.
289        let skip = all.len().saturating_sub(5000);
290        let _ = std::fs::write(&p, all[skip..].join("\n") + "\n");
291    }
292    written
293}
294
295#[cfg(test)]
296mod tests {
297    use super::*;
298
299    const T: &str = r#"{"type":"user","message":{"role":"user","content":"Can you fix the flaky build?"}}
300{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"Looking now."},{"type":"tool_use","name":"Bash","input":{"command":"cargo test because x"}}]}}
301{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"The root cause was a shared temp dir between tests. The fix is to give each test its own tempdir. Always run the suite with --test-threads=1 on this box. Done!"}]}}
302{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"The root cause was a shared temp dir between tests."}]}}"#;
303
304    #[test]
305    fn a_transcript_yields_its_durable_sentences_once() {
306        let got = facts(T, &BTreeSet::new());
307        assert_eq!(got.len(), 3, "{got:?}");
308        assert!(got[0].starts_with("The root cause was"));
309        assert!(got.iter().all(|f| !f.contains("Can you")));
310        assert!(
311            got.iter().all(|f| !f.contains("cargo test")),
312            "tool calls are not facts"
313        );
314    }
315
316    #[test]
317    fn already_kept_facts_are_skipped_and_the_count_is_bounded() {
318        let kept: BTreeSet<String> = [normal(
319            "The root cause was a shared temp dir between tests.",
320        )]
321        .into();
322        assert_eq!(facts(T, &kept).len(), 2);
323        let many: String = (0..20)
324            .map(|i| format!("Always pin tool number {i} to its tested version.\n"))
325            .collect();
326        let got = facts(&many, &BTreeSet::new());
327        assert_eq!(got.len(), MAX_PER_SESSION);
328        assert!(got.last().unwrap().contains("19"), "the newest are kept");
329    }
330
331    /// Unset runs the read. Only `off` stops it.
332    #[test]
333    fn extraction_runs_unless_turned_off() {
334        assert!(asked_by(None));
335        assert!(asked_by(Some("")));
336        assert!(asked_by(Some("on")));
337        assert!(asked_by(Some(" on ")));
338        assert!(asked_by(Some("1")));
339        assert!(!asked_by(Some("off")));
340        assert!(!asked_by(Some(" off ")));
341    }
342
343    #[test]
344    fn questions_code_and_short_lines_are_not_facts() {
345        assert!(!is_fact("Should we always do this before a release?"));
346        assert!(!is_fact("Always."));
347        assert!(!is_fact(
348            "Always run `rm -rf target` before a build here, it helps."
349        ));
350        assert!(is_fact(
351            "Never push to main without a green CI run on the branch."
352        ));
353    }
354}
ljos_cli/extract.rs

ljos_cli/
extract.rs
