ljos_cli/
extract.rs

1//! Facts a session leaves behind, kept without anyone asking. When a
2//! runner ends a session, the hook reads its transcript and keeps up to
3//! [`MAX_PER_SESSION`] sentences that state something durable: a rule
4//! ("always", "never", "must"), a cause ("root cause", "because", "turns
5//! out"), or a fix. Each is written at once as a `note` whose
6//! `source.basis` is `agent`, carrying `via:session-end`, the session,
7//! and the open sitting's issue. The read side goes by that stored basis:
8//! recall, search and the island label it an unverified agent note, and
9//! no prompt hook hands it on as a fact or a rule. With no sitting,
10//! packset has no place for a note, and the write is skipped. No model
11//! runs: the extraction is a bounded, deterministic read. A sentence
12//! already kept (by its normalized form) is not written again.
13//!
14//! The read runs at session end. `LJOS_EXTRACT=off` turns it off.
15//! A mined sentence is still nobody's claim: the note stays on the
16//! tracker, and no prompt hook injects it.
17
18use serde_json::Value;
19use std::collections::BTreeSet;
20
21/// The most facts one session writes.
22pub const MAX_PER_SESSION: usize = 5;
23/// The longest transcript read, in bytes, from its end.
24pub const MAX_TRANSCRIPT_BYTES: usize = 2 * 1024 * 1024;
25
26/// The last `cap` bytes of `path`, or the whole file when it is shorter.
27/// The head of a long transcript is not read. `None` when the file cannot
28/// be opened.
29#[must_use]
30pub fn read_tail(path: &std::path::Path, cap: usize) -> Option<String> {
31    use std::io::{Read, Seek, SeekFrom};
32    let mut file = std::fs::File::open(path).ok()?;
33    let len = file.metadata().ok()?.len();
34    let start = len.saturating_sub(cap as u64);
35    file.seek(SeekFrom::Start(start)).ok()?;
36    let mut buf = Vec::new();
37    file.take(len - start).read_to_end(&mut buf).ok()?;
38    Some(String::from_utf8_lossy(&buf).into_owned())
39}
40
41const CUES: &[&str] = &[
42    "always ",
43    "never ",
44    " must ",
45    "root cause",
46    "turns out",
47    "turned out",
48    "the fix ",
49    "fixed by",
50    " because ",
51    " requires ",
52    " instead of ",
53    "does not work",
54    "doesn't work",
55    "is broken",
56    "the reason ",
57    "make sure ",
58    "remember that",
59];
60
61/// A sentence's form for deduplication: lowercase words only.
62#[must_use]
63pub fn normal(s: &str) -> String {
64    s.to_lowercase()
65        .split(|c: char| !c.is_alphanumeric())
66        .filter(|w| !w.is_empty())
67        .collect::<Vec<_>>()
68        .join(" ")
69}
70
71fn texts_of(v: &Value, out: &mut Vec<String>) {
72    match v {
73        Value::String(s) => out.push(s.clone()),
74        Value::Array(a) => a.iter().for_each(|x| texts_of(x, out)),
75        Value::Object(o) => {
76            // Tool calls and results are not what the session concluded.
77            if matches!(
78                o.get("type").and_then(Value::as_str),
79                Some("tool_use" | "tool_result" | "thinking" | "image")
80            ) {
81                return;
82            }
83            for key in ["text", "content", "message"] {
84                if let Some(x) = o.get(key) {
85                    texts_of(x, out);
86                }
87            }
88        }
89        _ => {}
90    }
91}
92
93/// The prose of a transcript: JSONL with one message a line (Claude Code,
94/// Codex and most runners), or plain text.
95#[must_use]
96pub fn prose(transcript: &str) -> Vec<String> {
97    let mut out = Vec::new();
98    for line in transcript.lines() {
99        match serde_json::from_str::<Value>(line) {
100            Ok(v) => {
101                let role = v["type"]
102                    .as_str()
103                    .or_else(|| v["role"].as_str())
104                    .or_else(|| v["message"]["role"].as_str())
105                    .unwrap_or("");
106                if matches!(role, "user" | "assistant" | "") {
107                    texts_of(&v, &mut out);
108                }
109            }
110            Err(_) => out.push(line.to_string()),
111        }
112    }
113    out
114}
115
116fn sentences(text: &str) -> Vec<String> {
117    let mut out = Vec::new();
118    let mut in_fence = false;
119    for line in text.lines() {
120        if line.trim_start().starts_with("```") {
121            in_fence = !in_fence;
122            continue;
123        }
124        if in_fence {
125            continue;
126        }
127        let mut cur = String::new();
128        let mut chars = line.chars().peekable();
129        while let Some(c) = chars.next() {
130            cur.push(c);
131            // A stop ends a sentence only before a space or the line's end,
132            // so `build.rs` and `v0.29.1` stay whole.
133            if matches!(c, '.' | '!' | '?') && chars.peek().is_none_or(|n| n.is_whitespace()) {
134                out.push(std::mem::take(&mut cur));
135            }
136        }
137        if !cur.trim().is_empty() {
138            out.push(cur);
139        }
140    }
141    out.into_iter()
142        .map(|s| {
143            s.trim()
144                .trim_start_matches(['-', '*', '>', ' '])
145                .trim()
146                .to_string()
147        })
148        .collect()
149}
150
151/// Whether a sentence states something durable enough to keep.
152#[must_use]
153pub fn is_fact(s: &str) -> bool {
154    let n = s.chars().count();
155    if !(30..=240).contains(&n) || s.ends_with('?') || s.contains('`') || s.contains("://") {
156        return false;
157    }
158    let low = format!(" {} ", s.to_lowercase());
159    // A question, an instruction to this session, or a placeholder.
160    if low.starts_with(" can you") || low.starts_with(" please") || low.contains(" i will ") {
161        return false;
162    }
163    CUES.iter().any(|c| low.contains(c))
164}
165
166/// Up to [`MAX_PER_SESSION`] facts from a transcript, newest last, with no
167/// two of the same normalized form and none in `kept`.
168#[must_use]
169pub fn facts(transcript: &str, kept: &BTreeSet<String>) -> Vec<String> {
170    let mut seen = kept.clone();
171    let mut out: Vec<String> = Vec::new();
172    for text in prose(transcript) {
173        for s in sentences(&text) {
174            if !is_fact(&s) {
175                continue;
176            }
177            let key = normal(&s);
178            if seen.insert(key) {
179                out.push(s);
180            }
181        }
182    }
183    // The end of a session is what it concluded.
184    let skip = out.len().saturating_sub(MAX_PER_SESSION);
185    out.into_iter().skip(skip).collect()
186}
187
188fn ledger() -> Option<std::path::PathBuf> {
189    let base = std::env::var_os("XDG_STATE_HOME")
190        .map(std::path::PathBuf::from)
191        .or_else(|| {
192            std::env::var_os("HOME").map(|h| std::path::PathBuf::from(h).join(".local/state"))
193        })?;
194    Some(base.join("ljos").join("extracted"))
195}
196
197/// The normalized forms already kept on this machine.
198#[must_use]
199pub fn kept() -> BTreeSet<String> {
200    ledger()
201        .and_then(|p| std::fs::read_to_string(p).ok())
202        .map(|t| t.lines().map(str::to_string).collect())
203        .unwrap_or_default()
204}
205
206/// The pack's reason with the HTTP status left off, so a debug line does
207/// not repeat `400:`.
208fn plain_failure(err: &anyhow::Error) -> String {
209    let text = format!("{err:#}");
210    let parts: Vec<&str> = text.split(": ").collect();
211    for (i, part) in parts.iter().enumerate() {
212        let status = part.trim().parse::<u16>();
213        if status.is_ok_and(|code| (400..=599).contains(&code)) {
214            let rest = parts[i + 1..].join(": ");
215            if !rest.is_empty() {
216                return rest;
217            }
218        }
219    }
220    text
221}
222
223/// Whether session-end extraction runs. It does, unless `LJOS_EXTRACT=off`.
224#[must_use]
225pub fn asked_for() -> bool {
226    asked_by(std::env::var("LJOS_EXTRACT").ok().as_deref())
227}
228
229/// [`asked_for`] over the variable's value, so a test names it.
230#[must_use]
231pub fn asked_by(set: Option<&str>) -> bool {
232    !matches!(set.map(str::trim), Some("off"))
233}
234
235/// Hand the session finish to a detached process and return. The hook
236/// does not read the transcript and does not call the pack. The child
237/// fires the session's memories, writes the notes, and drains a queued
238/// tracker push. Returns 0: the count is not known until the child ends.
239pub fn at_session_end(input: &str, session: Option<&str>) -> usize {
240    schedule_finish(input, session);
241    0
242}
243
244/// Write the hook payload and start `ljos hook --finish-session` in its
245/// own session. A failure to start is one line. The hook does not do the
246/// work itself.
247pub fn schedule_finish(input: &str, session: Option<&str>) {
248    let dir = crate::runtime_dir();
249    let _ = std::fs::create_dir_all(&dir);
250    let _ = std::fs::remove_file(dir.join("session-finish.done"));
251    let stamp = std::time::SystemTime::now()
252        .duration_since(std::time::UNIX_EPOCH)
253        .map_or(0, |d| d.as_nanos());
254    let payload = dir.join(format!("session-finish-{stamp}.json"));
255    let body = serde_json::json!({
256        "session": session,
257        "input": input,
258    });
259    if std::fs::write(&payload, body.to_string()).is_err() {
260        eprintln!("ljos extract: the session finish could not be queued");
261        return;
262    }
263    let bin = std::env::current_exe()
264        .map(|p| p.display().to_string())
265        .unwrap_or_else(|_| "ljos".to_string());
266    let args = vec![
267        "hook".to_string(),
268        "--finish-session".to_string(),
269        payload.display().to_string(),
270    ];
271    let log = dir.join("session-finish.log");
272    if crate::detach(&bin, &args, &log).is_err() {
273        let _ = std::fs::remove_file(&payload);
274        eprintln!("ljos extract: the session finish could not be started");
275    }
276}
277
278/// The detached session finish. `path` is the job [`schedule_finish`] wrote.
279///
280/// # Errors
281///
282/// The job file cannot be read.
283pub fn run_finish_job(path: &std::path::Path) -> anyhow::Result<()> {
284    // This process is not the runner's hook. A pack read may take its
285    // own timeout. The wall budget stays off.
286    unsafe { std::env::remove_var("LJOS_IN_HOOK") };
287    let raw = std::fs::read_to_string(path)
288        .map_err(|err| anyhow::anyhow!("session finish: {} ({err})", path.display()))?;
289    let _ = std::fs::remove_file(path);
290    let job: Value = serde_json::from_str(&raw).unwrap_or(Value::Null);
291    let input = job["input"].as_str().unwrap_or("").to_string();
292    let session = job["session"].as_str().map(str::to_string);
293    let _ = crate::session_end(session.as_deref());
294    let lines = write_notes(&input, session.as_deref());
295    let _ = crate::drain_tracker_pushes();
296    mark_done(&lines);
297    Ok(())
298}
299
300fn mark_done(lines: &[String]) {
301    let path = crate::runtime_dir().join("session-finish.done");
302    let body = if lines.is_empty() {
303        "ok\n".to_string()
304    } else {
305        format!("{}\n", lines.join("\n"))
306    };
307    let _ = std::fs::write(path, body);
308}
309
310/// Read the session's transcript (the hook payload's `transcript_path`)
311/// from its tail, write each new sentence as an unverified note on the
312/// open sitting, and return the lines this printed. The prose is the
313/// seat's own reading, so the note is never shown as a fact. With no
314/// sitting the write is skipped: packset keeps a note only on a ticket.
315/// Quiet on every other failure.
316fn write_notes(input: &str, session: Option<&str>) -> Vec<String> {
317    if !asked_for() {
318        return Vec::new();
319    }
320    let Ok(v) = serde_json::from_str::<Value>(input) else {
321        return Vec::new();
322    };
323    let Some(path) = v["transcript_path"]
324        .as_str()
325        .or_else(|| v["transcriptPath"].as_str())
326    else {
327        return Vec::new();
328    };
329    let Some(text) = read_tail(std::path::Path::new(path), MAX_TRANSCRIPT_BYTES) else {
330        return Vec::new();
331    };
332    let mut lines = Vec::new();
333    let say = |lines: &mut Vec<String>, line: String| {
334        eprintln!("{line}");
335        lines.push(line);
336    };
337    let found = facts(&text, &kept());
338    let debug = std::env::var_os("LJOS_EXTRACT_DEBUG").is_some();
339    if debug {
340        say(
341            &mut lines,
342            format!("ljos extract: {} facts from {path}", found.len()),
343        );
344    }
345    if found.is_empty() {
346        return lines;
347    }
348    // Packset keeps a note only on a ticket, and has no form for one that
349    // names none. Skip before the post, one line, rather than collect a
350    // refusal per fact.
351    if crate::live_held_issue().is_none() {
352        say(
353            &mut lines,
354            format!("ljos extract: {}", crate::NOTE_WITHOUT_TICKET),
355        );
356        return lines;
357    }
358    let mut written = 0;
359    let mut keys = Vec::new();
360    for f in found {
361        let mut tags = vec!["via:session-end".to_string()];
362        if let Some(s) = session {
363            tags.push(format!("session:{s}"));
364        }
365        match crate::packset_write_agent_note_tagged(&f, &tags) {
366            Ok(_) => {
367                written += 1;
368                keys.push(normal(&f));
369            }
370            Err(e) => {
371                if debug {
372                    say(
373                        &mut lines,
374                        format!("ljos extract: {f:?} not written: {}", plain_failure(&e)),
375                    );
376                }
377            }
378        }
379    }
380    if debug {
381        say(&mut lines, format!("ljos extract: wrote {written}"));
382    }
383    if let (Some(p), false) = (ledger(), keys.is_empty()) {
384        if let Some(dir) = p.parent() {
385            let _ = std::fs::create_dir_all(dir);
386        }
387        let mut all: Vec<String> = kept().into_iter().collect();
388        all.extend(keys);
389        // Bounded: the newest 5000 forms.
390        let skip = all.len().saturating_sub(5000);
391        let _ = std::fs::write(&p, all[skip..].join("\n") + "\n");
392    }
393    lines
394}
395
396#[cfg(test)]
397mod tests {
398    use super::*;
399
400    const T: &str = r#"{"type":"user","message":{"role":"user","content":"Can you fix the flaky build?"}}
401{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"Looking now."},{"type":"tool_use","name":"Bash","input":{"command":"cargo test because x"}}]}}
402{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"The root cause was a shared temp dir between tests. The fix is to give each test its own tempdir. Always run the suite with --test-threads=1 on this box. Done!"}]}}
403{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"The root cause was a shared temp dir between tests."}]}}"#;
404
405    #[test]
406    fn a_transcript_yields_its_durable_sentences_once() {
407        let got = facts(T, &BTreeSet::new());
408        assert_eq!(got.len(), 3, "{got:?}");
409        assert!(got[0].starts_with("The root cause was"));
410        assert!(got.iter().all(|f| !f.contains("Can you")));
411        assert!(
412            got.iter().all(|f| !f.contains("cargo test")),
413            "tool calls are not facts"
414        );
415    }
416
417    #[test]
418    fn already_kept_facts_are_skipped_and_the_count_is_bounded() {
419        let kept: BTreeSet<String> = [normal(
420            "The root cause was a shared temp dir between tests.",
421        )]
422        .into();
423        assert_eq!(facts(T, &kept).len(), 2);
424        let many: String = (0..20)
425            .map(|i| format!("Always pin tool number {i} to its tested version.\n"))
426            .collect();
427        let got = facts(&many, &BTreeSet::new());
428        assert_eq!(got.len(), MAX_PER_SESSION);
429        assert!(got.last().unwrap().contains("19"), "the newest are kept");
430    }
431
432    /// Unset runs the read. Only `off` stops it.
433    #[test]
434    fn extraction_runs_unless_turned_off() {
435        assert!(asked_by(None));
436        assert!(asked_by(Some("")));
437        assert!(asked_by(Some("on")));
438        assert!(asked_by(Some(" on ")));
439        assert!(asked_by(Some("1")));
440        assert!(!asked_by(Some("off")));
441        assert!(!asked_by(Some(" off ")));
442    }
443
444    #[test]
445    fn a_long_transcript_is_read_from_its_tail() {
446        let dir = std::env::temp_dir().join(format!("ljos-tail-{}", std::process::id()));
447        let _ = std::fs::remove_dir_all(&dir);
448        std::fs::create_dir_all(&dir).unwrap();
449        let path = dir.join("t.jsonl");
450        let mut body =
451            String::from("Always keep this sentence that sits at the head of the file.\n");
452        let pad = format!("{}\n", "x".repeat(200));
453        while body.len() < 8_000 {
454            body.push_str(&pad);
455        }
456        body.push_str("Never push to main without a green CI run on the branch.\n");
457        std::fs::write(&path, &body).unwrap();
458        let got = read_tail(&path, 800).unwrap();
459        assert!(got.contains("Never push"), "{got}");
460        assert!(!got.contains("sits at the head"), "{got}");
461        assert!(got.len() <= 800, "{}", got.len());
462        let _ = std::fs::remove_dir_all(&dir);
463    }
464
465    #[test]
466    fn questions_code_and_short_lines_are_not_facts() {
467        assert!(!is_fact("Should we always do this before a release?"));
468        assert!(!is_fact("Always."));
469        assert!(!is_fact(
470            "Always run `rm -rf target` before a build here, it helps."
471        ));
472        assert!(is_fact(
473            "Never push to main without a green CI run on the branch."
474        ));
475    }
476}
ljos_cli/extract.rs

ljos_cli/
extract.rs
