Skip to main content

second_brain_api/
progress.rs

1use std::collections::BTreeSet;
2use std::io;
3use std::path::{Path, PathBuf};
4use std::sync::Arc;
5use std::time::{Duration, SystemTime};
6
7use anyhow::{Context, Result};
8use chrono::{DateTime, Utc};
9use second_brain_core::sqlite_store::{IngestedSession, SqliteStore};
10use second_brain_core::store::StoreAdmin;
11use second_brain_sync::peers;
12
13use crate::health::{DistillMode, DistillReport, Health, ProgressSeen, SyncReport};
14use crate::note_cover::distill_mode;
15
16pub const INTERVAL: Duration = Duration::from_secs(5 * 60);
17
18const CLAUDE_CODE_SOURCE: &str = "claude-code";
19
20pub struct ProgressCheck {
21    pub store: Arc<SqliteStore>,
22    /// Where `sync-peers` and the sync success stamp live.
23    pub data_dir: PathBuf,
24    pub health: Arc<Health>,
25    /// False only for a config read cleanly without an `[llm]` section; the
26    /// mode then also depends on recent LLM distillation, read each pass.
27    pub llm_configured: bool,
28}
29
30impl ProgressCheck {
31    /// Runs one check before returning, so health holds a full reading
32    /// when the caller announces it, then checks every `INTERVAL`.
33    pub async fn start(self) -> tokio::task::JoinHandle<()> {
34        let check = Arc::new(self);
35        check.clone().run_blocking().await;
36        tokio::spawn(async move {
37            let mut interval = tokio::time::interval(INTERVAL);
38            interval.tick().await;
39            loop {
40                interval.tick().await;
41                check.clone().run_blocking().await;
42            }
43        })
44    }
45
46    async fn run_blocking(self: Arc<Self>) {
47        if let Err(e) = tokio::task::spawn_blocking(move || self.run_once(Utc::now())).await {
48            tracing::error!("sync and distillation check panicked: {e}");
49        }
50    }
51
52    pub fn run_once(&self, now: DateTime<Utc>) {
53        let mut seen = ProgressSeen::default();
54        let mut errors = Vec::new();
55        match self.sync_seen() {
56            Ok(sync) => seen.sync = Some(sync),
57            Err(e) => errors.push(format!("{e:#}")),
58        }
59        let mode = distill_mode(&self.store, self.llm_configured, now).unwrap_or_else(|e| {
60            errors.push(format!("reading recent LLM distillation: {e:#}"));
61            DistillMode::Llm
62        });
63        match self.store.pending_distillation() {
64            Ok(p) => {
65                seen.distill = Some(DistillReport {
66                    mode,
67                    pending: p.count,
68                    oldest_at: p.oldest_created_at,
69                    ..DistillReport::default()
70                })
71            }
72            Err(e) => errors.push(format!("reading raws pending distillation: {e:#}")),
73        }
74        if let Some(since) = self.health.ingest_window_start(now) {
75            match self
76                .store
77                .ingested_sessions_since(CLAUDE_CODE_SOURCE, since)
78            {
79                Ok(rows) => {
80                    seen.ingested_sessions = sessions_started_since(rows, since, |path| {
81                        std::fs::metadata(path)?.created()
82                    })
83                }
84                Err(e) => errors.push(format!("reading recently ingested sessions: {e:#}")),
85            }
86        }
87        if !errors.is_empty() {
88            let error = errors.join("; ");
89            tracing::warn!("sync and distillation check failed: {error}");
90            seen.error = Some(error);
91        }
92        self.health.progress_checked(seen, now);
93    }
94
95    fn sync_seen(&self) -> Result<SyncReport> {
96        let peers = peers::configured_peers(&self.data_dir)
97            .with_context(|| format!("reading {}", peers::PEERS_FILE))?;
98        let cursors = self
99            .store
100            .get_all_sync_states()
101            .context("reading sync state")?;
102        let last_success_at = peers::last_success(&self.data_dir)?;
103        Ok(SyncReport {
104            checked_at: None,
105            peers: peers.map(|p| p.len()),
106            synced_before: !cursors.is_empty(),
107            last_success_at,
108            last_cursor_at: cursors.iter().map(|c| c.last_sync_at).max(),
109        })
110    }
111}
112
113/// Top-level session keys whose transcript was created at or after `since`,
114/// once each and sorted. A re-ingest rewrites `ingested_at` but not the
115/// file's creation time, so old sessions cannot inflate the count; a row
116/// whose creation time cannot be read (file gone, no birth time) is left out.
117/// None when no transcript's filesystem records birth times, since the count
118/// would then read zero with nothing to say why.
119fn sessions_started_since(
120    rows: Vec<IngestedSession>,
121    since: DateTime<Utc>,
122    created: impl Fn(&Path) -> io::Result<SystemTime>,
123) -> Option<Vec<String>> {
124    let born: Vec<(String, io::Result<SystemTime>)> = rows
125        .into_iter()
126        .filter(|row| !row.session_key.contains('/'))
127        .map(|row| {
128            let at = created(Path::new(&row.file_path));
129            (row.session_key, at)
130        })
131        .collect();
132    let no_birth_times = born.iter().all(|(_, at)| at.is_err())
133        && born.iter().any(|(_, at)| {
134            at.as_ref()
135                .is_err_and(|e| e.kind() == io::ErrorKind::Unsupported)
136        });
137    if no_birth_times {
138        return None;
139    }
140    let kept: BTreeSet<String> = born
141        .into_iter()
142        .filter(|(_, at)| {
143            at.as_ref()
144                .is_ok_and(|&at| DateTime::<Utc>::from(at) >= since)
145        })
146        .map(|(key, _)| key)
147        .collect();
148    Some(kept.into_iter().collect())
149}
150
151#[cfg(test)]
152mod tests {
153    use super::*;
154
155    fn row(key: &str, path: &str) -> IngestedSession {
156        IngestedSession {
157            session_key: key.to_string(),
158            file_path: path.to_string(),
159        }
160    }
161
162    #[test]
163    fn only_top_level_sessions_whose_transcript_was_created_since_the_bound_count() {
164        let since = DateTime::parse_from_rfc3339("2026-11-02T10:00:00Z")
165            .unwrap()
166            .with_timezone(&Utc);
167        let at = |secs: i64| SystemTime::from(since + chrono::Duration::seconds(secs));
168        let created = |path: &Path| match path.to_str().unwrap() {
169            "/p/sub.jsonl" | "/p/after.jsonl" | "/p/dup-a.jsonl" | "/p/dup-b.jsonl" => Ok(at(5)),
170            "/p/at.jsonl" => Ok(at(0)),
171            "/p/before.jsonl" => Ok(at(-1)),
172            "/p/dup-old.jsonl" => Ok(at(-60)),
173            "/p/no-btime.jsonl" => Err(io::Error::from(io::ErrorKind::Unsupported)),
174            _ => Err(io::Error::from(io::ErrorKind::NotFound)),
175        };
176        let cases: [(&str, Vec<IngestedSession>, Option<Vec<&str>>); 10] = [
177            (
178                "subagent",
179                vec![row("s/agent-1", "/p/sub.jsonl")],
180                Some(vec![]),
181            ),
182            (
183                "created before",
184                vec![row("s", "/p/before.jsonl")],
185                Some(vec![]),
186            ),
187            (
188                "created at the bound",
189                vec![row("s", "/p/at.jsonl")],
190                Some(vec!["s"]),
191            ),
192            (
193                "created after",
194                vec![row("s", "/p/after.jsonl")],
195                Some(vec!["s"]),
196            ),
197            (
198                "missing file",
199                vec![row("s", "/p/gone.jsonl")],
200                Some(vec![]),
201            ),
202            (
203                "unreadable btime",
204                vec![row("s", "/p/no-btime.jsonl")],
205                None,
206            ),
207            (
208                "unreadable btime beside a deleted transcript",
209                vec![row("s", "/p/no-btime.jsonl"), row("t", "/p/gone.jsonl")],
210                None,
211            ),
212            (
213                "unreadable btime beside a readable one",
214                vec![row("s", "/p/no-btime.jsonl"), row("t", "/p/after.jsonl")],
215                Some(vec!["t"]),
216            ),
217            (
218                "duplicate keys",
219                vec![
220                    row("s", "/p/dup-a.jsonl"),
221                    row("s", "/p/dup-b.jsonl"),
222                    row("s", "/p/dup-old.jsonl"),
223                ],
224                Some(vec!["s"]),
225            ),
226            (
227                "mixed, sorted",
228                vec![
229                    row("t", "/p/after.jsonl"),
230                    row("s", "/p/at.jsonl"),
231                    row("t/agent-2", "/p/sub.jsonl"),
232                    row("u", "/p/before.jsonl"),
233                ],
234                Some(vec!["s", "t"]),
235            ),
236        ];
237        for (name, rows, expected) in cases {
238            assert_eq!(
239                sessions_started_since(rows, since, created),
240                expected.map(|keys| keys.into_iter().map(String::from).collect::<Vec<_>>()),
241                "{name}"
242            );
243        }
244    }
245}