Skip to main content

second_brain_api/
progress.rs

1use std::collections::BTreeSet;
2use std::io;
3use std::path::{Path, PathBuf};
4use std::sync::Arc;
5use std::time::{Duration, SystemTime};
6
7use anyhow::{Context, Result};
8use chrono::{DateTime, Utc};
9use second_brain_core::sqlite_store::{IngestedSession, SqliteStore};
10use second_brain_core::store::StoreAdmin;
11use second_brain_sync::{dial, peers};
12
13use crate::health::{DistillMode, DistillReport, Health, ProgressSeen, SyncReport};
14use crate::note_cover::distill_mode;
15
16pub const INTERVAL: Duration = Duration::from_secs(5 * 60);
17
18const CLAUDE_CODE_SOURCE: &str = "claude-code";
19
20pub struct ProgressCheck {
21    pub store: Arc<SqliteStore>,
22    /// Where `sync-peers` and the sync success stamp live.
23    pub data_dir: PathBuf,
24    pub health: Arc<Health>,
25    /// False only for a config read cleanly without an `[llm]` section; the
26    /// mode then also depends on recent LLM distillation, read each pass.
27    pub llm_configured: bool,
28}
29
30impl ProgressCheck {
31    /// Runs one check before returning, so health holds a full reading
32    /// when the caller announces it, then checks every `INTERVAL`.
33    pub async fn start(self) -> tokio::task::JoinHandle<()> {
34        let check = Arc::new(self);
35        check.clone().run_blocking().await;
36        tokio::spawn(async move {
37            let mut interval = tokio::time::interval(INTERVAL);
38            interval.tick().await;
39            loop {
40                interval.tick().await;
41                check.clone().run_blocking().await;
42            }
43        })
44    }
45
46    async fn run_blocking(self: Arc<Self>) {
47        if let Err(e) = tokio::task::spawn_blocking(move || self.run_once(Utc::now())).await {
48            tracing::error!("sync and distillation check panicked: {e}");
49        }
50    }
51
52    pub fn run_once(&self, now: DateTime<Utc>) {
53        let mut seen = ProgressSeen::default();
54        let mut errors = Vec::new();
55        match self.sync_seen(&mut errors) {
56            Ok(sync) => seen.sync = Some(sync),
57            Err(e) => errors.push(format!("{e:#}")),
58        }
59        let mode = distill_mode(&self.store, self.llm_configured, now).unwrap_or_else(|e| {
60            errors.push(format!("reading recent LLM distillation: {e:#}"));
61            DistillMode::Llm
62        });
63        match self.store.pending_distillation() {
64            Ok(p) => {
65                seen.distill = Some(DistillReport {
66                    mode,
67                    pending: p.count,
68                    oldest_at: p.oldest_created_at,
69                    ..DistillReport::default()
70                })
71            }
72            Err(e) => errors.push(format!("reading raws pending distillation: {e:#}")),
73        }
74        if let Some(since) = self.health.ingest_window_start(now) {
75            match self
76                .store
77                .ingested_sessions_since(CLAUDE_CODE_SOURCE, since)
78            {
79                Ok(rows) => {
80                    seen.ingested_sessions = sessions_started_since(rows, since, |path| {
81                        std::fs::metadata(path)?.created()
82                    })
83                }
84                Err(e) => errors.push(format!("reading recently ingested sessions: {e:#}")),
85            }
86        }
87        if !errors.is_empty() {
88            let error = errors.join("; ");
89            tracing::warn!("sync and distillation check failed: {error}");
90            seen.error = Some(error);
91        }
92        self.health.progress_checked(seen, now);
93    }
94
95    /// An unreadable peer list is pushed to `errors` and leaves `peers`
96    /// unknown rather than dropping the cursors and last success with it.
97    fn sync_seen(&self, errors: &mut Vec<String>) -> Result<SyncReport> {
98        let partners = match dial::sync_partners(&self.store, &self.data_dir) {
99            Ok(partners) => Some(partners),
100            Err(e) => {
101                errors.push(format!(
102                    "reading the device list and {}: {e:#}",
103                    peers::PEERS_FILE
104                ));
105                None
106            }
107        };
108        let cursors = self
109            .store
110            .get_all_sync_states()
111            .context("reading sync state")?;
112        let last_success_at = peers::last_success(&self.data_dir)?;
113        Ok(SyncReport {
114            checked_at: None,
115            peers: partners,
116            synced_before: !cursors.is_empty(),
117            last_success_at,
118            last_cursor_at: cursors.iter().map(|c| c.last_sync_at).max(),
119            floor: None,
120            snapshot_merge: None,
121        })
122    }
123}
124
125/// Top-level session keys whose transcript was created at or after `since`,
126/// once each and sorted. A re-ingest rewrites `ingested_at` but not the
127/// file's creation time, so old sessions cannot inflate the count; a row
128/// whose creation time cannot be read (file gone, no birth time) is left out.
129/// None when no transcript's filesystem records birth times, since the count
130/// would then read zero with nothing to say why.
131fn sessions_started_since(
132    rows: Vec<IngestedSession>,
133    since: DateTime<Utc>,
134    created: impl Fn(&Path) -> io::Result<SystemTime>,
135) -> Option<Vec<String>> {
136    let born: Vec<(String, io::Result<SystemTime>)> = rows
137        .into_iter()
138        .filter(|row| !row.session_key.contains('/'))
139        .map(|row| {
140            let at = created(Path::new(&row.file_path));
141            (row.session_key, at)
142        })
143        .collect();
144    let no_birth_times = born.iter().all(|(_, at)| at.is_err())
145        && born.iter().any(|(_, at)| {
146            at.as_ref()
147                .is_err_and(|e| e.kind() == io::ErrorKind::Unsupported)
148        });
149    if no_birth_times {
150        return None;
151    }
152    let kept: BTreeSet<String> = born
153        .into_iter()
154        .filter(|(_, at)| {
155            at.as_ref()
156                .is_ok_and(|&at| DateTime::<Utc>::from(at) >= since)
157        })
158        .map(|(key, _)| key)
159        .collect();
160    Some(kept.into_iter().collect())
161}
162
163#[cfg(test)]
164mod tests {
165    use super::*;
166
167    fn row(key: &str, path: &str) -> IngestedSession {
168        IngestedSession {
169            session_key: key.to_string(),
170            file_path: path.to_string(),
171        }
172    }
173
174    #[test]
175    fn only_top_level_sessions_whose_transcript_was_created_since_the_bound_count() {
176        let since = DateTime::parse_from_rfc3339("2026-11-02T10:00:00Z")
177            .unwrap()
178            .with_timezone(&Utc);
179        let at = |secs: i64| SystemTime::from(since + chrono::Duration::seconds(secs));
180        let created = |path: &Path| match path.to_str().unwrap() {
181            "/p/sub.jsonl" | "/p/after.jsonl" | "/p/dup-a.jsonl" | "/p/dup-b.jsonl" => Ok(at(5)),
182            "/p/at.jsonl" => Ok(at(0)),
183            "/p/before.jsonl" => Ok(at(-1)),
184            "/p/dup-old.jsonl" => Ok(at(-60)),
185            "/p/no-btime.jsonl" => Err(io::Error::from(io::ErrorKind::Unsupported)),
186            _ => Err(io::Error::from(io::ErrorKind::NotFound)),
187        };
188        let cases: [(&str, Vec<IngestedSession>, Option<Vec<&str>>); 10] = [
189            (
190                "subagent",
191                vec![row("s/agent-1", "/p/sub.jsonl")],
192                Some(vec![]),
193            ),
194            (
195                "created before",
196                vec![row("s", "/p/before.jsonl")],
197                Some(vec![]),
198            ),
199            (
200                "created at the bound",
201                vec![row("s", "/p/at.jsonl")],
202                Some(vec!["s"]),
203            ),
204            (
205                "created after",
206                vec![row("s", "/p/after.jsonl")],
207                Some(vec!["s"]),
208            ),
209            (
210                "missing file",
211                vec![row("s", "/p/gone.jsonl")],
212                Some(vec![]),
213            ),
214            (
215                "unreadable btime",
216                vec![row("s", "/p/no-btime.jsonl")],
217                None,
218            ),
219            (
220                "unreadable btime beside a deleted transcript",
221                vec![row("s", "/p/no-btime.jsonl"), row("t", "/p/gone.jsonl")],
222                None,
223            ),
224            (
225                "unreadable btime beside a readable one",
226                vec![row("s", "/p/no-btime.jsonl"), row("t", "/p/after.jsonl")],
227                Some(vec!["t"]),
228            ),
229            (
230                "duplicate keys",
231                vec![
232                    row("s", "/p/dup-a.jsonl"),
233                    row("s", "/p/dup-b.jsonl"),
234                    row("s", "/p/dup-old.jsonl"),
235                ],
236                Some(vec!["s"]),
237            ),
238            (
239                "mixed, sorted",
240                vec![
241                    row("t", "/p/after.jsonl"),
242                    row("s", "/p/at.jsonl"),
243                    row("t/agent-2", "/p/sub.jsonl"),
244                    row("u", "/p/before.jsonl"),
245                ],
246                Some(vec!["s", "t"]),
247            ),
248        ];
249        for (name, rows, expected) in cases {
250            assert_eq!(
251                sessions_started_since(rows, since, created),
252                expected.map(|keys| keys.into_iter().map(String::from).collect::<Vec<_>>()),
253                "{name}"
254            );
255        }
256    }
257}