Skip to main content

supercode_harness/
runs.rs

1//! Observed-tier, READ-ONLY inventory of scheduled-job FIRES across the
2//! harnesses that keep a run store (Domain 11, concept 7).
3//!
4//! A run is one execution of a [`crate::jobs::ScheduledJob`], with its
5//! outcome and — where the harness leaves enough behind to recover it — the
6//! session the fire opened.
7//!
8//! * **Hermes** — `HERMES_HOME/cron/executions.db`, a profile-local SQLite
9//!   audit ledger (`cron/executions.py`: "the ledger records what is known
10//!   about each attempt; it is not a retry queue"). One `executions` row per
11//!   attempt: `id, job_id, source, process_id, pid, process_started_at,
12//!   status, claimed_at, started_at, finished_at, error`, status one of
13//!   `claimed | running | completed | failed | unknown`. A Hermes profile home
14//!   is a full HERMES_HOME, so each `profiles/<name>/` has its own ledger.
15//! * **OpenClaw** — `cron_run_logs` in the shared state database
16//!   (`<state dir>/state/openclaw.sqlite`) at the pinned 2026.7.1-2:
17//!   `store_key, job_id, seq, ts, status, error, …, session_id, session_key,
18//!   run_id, run_at_ms, duration_ms, …, entry_json`, status one of
19//!   `ok | error | skipped`. `store_key` is `path.resolve(cron.store)` — the
20//!   legacy `cron/jobs.json` path used purely as a per-store partition key;
21//!   at the pin no such file exists, and neither does a `cron/runs/*.jsonl`
22//!   run log (that shape is what `openclaw doctor --fix` imports FROM).
23//! * **Claude Code** — has no run store at all. A `CronCreate` fire is an
24//!   ordinary turn inside the session that created the job, so `runs.list` /
25//!   `runs.get` refuse for `claude-code` rather than inventing a fire record
26//!   from turns. See [`RUN_HARNESSES`].
27//!
28//! Nothing here writes, claims, retries, or prunes. Every store is opened
29//! `SQLITE_OPEN_READ_ONLY` — these are live databases owned by a running
30//! scheduler.
31//!
32//! **Status words are the harness's own.** Hermes says `completed`/`failed`,
33//! OpenClaw says `ok`/`error`; renaming either onto a shared vocabulary would
34//! discard the distinction Hermes draws between `failed` (a terminal result
35//! it wrote) and `unknown` (an attempt whose owner died before writing one).
36//!
37//! **Delivery (ORCH-13).** Where a fire's output went is read from each
38//! harness's own delivery record:
39//!
40//! * **OpenClaw** writes it onto the run-log row itself — `delivery_status`,
41//!   `delivery_error`, `delivered` — and declares the destination on the job
42//!   (`cron_jobs.delivery_channel` / `delivery_to`), so the row's `target` is
43//!   joined from there: the run log records the OUTCOME, never the address.
44//! * **Hermes** keeps a separate `delivery_obligations` ledger inside
45//!   `state.db` (`gateway/delivery_ledger.py`), keyed by the CONVERSATION's
46//!   `session_key` and the platform surface — not by job or fire. So a fire is
47//!   matched to an obligation the way [`join_hermes_session`] matches a
48//!   session: by the fire's own `[claimed_at, finished_at]` window, on the
49//!   fire's own surface. See [`hermes_delivery`] for the two questions asked,
50//!   in order, and for why an unanchored ledger instant answers `None`.
51//!
52//! `None` stays honest: a fire whose delivery nothing recorded says so rather
53//! than borrowing a neighbouring fire's outcome.
54
55use std::path::{Path, PathBuf};
56
57use rusqlite::Connection;
58use serde::{Deserialize, Serialize};
59use serde_json::{Map, Value};
60
61use crate::{HarnessHomes, HarnessId, Result};
62
63/// Harnesses that keep a run store at all. Every other harness answers
64/// `runs.list` / `runs.get` with `UnsupportedAction`, never an empty list —
65/// an absent store and an empty history are different answers.
66pub const RUN_HARNESSES: &[&str] = &[
67    HarnessId::HERMES,
68    HarnessId::OPENCLAW,
69    HarnessId::ORCHESTRATOR,
70];
71
72/// Hard stop on how far a compression chain is followed from a fire's own
73/// session to the readable tip. Hermes chains are short; a cycle in a
74/// corrupted store must not spin.
75const COMPRESSION_CHAIN_LIMIT: usize = 32;
76
77/// One fire of one scheduled job, projected onto the uniform Domain 11 row.
78#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
79pub struct HarnessRun {
80    /// Harness-native run id: Hermes's `executions.id` (a uuid hex),
81    /// OpenClaw's `run_id` — or `<job_id>#<seq>` when a run-log row predates
82    /// run ids, since the `(job_id, seq)` pair is that store's own key.
83    pub id: String,
84    /// Owning harness.
85    pub harness: String,
86    /// The scheduled job this fire belongs to.
87    pub job_id: String,
88    /// The harness's own outcome word: Hermes `claimed | running | completed
89    /// | failed | unknown`, OpenClaw `ok | error | skipped`.
90    pub status: String,
91    /// When the scheduler claimed the fire. Hermes only — OpenClaw's run log
92    /// is written once, at finish, and records no claim.
93    pub claimed_at: Option<String>,
94    /// When the fire began executing.
95    pub started_at: Option<String>,
96    /// When the fire reached a terminal state.
97    pub finished_at: Option<String>,
98    /// The failure the harness recorded, verbatim.
99    pub error: Option<String>,
100    /// The session this fire opened, when it is recoverable: OpenClaw records
101    /// it on the row; Hermes does not, so it is recovered by matching
102    /// `cron_<job_id>_<YYYYMMDD_HHMMSS>` session ids inside the fire's own
103    /// window (see [`join_hermes_session`]). `None` means no session is
104    /// recoverable — never a guess.
105    pub session_id: Option<String>,
106    /// Where this fire's output went, when the harness recorded a delivery
107    /// for it. `None` means nothing in the harness's delivery record matches
108    /// this fire — never that the delivery failed.
109    pub delivery: Option<RunDelivery>,
110}
111
112/// A fire's delivery outcome (ORCH-13).
113#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
114pub struct RunDelivery {
115    /// Where the output was addressed: Hermes's obligation surface
116    /// `<platform>:<chat_id>[:<thread_id>]`, or OpenClaw's job-declared
117    /// `<channel>[:<to>]`.
118    pub target: Option<String>,
119    /// The harness's own state word: Hermes `pending | attempting |
120    /// delivered | failed | abandoned`, OpenClaw `delivery_status`.
121    pub state: Option<String>,
122    /// Delivery attempts recorded. Hermes only — OpenClaw's run log counts
123    /// no attempts.
124    pub attempts: Option<u64>,
125    /// The last delivery failure, verbatim.
126    pub last_error: Option<String>,
127    /// When the harness stamped the delivery as done. Hermes only: it is the
128    /// obligation's `updated_at` on a `delivered` row (the ledger writes no
129    /// separate delivered-at column). OpenClaw's run log records `delivered`
130    /// as a flag with no instant of its own, so it stays empty there.
131    pub delivered_at: Option<String>,
132}
133
134impl HarnessRun {
135    /// The observed row of a typed world [`Fire`] (`docs/ONTOLOGY.md` §2.7):
136    /// the join to its session and delivery is the caller's, as it is for the
137    /// ledger read, so `runs list` and the world never disagree about a fire.
138    pub fn from_fire(
139        harness: &str,
140        fire: &supercode_interchange::world::Fire,
141        session_id: Option<String>,
142        delivery: Option<RunDelivery>,
143    ) -> Self {
144        Self {
145            id: fire.id.clone(),
146            harness: harness.into(),
147            job_id: fire.job_id.clone(),
148            status: fire.status.hermes_word().to_string(),
149            claimed_at: Some(fire.claimed_at.clone()),
150            started_at: fire.started_at.clone(),
151            finished_at: fire.finished_at.clone(),
152            error: fire.error.clone(),
153            session_id: session_id.or_else(|| fire.session_id.clone()),
154            delivery,
155        }
156    }
157}
158
159/// One store the listing consulted, and what it found there.
160#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
161pub struct RunSource {
162    /// Harness the store belongs to.
163    pub harness: String,
164    /// Absolute path consulted.
165    pub path: PathBuf,
166    /// `read` | `absent_store` | `unreadable`.
167    pub state: String,
168    /// Hermes profile home this ledger belongs to.
169    pub profile: Option<String>,
170    /// Why a store is `unreadable`.
171    pub detail: Option<String>,
172}
173
174impl RunSource {
175    fn store(harness: &str, path: PathBuf, state: &str, profile: Option<String>) -> Self {
176        Self {
177            harness: harness.to_string(),
178            path,
179            state: state.to_string(),
180            profile,
181            detail: None,
182        }
183    }
184}
185
186/// Result of a `runs.list`: the rows plus every store that was consulted.
187#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
188pub struct RunsListing {
189    /// Uniform rows, harness-major then store order, newest fire first
190    /// within a store.
191    pub runs: Vec<HarnessRun>,
192    /// Stores consulted, including the ones that were absent.
193    pub sources: Vec<RunSource>,
194}
195
196/// Filters for a run-history read.
197#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
198#[serde(default)]
199pub struct RunsQuery {
200    /// Restrict to one harness. Absent means every harness in
201    /// [`RUN_HARNESSES`].
202    pub harness: Option<String>,
203    /// Restrict to one job's fires.
204    pub job: Option<String>,
205    /// Cap on rows. Applied per store as the read's own `LIMIT` (so a long
206    /// history is never fully materialized) and again to the merged listing.
207    pub limit: Option<usize>,
208    /// Storage roots to read.
209    pub homes: HarnessHomes,
210}
211
212/// Whether `harness` keeps a run store.
213pub fn supports_runs(harness: &str) -> bool {
214    RUN_HARNESSES.contains(&harness)
215}
216
217/// Read every fire the query selects.
218///
219/// Read-only: every store is opened `SQLITE_OPEN_READ_ONLY`, and no fire is
220/// claimed, retried, or pruned.
221pub fn list_runs(query: &RunsQuery) -> Result<RunsListing> {
222    let (rows, sources) = collect(query);
223    let mut runs: Vec<HarnessRun> = rows.into_iter().map(|(run, _)| run).collect();
224    if let Some(limit) = query.limit {
225        runs.truncate(limit);
226    }
227    Ok(RunsListing { runs, sources })
228}
229
230/// Read one fire by harness and id, with the verbatim native record beside
231/// the uniform row. `Ok(None)` means the harness's stores hold no such run.
232pub fn get_run(
233    harness: &str,
234    id: &str,
235    homes: &HarnessHomes,
236) -> Result<Option<(HarnessRun, Value)>> {
237    let (rows, _) = collect(&RunsQuery {
238        harness: Some(harness.to_string()),
239        homes: homes.clone(),
240        ..RunsQuery::default()
241    });
242    Ok(rows.into_iter().find(|(run, _)| run.id == id))
243}
244
245/// Every store the query selects, in harness-major order, each row paired
246/// with the harness's own record so `get` never re-reads (and so the two
247/// verbs can never disagree about a fire).
248fn collect(query: &RunsQuery) -> (Vec<(HarnessRun, Value)>, Vec<RunSource>) {
249    let mut rows = Vec::new();
250    let mut sources = Vec::new();
251    let wanted = query.harness.as_deref();
252    if wanted.is_none_or(|harness| harness == HarnessId::HERMES) {
253        collect_hermes(query, &mut rows, &mut sources);
254    }
255    if wanted.is_none_or(|harness| harness == HarnessId::OPENCLAW) {
256        collect_openclaw(query, &mut rows, &mut sources);
257    }
258    if wanted.is_none_or(|harness| harness == HarnessId::ORCHESTRATOR) {
259        collect_hermes_shaped(
260            HarnessId::ORCHESTRATOR,
261            orchestrator_ledgers(&query.homes),
262            query,
263            &mut rows,
264            &mut sources,
265        );
266    }
267    (rows, sources)
268}
269
270/// Open a harness store strictly read-only. These are live databases owned by
271/// a running scheduler; no connection here may ever be handed a write API.
272///
273/// Same fallback as `jobs::openclaw_sqlite_records`: a WAL-mode store whose
274/// `-shm` sidecar is missing refuses a plain read-only open, so the immutable
275/// URI form is tried second.
276fn open_read_only(path: &Path) -> std::result::Result<Connection, rusqlite::Error> {
277    Connection::open_with_flags(
278        path,
279        rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX,
280    )
281    .or_else(|_| {
282        Connection::open_with_flags(
283            format!("file:{}?immutable=1", path.display()),
284            rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY
285                | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX
286                | rusqlite::OpenFlags::SQLITE_OPEN_URI,
287        )
288    })
289}
290
291fn table_exists(conn: &Connection, table: &str) -> bool {
292    conn.query_row(
293        "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1",
294        [table],
295        |row| row.get::<_, i64>(0),
296    )
297    .is_ok()
298}
299
300/// Render one SQLite value the way the store wrote it, for the native record.
301fn native_value(value: rusqlite::types::ValueRef<'_>) -> Value {
302    match value {
303        rusqlite::types::ValueRef::Null => Value::Null,
304        rusqlite::types::ValueRef::Integer(i) => Value::from(i),
305        rusqlite::types::ValueRef::Real(f) => serde_json::Number::from_f64(f)
306            .map(Value::Number)
307            .unwrap_or(Value::Null),
308        rusqlite::types::ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
309        rusqlite::types::ValueRef::Blob(_) => Value::Null,
310    }
311}
312
313/// Every column of one row, keyed by the store's own column names.
314fn native_row(row: &rusqlite::Row<'_>, columns: &[&str]) -> Value {
315    let mut map = Map::new();
316    for (index, name) in columns.iter().enumerate() {
317        let value = row
318            .get_ref(index)
319            .map_or(Value::Null, |value| native_value(value));
320        map.insert((*name).to_string(), value);
321    }
322    Value::Object(map)
323}
324
325// ---------------------------------------------------------------------------
326// Hermes — `cron/executions.db`, joined to `state.db` sessions
327// ---------------------------------------------------------------------------
328
329/// One Hermes execution ledger, with the profile home it belongs to and the
330/// `state.db` whose sessions its fires opened.
331struct HermesLedger {
332    executions: PathBuf,
333    sessions: PathBuf,
334    /// The job store beside this ledger. A fire's delivery SURFACE is
335    /// declared on the job, not on the execution row, so the obligation
336    /// match (ORCH-13) needs it.
337    jobs: PathBuf,
338    profile: Option<String>,
339}
340
341/// Every `cron/executions.db` a Hermes install can hold.
342///
343/// A Hermes profile home IS a full HERMES_HOME (`hermes_constants.get_hermes_home`
344/// resolves the context-local profile override first), so the root home and
345/// every `profiles/<name>/` carry their own ledger AND their own `state.db`.
346/// A profile that has no `state.db` of its own falls back to the root store,
347/// where its rows carry `profile_name = <name>`.
348fn hermes_ledgers(homes: &HarnessHomes) -> Vec<HermesLedger> {
349    // `HarnessHomes::hermes` addresses `state.db`; the cron store is its
350    // sibling under the same HERMES_HOME.
351    let root = homes
352        .hermes
353        .parent()
354        .map_or_else(|| PathBuf::from("."), Path::to_path_buf);
355    let mut ledgers = vec![HermesLedger {
356        executions: root.join("cron/executions.db"),
357        sessions: homes.hermes.clone(),
358        jobs: root.join("cron/jobs.json"),
359        profile: None,
360    }];
361    if let Ok(entries) = std::fs::read_dir(root.join("profiles")) {
362        let mut found: Vec<HermesLedger> = entries
363            .flatten()
364            .filter(|entry| entry.path().is_dir())
365            .map(|entry| {
366                let home = entry.path();
367                let own = home.join("state.db");
368                HermesLedger {
369                    executions: home.join("cron/executions.db"),
370                    sessions: if own.is_file() {
371                        own
372                    } else {
373                        homes.hermes.clone()
374                    },
375                    jobs: home.join("cron/jobs.json"),
376                    profile: Some(entry.file_name().to_string_lossy().into_owned()),
377                }
378            })
379            .collect();
380        found.sort_by(|left, right| left.profile.cmp(&right.profile));
381        ledgers.extend(found);
382    }
383    ledgers
384}
385
386/// Every ledger an orchestrator home holds.
387///
388/// Identical in shape to [`hermes_ledgers`] because the folder is: each
389/// profile folder is a complete home with its own `cron/executions.db`,
390/// `cron/jobs.json` and `state.db` (`docs/ORCHESTRATOR-IR.md` §6). Bindings
391/// and obligations live in the profile's own store, so — unlike Hermes's
392/// multiplexed gateway — there is no fallback to a root store.
393fn orchestrator_ledgers(homes: &HarnessHomes) -> Vec<HermesLedger> {
394    crate::orchestrator_profile_dirs(&homes.orchestrator)
395        .into_iter()
396        .map(|(name, dir)| HermesLedger {
397            executions: dir.join("cron/executions.db"),
398            sessions: dir.join("state.db"),
399            jobs: dir.join("cron/jobs.json"),
400            profile: (name != "default").then_some(name),
401        })
402        .collect()
403}
404
405const HERMES_EXECUTION_COLUMNS: &[&str] = &[
406    "id",
407    "job_id",
408    "source",
409    "process_id",
410    "pid",
411    "process_started_at",
412    "status",
413    "claimed_at",
414    "started_at",
415    "finished_at",
416    "error",
417];
418
419fn collect_hermes(
420    query: &RunsQuery,
421    rows: &mut Vec<(HarnessRun, Value)>,
422    sources: &mut Vec<RunSource>,
423) {
424    collect_hermes_shaped(
425        HarnessId::HERMES,
426        hermes_ledgers(&query.homes),
427        query,
428        rows,
429        sources,
430    );
431}
432
433/// Read every fire from a set of Hermes-SHAPED ledgers.
434///
435/// Hermes and the orchestrator keep the same `executions` table, the same
436/// `delivery_obligations` ledger and the same job store beside them, so the
437/// harness id and the ledger list are parameters and the reader is one
438/// implementation (ORC-7). ORCH-13's delivery join runs unchanged for both.
439fn collect_hermes_shaped(
440    harness: &str,
441    ledgers: Vec<HermesLedger>,
442    query: &RunsQuery,
443    rows: &mut Vec<(HarnessRun, Value)>,
444    sources: &mut Vec<RunSource>,
445) {
446    for ledger in ledgers {
447        if !ledger.executions.is_file() {
448            sources.push(RunSource::store(
449                harness,
450                ledger.executions.clone(),
451                "absent_store",
452                ledger.profile.clone(),
453            ));
454            continue;
455        }
456        let connection = match open_read_only(&ledger.executions) {
457            Ok(connection) => connection,
458            Err(error) => {
459                sources.push(RunSource {
460                    detail: Some(error.to_string()),
461                    ..RunSource::store(
462                        harness,
463                        ledger.executions.clone(),
464                        "unreadable",
465                        ledger.profile.clone(),
466                    )
467                });
468                continue;
469            }
470        };
471        if !table_exists(&connection, "executions") {
472            sources.push(RunSource {
473                detail: Some("no `executions` table — not a Hermes cron ledger".into()),
474                ..RunSource::store(
475                    harness,
476                    ledger.executions.clone(),
477                    "unreadable",
478                    ledger.profile.clone(),
479                )
480            });
481            continue;
482        }
483        match read_hermes_ledger(harness, &connection, &ledger, query) {
484            Ok(found) => {
485                sources.push(RunSource::store(
486                    harness,
487                    ledger.executions.clone(),
488                    "read",
489                    ledger.profile.clone(),
490                ));
491                rows.extend(found);
492            }
493            Err(error) => sources.push(RunSource {
494                detail: Some(error.to_string()),
495                ..RunSource::store(
496                    harness,
497                    ledger.executions.clone(),
498                    "unreadable",
499                    ledger.profile.clone(),
500                )
501            }),
502        }
503    }
504}
505
506fn read_hermes_ledger(
507    harness: &str,
508    connection: &Connection,
509    ledger: &HermesLedger,
510    query: &RunsQuery,
511) -> std::result::Result<Vec<(HarnessRun, Value)>, rusqlite::Error> {
512    // The ledger's own ordering index is `(job_id, claimed_at DESC, id DESC)`;
513    // read newest-claimed first so a `limit` keeps the recent fires.
514    let sql = format!(
515        "SELECT {} FROM executions {} ORDER BY claimed_at DESC, id DESC {}",
516        HERMES_EXECUTION_COLUMNS.join(", "),
517        if query.job.is_some() {
518            "WHERE job_id = ?1"
519        } else {
520            ""
521        },
522        query
523            .limit
524            .map_or_else(String::new, |limit| format!("LIMIT {limit}")),
525    );
526    let mut statement = connection.prepare(&sql)?;
527    let read = |row: &rusqlite::Row<'_>| -> rusqlite::Result<(HermesExecution, Value)> {
528        Ok((
529            HermesExecution {
530                id: row.get::<_, Option<String>>(0)?.unwrap_or_default(),
531                job_id: row.get::<_, Option<String>>(1)?.unwrap_or_default(),
532                status: row.get::<_, Option<String>>(6)?.unwrap_or_default(),
533                claimed_at: row.get(7)?,
534                started_at: row.get(8)?,
535                finished_at: row.get(9)?,
536                error: row.get(10)?,
537            },
538            native_row(row, HERMES_EXECUTION_COLUMNS),
539        ))
540    };
541    let executions: Vec<(HermesExecution, Value)> = match query.job.as_deref() {
542        Some(job) => statement
543            .query_map([job], read)?
544            .collect::<rusqlite::Result<_>>()?,
545        None => statement
546            .query_map([], read)?
547            .collect::<rusqlite::Result<_>>()?,
548    };
549    // Sessions are joined from the ledger's own home, once per read; the job
550    // store beside the ledger is read once for the delivery surfaces.
551    let sessions = open_read_only(&ledger.sessions).ok();
552    let surfaces = hermes_delivery_surfaces(&ledger.jobs);
553    Ok(executions
554        .into_iter()
555        .map(|(execution, native)| {
556            let session_id = sessions
557                .as_ref()
558                .and_then(|connection| join_hermes_session(connection, &execution));
559            let delivery = sessions.as_ref().and_then(|connection| {
560                hermes_delivery(
561                    connection,
562                    &execution,
563                    session_id.as_deref(),
564                    surfaces.get(execution.job_id.as_str()),
565                )
566            });
567            (
568                HarnessRun {
569                    id: execution.id,
570                    harness: harness.into(),
571                    job_id: execution.job_id,
572                    status: execution.status,
573                    claimed_at: execution.claimed_at,
574                    started_at: execution.started_at,
575                    finished_at: execution.finished_at,
576                    error: execution.error,
577                    session_id,
578                    delivery,
579                },
580                native,
581            )
582        })
583        .collect())
584}
585
586/// The platform surface each job in `store` delivers to, when the job names
587/// one that a `delivery_obligations` row could carry.
588///
589/// Only two of Hermes's `deliver` values name a chat: `origin` (the creating
590/// conversation, recorded on the job as `origin{platform, chat_id}`) and the
591/// explicit `<platform>:<chat>[:<thread>]` form `hermes send --to` spells.
592/// `local` and `home` deliver nowhere a platform ledger would see, so those
593/// jobs get no surface and their fires answer `None` — matching them on the
594/// job's origin anyway would attribute a chat delivery to a run that never
595/// made one.
596fn hermes_delivery_surfaces(store: &Path) -> std::collections::BTreeMap<String, (String, String)> {
597    let mut surfaces = std::collections::BTreeMap::new();
598    for record in crate::jobs::read_job_array(store) {
599        let Some(job_id) = crate::jobs::record_id(&record) else {
600            continue;
601        };
602        let deliver = record
603            .get("deliver")
604            .and_then(Value::as_str)
605            .unwrap_or_default();
606        let surface = if deliver == "origin" {
607            let platform = record.pointer("/origin/platform").and_then(Value::as_str);
608            let chat = record.pointer("/origin/chat_id").and_then(Value::as_str);
609            platform.zip(chat)
610        } else {
611            let mut parts = deliver.splitn(3, ':');
612            parts.next().zip(parts.next())
613        };
614        if let Some((platform, chat)) = surface {
615            if !platform.is_empty() && !chat.is_empty() {
616                surfaces.insert(job_id, (platform.to_string(), chat.to_string()));
617            }
618        }
619    }
620    surfaces
621}
622
623/// Read the delivery Hermes recorded for one fire, from the
624/// `delivery_obligations` ledger inside the same `state.db`.
625///
626/// The ledger is the GATEWAY's, not the scheduler's: `gateway/delivery_ledger.py`
627/// records one row per outbound final response, keyed by the conversation's
628/// `session_key` and the platform surface it was addressed to. Nothing in it
629/// names a job or a fire. So the fire's own `[claimed_at, finished_at]` window
630/// does the work, and two questions are asked in order:
631///
632/// 1. **By session key** — when the fire opened a session that carries one,
633///    the obligations recorded for that key inside the window are this fire's.
634/// 2. **By surface** — otherwise, the obligations addressed to the job's own
635///    delivery surface (`<platform>, <chat_id>`) inside the window. A Hermes
636///    cron session carries no `session_key` at all (the scheduler clears the
637///    routing vars before the run), so this is the usual path for a fire.
638///
639/// The newest obligation in the window wins: a fire that retried its send
640/// wrote more than one, and the last is its outcome.
641///
642/// **Both instants must be anchored.** Executions are written as
643/// `hermes_time.now().isoformat()` — an offset-bearing instant — while
644/// obligations are written as `time.time()`, UTC epoch seconds. An execution
645/// string with no offset cannot be placed on that epoch line without assuming
646/// a timezone supercode has no business guessing, so such a fire answers
647/// `None` rather than matching on an invented instant.
648fn hermes_delivery(
649    connection: &Connection,
650    execution: &HermesExecution,
651    session_id: Option<&str>,
652    surface: Option<&(String, String)>,
653) -> Option<RunDelivery> {
654    if !table_exists(connection, "delivery_obligations") {
655        return None;
656    }
657    let from = execution.claimed_at.as_deref().and_then(hermes_epoch)?;
658    let to = execution
659        .finished_at
660        .as_deref()
661        .and_then(hermes_epoch)
662        .unwrap_or(f64::MAX);
663    let session_key = session_id.and_then(|session_id| {
664        connection
665            .query_row(
666                "SELECT session_key FROM sessions WHERE id = ?1",
667                [session_id],
668                |row| row.get::<_, Option<String>>(0),
669            )
670            .ok()
671            .flatten()
672            .filter(|key| !key.is_empty())
673    });
674    let by_key = session_key.and_then(|key| {
675        read_obligation(
676            connection,
677            "session_key = ?1",
678            rusqlite::params![key, from, to],
679        )
680    });
681    by_key.or_else(|| {
682        let (platform, chat_id) = surface?;
683        read_obligation(
684            connection,
685            "platform = ?1 AND chat_id = ?4",
686            rusqlite::params![platform, from, to, chat_id],
687        )
688    })
689}
690
691/// The newest obligation matching `predicate` inside `[?2, ?3]`.
692fn read_obligation(
693    connection: &Connection,
694    predicate: &str,
695    params: &[&dyn rusqlite::ToSql],
696) -> Option<RunDelivery> {
697    let sql = format!(
698        "SELECT platform, chat_id, thread_id, state, attempts, last_error, updated_at \
699         FROM delivery_obligations \
700         WHERE {predicate} AND created_at >= ?2 AND created_at <= ?3 \
701         ORDER BY created_at DESC LIMIT 1"
702    );
703    connection
704        .query_row(&sql, params, |row| {
705            let platform: String = row.get(0)?;
706            let chat_id: String = row.get(1)?;
707            let thread_id: Option<String> = row.get(2)?;
708            let state: Option<String> = row.get(3)?;
709            let updated_at: Option<f64> = row.get(6)?;
710            Ok(RunDelivery {
711                target: Some(match thread_id.filter(|thread| !thread.is_empty()) {
712                    Some(thread) => format!("{platform}:{chat_id}:{thread}"),
713                    None => format!("{platform}:{chat_id}"),
714                }),
715                // Only a `delivered` row carries an instant of delivery: the
716                // ledger stamps `updated_at` on every transition, so reading
717                // it on a `failed` row would date the failure, not a send.
718                delivered_at: updated_at
719                    .filter(|_| state.as_deref() == Some("delivered"))
720                    .map(|seconds| crate::sidecar::ms_to_rfc3339((seconds * 1000.0) as i64)),
721                state,
722                attempts: row
723                    .get::<_, Option<i64>>(4)?
724                    .map(|attempts| attempts as u64),
725                last_error: row
726                    .get::<_, Option<String>>(5)?
727                    .filter(|error| !error.is_empty()),
728            })
729        })
730        .ok()
731}
732
733/// An offset-bearing Hermes instant as UTC epoch seconds, or `None` when the
734/// string carries no offset (see [`hermes_delivery`]).
735fn hermes_epoch(iso: &str) -> Option<f64> {
736    let (instant, offset) = split_offset(iso)?;
737    let (date, time) = instant.split_once('T')?;
738    let mut date = date.splitn(3, '-');
739    let year: i64 = date.next()?.parse().ok()?;
740    let month: i64 = date.next()?.parse().ok()?;
741    let day: i64 = date.next()?.parse().ok()?;
742    let mut clock = time.splitn(3, ':');
743    let hour: i64 = clock.next()?.parse().ok()?;
744    let minute: i64 = clock.next()?.parse().ok()?;
745    let seconds: f64 = clock.next()?.parse().ok()?;
746    // Days from the civil date (Howard Hinnant's `days_from_civil`), the
747    // inverse of `jobs::iso_from_ms`'s civil-from-days.
748    let year = year - i64::from(month <= 2);
749    let era = year.div_euclid(400);
750    let yoe = year - era * 400;
751    let doy = (153 * (if month > 2 { month - 3 } else { month + 9 }) + 2) / 5 + day - 1;
752    let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
753    let days = era * 146_097 + doe - 719_468;
754    Some((days * 86_400 + hour * 3_600 + minute * 60) as f64 + seconds - offset)
755}
756
757/// Split `<instant><offset>` into the naive part and the offset in seconds.
758/// `Z` is zero; a bare instant has no offset and cannot be placed on the
759/// epoch line at all.
760fn split_offset(iso: &str) -> Option<(&str, f64)> {
761    if let Some(instant) = iso.strip_suffix('Z') {
762        return Some((instant, 0.0));
763    }
764    // The sign is only an offset after the time part, never the date's own
765    // separators, so search from the `T`.
766    let time_at = iso.find('T')?;
767    let sign_at = iso[time_at..]
768        .find(['+', '-'])
769        .map(|index| index + time_at)?;
770    let (instant, offset) = iso.split_at(sign_at);
771    let (hours, minutes) = offset[1..].split_once(':')?;
772    let seconds = hours.parse::<f64>().ok()? * 3_600.0 + minutes.parse::<f64>().ok()? * 60.0;
773    Some((
774        instant,
775        if offset.starts_with('-') {
776            -seconds
777        } else {
778            seconds
779        },
780    ))
781}
782
783/// The fields of one `executions` row the uniform projection and the session
784/// join need.
785struct HermesExecution {
786    id: String,
787    job_id: String,
788    status: String,
789    claimed_at: Option<String>,
790    started_at: Option<String>,
791    finished_at: Option<String>,
792    error: Option<String>,
793}
794
795/// Sortable `YYYYMMDDHHMMSS` key for a Hermes wall-clock instant.
796///
797/// The ledger writes `_hermes_now().isoformat()` and the scheduler mints the
798/// fire session id with `_hermes_now().strftime('%Y%m%d_%H%M%S')` — the SAME
799/// clock, so comparing the two as naive local instants needs no timezone
800/// arithmetic and cannot be wrong by an offset. Sub-second precision is
801/// dropped on both sides (the session id has none).
802fn hermes_instant_key(iso: &str) -> Option<u64> {
803    let digits: String = iso
804        .chars()
805        .take_while(|c| *c != '+' && *c != 'Z')
806        .filter(char::is_ascii_digit)
807        .collect();
808    (digits.len() >= 14).then(|| digits[..14].parse().ok())?
809}
810
811/// `cron_<job>_<YYYYMMDD_HHMMSS>` → the same sortable key.
812fn hermes_session_key(session_id: &str, job_id: &str) -> Option<u64> {
813    if crate::session::hermes_cron_job_id(session_id).as_deref() != Some(job_id) {
814        return None;
815    }
816    let stamp = session_id.rsplit_once('_')?;
817    let date = stamp.0.rsplit_once('_')?.1;
818    format!("{date}{}", stamp.1).parse().ok()
819}
820
821/// Recover the session a Hermes fire opened.
822///
823/// Hermes writes no link from an execution row to its session: the scheduler
824/// mints `cron_<job_id>_<YYYYMMDD_HHMMSS>` inside the run body, after the
825/// claim and before the terminal write. So the candidates are the sessions
826/// whose id carries this job's prefix, and the fire's own window picks one:
827///
828/// * A terminated fire owns the candidates in `[claimed_at, finished_at]`,
829///   and the NEWEST of those is the session that produced the outcome (a
830///   retried script can open more than one).
831/// * An unterminated fire (`claimed` / `running`) has an open-ended window,
832///   so the OLDEST candidate at or after the claim is taken — the newest
833///   there would be a later fire's session.
834///
835/// No candidate in the window means `None`: an honest "not recoverable"
836/// rather than the nearest-looking session.
837fn join_hermes_session(connection: &Connection, execution: &HermesExecution) -> Option<String> {
838    let claimed = execution
839        .claimed_at
840        .as_deref()
841        .and_then(hermes_instant_key)?;
842    let finished = execution
843        .finished_at
844        .as_deref()
845        .and_then(hermes_instant_key);
846    let prefix = format!("cron_{}_", execution.job_id);
847    let mut statement = connection
848        .prepare("SELECT id FROM sessions WHERE substr(id, 1, ?1) = ?2")
849        .ok()?;
850    let candidates: Vec<(u64, String)> = statement
851        .query_map(
852            rusqlite::params![prefix.chars().count() as i64, prefix],
853            |row| row.get::<_, String>(0),
854        )
855        .ok()?
856        .flatten()
857        .filter_map(|id| {
858            let key = hermes_session_key(&id, &execution.job_id)?;
859            (key >= claimed && finished.is_none_or(|finished| key <= finished)).then_some((key, id))
860        })
861        .collect();
862    let chosen = match finished {
863        Some(_) => candidates.into_iter().max_by_key(|(key, _)| *key),
864        None => candidates.into_iter().min_by_key(|(key, _)| *key),
865    }?;
866    Some(compression_tip(connection, chosen.1))
867}
868
869/// Follow a Hermes compression chain from a fire's own session to the session
870/// that still holds the conversation.
871///
872/// A compressed session ends with `end_reason = 'compression'` and its
873/// continuation is the row whose `parent_session_id` points back at it — the
874/// same tri-semantic lineage `hermes_lineage_kind` classifies as `compaction`.
875/// A fire whose session was compressed mid-run is therefore only readable at
876/// the chain's tip, so that is what the row reports.
877fn compression_tip(connection: &Connection, start: String) -> String {
878    let mut current = start;
879    for _ in 0..COMPRESSION_CHAIN_LIMIT {
880        let compressed = connection
881            .query_row(
882                "SELECT end_reason FROM sessions WHERE id = ?1",
883                [&current],
884                |row| row.get::<_, Option<String>>(0),
885            )
886            .ok()
887            .flatten()
888            .is_some_and(|reason| reason == "compression");
889        if !compressed {
890            return current;
891        }
892        let next: Option<String> = connection
893            .query_row(
894                "SELECT id FROM sessions WHERE parent_session_id = ?1 \
895                 ORDER BY started_at DESC, id DESC LIMIT 1",
896                [&current],
897                |row| row.get(0),
898            )
899            .ok();
900        match next {
901            // A compressed session whose continuation is missing is still the
902            // best answer available: the fire did open it.
903            None => return current,
904            Some(next) => current = next,
905        }
906    }
907    current
908}
909
910// ---------------------------------------------------------------------------
911// OpenClaw — `cron_run_logs` in the shared state DB
912// ---------------------------------------------------------------------------
913
914/// The shared OpenClaw state database, at the pinned 2026.7.1-2 path
915/// (`src/state/openclaw-state-db.paths.ts`: the state root — `OPENCLAW_STATE_DIR`
916/// or `~/.openclaw` — plus `state/openclaw.sqlite`).
917fn openclaw_state_db(homes: &HarnessHomes) -> PathBuf {
918    homes.openclaw.join("state/openclaw.sqlite")
919}
920
921const OPENCLAW_RUN_LOG_COLUMNS: &[&str] = &[
922    "store_key",
923    "job_id",
924    "seq",
925    "ts",
926    "status",
927    "error",
928    "summary",
929    "delivery_status",
930    "delivery_error",
931    "delivered",
932    "session_id",
933    "session_key",
934    "run_id",
935    "run_at_ms",
936    "duration_ms",
937];
938
939fn collect_openclaw(
940    query: &RunsQuery,
941    rows: &mut Vec<(HarnessRun, Value)>,
942    sources: &mut Vec<RunSource>,
943) {
944    let state_db = openclaw_state_db(&query.homes);
945    if !state_db.is_file() {
946        sources.push(RunSource::store(
947            HarnessId::OPENCLAW,
948            state_db.clone(),
949            "absent_store",
950            None,
951        ));
952    } else {
953        match open_read_only(&state_db).and_then(|connection| {
954            if table_exists(&connection, "cron_run_logs") {
955                let targets = openclaw_delivery_targets(&connection);
956                read_openclaw_run_logs(&connection, query, &targets)
957            } else {
958                Ok(Vec::new())
959            }
960        }) {
961            Ok(found) => {
962                sources.push(RunSource::store(
963                    HarnessId::OPENCLAW,
964                    state_db.clone(),
965                    "read",
966                    None,
967                ));
968                rows.extend(found);
969            }
970            Err(error) => sources.push(RunSource {
971                detail: Some(error.to_string()),
972                ..RunSource::store(HarnessId::OPENCLAW, state_db.clone(), "unreadable", None)
973            }),
974        }
975    }
976}
977
978/// Where each job announces, from the store's own `cron_jobs` delivery
979/// columns. A run-log row records whether a delivery happened, never where it
980/// went, so the address comes from the job the fire belongs to.
981fn openclaw_delivery_targets(
982    connection: &Connection,
983) -> std::collections::BTreeMap<String, String> {
984    let mut targets = std::collections::BTreeMap::new();
985    if !table_exists(connection, "cron_jobs") {
986        return targets;
987    }
988    let Ok(mut statement) =
989        connection.prepare("SELECT job_id, delivery_channel, delivery_to FROM cron_jobs")
990    else {
991        return targets;
992    };
993    let Ok(rows) = statement.query_map([], |row| {
994        Ok((
995            row.get::<_, String>(0)?,
996            row.get::<_, Option<String>>(1)?,
997            row.get::<_, Option<String>>(2)?,
998        ))
999    }) else {
1000        return targets;
1001    };
1002    for (job_id, channel, to) in rows.flatten() {
1003        let channel = channel.filter(|value| !value.is_empty());
1004        let to = to.filter(|value| !value.is_empty());
1005        let target = match (channel, to) {
1006            (Some(channel), Some(to)) => Some(format!("{channel}:{to}")),
1007            (Some(only), None) | (None, Some(only)) => Some(only),
1008            (None, None) => None,
1009        };
1010        if let Some(target) = target {
1011            targets.insert(job_id, target);
1012        }
1013    }
1014    targets
1015}
1016
1017fn read_openclaw_run_logs(
1018    connection: &Connection,
1019    query: &RunsQuery,
1020    targets: &std::collections::BTreeMap<String, String>,
1021) -> std::result::Result<Vec<(HarnessRun, Value)>, rusqlite::Error> {
1022    // `idx_cron_run_logs_job_status` orders by `(ts DESC, seq DESC)`; the
1023    // store's own newest-first order.
1024    let sql = format!(
1025        "SELECT {} FROM cron_run_logs {} ORDER BY ts DESC, seq DESC {}",
1026        OPENCLAW_RUN_LOG_COLUMNS.join(", "),
1027        if query.job.is_some() {
1028            "WHERE job_id = ?1"
1029        } else {
1030            ""
1031        },
1032        query
1033            .limit
1034            .map_or_else(String::new, |limit| format!("LIMIT {limit}")),
1035    );
1036    let mut statement = connection.prepare(&sql)?;
1037    let read = |row: &rusqlite::Row<'_>| -> rusqlite::Result<(HarnessRun, Value)> {
1038        let native = native_row(row, OPENCLAW_RUN_LOG_COLUMNS);
1039        let job_id = row.get::<_, Option<String>>(1)?.unwrap_or_default();
1040        let delivery = openclaw_delivery(
1041            targets.get(job_id.as_str()).cloned(),
1042            row.get(7)?,
1043            row.get(8)?,
1044            row.get(9)?,
1045        );
1046        Ok((
1047            openclaw_row(
1048                job_id,
1049                row.get(12)?,
1050                row.get::<_, Option<i64>>(2)?,
1051                row.get(4)?,
1052                row.get(5)?,
1053                row.get(13)?,
1054                row.get(3)?,
1055                row.get(10)?,
1056                delivery,
1057            ),
1058            native,
1059        ))
1060    };
1061    match query.job.as_deref() {
1062        Some(job) => statement.query_map([job], read)?.collect(),
1063        None => statement.query_map([], read)?.collect(),
1064    }
1065}
1066
1067/// Project one OpenClaw run-log entry, from either of its two stores.
1068///
1069/// `run_at_ms` is when the fire began and `ts` is the entry's own timestamp,
1070/// written once the run finished (`parseCronRunLogEntryObject` accepts only
1071/// `action: "finished"` records), so those are the row's start and finish.
1072/// OpenClaw records no claim, so `claimed_at` is honestly empty.
1073#[allow(clippy::too_many_arguments)]
1074fn openclaw_row(
1075    job_id: String,
1076    run_id: Option<String>,
1077    seq: Option<i64>,
1078    status: Option<String>,
1079    error: Option<String>,
1080    run_at_ms: Option<i64>,
1081    ts: Option<i64>,
1082    session_id: Option<String>,
1083    delivery: Option<RunDelivery>,
1084) -> HarnessRun {
1085    let id = run_id
1086        .filter(|run_id| !run_id.is_empty())
1087        .unwrap_or_else(|| match seq {
1088            Some(seq) => format!("{job_id}#{seq}"),
1089            None => job_id.clone(),
1090        });
1091    HarnessRun {
1092        id,
1093        harness: HarnessId::OPENCLAW.into(),
1094        job_id,
1095        status: status.unwrap_or_default(),
1096        claimed_at: None,
1097        started_at: run_at_ms.map(crate::sidecar::ms_to_rfc3339),
1098        finished_at: ts.map(crate::sidecar::ms_to_rfc3339),
1099        error: error.filter(|error| !error.is_empty()),
1100        session_id: session_id.filter(|session| !session.is_empty()),
1101        delivery,
1102    }
1103}
1104
1105/// Project one run-log row's delivery columns.
1106///
1107/// `delivery_status` is the harness's own word; when a row carries only the
1108/// `delivered` flag, that flag becomes the state, so the fact is never lost
1109/// for want of a status string. A row with none of the three recorded no
1110/// delivery at all and answers `None` — the job's declared target alone is
1111/// not evidence that anything was sent.
1112fn openclaw_delivery(
1113    target: Option<String>,
1114    status: Option<String>,
1115    error: Option<String>,
1116    delivered: Option<i64>,
1117) -> Option<RunDelivery> {
1118    let status = status.filter(|status| !status.is_empty());
1119    let error = error.filter(|error| !error.is_empty());
1120    if status.is_none() && error.is_none() && delivered.is_none() {
1121        return None;
1122    }
1123    Some(RunDelivery {
1124        target,
1125        state: status.or_else(|| {
1126            delivered.map(|delivered| {
1127                if delivered == 0 {
1128                    "not-delivered".to_string()
1129                } else {
1130                    "delivered".to_string()
1131                }
1132            })
1133        }),
1134        // OpenClaw's run log counts no delivery attempts, and stamps no
1135        // instant on `delivered` — the flag rides the finish record.
1136        attempts: None,
1137        last_error: error,
1138        delivered_at: None,
1139    })
1140}