use std::path::{Path, PathBuf};
use rusqlite::Connection;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use crate::{HarnessHomes, HarnessId, Result};
pub const RUN_HARNESSES: &[&str] = &[
HarnessId::HERMES,
HarnessId::OPENCLAW,
HarnessId::ORCHESTRATOR,
];
const COMPRESSION_CHAIN_LIMIT: usize = 32;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct HarnessRun {
pub id: String,
pub harness: String,
pub job_id: String,
pub status: String,
pub claimed_at: Option<String>,
pub started_at: Option<String>,
pub finished_at: Option<String>,
pub error: Option<String>,
pub session_id: Option<String>,
pub delivery: Option<RunDelivery>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RunDelivery {
pub target: Option<String>,
pub state: Option<String>,
pub attempts: Option<u64>,
pub last_error: Option<String>,
pub delivered_at: Option<String>,
}
impl HarnessRun {
pub fn from_fire(
harness: &str,
fire: &supercode_interchange::world::Fire,
session_id: Option<String>,
delivery: Option<RunDelivery>,
) -> Self {
Self {
id: fire.id.clone(),
harness: harness.into(),
job_id: fire.job_id.clone(),
status: fire.status.hermes_word().to_string(),
claimed_at: Some(fire.claimed_at.clone()),
started_at: fire.started_at.clone(),
finished_at: fire.finished_at.clone(),
error: fire.error.clone(),
session_id: session_id.or_else(|| fire.session_id.clone()),
delivery,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RunSource {
pub harness: String,
pub path: PathBuf,
pub state: String,
pub profile: Option<String>,
pub detail: Option<String>,
}
impl RunSource {
fn store(harness: &str, path: PathBuf, state: &str, profile: Option<String>) -> Self {
Self {
harness: harness.to_string(),
path,
state: state.to_string(),
profile,
detail: None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RunsListing {
pub runs: Vec<HarnessRun>,
pub sources: Vec<RunSource>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
pub struct RunsQuery {
pub harness: Option<String>,
pub job: Option<String>,
pub limit: Option<usize>,
pub homes: HarnessHomes,
}
pub fn supports_runs(harness: &str) -> bool {
RUN_HARNESSES.contains(&harness)
}
pub fn list_runs(query: &RunsQuery) -> Result<RunsListing> {
let (rows, sources) = collect(query);
let mut runs: Vec<HarnessRun> = rows.into_iter().map(|(run, _)| run).collect();
if let Some(limit) = query.limit {
runs.truncate(limit);
}
Ok(RunsListing { runs, sources })
}
pub fn get_run(
harness: &str,
id: &str,
homes: &HarnessHomes,
) -> Result<Option<(HarnessRun, Value)>> {
let (rows, _) = collect(&RunsQuery {
harness: Some(harness.to_string()),
homes: homes.clone(),
..RunsQuery::default()
});
Ok(rows.into_iter().find(|(run, _)| run.id == id))
}
fn collect(query: &RunsQuery) -> (Vec<(HarnessRun, Value)>, Vec<RunSource>) {
let mut rows = Vec::new();
let mut sources = Vec::new();
let wanted = query.harness.as_deref();
if wanted.is_none_or(|harness| harness == HarnessId::HERMES) {
collect_hermes(query, &mut rows, &mut sources);
}
if wanted.is_none_or(|harness| harness == HarnessId::OPENCLAW) {
collect_openclaw(query, &mut rows, &mut sources);
}
if wanted.is_none_or(|harness| harness == HarnessId::ORCHESTRATOR) {
collect_hermes_shaped(
HarnessId::ORCHESTRATOR,
orchestrator_ledgers(&query.homes),
query,
&mut rows,
&mut sources,
);
}
(rows, sources)
}
fn open_read_only(path: &Path) -> std::result::Result<Connection, rusqlite::Error> {
Connection::open_with_flags(
path,
rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX,
)
.or_else(|_| {
Connection::open_with_flags(
format!("file:{}?immutable=1", path.display()),
rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY
| rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX
| rusqlite::OpenFlags::SQLITE_OPEN_URI,
)
})
}
fn table_exists(conn: &Connection, table: &str) -> bool {
conn.query_row(
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1",
[table],
|row| row.get::<_, i64>(0),
)
.is_ok()
}
fn native_value(value: rusqlite::types::ValueRef<'_>) -> Value {
match value {
rusqlite::types::ValueRef::Null => Value::Null,
rusqlite::types::ValueRef::Integer(i) => Value::from(i),
rusqlite::types::ValueRef::Real(f) => serde_json::Number::from_f64(f)
.map(Value::Number)
.unwrap_or(Value::Null),
rusqlite::types::ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
rusqlite::types::ValueRef::Blob(_) => Value::Null,
}
}
fn native_row(row: &rusqlite::Row<'_>, columns: &[&str]) -> Value {
let mut map = Map::new();
for (index, name) in columns.iter().enumerate() {
let value = row
.get_ref(index)
.map_or(Value::Null, |value| native_value(value));
map.insert((*name).to_string(), value);
}
Value::Object(map)
}
struct HermesLedger {
executions: PathBuf,
sessions: PathBuf,
jobs: PathBuf,
profile: Option<String>,
}
fn hermes_ledgers(homes: &HarnessHomes) -> Vec<HermesLedger> {
let root = homes
.hermes
.parent()
.map_or_else(|| PathBuf::from("."), Path::to_path_buf);
let mut ledgers = vec![HermesLedger {
executions: root.join("cron/executions.db"),
sessions: homes.hermes.clone(),
jobs: root.join("cron/jobs.json"),
profile: None,
}];
if let Ok(entries) = std::fs::read_dir(root.join("profiles")) {
let mut found: Vec<HermesLedger> = entries
.flatten()
.filter(|entry| entry.path().is_dir())
.map(|entry| {
let home = entry.path();
let own = home.join("state.db");
HermesLedger {
executions: home.join("cron/executions.db"),
sessions: if own.is_file() {
own
} else {
homes.hermes.clone()
},
jobs: home.join("cron/jobs.json"),
profile: Some(entry.file_name().to_string_lossy().into_owned()),
}
})
.collect();
found.sort_by(|left, right| left.profile.cmp(&right.profile));
ledgers.extend(found);
}
ledgers
}
fn orchestrator_ledgers(homes: &HarnessHomes) -> Vec<HermesLedger> {
crate::orchestrator_profile_dirs(&homes.orchestrator)
.into_iter()
.map(|(name, dir)| HermesLedger {
executions: dir.join("cron/executions.db"),
sessions: dir.join("state.db"),
jobs: dir.join("cron/jobs.json"),
profile: (name != "default").then_some(name),
})
.collect()
}
const HERMES_EXECUTION_COLUMNS: &[&str] = &[
"id",
"job_id",
"source",
"process_id",
"pid",
"process_started_at",
"status",
"claimed_at",
"started_at",
"finished_at",
"error",
];
fn collect_hermes(
query: &RunsQuery,
rows: &mut Vec<(HarnessRun, Value)>,
sources: &mut Vec<RunSource>,
) {
collect_hermes_shaped(
HarnessId::HERMES,
hermes_ledgers(&query.homes),
query,
rows,
sources,
);
}
fn collect_hermes_shaped(
harness: &str,
ledgers: Vec<HermesLedger>,
query: &RunsQuery,
rows: &mut Vec<(HarnessRun, Value)>,
sources: &mut Vec<RunSource>,
) {
for ledger in ledgers {
if !ledger.executions.is_file() {
sources.push(RunSource::store(
harness,
ledger.executions.clone(),
"absent_store",
ledger.profile.clone(),
));
continue;
}
let connection = match open_read_only(&ledger.executions) {
Ok(connection) => connection,
Err(error) => {
sources.push(RunSource {
detail: Some(error.to_string()),
..RunSource::store(
harness,
ledger.executions.clone(),
"unreadable",
ledger.profile.clone(),
)
});
continue;
}
};
if !table_exists(&connection, "executions") {
sources.push(RunSource {
detail: Some("no `executions` table — not a Hermes cron ledger".into()),
..RunSource::store(
harness,
ledger.executions.clone(),
"unreadable",
ledger.profile.clone(),
)
});
continue;
}
match read_hermes_ledger(harness, &connection, &ledger, query) {
Ok(found) => {
sources.push(RunSource::store(
harness,
ledger.executions.clone(),
"read",
ledger.profile.clone(),
));
rows.extend(found);
}
Err(error) => sources.push(RunSource {
detail: Some(error.to_string()),
..RunSource::store(
harness,
ledger.executions.clone(),
"unreadable",
ledger.profile.clone(),
)
}),
}
}
}
fn read_hermes_ledger(
harness: &str,
connection: &Connection,
ledger: &HermesLedger,
query: &RunsQuery,
) -> std::result::Result<Vec<(HarnessRun, Value)>, rusqlite::Error> {
let sql = format!(
"SELECT {} FROM executions {} ORDER BY claimed_at DESC, id DESC {}",
HERMES_EXECUTION_COLUMNS.join(", "),
if query.job.is_some() {
"WHERE job_id = ?1"
} else {
""
},
query
.limit
.map_or_else(String::new, |limit| format!("LIMIT {limit}")),
);
let mut statement = connection.prepare(&sql)?;
let read = |row: &rusqlite::Row<'_>| -> rusqlite::Result<(HermesExecution, Value)> {
Ok((
HermesExecution {
id: row.get::<_, Option<String>>(0)?.unwrap_or_default(),
job_id: row.get::<_, Option<String>>(1)?.unwrap_or_default(),
status: row.get::<_, Option<String>>(6)?.unwrap_or_default(),
claimed_at: row.get(7)?,
started_at: row.get(8)?,
finished_at: row.get(9)?,
error: row.get(10)?,
},
native_row(row, HERMES_EXECUTION_COLUMNS),
))
};
let executions: Vec<(HermesExecution, Value)> = match query.job.as_deref() {
Some(job) => statement
.query_map([job], read)?
.collect::<rusqlite::Result<_>>()?,
None => statement
.query_map([], read)?
.collect::<rusqlite::Result<_>>()?,
};
let sessions = open_read_only(&ledger.sessions).ok();
let surfaces = hermes_delivery_surfaces(&ledger.jobs);
Ok(executions
.into_iter()
.map(|(execution, native)| {
let session_id = sessions
.as_ref()
.and_then(|connection| join_hermes_session(connection, &execution));
let delivery = sessions.as_ref().and_then(|connection| {
hermes_delivery(
connection,
&execution,
session_id.as_deref(),
surfaces.get(execution.job_id.as_str()),
)
});
(
HarnessRun {
id: execution.id,
harness: harness.into(),
job_id: execution.job_id,
status: execution.status,
claimed_at: execution.claimed_at,
started_at: execution.started_at,
finished_at: execution.finished_at,
error: execution.error,
session_id,
delivery,
},
native,
)
})
.collect())
}
fn hermes_delivery_surfaces(store: &Path) -> std::collections::BTreeMap<String, (String, String)> {
let mut surfaces = std::collections::BTreeMap::new();
for record in crate::jobs::read_job_array(store) {
let Some(job_id) = crate::jobs::record_id(&record) else {
continue;
};
let deliver = record
.get("deliver")
.and_then(Value::as_str)
.unwrap_or_default();
let surface = if deliver == "origin" {
let platform = record.pointer("/origin/platform").and_then(Value::as_str);
let chat = record.pointer("/origin/chat_id").and_then(Value::as_str);
platform.zip(chat)
} else {
let mut parts = deliver.splitn(3, ':');
parts.next().zip(parts.next())
};
if let Some((platform, chat)) = surface {
if !platform.is_empty() && !chat.is_empty() {
surfaces.insert(job_id, (platform.to_string(), chat.to_string()));
}
}
}
surfaces
}
fn hermes_delivery(
connection: &Connection,
execution: &HermesExecution,
session_id: Option<&str>,
surface: Option<&(String, String)>,
) -> Option<RunDelivery> {
if !table_exists(connection, "delivery_obligations") {
return None;
}
let from = execution.claimed_at.as_deref().and_then(hermes_epoch)?;
let to = execution
.finished_at
.as_deref()
.and_then(hermes_epoch)
.unwrap_or(f64::MAX);
let session_key = session_id.and_then(|session_id| {
connection
.query_row(
"SELECT session_key FROM sessions WHERE id = ?1",
[session_id],
|row| row.get::<_, Option<String>>(0),
)
.ok()
.flatten()
.filter(|key| !key.is_empty())
});
let by_key = session_key.and_then(|key| {
read_obligation(
connection,
"session_key = ?1",
rusqlite::params![key, from, to],
)
});
by_key.or_else(|| {
let (platform, chat_id) = surface?;
read_obligation(
connection,
"platform = ?1 AND chat_id = ?4",
rusqlite::params![platform, from, to, chat_id],
)
})
}
fn read_obligation(
connection: &Connection,
predicate: &str,
params: &[&dyn rusqlite::ToSql],
) -> Option<RunDelivery> {
let sql = format!(
"SELECT platform, chat_id, thread_id, state, attempts, last_error, updated_at \
FROM delivery_obligations \
WHERE {predicate} AND created_at >= ?2 AND created_at <= ?3 \
ORDER BY created_at DESC LIMIT 1"
);
connection
.query_row(&sql, params, |row| {
let platform: String = row.get(0)?;
let chat_id: String = row.get(1)?;
let thread_id: Option<String> = row.get(2)?;
let state: Option<String> = row.get(3)?;
let updated_at: Option<f64> = row.get(6)?;
Ok(RunDelivery {
target: Some(match thread_id.filter(|thread| !thread.is_empty()) {
Some(thread) => format!("{platform}:{chat_id}:{thread}"),
None => format!("{platform}:{chat_id}"),
}),
delivered_at: updated_at
.filter(|_| state.as_deref() == Some("delivered"))
.map(|seconds| crate::sidecar::ms_to_rfc3339((seconds * 1000.0) as i64)),
state,
attempts: row
.get::<_, Option<i64>>(4)?
.map(|attempts| attempts as u64),
last_error: row
.get::<_, Option<String>>(5)?
.filter(|error| !error.is_empty()),
})
})
.ok()
}
fn hermes_epoch(iso: &str) -> Option<f64> {
let (instant, offset) = split_offset(iso)?;
let (date, time) = instant.split_once('T')?;
let mut date = date.splitn(3, '-');
let year: i64 = date.next()?.parse().ok()?;
let month: i64 = date.next()?.parse().ok()?;
let day: i64 = date.next()?.parse().ok()?;
let mut clock = time.splitn(3, ':');
let hour: i64 = clock.next()?.parse().ok()?;
let minute: i64 = clock.next()?.parse().ok()?;
let seconds: f64 = clock.next()?.parse().ok()?;
let year = year - i64::from(month <= 2);
let era = year.div_euclid(400);
let yoe = year - era * 400;
let doy = (153 * (if month > 2 { month - 3 } else { month + 9 }) + 2) / 5 + day - 1;
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
let days = era * 146_097 + doe - 719_468;
Some((days * 86_400 + hour * 3_600 + minute * 60) as f64 + seconds - offset)
}
fn split_offset(iso: &str) -> Option<(&str, f64)> {
if let Some(instant) = iso.strip_suffix('Z') {
return Some((instant, 0.0));
}
let time_at = iso.find('T')?;
let sign_at = iso[time_at..]
.find(['+', '-'])
.map(|index| index + time_at)?;
let (instant, offset) = iso.split_at(sign_at);
let (hours, minutes) = offset[1..].split_once(':')?;
let seconds = hours.parse::<f64>().ok()? * 3_600.0 + minutes.parse::<f64>().ok()? * 60.0;
Some((
instant,
if offset.starts_with('-') {
-seconds
} else {
seconds
},
))
}
struct HermesExecution {
id: String,
job_id: String,
status: String,
claimed_at: Option<String>,
started_at: Option<String>,
finished_at: Option<String>,
error: Option<String>,
}
fn hermes_instant_key(iso: &str) -> Option<u64> {
let digits: String = iso
.chars()
.take_while(|c| *c != '+' && *c != 'Z')
.filter(char::is_ascii_digit)
.collect();
(digits.len() >= 14).then(|| digits[..14].parse().ok())?
}
fn hermes_session_key(session_id: &str, job_id: &str) -> Option<u64> {
if crate::session::hermes_cron_job_id(session_id).as_deref() != Some(job_id) {
return None;
}
let stamp = session_id.rsplit_once('_')?;
let date = stamp.0.rsplit_once('_')?.1;
format!("{date}{}", stamp.1).parse().ok()
}
fn join_hermes_session(connection: &Connection, execution: &HermesExecution) -> Option<String> {
let claimed = execution
.claimed_at
.as_deref()
.and_then(hermes_instant_key)?;
let finished = execution
.finished_at
.as_deref()
.and_then(hermes_instant_key);
let prefix = format!("cron_{}_", execution.job_id);
let mut statement = connection
.prepare("SELECT id FROM sessions WHERE substr(id, 1, ?1) = ?2")
.ok()?;
let candidates: Vec<(u64, String)> = statement
.query_map(
rusqlite::params![prefix.chars().count() as i64, prefix],
|row| row.get::<_, String>(0),
)
.ok()?
.flatten()
.filter_map(|id| {
let key = hermes_session_key(&id, &execution.job_id)?;
(key >= claimed && finished.is_none_or(|finished| key <= finished)).then_some((key, id))
})
.collect();
let chosen = match finished {
Some(_) => candidates.into_iter().max_by_key(|(key, _)| *key),
None => candidates.into_iter().min_by_key(|(key, _)| *key),
}?;
Some(compression_tip(connection, chosen.1))
}
fn compression_tip(connection: &Connection, start: String) -> String {
let mut current = start;
for _ in 0..COMPRESSION_CHAIN_LIMIT {
let compressed = connection
.query_row(
"SELECT end_reason FROM sessions WHERE id = ?1",
[¤t],
|row| row.get::<_, Option<String>>(0),
)
.ok()
.flatten()
.is_some_and(|reason| reason == "compression");
if !compressed {
return current;
}
let next: Option<String> = connection
.query_row(
"SELECT id FROM sessions WHERE parent_session_id = ?1 \
ORDER BY started_at DESC, id DESC LIMIT 1",
[¤t],
|row| row.get(0),
)
.ok();
match next {
None => return current,
Some(next) => current = next,
}
}
current
}
fn openclaw_state_db(homes: &HarnessHomes) -> PathBuf {
homes.openclaw.join("state/openclaw.sqlite")
}
const OPENCLAW_RUN_LOG_COLUMNS: &[&str] = &[
"store_key",
"job_id",
"seq",
"ts",
"status",
"error",
"summary",
"delivery_status",
"delivery_error",
"delivered",
"session_id",
"session_key",
"run_id",
"run_at_ms",
"duration_ms",
];
fn collect_openclaw(
query: &RunsQuery,
rows: &mut Vec<(HarnessRun, Value)>,
sources: &mut Vec<RunSource>,
) {
let state_db = openclaw_state_db(&query.homes);
if !state_db.is_file() {
sources.push(RunSource::store(
HarnessId::OPENCLAW,
state_db.clone(),
"absent_store",
None,
));
} else {
match open_read_only(&state_db).and_then(|connection| {
if table_exists(&connection, "cron_run_logs") {
let targets = openclaw_delivery_targets(&connection);
read_openclaw_run_logs(&connection, query, &targets)
} else {
Ok(Vec::new())
}
}) {
Ok(found) => {
sources.push(RunSource::store(
HarnessId::OPENCLAW,
state_db.clone(),
"read",
None,
));
rows.extend(found);
}
Err(error) => sources.push(RunSource {
detail: Some(error.to_string()),
..RunSource::store(HarnessId::OPENCLAW, state_db.clone(), "unreadable", None)
}),
}
}
}
fn openclaw_delivery_targets(
connection: &Connection,
) -> std::collections::BTreeMap<String, String> {
let mut targets = std::collections::BTreeMap::new();
if !table_exists(connection, "cron_jobs") {
return targets;
}
let Ok(mut statement) =
connection.prepare("SELECT job_id, delivery_channel, delivery_to FROM cron_jobs")
else {
return targets;
};
let Ok(rows) = statement.query_map([], |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, Option<String>>(1)?,
row.get::<_, Option<String>>(2)?,
))
}) else {
return targets;
};
for (job_id, channel, to) in rows.flatten() {
let channel = channel.filter(|value| !value.is_empty());
let to = to.filter(|value| !value.is_empty());
let target = match (channel, to) {
(Some(channel), Some(to)) => Some(format!("{channel}:{to}")),
(Some(only), None) | (None, Some(only)) => Some(only),
(None, None) => None,
};
if let Some(target) = target {
targets.insert(job_id, target);
}
}
targets
}
fn read_openclaw_run_logs(
connection: &Connection,
query: &RunsQuery,
targets: &std::collections::BTreeMap<String, String>,
) -> std::result::Result<Vec<(HarnessRun, Value)>, rusqlite::Error> {
let sql = format!(
"SELECT {} FROM cron_run_logs {} ORDER BY ts DESC, seq DESC {}",
OPENCLAW_RUN_LOG_COLUMNS.join(", "),
if query.job.is_some() {
"WHERE job_id = ?1"
} else {
""
},
query
.limit
.map_or_else(String::new, |limit| format!("LIMIT {limit}")),
);
let mut statement = connection.prepare(&sql)?;
let read = |row: &rusqlite::Row<'_>| -> rusqlite::Result<(HarnessRun, Value)> {
let native = native_row(row, OPENCLAW_RUN_LOG_COLUMNS);
let job_id = row.get::<_, Option<String>>(1)?.unwrap_or_default();
let delivery = openclaw_delivery(
targets.get(job_id.as_str()).cloned(),
row.get(7)?,
row.get(8)?,
row.get(9)?,
);
Ok((
openclaw_row(
job_id,
row.get(12)?,
row.get::<_, Option<i64>>(2)?,
row.get(4)?,
row.get(5)?,
row.get(13)?,
row.get(3)?,
row.get(10)?,
delivery,
),
native,
))
};
match query.job.as_deref() {
Some(job) => statement.query_map([job], read)?.collect(),
None => statement.query_map([], read)?.collect(),
}
}
#[allow(clippy::too_many_arguments)]
fn openclaw_row(
job_id: String,
run_id: Option<String>,
seq: Option<i64>,
status: Option<String>,
error: Option<String>,
run_at_ms: Option<i64>,
ts: Option<i64>,
session_id: Option<String>,
delivery: Option<RunDelivery>,
) -> HarnessRun {
let id = run_id
.filter(|run_id| !run_id.is_empty())
.unwrap_or_else(|| match seq {
Some(seq) => format!("{job_id}#{seq}"),
None => job_id.clone(),
});
HarnessRun {
id,
harness: HarnessId::OPENCLAW.into(),
job_id,
status: status.unwrap_or_default(),
claimed_at: None,
started_at: run_at_ms.map(crate::sidecar::ms_to_rfc3339),
finished_at: ts.map(crate::sidecar::ms_to_rfc3339),
error: error.filter(|error| !error.is_empty()),
session_id: session_id.filter(|session| !session.is_empty()),
delivery,
}
}
fn openclaw_delivery(
target: Option<String>,
status: Option<String>,
error: Option<String>,
delivered: Option<i64>,
) -> Option<RunDelivery> {
let status = status.filter(|status| !status.is_empty());
let error = error.filter(|error| !error.is_empty());
if status.is_none() && error.is_none() && delivered.is_none() {
return None;
}
Some(RunDelivery {
target,
state: status.or_else(|| {
delivered.map(|delivered| {
if delivered == 0 {
"not-delivered".to_string()
} else {
"delivered".to_string()
}
})
}),
attempts: None,
last_error: error,
delivered_at: None,
})
}