supercode_interchange/orchestration/codec/

decode.rs

//! Strict decoders and canonical encoders for every record's FILE form
//! (`ir.mjs`). O-records refuse an unknown key by file and key; H-records
//! keep what they do not model in `residue`.

use std::collections::BTreeMap;

use serde_json::{Map, Value};

use crate::ontology::{HarnessId, Residue, SecretRef, SurfaceKey};
use crate::orchestration::{
    ChannelConfig, EnvValue, Fire, FireStatus, Job, JobOrigin, Obligation, ObligationSource,
    ObligationState, OutboundContent, PermissionDefault, PermissionPolicy, PermissionUnattended,
    Posted, Repeat, Route, RouteMatch, Schedule, Target, WebhookSubscription, WorkerSpec,
};
use crate::Result;

/// A load refusal naming the file and the key (`ir.mjs::LoadError`).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LoadError {
    /// The file.
    pub file: String,
    /// The key within it, when one applies.
    pub key: Option<String>,
    /// What was wrong.
    pub message: String,
}

impl std::fmt::Display for LoadError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match &self.key {
            Some(key) => write!(f, "{} [{key}]: {}", self.file, self.message),
            None => write!(f, "{}: {}", self.file, self.message),
        }
    }
}

impl From<LoadError> for crate::Error {
    fn from(error: LoadError) -> Self {
        crate::Error::Other(error.to_string())
    }
}

pub(crate) fn load_error(file: &str, key: &str, message: impl Into<String>) -> crate::Error {
    LoadError {
        file: file.to_string(),
        key: if key.is_empty() {
            None
        } else {
            Some(key.to_string())
        },
        message: message.into(),
    }
    .into()
}

fn expect_keys(file: &str, key: &str, obj: &Value, allowed: &[&str]) -> Result<()> {
    let Some(map) = obj.as_object() else {
        return Err(load_error(file, key, "expected an object"));
    };
    for k in map.keys() {
        if !allowed.contains(&k.as_str()) {
            let path = if key.is_empty() {
                k.clone()
            } else {
                format!("{key}.{k}")
            };
            return Err(load_error(file, &path, "unknown key"));
        }
    }
    Ok(())
}

fn req_str(file: &str, key: &str, v: Option<&Value>) -> Result<String> {
    match v {
        Some(Value::String(s)) if !s.is_empty() => Ok(s.clone()),
        _ => Err(load_error(file, key, "expected a non-empty string")),
    }
}

fn opt_str(file: &str, key: &str, v: Option<&Value>) -> Result<Option<String>> {
    match v {
        None | Some(Value::Null) => Ok(None),
        Some(Value::String(s)) => Ok(Some(s.clone())),
        _ => Err(load_error(file, key, "expected a string")),
    }
}

fn opt_bool(
    file: &str,
    key: &str,
    v: Option<&Value>,
    default: Option<bool>,
) -> Result<Option<bool>> {
    match v {
        None | Some(Value::Null) => Ok(default),
        Some(Value::Bool(b)) => Ok(Some(*b)),
        _ => Err(load_error(file, key, "expected a boolean")),
    }
}

fn residue_of(obj: &Map<String, Value>, mapped: &[&str]) -> Residue {
    let mut out = Residue::default();
    for (k, v) in obj {
        if !mapped.contains(&k.as_str()) {
            out.keep(k.clone(), v.clone());
        }
    }
    out
}

/// Row residue: SQL NULLs are the column's absence, not a value worth carrying.
fn row_residue_of(row: &Map<String, Value>, mapped: &[&str]) -> Residue {
    let mut out = Residue::default();
    for (k, v) in row {
        if !mapped.contains(&k.as_str()) && !v.is_null() {
            out.keep(k.clone(), v.clone());
        }
    }
    out
}

/// `String(value)` as JavaScript spells it: an integral REAL prints without a fraction.
fn text_of(v: &Value) -> Option<String> {
    match v {
        Value::String(s) => Some(s.clone()),
        Value::Number(n) => Some(match n.as_f64() {
            Some(f) if n.is_f64() && f.fract() == 0.0 && f.abs() < 1e21 => format!("{}", f as i64),
            _ => n.to_string(),
        }),
        _ => None,
    }
}

fn number_of(v: &Value) -> Option<f64> {
    match v {
        Value::Number(n) => n.as_f64(),
        Value::String(s) => s.parse().ok(),
        _ => None,
    }
}

// ------------------------------------------------------------- surface keys

/// `dm` | `group` | `channel` | `thread`.
pub const CHAT_TYPES: &[&str] = &["dm", "group", "channel", "thread"];

/// The map key of a binding: `<platform>|<chat_type>|<chat_id>|<thread_id>|<participant_id>`.
pub fn surface_key_string(key: &SurfaceKey) -> String {
    format!(
        "{}|{}|{}|{}|{}",
        key.platform.as_deref().unwrap_or(""),
        key.kind.as_deref().unwrap_or(""),
        key.chat_id.as_deref().unwrap_or(""),
        key.thread_id.as_deref().unwrap_or(""),
        key.participant_id.as_deref().unwrap_or("")
    )
}

/// A surface key from the IR's own JSON (`{platform, chat_type, chat_id, thread_id?, participant_id?}`).
pub fn decode_surface_key(file: &str, key: &str, raw: &Value) -> Result<SurfaceKey> {
    let map = raw
        .as_object()
        .ok_or_else(|| load_error(file, key, "expected an object"))?;
    for k in map.keys() {
        if ![
            "platform",
            "chat_type",
            "kind",
            "chat_id",
            "thread_id",
            "participant_id",
            "key",
        ]
        .contains(&k.as_str())
        {
            return Err(load_error(file, &format!("{key}.{k}"), "unknown key"));
        }
    }
    let chat_type = map
        .get("chat_type")
        .or_else(|| map.get("kind"))
        .and_then(Value::as_str)
        .map(str::to_string);
    if !chat_type
        .as_deref()
        .is_some_and(|c| CHAT_TYPES.contains(&c))
    {
        return Err(load_error(
            file,
            key,
            "expected platform, chat_type, chat_id",
        ));
    }
    Ok(SurfaceKey {
        key: None,
        platform: Some(req_str(
            file,
            &format!("{key}.platform"),
            map.get("platform"),
        )?),
        kind: chat_type,
        chat_id: map.get("chat_id").and_then(text_of),
        thread_id: map
            .get("thread_id")
            .and_then(text_of)
            .filter(|s| !s.is_empty()),
        participant_id: map
            .get("participant_id")
            .and_then(text_of)
            .filter(|s| !s.is_empty()),
    })
}

/// The orchestration's JSON of a surface key (`kind`, as the ontology spells it; absent optionals omitted).
pub fn encode_surface_key(key: &SurfaceKey) -> Value {
    let mut out = Map::new();
    out.insert(
        "platform".into(),
        Value::String(key.platform.clone().unwrap_or_default()),
    );
    out.insert(
        "kind".into(),
        Value::String(key.kind.clone().unwrap_or_default()),
    );
    if let Some(c) = &key.chat_id {
        out.insert("chat_id".into(), Value::String(c.clone()));
    }
    if let Some(t) = &key.thread_id {
        out.insert("thread_id".into(), Value::String(t.clone()));
    }
    if let Some(p) = &key.participant_id {
        out.insert("participant_id".into(), Value::String(p.clone()));
    }
    Value::Object(out)
}

// ------------------------------------------------------------------ targets

/// Hermes `deliver` word → Target: `origin | local | home | <platform>[:<chat_id>[:<thread_id>]]`.
pub fn parse_target(
    file: &str,
    key: &str,
    word: Option<&Value>,
    extra: Option<&Value>,
) -> Result<Option<Target>> {
    let word = match word {
        None | Some(Value::Null) => return Ok(None),
        Some(Value::String(s)) if !s.is_empty() => s.as_str(),
        _ => return Err(load_error(file, key, "expected a delivery word")),
    };
    Ok(Some(match word {
        "origin" => Target::Origin,
        "local" => Target::Local,
        "home" => Target::Home,
        _ => {
            let parts: Vec<&str> = word.split(':').collect();
            let extra_get = |name: &str| extra.and_then(|e| e.get(name)).and_then(text_of);
            let chat_id = parts
                .get(1)
                .map(|s| s.to_string())
                .or_else(|| extra_get("chat_id"))
                .filter(|s| !s.is_empty());
            let thread_id = parts
                .get(2)
                .map(|s| s.to_string())
                .or_else(|| extra_get("thread_id"))
                .filter(|s| !s.is_empty());
            Target::Explicit {
                platform: parts[0].to_string(),
                chat_id,
                thread_id,
            }
        }
    }))
}

// ---------------------------------------------------------------- O-records

/// A worker env value: a string, `${NAME}` (a secret, Hermes's reference form) or a literal.
pub fn decode_env_value(file: &str, key: &str, v: &Value) -> Result<EnvValue> {
    match v {
        Value::String(s) => Ok(match placeholder_ref(s) {
            Some(name) => EnvValue::Secret(SecretRef::Dotenv(name.to_string())),
            None => EnvValue::Literal(s.clone()),
        }),
        _ => Err(load_error(
            file,
            key,
            "expected a string (a secret is written `${NAME}`)",
        )),
    }
}

/// `worker:` block of config.yaml.
pub fn decode_worker(file: &str, raw: Option<&Value>) -> Result<Option<WorkerSpec>> {
    let Some(raw) = raw.filter(|v| !v.is_null()) else {
        return Ok(None);
    };
    expect_keys(
        file,
        "worker",
        raw,
        &[
            "harness",
            "model",
            "reasoning_effort",
            "preset",
            "cwd",
            "env",
            "permission",
            "home",
            "surface",
            "capacity",
        ],
    )?;
    let cwd = match raw.get("cwd") {
        None => ".".to_string(),
        v => req_str(file, "worker.cwd", v)?,
    };
    if cwd.starts_with('/') || cwd.split('/').any(|p| p == "..") {
        return Err(load_error(
            file,
            "worker.cwd",
            "must be a relative path inside the profile",
        ));
    }
    let mut env = BTreeMap::new();
    if let Some(e) = raw.get("env") {
        let map = e
            .as_object()
            .ok_or_else(|| load_error(file, "worker.env", "expected a map"))?;
        for (k, v) in map {
            env.insert(
                k.clone(),
                decode_env_value(file, &format!("worker.env.{k}"), v)?,
            );
        }
    }
    let mut permission = PermissionPolicy::default();
    if let Some(p) = raw.get("permission") {
        expect_keys(
            file,
            "worker.permission",
            p,
            &["timeout_seconds", "default", "unattended"],
        )?;
        if let Some(t) = p.get("timeout_seconds") {
            permission.timeout_seconds = t.as_u64().filter(|v| *v >= 1).ok_or_else(|| {
                load_error(
                    file,
                    "worker.permission.timeout_seconds",
                    "expected an integer >= 1",
                )
            })? as u32;
        }
        if let Some(d) = p.get("default") {
            permission.default = match d.as_str() {
                Some("deny") => PermissionDefault::Deny,
                Some("allow") => PermissionDefault::Allow,
                _ => {
                    return Err(load_error(
                        file,
                        "worker.permission.default",
                        "expected deny|allow",
                    ))
                }
            };
        }
        if let Some(u) = p.get("unattended") {
            permission.unattended = match u.as_str() {
                Some("deny") => PermissionUnattended::Deny,
                Some("approve") => PermissionUnattended::Approve,
                _ => {
                    return Err(load_error(
                        file,
                        "worker.permission.unattended",
                        "expected deny|approve",
                    ))
                }
            };
        }
    }
    let home = match opt_str(file, "worker.home", raw.get("home"))?.as_deref() {
        None | Some("profile") => crate::orchestration::WorkerHome::Profile,
        Some("user") => crate::orchestration::WorkerHome::User,
        Some(_) => {
            return Err(load_error(
                file,
                "worker.home",
                "expected `profile` or `user`",
            ))
        }
    };
    let surface = match opt_str(file, "worker.surface", raw.get("surface"))?.as_deref() {
        None | Some("headless") => crate::orchestration::WorkerSurface::Headless,
        Some("pane") => crate::orchestration::WorkerSurface::Pane,
        Some(_) => {
            return Err(load_error(
                file,
                "worker.surface",
                "expected `headless` or `pane`",
            ))
        }
    };
    let capacity = match raw.get("capacity") {
        None => None,
        Some(v) => Some(
            v.as_u64()
                .filter(|n| *n >= 1 && *n <= u64::from(u32::MAX))
                .ok_or_else(|| load_error(file, "worker.capacity", "expected an integer >= 1"))?
                as u32,
        ),
    };
    Ok(Some(WorkerSpec {
        harness: HarnessId::new(req_str(file, "worker.harness", raw.get("harness"))?),
        model: opt_str(file, "worker.model", raw.get("model"))?,
        reasoning_effort: opt_str(file, "worker.reasoning_effort", raw.get("reasoning_effort"))?,
        preset: opt_str(file, "worker.preset", raw.get("preset"))?,
        cwd,
        env,
        permission,
        home,
        surface,
        capacity,
    }))
}

// ---------------------------------------------------------------- H-records

const JOB_MAPPED: &[&str] = &[
    "id",
    "schedule",
    "prompt",
    "workdir",
    "model",
    "skills",
    "context_from",
    "deliver",
    "failure_deliver",
    "origin",
    "attach_to_session",
    "machine",
    "repeat",
    "enabled",
    "next_run_at",
    "last_run_at",
    "last_status",
    "created_at",
];

/// Hermes 0.21.0's canonical key order for a job record.
pub const HERMES_JOB_ORDER: &[&str] = &[
    "id",
    "schedule",
    "prompt",
    "skills",
    "script",
    "no_agent",
    "model",
    "provider",
    "workdir",
    "enabled_toolsets",
    "context_from",
    "deliver",
    "failure_deliver",
    "attach_to_session",
    "origin",
    "repeat",
    "enabled",
    "next_run_at",
    "last_run_at",
    "last_status",
    "created_at",
    "fire_claim",
];

/// `context_from`: a job id, a list of job ids, or `self` (`cron/jobs.py:2486-2492`).
pub fn decode_context_from(
    file: &str,
    key: &str,
    v: Option<&Value>,
) -> Result<Option<Vec<String>>> {
    let items: Vec<String> = match v {
        None | Some(Value::Null) => return Ok(None),
        Some(Value::String(s)) => vec![s.clone()],
        Some(Value::Array(a)) => a
            .iter()
            .map(|x| match x {
                Value::String(s) => s.clone(),
                other => other.to_string(),
            })
            .collect(),
        _ => {
            return Err(load_error(
                file,
                key,
                "expected a job id, a list of job ids, or \"self\"",
            ))
        }
    };
    let refs: Vec<String> = items
        .into_iter()
        .map(|s| s.trim().to_string())
        .filter(|s| !s.is_empty())
        .collect();
    Ok(if refs.is_empty() { None } else { Some(refs) })
}

/// Hermes 0.21.0 stores `repeat` as `{times: N | null, completed: M}`; older bool/number forms load.
pub fn decode_repeat(file: &str, key: &str, raw: Option<&Value>) -> Result<Option<Repeat>> {
    Ok(match raw {
        None | Some(Value::Null) => None,
        Some(Value::Bool(true)) => Some(Repeat {
            times: None,
            completed: 0,
        }),
        Some(Value::Bool(false)) => Some(Repeat {
            times: Some(1),
            completed: 0,
        }),
        Some(Value::Number(n)) => Some(Repeat {
            times: n.as_f64().filter(|f| *f > 0.0).map(|f| f.floor() as u32),
            completed: 0,
        }),
        Some(Value::Object(map)) => {
            let times = match map.get("times") {
                None | Some(Value::Null) => None,
                Some(Value::Number(n)) if n.as_u64().is_some_and(|v| v >= 1) => {
                    Some(n.as_u64().unwrap() as u32)
                }
                _ => {
                    return Err(load_error(
                        file,
                        &format!("{key}.times"),
                        "expected null or an integer >= 1",
                    ))
                }
            };
            let completed = match map.get("completed") {
                None => 0,
                Some(Value::Number(n)) if n.as_u64().is_some() => n.as_u64().unwrap() as u32,
                _ => {
                    return Err(load_error(
                        file,
                        &format!("{key}.completed"),
                        "expected an integer >= 0",
                    ))
                }
            };
            Some(Repeat { times, completed })
        }
        _ => {
            return Err(load_error(
                file,
                key,
                "expected null, {times, completed}, a number or a boolean",
            ))
        }
    })
}

/// One job record of `cron/jobs.json`.
pub fn decode_job(file: &str, raw: &Value) -> Result<Job> {
    let map = raw
        .as_object()
        .ok_or_else(|| load_error(file, "", "expected a job object"))?;
    let id = req_str(file, "id", map.get("id"))?;
    let k = |s: &str| format!("{id}.{s}");
    let sched = map
        .get("schedule")
        .and_then(Value::as_object)
        .ok_or_else(|| load_error(file, &k("schedule"), "expected an object"))?;
    let schedule = match sched.get("kind").and_then(Value::as_str) {
        Some("once") => Schedule::Once {
            run_at: req_str(file, &k("schedule.run_at"), sched.get("run_at"))?,
        },
        Some("interval") => {
            let minutes = sched
                .get("minutes")
                .and_then(Value::as_f64)
                .filter(|m| *m > 0.0)
                .ok_or_else(|| {
                    load_error(file, &k("schedule.minutes"), "expected a positive number")
                })?;
            Schedule::Interval { minutes }
        }
        Some("cron") => Schedule::Cron {
            expr: req_str(file, &k("schedule.expr"), sched.get("expr"))?,
            tz: opt_str(file, &k("schedule.tz"), sched.get("tz"))?,
        },
        _ => {
            return Err(load_error(
                file,
                &k("schedule.kind"),
                "expected once|interval|cron",
            ))
        }
    };
    let schedule_residue = residue_of(sched, &["kind", "run_at", "minutes", "expr", "tz"]);
    let mut residue = residue_of(map, JOB_MAPPED);
    if !schedule_residue.is_empty() {
        residue.keep(
            "__schedule",
            Value::Object(schedule_residue.0.into_iter().collect()),
        );
    }
    let origin = match map.get("origin") {
        Some(Value::Object(o)) => {
            let r = residue_of(o, &["platform", "chat_id", "thread_id"]);
            if !r.is_empty() {
                residue.keep("__origin", Value::Object(r.0.into_iter().collect()));
            }
            Some(JobOrigin {
                platform: req_str(file, &k("origin.platform"), o.get("platform"))?,
                chat_type: None,
                chat_id: opt_str(file, &k("origin.chat_id"), o.get("chat_id"))?,
                thread_id: opt_str(file, &k("origin.thread_id"), o.get("thread_id"))?,
            })
        }
        _ => None,
    };
    Ok(Job {
        schedule,
        prompt: opt_str(file, &k("prompt"), map.get("prompt"))?,
        workdir: opt_str(file, &k("workdir"), map.get("workdir"))?,
        model: opt_str(file, &k("model"), map.get("model"))?,
        skills: map
            .get("skills")
            .and_then(Value::as_array)
            .map(|a| {
                a.iter()
                    .map(|v| {
                        v.as_str()
                            .map(str::to_string)
                            .unwrap_or_else(|| v.to_string())
                    })
                    .collect()
            })
            .unwrap_or_default(),
        context_from: decode_context_from(file, &k("context_from"), map.get("context_from"))?,
        deliver: parse_target(file, &k("deliver"), map.get("deliver"), None)?
            .unwrap_or(Target::Local),
        failure_deliver: parse_target(
            file,
            &k("failure_deliver"),
            map.get("failure_deliver"),
            None,
        )?,
        origin,
        attach_to_session: opt_bool(
            file,
            &k("attach_to_session"),
            map.get("attach_to_session"),
            None,
        )?,
        machine: opt_str(file, &k("machine"), map.get("machine"))?,
        repeat: decode_repeat(file, &k("repeat"), map.get("repeat"))?,
        enabled: opt_bool(file, &k("enabled"), map.get("enabled"), Some(true))?.unwrap_or(true),
        next_run_at: opt_str(file, &k("next_run_at"), map.get("next_run_at"))?,
        last_run_at: opt_str(file, &k("last_run_at"), map.get("last_run_at"))?,
        last_status: opt_str(file, &k("last_status"), map.get("last_status"))?,
        created_at: opt_str(file, &k("created_at"), map.get("created_at"))?,
        residue,
        id,
    })
}

/// A job record in Hermes's canonical key order, mapped fields then residue (`ir.mjs::encodeJob`).
/// Returned as an ordered key list because `serde_json`'s object order is a cargo feature.
pub fn encode_job(job: &Job) -> Vec<(String, Value)> {
    let mut sched = Map::new();
    sched.insert("kind".into(), Value::String(job.schedule.kind().into()));
    match &job.schedule {
        Schedule::Once { run_at } => {
            sched.insert("run_at".into(), Value::String(run_at.clone()));
        }
        Schedule::Interval { minutes } => {
            sched.insert("minutes".into(), serde_json::json!(*minutes));
        }
        Schedule::Cron { expr, tz } => {
            sched.insert("expr".into(), Value::String(expr.clone()));
            if let Some(tz) = tz {
                sched.insert("tz".into(), Value::String(tz.clone()));
            }
        }
    }
    if let Some(Value::Object(extra)) = job.residue.0.get("__schedule") {
        for (k, v) in extra {
            sched.insert(k.clone(), v.clone());
        }
    }
    let origin = job
        .origin
        .as_ref()
        .map(|o| {
            let mut m = Map::new();
            m.insert("platform".into(), Value::String(o.platform.clone()));
            m.insert(
                "chat_id".into(),
                o.chat_id.clone().map(Value::String).unwrap_or(Value::Null),
            );
            m.insert(
                "thread_id".into(),
                o.thread_id
                    .clone()
                    .map(Value::String)
                    .unwrap_or(Value::Null),
            );
            if let Some(Value::Object(extra)) = job.residue.0.get("__origin") {
                for (k, v) in extra {
                    m.insert(k.clone(), v.clone());
                }
            }
            Value::Object(m)
        })
        .unwrap_or(Value::Null);
    let opt = |s: &Option<String>| s.clone().map(Value::String).unwrap_or(Value::Null);
    let mut mapped: Vec<(String, Value)> = vec![
        ("id".into(), Value::String(job.id.clone())),
        ("schedule".into(), Value::Object(sched)),
        ("prompt".into(), opt(&job.prompt)),
        (
            "skills".into(),
            Value::Array(
                job.skills
                    .iter()
                    .map(|s| Value::String(s.clone()))
                    .collect(),
            ),
        ),
        ("model".into(), opt(&job.model)),
        ("workdir".into(), opt(&job.workdir)),
        (
            "context_from".into(),
            job.context_from
                .as_ref()
                .map(|l| Value::Array(l.iter().map(|s| Value::String(s.clone())).collect()))
                .unwrap_or(Value::Null),
        ),
        ("deliver".into(), Value::String(job.deliver.render())),
        (
            "failure_deliver".into(),
            job.failure_deliver
                .as_ref()
                .map(|t| Value::String(t.render()))
                .unwrap_or(Value::Null),
        ),
        (
            "attach_to_session".into(),
            job.attach_to_session
                .map(Value::Bool)
                .unwrap_or(Value::Null),
        ),
        ("origin".into(), origin),
        (
            "repeat".into(),
            job.repeat
                .as_ref()
                .map(|r| serde_json::json!({"times": r.times, "completed": r.completed}))
                .unwrap_or(Value::Null),
        ),
        ("enabled".into(), Value::Bool(job.enabled)),
        ("next_run_at".into(), opt(&job.next_run_at)),
        ("last_run_at".into(), opt(&job.last_run_at)),
        ("last_status".into(), opt(&job.last_status)),
        ("created_at".into(), opt(&job.created_at)),
    ];
    // supercode's own key, written only where a job names a machine, so a Hermes job file keeps its bytes
    if let Some(machine) = &job.machine {
        mapped.push(("machine".into(), Value::String(machine.clone())));
    }
    let mut residue: Vec<(String, Value)> = job
        .residue
        .0
        .iter()
        // `__schedule` and `__origin` are re-emitted as structure above and
        // rebuilt on decode; every OTHER `__` residue key (a foreign codec's,
        // e.g. OpenClaw's `__delivery` / `__session_target` / `__payload`)
        // has no home in the Hermes shape and rides as a literal key, which is
        // what carries it losslessly through our folder
        .filter(|(k, _)| k.as_str() != "__schedule" && k.as_str() != "__origin")
        .map(|(k, v)| (k.clone(), v.clone()))
        .collect();
    let mut out: Vec<(String, Value)> = Vec::new();
    for key in HERMES_JOB_ORDER {
        if let Some(pos) = mapped.iter().position(|(k, _)| k == key) {
            out.push(mapped.remove(pos));
        } else if let Some(pos) = residue.iter().position(|(k, _)| k == key) {
            out.push(residue.remove(pos));
        }
    }
    out.extend(mapped);
    out.extend(residue);
    out
}

/// Fire status words on the Hermes boundary.
pub fn decode_fire_row(file: &str, row: &Map<String, Value>) -> Result<Fire> {
    let id = req_str(file, "id", row.get("id"))?;
    let status_word = row.get("status").and_then(Value::as_str).unwrap_or("");
    let status = FireStatus::from_hermes_word(status_word).ok_or_else(|| {
        load_error(
            file,
            &format!("{id}.status"),
            format!(
                "unknown fire status {}",
                serde_json::to_string(status_word).unwrap()
            ),
        )
    })?;
    Ok(Fire {
        job_id: req_str(file, &format!("{id}.job_id"), row.get("job_id"))?,
        session_id: row
            .get("session_id")
            .and_then(text_of)
            .filter(|s| !s.is_empty()),
        status,
        claimed_at: req_str(file, &format!("{id}.claimed_at"), row.get("claimed_at"))?,
        started_at: opt_str(file, &format!("{id}.started_at"), row.get("started_at"))?,
        finished_at: opt_str(file, &format!("{id}.finished_at"), row.get("finished_at"))?,
        error: opt_str(file, &format!("{id}.error"), row.get("error"))?,
        obligation_id: row
            .get("obligation_id")
            .and_then(text_of)
            .filter(|s| !s.is_empty()),
        residue: {
            let mut residue = row_residue_of(
                row,
                &[
                    "id",
                    "job_id",
                    "status",
                    "claimed_at",
                    "started_at",
                    "finished_at",
                    "error",
                    "residue_json",
                    "session_id",
                    "obligation_id",
                ],
            );
            // our folder's ledger keeps what another harness's run log had
            // and Hermes's columns do not (ORC-12 finding 5)
            if let Some(extra) = row
                .get("residue_json")
                .and_then(text_of)
                .and_then(|t| serde_json::from_str::<Map<String, Value>>(&t).ok())
            {
                for (k, v) in extra {
                    residue.keep(k, v);
                }
            }
            residue
        },
        id,
    })
}

/// The `executions` row of a fire (Hermes's own status word; residue fills the process columns).
pub fn encode_fire_row(fire: &Fire) -> Vec<Value> {
    let r = &fire.residue.0;
    let opt = |s: &Option<String>| s.clone().map(Value::String).unwrap_or(Value::Null);
    vec![
        Value::String(fire.id.clone()),
        Value::String(fire.job_id.clone()),
        r.get("source")
            .cloned()
            .unwrap_or(Value::String("scheduler".into())),
        r.get("process_id")
            .cloned()
            .unwrap_or(Value::String(String::new())),
        r.get("pid").cloned().unwrap_or(Value::from(0)),
        r.get("process_started_at").cloned().unwrap_or(Value::Null),
        Value::String(fire.status.hermes_word().into()),
        Value::String(fire.claimed_at.clone()),
        opt(&fire.started_at),
        opt(&fire.finished_at),
        opt(&fire.error),
    ]
}

/// The column list `encode_fire_row` fills, in order.
pub const EXECUTION_COLUMNS: &[&str] = &[
    "id",
    "job_id",
    "source",
    "process_id",
    "pid",
    "process_started_at",
    "status",
    "claimed_at",
    "started_at",
    "finished_at",
    "error",
];

/// One `delivery_obligations` row.
pub fn decode_obligation_row(file: &str, row: &Map<String, Value>) -> Result<Obligation> {
    let id = req_str(file, "obligation_id", row.get("obligation_id"))?;
    let state_word = row.get("state").and_then(Value::as_str).unwrap_or("");
    let state = ObligationState::from_hermes_word(state_word).ok_or_else(|| {
        load_error(
            file,
            &format!("{id}.state"),
            format!(
                "unknown obligation state {}",
                serde_json::to_string(state_word).unwrap()
            ),
        )
    })?;
    let session_key = row
        .get("session_key")
        .and_then(Value::as_str)
        .filter(|s| !s.is_empty())
        .map(str::to_string);
    let parsed = session_key
        .as_deref()
        .and_then(crate::ontology::parse_hermes_session_key)
        .map(|(k, _)| k);
    let target = SurfaceKey {
        key: None,
        platform: Some(req_str(
            file,
            &format!("{id}.platform"),
            row.get("platform"),
        )?),
        kind: parsed.as_ref().and_then(|k| k.kind.clone()),
        chat_id: Some(row.get("chat_id").and_then(text_of).unwrap_or_default()),
        thread_id: row
            .get("thread_id")
            .and_then(text_of)
            .filter(|s| !s.is_empty()),
        participant_id: None,
    };
    let text = |k: &str| row.get(k).and_then(text_of);
    let created_at = text("created_at").unwrap_or_else(|| "undefined".into());
    let updated_at = text("updated_at").unwrap_or_else(|| created_at.clone());
    Ok(Obligation {
        target,
        session_key,
        content: OutboundContent {
            text: row.get("content").and_then(text_of).unwrap_or_default(),
            attachments: None,
            reply_to: None,
            format: None,
        },
        state,
        attempts: row.get("attempts").and_then(number_of).unwrap_or(0.0) as u64,
        last_error: row.get("last_error").and_then(text_of),
        delivered_at: if state == ObligationState::Sent {
            Some(updated_at.clone())
        } else {
            None
        },
        created_at,
        updated_at,
        // our own folder's store carries the two facts Hermes's has no column
        // for (ORC-9 finding 6): the platform's handle for the sent message,
        // and where the obligation came from. A Hermes store has neither.
        posted: row
            .get("posted_message_id")
            .and_then(text_of)
            .filter(|s| !s.is_empty())
            .map(|message_id| Posted { message_id }),
        source: row
            .get("source_json")
            .and_then(text_of)
            .and_then(|text| serde_json::from_str::<ObligationSource>(&text).ok())
            .unwrap_or(ObligationSource::Turn { key: None }),
        residue: row_residue_of(
            row,
            &[
                "obligation_id",
                "session_key",
                "platform",
                "chat_id",
                "thread_id",
                "content",
                "state",
                "attempts",
                "created_at",
                "updated_at",
                "last_error",
                "posted_message_id",
                "source_json",
            ],
        ),
        id,
    })
}

/// Our folder's two columns beyond Hermes's `delivery_obligations` shape.
pub const FOLDER_OBLIGATION_EXTRA_COLUMNS: &[&str] = &["posted_message_id", "source_json"];

/// The values for [`FOLDER_OBLIGATION_EXTRA_COLUMNS`].
pub fn encode_obligation_folder_extras(o: &Obligation) -> Vec<Value> {
    vec![
        o.posted
            .as_ref()
            .map(|p| Value::String(p.message_id.clone()))
            .unwrap_or(Value::Null),
        Value::String(serde_json::to_string(&o.source).unwrap()),
    ]
}

/// The `delivery_obligations` row of an obligation, in `OBLIGATION_COLUMNS` order.
pub fn encode_obligation_row(o: &Obligation) -> Vec<Value> {
    let r = &o.residue.0;
    let num = |s: &str| {
        s.parse::<f64>()
            .map(|f| serde_json::json!(f))
            .unwrap_or(Value::from(0))
    };
    vec![
        Value::String(o.id.clone()),
        Value::String(o.session_key.clone().unwrap_or_default()),
        Value::String(o.target.platform.clone().unwrap_or_default()),
        Value::String(o.target.chat_id.clone().unwrap_or_default()),
        o.target
            .thread_id
            .clone()
            .map(Value::String)
            .unwrap_or(Value::Null),
        Value::String(o.content.text.clone()),
        Value::String(o.state.hermes_word().into()),
        Value::from(o.attempts),
        num(&o.created_at),
        num(o
            .delivered_at
            .as_deref()
            .map(|_| o.updated_at.as_str())
            .unwrap_or(&o.updated_at)),
        r.get("owner_pid").cloned().unwrap_or(Value::Null),
        r.get("owner_started_at").cloned().unwrap_or(Value::Null),
        o.last_error
            .clone()
            .map(Value::String)
            .unwrap_or(Value::Null),
        r.get("adapter_profile").cloned().unwrap_or(Value::Null),
    ]
}

/// The column list `encode_obligation_row` fills, in order.
pub const OBLIGATION_COLUMNS: &[&str] = &[
    "obligation_id",
    "session_key",
    "platform",
    "chat_id",
    "thread_id",
    "content",
    "state",
    "attempts",
    "created_at",
    "updated_at",
    "owner_pid",
    "owner_started_at",
    "last_error",
    "adapter_profile",
];

const ROUTE_MATCH: &[&str] = &["platform", "guild_id", "chat_id", "thread_id"];

/// One `profile_routes[]` entry.
pub fn decode_route(file: &str, index: usize, raw: &Value) -> Result<Route> {
    let map = raw.as_object().ok_or_else(|| {
        load_error(
            file,
            &format!("profile_routes[{index}]"),
            "expected an object",
        )
    })?;
    let text = |k: &str| map.get(k).filter(|v| !v.is_null()).and_then(text_of);
    Ok(Route {
        name: opt_str(
            file,
            &format!("profile_routes[{index}].name"),
            map.get("name"),
        )?,
        matches: RouteMatch {
            platform: req_str(
                file,
                &format!("profile_routes[{index}].platform"),
                map.get("platform"),
            )?,
            guild_id: text("guild_id"),
            chat_id: text("chat_id"),
            thread_id: text("thread_id"),
        },
        profile: req_str(
            file,
            &format!("profile_routes[{index}].profile"),
            map.get("profile"),
        )?,
        residue: residue_of(
            map,
            &[
                "name",
                "profile",
                "platform",
                "guild_id",
                "chat_id",
                "thread_id",
            ],
        ),
    })
}

/// A route back in Hermes's form.
pub fn encode_route(r: &Route) -> Value {
    let mut out = Map::new();
    if let Some(n) = &r.name {
        out.insert("name".into(), Value::String(n.clone()));
    }
    let m = &r.matches;
    for (k, v) in [
        ("platform", Some(&m.platform)),
        ("guild_id", m.guild_id.as_ref()),
        ("chat_id", m.chat_id.as_ref()),
        ("thread_id", m.thread_id.as_ref()),
    ] {
        if let Some(v) = v {
            out.insert(k.into(), Value::String(v.clone()));
        }
    }
    let _ = ROUTE_MATCH;
    out.insert("profile".into(), Value::String(r.profile.clone()));
    for (k, v) in &r.residue.0 {
        out.insert(k.clone(), v.clone());
    }
    Value::Object(out)
}

/// Whether a config key names a credential (`ir.mjs::CREDENTIAL_KEY`).
pub fn is_credential_key(key: &str) -> bool {
    let k = key.to_ascii_lowercase();
    [
        "token",
        "secret",
        "key",
        "password",
        "passwd",
        "api_key",
        "app_secret",
        "signing",
        "webhook_url",
        "private",
    ]
    .iter()
    .any(|w| k.contains(w))
}

/// The `.env` name inside a `${NAME}` reference, Hermes's own form (docs/plans/hermes-compat.md row 5).
///
/// Our own folder never inlines a secret: where a Hermes home holds the value,
/// we write the reference and keep the value in `.env`. A reader that took
/// the placeholder for a value would put the placeholder TEXT in the vault and
/// write it back as the secret on the next save.
pub fn placeholder_ref(text: &str) -> Option<&str> {
    let name = text.strip_prefix("${")?.strip_suffix('}')?;
    let mut chars = name.chars();
    let first = chars.next()?;
    ((first.is_ascii_alphabetic() || first == '_')
        && chars.all(|c| c.is_ascii_alphanumeric() || c == '_'))
    .then_some(name)
}

/// A secret reference as Hermes writes one in `config.yaml`: `${NAME}`, expanded from the home's `.env`
/// (which Hermes loads into its environment) and then the process environment.
pub fn render_ref(r: &SecretRef) -> Value {
    Value::String(format!("${{{}}}", r.name()))
}

/// The `.env` name for a credential.
pub fn credential_ref_name(platform: &str, key: &str) -> String {
    format!("{platform}_{key}")
        .chars()
        .map(|c| {
            if c.is_ascii_alphanumeric() {
                c.to_ascii_uppercase()
            } else {
                '_'
            }
        })
        .collect()
}

/// One `platforms.<name>` block; credential VALUES go to the vault, refs into the record.
pub fn decode_channel(
    file: &str,
    platform: &str,
    raw: &Value,
    vault: &mut BTreeMap<String, String>,
) -> Result<ChannelConfig> {
    let map = raw
        .as_object()
        .ok_or_else(|| load_error(file, &format!("platforms.{platform}"), "expected an object"))?;
    let mut ch = ChannelConfig {
        platform: platform.to_string(),
        enabled: opt_bool(
            file,
            &format!("platforms.{platform}.enabled"),
            map.get("enabled"),
            Some(true),
        )?
        .unwrap_or(true),
        credentials: BTreeMap::new(),
        extra: BTreeMap::new(),
    };
    fn walk(
        map: &Map<String, Value>,
        prefix: &str,
        platform: &str,
        ch: &mut ChannelConfig,
        vault: &mut BTreeMap<String, String>,
    ) {
        for (k, v) in map {
            if k == "enabled" && prefix.is_empty() {
                continue;
            }
            let name = format!("{prefix}{k}");
            if let Value::String(s) = v {
                if let Some(r) = placeholder_ref(s) {
                    ch.credentials
                        .insert(name, SecretRef::Dotenv(r.to_string()));
                    continue;
                }
            }
            if let Some(obj) = v.as_object() {
                if k == "extra" && prefix.is_empty() {
                    walk(obj, "extra.", platform, ch, vault);
                    continue;
                }
            }
            if let Value::String(s) = v {
                if is_credential_key(k) {
                    let r = credential_ref_name(platform, &name);
                    vault.insert(r.clone(), s.clone());
                    ch.credentials.insert(name, SecretRef::Dotenv(r));
                    continue;
                }
            }
            ch.extra.insert(name, v.clone());
        }
    }
    walk(map, "", platform, &mut ch, vault);
    Ok(ch)
}

/// A platform block; with a vault (a Hermes export) values are inlined, without one the refs are written.
pub fn encode_channel(ch: &ChannelConfig, vault: Option<&BTreeMap<String, String>>) -> Value {
    let mut out = Map::new();
    out.insert("enabled".into(), Value::Bool(ch.enabled));
    let mut extra = Map::new();
    for (k, v) in &ch.extra {
        match k.strip_prefix("extra.") {
            Some(inner) => {
                extra.insert(inner.to_string(), v.clone());
            }
            None => {
                out.insert(k.clone(), v.clone());
            }
        }
    }
    for (k, r) in &ch.credentials {
        let rendered = match vault.and_then(|v| v.get(r.name())) {
            Some(value) => Value::String(value.clone()),
            None => render_ref(r),
        };
        match k.strip_prefix("extra.") {
            Some(inner) => {
                extra.insert(inner.to_string(), rendered);
            }
            None => {
                out.insert(k.clone(), rendered);
            }
        }
    }
    if !extra.is_empty() {
        out.insert("extra".into(), Value::Object(extra));
    }
    Value::Object(out)
}

const SUB_MAPPED: &[&str] = &[
    "events",
    "prompt",
    "skills",
    "deliver",
    "deliver_extra",
    "secret",
    "description",
    "created_at",
];

/// One `webhook_subscriptions.json` entry; the secret's value goes to the vault.
pub fn decode_subscription(
    file: &str,
    name: &str,
    raw: &Value,
    vault: &mut BTreeMap<String, String>,
) -> Result<WebhookSubscription> {
    let map = raw
        .as_object()
        .ok_or_else(|| load_error(file, name, "expected an object"))?;
    let mut sub = WebhookSubscription {
        name: name.to_string(),
        secret: None,
        events: map.get("events").and_then(Value::as_array).map(|a| {
            a.iter()
                .map(|v| {
                    v.as_str()
                        .map(str::to_string)
                        .unwrap_or_else(|| v.to_string())
                })
                .collect()
        }),
        prompt_template: opt_str(file, &format!("{name}.prompt"), map.get("prompt"))?
            .unwrap_or_default(),
        deliver: parse_target(
            file,
            &format!("{name}.deliver"),
            map.get("deliver"),
            map.get("deliver_extra"),
        )?,
        skills: map
            .get("skills")
            .and_then(Value::as_array)
            .map(|a| {
                a.iter()
                    .map(|v| {
                        v.as_str()
                            .map(str::to_string)
                            .unwrap_or_else(|| v.to_string())
                    })
                    .collect()
            })
            .unwrap_or_default(),
        description: opt_str(file, &format!("{name}.description"), map.get("description"))?,
        created_at: opt_str(file, &format!("{name}.created_at"), map.get("created_at"))?,
        residue: residue_of(map, SUB_MAPPED),
    };
    match map.get("secret") {
        // A Hermes home inlines the value here or writes `${NAME}`; our own
        // folder writes `${NAME}` and keeps the value in `.env`.
        Some(Value::String(secret)) => {
            let r = match placeholder_ref(secret) {
                Some(name) => name.to_string(),
                None => {
                    let r = credential_ref_name("webhook", &format!("{name}_secret"));
                    vault.insert(r.clone(), secret.clone());
                    r
                }
            };
            sub.secret = Some(SecretRef::Dotenv(r));
        }
        Some(Value::Null) | None => {}
        Some(_) => {
            return Err(load_error(
                file,
                &format!("{name}.secret"),
                "expected a string (a secret is written `${NAME}`)",
            ))
        }
    }
    Ok(sub)
}

/// A subscription back in Hermes's form; the secret inlined when a vault is given, else a placeholder.
pub fn encode_subscription(
    sub: &WebhookSubscription,
    vault: Option<&BTreeMap<String, String>>,
) -> Value {
    let mut out = Map::new();
    if let Some(d) = &sub.description {
        out.insert("description".into(), Value::String(d.clone()));
    }
    if let Some(e) = &sub.events {
        out.insert(
            "events".into(),
            Value::Array(e.iter().map(|s| Value::String(s.clone())).collect()),
        );
    }
    out.insert("prompt".into(), Value::String(sub.prompt_template.clone()));
    out.insert(
        "skills".into(),
        Value::Array(
            sub.skills
                .iter()
                .map(|s| Value::String(s.clone()))
                .collect(),
        ),
    );
    if let Some(t) = &sub.deliver {
        match t {
            Target::Explicit {
                platform,
                chat_id,
                thread_id,
            } => {
                out.insert("deliver".into(), Value::String(platform.clone()));
                let mut extra = Map::new();
                if let Some(c) = chat_id {
                    extra.insert("chat_id".into(), Value::String(c.clone()));
                }
                if let Some(th) = thread_id {
                    extra.insert("thread_id".into(), Value::String(th.clone()));
                }
                if !extra.is_empty() {
                    out.insert("deliver_extra".into(), Value::Object(extra));
                }
            }
            other => {
                out.insert("deliver".into(), Value::String(other.render()));
            }
        }
    }
    if let Some(s) = &sub.secret {
        let value = vault
            .and_then(|v| v.get(s.name()).cloned())
            .unwrap_or_else(|| format!("${{{}}}", s.name()));
        out.insert("secret".into(), Value::String(value));
    }
    if let Some(c) = &sub.created_at {
        out.insert("created_at".into(), Value::String(c.clone()));
    }
    for (k, v) in &sub.residue.0 {
        out.insert(k.clone(), v.clone());
    }
    Value::Object(out)
}