Skip to main content

host/
journal.rs

1//! The shell job journal (session value `ultron.module/jobs`) and job-done events, ported from
2//! ultron/rlm/shell-jobs.ts (`#load`, `#persist`, `jobSummary`, `#fileTail`) and
3//! experimental/session-worker.ts `jobEvent`.
4//!
5//! TypeScript: processes do not outlive the worker that owned them, so on restore a job the journal still lists as
6//! running becomes `interrupted` ("worker restarted") and the journal is rewritten. The Rust host keeps a job's
7//! process running past its worker (see `Host::with_journal`): a record that names its keeper process (`pid`,
8//! `pidStart`) is left running here ([`restore_durable`]) and settled by the host from the process and its exit file.
9
10use serde_json::{Value, json};
11use std::fs::File;
12use std::io::{Read, Seek, SeekFrom};
13
14pub const JOURNAL_NAMESPACE: &str = "ultron.module/jobs";
15const JOURNAL_VERSION: u64 = 1;
16const JOB_TEXT_BYTES: u64 = 16 * 1024;
17
18/// A shell job's state, as journals and replies name it.
19#[derive(Clone, Copy, PartialEq, Eq, Debug)]
20pub enum Status {
21    Running,
22    Completed,
23    Failed,
24    Cancelled,
25    TimedOut,
26    /// A TypeScript journal's running job after a restart (no process to follow).
27    Interrupted,
28    /// Its process ended while no worker followed it and left no exit status.
29    Lost,
30}
31
32impl Status {
33    pub fn as_str(self) -> &'static str {
34        match self {
35            Status::Running => "running",
36            Status::Completed => "completed",
37            Status::Failed => "failed",
38            Status::Cancelled => "cancelled",
39            Status::TimedOut => "timed_out",
40            Status::Interrupted => "interrupted",
41            Status::Lost => "lost",
42        }
43    }
44
45    fn from_name(name: &str) -> Status {
46        match name {
47            "running" => Status::Running,
48            "completed" => Status::Completed,
49            "cancelled" => Status::Cancelled,
50            "timed_out" => Status::TimedOut,
51            "interrupted" => Status::Interrupted,
52            "lost" => Status::Lost,
53            _ => Status::Failed,
54        }
55    }
56}
57
58/// One journaled job (camelCase on the wire, as TypeScript stores it).
59#[derive(Clone, Debug, PartialEq)]
60pub struct JobRecord {
61    pub id: String,
62    pub lane: String,
63    pub root_id: Option<String>,
64    pub command: String,
65    pub cwd: String,
66    pub timeout: Option<f64>,
67    pub status: Status,
68    pub exit_code: Option<i64>,
69    pub started_at: u64,
70    pub ended_at: Option<u64>,
71    pub output_path: String,
72    pub output_bytes: u64,
73    pub retained_bytes: u64,
74    pub tail: String,
75    pub error: Option<String>,
76    /// The job's keeper process (its process group leader) and that process's start time in clock ticks since boot,
77    /// so a reopened worker can tell the job's own process from a reused pid. Absent in TypeScript journals.
78    pub pid: Option<i64>,
79    pub pid_start: Option<u64>,
80}
81
82fn text_of(value: &Value, key: &str) -> String {
83    value.get(key).and_then(Value::as_str).unwrap_or_default().to_owned()
84}
85
86fn int_of(value: &Value, key: &str) -> Option<u64> {
87    value
88        .get(key)
89        .and_then(|v| v.as_u64().or_else(|| v.as_f64().map(|f| f as u64)))
90}
91
92/// A number as JSON.stringify writes it: integral values without a fraction.
93fn js_number(value: f64) -> Value {
94    jsrt::json::number(value)
95}
96
97impl JobRecord {
98    pub fn from_value(value: &Value) -> JobRecord {
99        JobRecord {
100            id: text_of(value, "id"),
101            lane: text_of(value, "lane"),
102            root_id: value.get("rootId").and_then(Value::as_str).map(str::to_owned),
103            command: text_of(value, "command"),
104            cwd: text_of(value, "cwd"),
105            timeout: value.get("timeout").and_then(Value::as_f64),
106            status: Status::from_name(&text_of(value, "status")),
107            exit_code: value.get("exitCode").and_then(Value::as_i64),
108            started_at: int_of(value, "startedAt").unwrap_or(0),
109            ended_at: int_of(value, "endedAt"),
110            output_path: text_of(value, "outputPath"),
111            output_bytes: int_of(value, "outputBytes").unwrap_or(0),
112            retained_bytes: int_of(value, "retainedBytes").unwrap_or(0),
113            tail: text_of(value, "tail"),
114            error: value.get("error").and_then(Value::as_str).map(str::to_owned),
115            pid: value.get("pid").and_then(Value::as_i64),
116            pid_start: int_of(value, "pidStart"),
117        }
118    }
119
120    pub fn to_value(&self) -> Value {
121        let mut out = json!({
122            "id": self.id,
123            "lane": self.lane,
124            "rootId": self.root_id,
125            "command": self.command,
126            "cwd": self.cwd,
127            "timeout": self.timeout.map(js_number),
128            "status": self.status.as_str(),
129            "exitCode": self.exit_code,
130            "startedAt": self.started_at,
131            "endedAt": self.ended_at,
132            "outputPath": self.output_path,
133            "outputBytes": self.output_bytes,
134            "retainedBytes": self.retained_bytes,
135            "tail": self.tail,
136        });
137        if let Some(error) = &self.error {
138            out["error"] = json!(error);
139        }
140        if let Some(pid) = self.pid {
141            out["pid"] = json!(pid);
142        }
143        if let Some(start) = self.pid_start {
144            out["pidStart"] = json!(start);
145        }
146        out
147    }
148}
149
150fn read_text(path: &str, cursor: u64, max: u64) -> String {
151    let mut buffer = Vec::new();
152    if let Ok(mut file) = File::open(path)
153        && file.seek(SeekFrom::Start(cursor)).is_ok()
154    {
155        let _ = file.take(max).read_to_end(&mut buffer);
156    }
157    String::from_utf8_lossy(&buffer).into_owned()
158}
159
160/// The last `max` bytes of `text`, starting on a character boundary.
161fn keep_tail(text: &str, max: usize) -> &str {
162    let mut start = text.len().saturating_sub(max);
163    while !text.is_char_boundary(start) {
164        start += 1;
165    }
166    &text[start..]
167}
168
169/// The last bytes of a job's spill file, when the file holds all of its output.
170fn file_tail(record: &JobRecord) -> String {
171    if record.retained_bytes != record.output_bytes || record.retained_bytes == 0 {
172        return String::new();
173    }
174    let half = JOB_TEXT_BYTES / 2;
175    let text = read_text(
176        &record.output_path,
177        record.retained_bytes.saturating_sub(half + 4),
178        half + 4,
179    );
180    keep_tail(&text, half as usize).to_owned()
181}
182
183/// The jobs restored from a journal, and the document to write back when restoring changed a record.
184pub struct Restored {
185    pub records: Vec<JobRecord>,
186    pub persist: Option<Value>,
187}
188
189/// Restore the journal (`saved` is the session value, None when unset) at time `now` (ms) for a host that keeps job
190/// processes past their worker: a running record that names its keeper process stays `running` for the host to
191/// settle; one without (a TypeScript journal) becomes `interrupted` ("worker restarted"), as TypeScript restores
192/// every running job.
193pub fn restore_durable(saved: Option<&Value>, now: u64) -> Result<Restored, String> {
194    let jobs = match saved {
195        Some(Value::Object(document)) => {
196            if let Some(version) = document.get("version")
197                && version.as_f64() != Some(JOURNAL_VERSION as f64)
198            {
199                return Err(format!(
200                    "Cannot resume: session value {JOURNAL_NAMESPACE} has format version {version}, but this Ultron reads only format version {JOURNAL_VERSION}. It was left unchanged; open the session with the Ultron version that wrote it, or upgrade."
201                ));
202            }
203            document
204                .get("jobs")
205                .and_then(Value::as_array)
206                .cloned()
207                .unwrap_or_default()
208        }
209        _ => Vec::new(),
210    };
211    let mut changed = false;
212    let records = jobs
213        .iter()
214        .map(|saved| {
215            let mut job = JobRecord::from_value(saved);
216            if job.tail.is_empty() {
217                job.tail = file_tail(&job);
218            }
219            if job.status == Status::Running && job.pid.is_none() {
220                changed = true;
221                job.status = Status::Interrupted;
222                job.ended_at = Some(job.ended_at.unwrap_or(now));
223                job.error = Some("worker restarted".into());
224            }
225            job
226        })
227        .collect::<Vec<_>>();
228    let persist = changed.then(|| persisted(&records));
229    Ok(Restored { records, persist })
230}
231
232/// The journal document: every record with its tail left out (it would be rewritten with every job).
233pub fn persisted(records: &[JobRecord]) -> Value {
234    let jobs: Vec<Value> = records
235        .iter()
236        .map(|record| {
237            let mut value = record.to_value();
238            value["tail"] = json!("");
239            value
240        })
241        .collect();
242    json!({ "version": JOURNAL_VERSION, "jobs": jobs })
243}
244
245/// One-line completion summary: the last three non-empty output lines, at most `max_chars` UTF-16 units.
246pub fn job_summary(record: &JobRecord, max_chars: usize) -> String {
247    let lines: Vec<&str> = record
248        .tail
249        .split('\n')
250        .map(str::trim)
251        .filter(|line| !line.is_empty())
252        .collect();
253    let text = lines[lines.len().saturating_sub(3)..].join(" | ");
254    let status = match (record.status, &record.error) {
255        (Status::Completed, _) => format!(
256            "exit {}",
257            record.exit_code.map_or("null".to_owned(), |code| code.to_string())
258        ),
259        (status, Some(error)) if !error.is_empty() => format!("{}: {error}", status.as_str()),
260        (status, _) => status.as_str().to_owned(),
261    };
262    let units: Vec<u16> = text.encode_utf16().collect();
263    let body = if units.len() > max_chars {
264        format!("…{}", String::from_utf16_lossy(&units[units.len() - (max_chars - 1)..]))
265    } else {
266        text
267    };
268    if body.is_empty() {
269        status
270    } else {
271        format!("{status}; {body}")
272    }
273}
274
275/// The `job_done` event for a job that ended while nobody waited on it and its root turn was not aborted.
276pub fn job_event(record: &JobRecord, awaited: bool, root_aborted: bool) -> Option<Value> {
277    if awaited || root_aborted {
278        return None;
279    }
280    let mut event = json!({
281        "kind": "job_done",
282        "id": record.id,
283        "status": record.status.as_str(),
284        "summary": job_summary(record, 240),
285        "fetch": format!("await rlm.job(\"{}\")", record.id),
286        "lane": record.lane,
287    });
288    if let Some(root) = &record.root_id {
289        event["rootId"] = json!(root);
290    }
291    Some(event)
292}