kernel_apis/durable_work.rs
1//! Durable work: schedules, the session goal and shell jobs as durable tasks that survive worker restarts. This is
2//! the one entry point a composition root calls.
3//!
4//! On a worker's open, after its task host (the task journal's owner) and its host with a jobs journal exist:
5//!
6//! ```text
7//! let host = Arc::new(Host::new(cwd, jobs_dir, spill_dir, guard) // shell jobs: recovered here,
8//! .with_journal(Arc::new(FileDocumentStore::new(jobs_journal)))?); // from their processes
9//! let schedules = Arc::new(SchedulesModule::new(store("schedules"), None));
10//! let goal = Arc::new(GoalDriver::new(store("goal"),
11//! Arc::new(TaskGoalJobs::new(tasks.clone(), host.clone())), clock));
12//! let apis = KernelApis::new(inner).with_host(tasks.clone(), lane)
13//! .with(schedules.clone()).with(goal.clone()); // schedules.*, goals.*, goal.*
14//! let work = DurableWork::open(DurableWorkOptions::new(tasks.clone())
15//! .schedules(schedules).goal(goal).jobs(host))?; // recover + start the tick
16//! ...
17//! work.close(); // session close: stop the tick, stop starting goal jobs, stop shell jobs
18//! work.hand_over(); // a successor worker takes over: the same, but shell jobs keep running for it
19//! ```
20//!
21//! [`DurableWork::open`] reconciles schedules with the task journal (goal links of admitted firings), reconciles the
22//! goal (adopts an admitted run, restarts an interrupted one, settles one that ended), and starts the
23//! [`ScheduleTicker`], which fires due slots on its own, including those missed while no worker ran (coalesced into
24//! one run). The task end observer still calls [`GoalDriver::task_ended`], and `/goal` still goes to
25//! [`GoalDriver::command`].
26
27use crate::goal::GoalDriver;
28use crate::schedule_tick::{ScheduleTicker, SystemTickClock, TickClock, TickObserver};
29use crate::schedules::SchedulesModule;
30use crate::tasks::TaskHost;
31use kernel::{HostHandler, Signal};
32use serde_json::{Map, Value};
33use std::sync::{Arc, Mutex};
34
35/// What [`DurableWork::open`] takes. Every part is optional except the task host.
36pub struct DurableWorkOptions {
37 /// The task journal's owner: firings and goal runs are admitted through it.
38 pub tasks: Arc<dyn TaskHost>,
39 pub schedules: Option<Arc<SchedulesModule>>,
40 pub goal: Option<Arc<GoalDriver>>,
41 /// The host whose shell jobs this worker owns (recovered by `Host::with_journal`); stopped or handed over on
42 /// close.
43 pub jobs: Option<Arc<host::Host>>,
44 pub clock: Arc<dyn TickClock>,
45 /// Sees every tick's report (`{fired, failed}`), e.g. to wake the session when a firing was admitted.
46 pub on_tick: Option<TickObserver>,
47}
48
49impl DurableWorkOptions {
50 pub fn new(tasks: Arc<dyn TaskHost>) -> DurableWorkOptions {
51 DurableWorkOptions {
52 tasks,
53 schedules: None,
54 goal: None,
55 jobs: None,
56 clock: Arc::new(SystemTickClock),
57 on_tick: None,
58 }
59 }
60
61 pub fn schedules(mut self, schedules: Arc<SchedulesModule>) -> Self {
62 self.schedules = Some(schedules);
63 self
64 }
65
66 pub fn goal(mut self, goal: Arc<GoalDriver>) -> Self {
67 self.goal = Some(goal);
68 self
69 }
70
71 pub fn jobs(mut self, jobs: Arc<host::Host>) -> Self {
72 self.jobs = Some(jobs);
73 self
74 }
75
76 pub fn clock(mut self, clock: Arc<dyn TickClock>) -> Self {
77 self.clock = clock;
78 self
79 }
80
81 pub fn on_tick(mut self, observer: TickObserver) -> Self {
82 self.on_tick = Some(observer);
83 self
84 }
85}
86
87/// What recovery found on open.
88#[derive(Debug, Clone, PartialEq)]
89pub struct Recovered {
90 /// The session goal after reconciliation (`None`: no goal).
91 pub goal: Option<Value>,
92 /// The earliest time a schedule fires (`None`: none will).
93 pub next_fire: Option<i64>,
94 /// The shell jobs as `rlm.jobs()` lists them, newest first.
95 pub jobs: Vec<Value>,
96}
97
98pub struct DurableWork {
99 ticker: Mutex<Option<ScheduleTicker>>,
100 goal: Option<Arc<GoalDriver>>,
101 jobs: Option<Arc<host::Host>>,
102 recovered: Recovered,
103}
104
105impl DurableWork {
106 /// Recovers schedules, the goal and the jobs list, then starts the schedule tick.
107 pub fn open(options: DurableWorkOptions) -> Result<DurableWork, String> {
108 let mut next_fire = None;
109 if let Some(schedules) = &options.schedules {
110 schedules.start(options.tasks.as_ref())?;
111 next_fire = schedules.next_fire()?;
112 }
113 let goal = match &options.goal {
114 Some(driver) => driver.recover()?,
115 None => None,
116 };
117 let jobs = match &options.jobs {
118 Some(host) => host
119 .handle("shell.list", Map::new(), &Signal::new())?
120 .as_array()
121 .cloned()
122 .unwrap_or_default(),
123 None => Vec::new(),
124 };
125 let ticker = options
126 .schedules
127 .map(|schedules| ScheduleTicker::start(schedules, options.tasks, options.clock, options.on_tick));
128 Ok(DurableWork {
129 ticker: Mutex::new(ticker),
130 goal: options.goal,
131 jobs: options.jobs,
132 recovered: Recovered { goal, next_fire, jobs },
133 })
134 }
135
136 pub fn recovered(&self) -> &Recovered {
137 &self.recovered
138 }
139
140 /// Re-reads the next fire time now (after the clock moved or the schedules document changed elsewhere).
141 pub fn wake(&self) {
142 if let Some(ticker) = jsrt::sync::lock(&self.ticker).as_ref() {
143 ticker.wake();
144 }
145 }
146
147 fn stop(&self) {
148 let ticker = jsrt::sync::lock(&self.ticker).take();
149 if let Some(ticker) = ticker {
150 ticker.close();
151 }
152 if let Some(goal) = &self.goal {
153 goal.close();
154 }
155 }
156
157 /// Session close: stops the tick (a tick in progress finishes), stops starting goal jobs, and stops every
158 /// running shell job. Durable state stays for the next open. Idempotent.
159 pub fn close(&self) {
160 self.stop();
161 if let Some(jobs) = &self.jobs {
162 jobs.stop_jobs();
163 }
164 }
165
166 /// A successor worker takes over the session: as [`DurableWork::close`], but running shell jobs keep running
167 /// and are left journaled as running for the successor to follow.
168 pub fn hand_over(&self) {
169 self.stop();
170 if let Some(jobs) = &self.jobs {
171 jobs.release_jobs();
172 }
173 }
174}
175
176impl Drop for DurableWork {
177 fn drop(&mut self) {
178 self.stop();
179 }
180}