Skip to main content

kernel_apis/
durable_work.rs

1//! Durable work: schedules, the session goal and shell jobs as durable tasks that survive worker restarts. This is
2//! the one entry point a composition root calls.
3//!
4//! On a worker's open, after its task host (the task journal's owner) and its host with a jobs journal exist:
5//!
6//! ```text
7//! let host = Arc::new(Host::new(cwd, jobs_dir, spill_dir, guard)            // shell jobs: recovered here,
8//!     .with_journal(Arc::new(FileDocumentStore::new(jobs_journal)))?);         //   from their processes
9//! let schedules = Arc::new(SchedulesModule::new(store("schedules"), None));
10//! let goal = Arc::new(GoalDriver::new(store("goal"),
11//!     Arc::new(TaskGoalJobs::new(tasks.clone(), host.clone())), clock));
12//! let apis = KernelApis::new(inner).with_host(tasks.clone(), lane)
13//!     .with(schedules.clone()).with(goal.clone());                           // schedules.*, goals.*, goal.*
14//! let work = DurableWork::open(DurableWorkOptions::new(tasks.clone())
15//!     .schedules(schedules).goal(goal).jobs(host))?;                         // recover + start the tick
16//! ...
17//! work.close();      // session close: stop the tick, stop starting goal jobs, stop shell jobs
18//! work.hand_over();  // a successor worker takes over: the same, but shell jobs keep running for it
19//! ```
20//!
21//! [`DurableWork::open`] reconciles schedules with the task journal (goal links of admitted firings), reconciles the
22//! goal (adopts an admitted run, restarts an interrupted one, settles one that ended), and starts the
23//! [`ScheduleTicker`], which fires due slots on its own, including those missed while no worker ran (coalesced into
24//! one run). The task end observer still calls [`GoalDriver::task_ended`], and `/goal` still goes to
25//! [`GoalDriver::command`].
26
27use crate::goal::GoalDriver;
28use crate::schedule_tick::{ScheduleTicker, SystemTickClock, TickClock, TickObserver};
29use crate::schedules::SchedulesModule;
30use crate::tasks::TaskHost;
31use kernel::{HostHandler, Signal};
32use serde_json::{Map, Value};
33use std::sync::{Arc, Mutex};
34
35/// What [`DurableWork::open`] takes. Every part is optional except the task host.
36pub struct DurableWorkOptions {
37    /// The task journal's owner: firings and goal runs are admitted through it.
38    pub tasks: Arc<dyn TaskHost>,
39    pub schedules: Option<Arc<SchedulesModule>>,
40    pub goal: Option<Arc<GoalDriver>>,
41    /// The host whose shell jobs this worker owns (recovered by `Host::with_journal`); stopped or handed over on
42    /// close.
43    pub jobs: Option<Arc<host::Host>>,
44    pub clock: Arc<dyn TickClock>,
45    /// Sees every tick's report (`{fired, failed}`), e.g. to wake the session when a firing was admitted.
46    pub on_tick: Option<TickObserver>,
47}
48
49impl DurableWorkOptions {
50    pub fn new(tasks: Arc<dyn TaskHost>) -> DurableWorkOptions {
51        DurableWorkOptions {
52            tasks,
53            schedules: None,
54            goal: None,
55            jobs: None,
56            clock: Arc::new(SystemTickClock),
57            on_tick: None,
58        }
59    }
60
61    pub fn schedules(mut self, schedules: Arc<SchedulesModule>) -> Self {
62        self.schedules = Some(schedules);
63        self
64    }
65
66    pub fn goal(mut self, goal: Arc<GoalDriver>) -> Self {
67        self.goal = Some(goal);
68        self
69    }
70
71    pub fn jobs(mut self, jobs: Arc<host::Host>) -> Self {
72        self.jobs = Some(jobs);
73        self
74    }
75
76    pub fn clock(mut self, clock: Arc<dyn TickClock>) -> Self {
77        self.clock = clock;
78        self
79    }
80
81    pub fn on_tick(mut self, observer: TickObserver) -> Self {
82        self.on_tick = Some(observer);
83        self
84    }
85}
86
87/// What recovery found on open.
88#[derive(Debug, Clone, PartialEq)]
89pub struct Recovered {
90    /// The session goal after reconciliation (`None`: no goal).
91    pub goal: Option<Value>,
92    /// The earliest time a schedule fires (`None`: none will).
93    pub next_fire: Option<i64>,
94    /// The shell jobs as `rlm.jobs()` lists them, newest first.
95    pub jobs: Vec<Value>,
96}
97
98pub struct DurableWork {
99    ticker: Mutex<Option<ScheduleTicker>>,
100    goal: Option<Arc<GoalDriver>>,
101    jobs: Option<Arc<host::Host>>,
102    recovered: Recovered,
103}
104
105impl DurableWork {
106    /// Recovers schedules, the goal and the jobs list, then starts the schedule tick.
107    pub fn open(options: DurableWorkOptions) -> Result<DurableWork, String> {
108        let mut next_fire = None;
109        if let Some(schedules) = &options.schedules {
110            schedules.start(options.tasks.as_ref())?;
111            next_fire = schedules.next_fire()?;
112        }
113        let goal = match &options.goal {
114            Some(driver) => driver.recover()?,
115            None => None,
116        };
117        let jobs = match &options.jobs {
118            Some(host) => host
119                .handle("shell.list", Map::new(), &Signal::new())?
120                .as_array()
121                .cloned()
122                .unwrap_or_default(),
123            None => Vec::new(),
124        };
125        let ticker = options
126            .schedules
127            .map(|schedules| ScheduleTicker::start(schedules, options.tasks, options.clock, options.on_tick));
128        Ok(DurableWork {
129            ticker: Mutex::new(ticker),
130            goal: options.goal,
131            jobs: options.jobs,
132            recovered: Recovered { goal, next_fire, jobs },
133        })
134    }
135
136    pub fn recovered(&self) -> &Recovered {
137        &self.recovered
138    }
139
140    /// Re-reads the next fire time now (after the clock moved or the schedules document changed elsewhere).
141    pub fn wake(&self) {
142        if let Some(ticker) = jsrt::sync::lock(&self.ticker).as_ref() {
143            ticker.wake();
144        }
145    }
146
147    fn stop(&self) {
148        let ticker = jsrt::sync::lock(&self.ticker).take();
149        if let Some(ticker) = ticker {
150            ticker.close();
151        }
152        if let Some(goal) = &self.goal {
153            goal.close();
154        }
155    }
156
157    /// Session close: stops the tick (a tick in progress finishes), stops starting goal jobs, and stops every
158    /// running shell job. Durable state stays for the next open. Idempotent.
159    pub fn close(&self) {
160        self.stop();
161        if let Some(jobs) = &self.jobs {
162            jobs.stop_jobs();
163        }
164    }
165
166    /// A successor worker takes over the session: as [`DurableWork::close`], but running shell jobs keep running
167    /// and are left journaled as running for the successor to follow.
168    pub fn hand_over(&self) {
169        self.stop();
170        if let Some(jobs) = &self.jobs {
171            jobs.release_jobs();
172        }
173    }
174}
175
176impl Drop for DurableWork {
177    fn drop(&mut self) {
178        self.stop();
179    }
180}