Skip to main content

kernel_apis/
schedule_tick.rs

1//! The schedule tick: fires due schedules on its own, without anything else waking the session (the TS
2//! `ScheduleModule` interval timer, `tickIntervalMs`).
3//!
4//! A [`ScheduleTicker`] thread sleeps until the earliest next fire time ([`SchedulesModule::next_fire`]) or until it
5//! is woken (a committed schedule change, [`ScheduleTicker::wake`]), then runs [`SchedulesModule::tick`]. The next
6//! fire time is derived from durable schedule fields, so a reopened ticker recomputes it: slots missed while no
7//! worker ran are due at once and coalesce into one run of the latest slot (TS semantics). Each fire is admitted as a
8//! task whose key is its slot (`schedule:<id>:<slot>`) after the slot is claimed durably, so a crash between firing
9//! and delivery resumes the claimed slot with the same key and the slot runs exactly once.
10//!
11//! Time comes from a [`TickClock`]: [`SystemTickClock`] in the product, [`ManualTickClock`] in tests, which never
12//! sleeps on real time (its waits end only when woken) so tests advance it and wake the ticker.
13
14use crate::schedules::SchedulesModule;
15use crate::tasks::TaskHost;
16use serde_json::Value;
17use std::sync::atomic::{AtomicI64, Ordering};
18use std::sync::{Arc, Condvar, Mutex};
19use std::thread::JoinHandle;
20use std::time::Duration;
21
22/// After a tick that failed (a store write failed) or settled nothing, wait this long before trying again.
23pub const RETRY_MS: i64 = 1000;
24
25/// The ticker's time source.
26pub trait TickClock: Send + Sync {
27    /// Now, in ms on the schedules' clock.
28    fn now(&self) -> i64;
29    /// How long to block, at most, before `deadline` may have passed; `None` blocks until woken.
30    fn wait_limit(&self, deadline: i64) -> Option<Duration>;
31}
32
33/// Wall-clock time; waits end at the deadline.
34pub struct SystemTickClock;
35
36impl TickClock for SystemTickClock {
37    fn now(&self) -> i64 {
38        (crate::util::system_clock())()
39    }
40
41    fn wait_limit(&self, deadline: i64) -> Option<Duration> {
42        Some(Duration::from_millis(deadline.saturating_sub(self.now()).max(0) as u64))
43    }
44}
45
46/// A clock that moves only when told to; a ticker on it waits until woken.
47pub struct ManualTickClock {
48    now: AtomicI64,
49}
50
51impl ManualTickClock {
52    pub fn new(now: i64) -> Arc<ManualTickClock> {
53        Arc::new(ManualTickClock {
54            now: AtomicI64::new(now),
55        })
56    }
57
58    pub fn set(&self, now: i64) {
59        self.now.store(now, Ordering::SeqCst);
60    }
61
62    pub fn advance(&self, ms: i64) -> i64 {
63        self.now.fetch_add(ms, Ordering::SeqCst) + ms
64    }
65
66    /// This clock as the schedules module's clock.
67    pub fn as_clock(self: &Arc<Self>) -> crate::util::Clock {
68        let clock = self.clone();
69        Arc::new(move || clock.now())
70    }
71}
72
73impl TickClock for ManualTickClock {
74    fn now(&self) -> i64 {
75        self.now.load(Ordering::SeqCst)
76    }
77
78    fn wait_limit(&self, _deadline: i64) -> Option<Duration> {
79        None
80    }
81}
82
83/// Observes every tick's report (`{fired, failed}`) or error.
84pub type TickObserver = Arc<dyn Fn(&Result<Value, String>) + Send + Sync>;
85
86#[derive(Default)]
87struct Flags {
88    woken: bool,
89    closed: bool,
90}
91
92struct Shared {
93    flags: Mutex<Flags>,
94    bell: Condvar,
95}
96
97impl Shared {
98    fn wake(&self) {
99        jsrt::sync::lock(&self.flags).woken = true;
100        self.bell.notify_all();
101    }
102}
103
104pub struct ScheduleTicker {
105    shared: Arc<Shared>,
106    module: Arc<SchedulesModule>,
107    thread: Mutex<Option<JoinHandle<()>>>,
108}
109
110impl ScheduleTicker {
111    /// Starts the tick thread over `module` (already started on `host`), admitting firings through `host`.
112    /// `observer` sees each tick's report.
113    pub fn start(
114        module: Arc<SchedulesModule>,
115        host: Arc<dyn TaskHost>,
116        clock: Arc<dyn TickClock>,
117        observer: Option<TickObserver>,
118    ) -> ScheduleTicker {
119        let shared = Arc::new(Shared {
120            flags: Mutex::default(),
121            bell: Condvar::new(),
122        });
123        let waker = shared.clone();
124        module.on_change(Some(Arc::new(move || waker.wake())));
125        let (thread_shared, thread_module) = (shared.clone(), module.clone());
126        let thread = std::thread::Builder::new()
127            .name("ultron-schedule-tick".into())
128            .spawn(move || run(&thread_shared, &thread_module, host.as_ref(), clock.as_ref(), observer))
129            .ok();
130        ScheduleTicker {
131            shared,
132            module,
133            thread: Mutex::new(thread),
134        }
135    }
136
137    /// Re-read the next fire time now (the clock moved, or something outside the module changed).
138    pub fn wake(&self) {
139        self.shared.wake();
140    }
141
142    /// Stops the ticker and waits for its thread; a tick in progress finishes first. Idempotent.
143    pub fn close(&self) {
144        jsrt::sync::lock(&self.shared.flags).closed = true;
145        self.shared.bell.notify_all();
146        self.module.on_change(None);
147        let thread = jsrt::sync::lock(&self.thread).take();
148        if let Some(thread) = thread {
149            let _ = thread.join();
150        }
151    }
152}
153
154impl Drop for ScheduleTicker {
155    fn drop(&mut self) {
156        self.close();
157    }
158}
159
160fn run(
161    shared: &Shared,
162    module: &SchedulesModule,
163    host: &dyn TaskHost,
164    clock: &dyn TickClock,
165    observer: Option<TickObserver>,
166) {
167    loop {
168        if jsrt::sync::lock(&shared.flags).closed {
169            return;
170        }
171        let deadline = match module.next_fire() {
172            Ok(Some(next)) if next <= clock.now() => {
173                let report = module.tick(host);
174                // A tick that settled nothing (or failed) must not be repeated at once: wait for a change or retry.
175                let settled = report.as_ref().is_ok_and(|report| {
176                    ["fired", "failed"]
177                        .iter()
178                        .any(|key| report[*key].as_array().is_some_and(|items| !items.is_empty()))
179                });
180                if let Some(observer) = &observer {
181                    observer(&report);
182                }
183                if settled {
184                    continue;
185                }
186                Some(clock.now().saturating_add(RETRY_MS))
187            }
188            Ok(next) => next,
189            Err(_) => Some(clock.now().saturating_add(RETRY_MS)),
190        };
191        let mut flags = jsrt::sync::lock(&shared.flags);
192        while !flags.woken && !flags.closed {
193            let limit = deadline.map(|deadline| clock.wait_limit(deadline));
194            match limit {
195                // No schedule fires and none is due: sleep until something changes.
196                None | Some(None) => {
197                    flags = shared
198                        .bell
199                        .wait(flags)
200                        .unwrap_or_else(std::sync::PoisonError::into_inner);
201                }
202                Some(Some(limit)) if limit.is_zero() => break,
203                Some(Some(limit)) => {
204                    flags = shared
205                        .bell
206                        .wait_timeout(flags, limit)
207                        .unwrap_or_else(std::sync::PoisonError::into_inner)
208                        .0;
209                }
210            }
211        }
212        flags.woken = false;
213    }
214}