1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
//! The shard's timers: the millisecond wheel, the microsecond wheel of the
//! precise sleeps, their times on the runtime's clock, firing what came due,
//! the wait's high-resolution timer for a precise deadline, and the park
//! path's part in a paused clock (publishing the earliest deadline, trying
//! the jump, waiting for one in progress).
use core::cell::RefCell;
use std::time::{Duration, Instant};
use super::Shard;
use crate::sync::internal::Ordering;
use crate::time::units::{MICROS, MILLIS, units};
use crate::time::wheel::Wheel;
impl Shard {
/// The shard's millisecond wheel.
#[cfg(test)]
pub(crate) fn wheel(&self) -> &RefCell<Wheel> {
&self.wheel
}
/// The shard's millisecond wheel, or its microsecond wheel when
/// `precise`.
#[inline]
pub(crate) fn wheel_of(&self, precise: bool) -> &RefCell<Wheel> {
if precise { &self.precise } else { &self.wheel }
}
/// The deadline `at` (nanoseconds after the epoch, the clock's origin)
/// in the time of the wheel `precise` picks, rounded up: a timer never
/// fires before its deadline.
#[inline]
pub(crate) fn time_of(&self, precise: bool, at: u64) -> u64 {
let per_second = if precise { MICROS } else { MILLIS };
units(at, per_second, true)
}
/// The wheel's time now on the runtime's clock, rounded down: the wheel
/// never runs ahead of the clock.
pub(super) fn wheel_now(&self) -> u64 {
units(crate::time::clock::now_nanos(), MILLIS, false)
}
/// The microsecond wheel's time now, rounded down as `wheel_now`.
fn precise_now(&self) -> u64 {
units(crate::time::clock::now_nanos(), MICROS, false)
}
/// The earliest time either wheel has work at, and whether it is the
/// microsecond wheel's.
fn next_deadline(&self) -> Option<(Instant, bool)> {
let coarse = self
.wheel
.borrow()
.next_deadline()
.map(|ms| self.epoch + Duration::from_millis(ms));
let precise = self.precise.borrow();
let fine = if precise.armed() == 0 {
None
} else {
precise
.next_deadline()
.map(|us| self.epoch + Duration::from_micros(us))
};
match (coarse, fine) {
(Some(coarse), Some(fine)) if fine < coarse => Some((fine, true)),
(Some(coarse), _) => Some((coarse, false)),
(None, fine) => fine.map(|fine| (fine, true)),
}
}
/// Before a wait that may sleep (`parking`) until a precise deadline,
/// `after` from now: a poller whose own timeout counts milliseconds
/// (epoll), or lets the system coalesce it (kqueue), arms its
/// high-resolution timer for it. A ring's timeout and a port's timer are
/// high-resolution already: their wait ends there by itself.
fn arm_precise(&self, parking: bool, next: Option<(Instant, bool)>, after: Option<Duration>) {
#[cfg(readiness)]
if parking
&& let (Some((at, true)), Some(after)) = (next, after)
&& let super::Driver::Poll(poller) = &self.driver
{
poller.borrow_mut().arm_precise(at, after);
}
#[cfg(not(readiness))]
let _ = (parking, next, after);
}
/// How long the shard's wait may last for its next timer, if it has one.
///
/// On a paused clock the timer's wait is not real time: a due timer
/// fires at once, and otherwise the wait is unbounded, for a waker or for
/// the clock to move. A shard about to park (`parking`, its deadline
/// published by `before_park`) tries to move the clock to the runtime's
/// earliest deadline (`paused::try_advance`); if that makes its own timer
/// due, it does not wait.
pub(super) fn timer_wait(&self, parking: bool) -> Option<Duration> {
// From the clock's own nanoseconds, so the wait ends at the
// deadline's millisecond (or microsecond) and not one early.
let earliest = self.next_deadline();
let next = earliest.map(|(at, _)| at);
if !crate::time::clock::any_altered() {
let wait = next.map(|at| at.saturating_duration_since(Instant::now()));
self.arm_precise(parking, earliest, wait);
return wait;
}
let Some(inner) = crate::rt::context::runtime() else {
let wait = next.map(|at| at.saturating_duration_since(crate::time::clock::now()));
self.arm_precise(parking, earliest, wait);
return wait;
};
let now = inner.clock.now();
if !inner.clock.is_paused() {
let wait = next.map(|at| at.saturating_duration_since(now));
self.arm_precise(parking, earliest, wait);
return wait;
}
if next.is_some_and(|at| at <= now) {
return Some(Duration::ZERO);
}
if !parking {
return None;
}
if crate::rt::paused::try_advance(&inner, Some(&self.shared)) {
Some(Duration::ZERO)
} else {
None
}
}
/// Before a park, for a paused clock: publishes the shard's earliest
/// deadline and counts the park, both before the shard says it parks, so
/// a shard deciding a jump that sees this one parked sees its deadline
/// (`proofs/tla/clock/`, publish_late). Nothing to do while no clock is
/// paused.
pub(super) fn before_park(&self) {
if crate::time::clock::any_altered() {
let at = self
.next_deadline()
.map_or(u64::MAX, |(at, _)| crate::time::clock::to_nanos(at));
self.shared.timer_at.store(at, Ordering::SeqCst);
self.shared.parks.fetch_add(1, Ordering::SeqCst);
}
}
/// After a park, slept or given up for work that came: a jump of a paused
/// clock in progress ends before this shard runs and reads the clock
/// (`proofs/tla/clock/`, busy_no_wait and no_wait_advance).
pub(super) fn after_park(&self) {
if crate::time::clock::any_altered()
&& let Some(inner) = crate::rt::context::runtime()
{
inner.clock.wait_advance();
}
}
/// Fires the due timers from inside a task's poll (`time::advance`): the
/// tasks they wake go to the queue in deadline order, ahead of a task
/// that yields next, and not to the LIFO slot.
pub(crate) fn fire_timers_now(&self) {
crate::rt::context::with_in_task(false, || self.fire_timers());
}
#[inline]
pub(super) fn fire_timers(&self) {
if self.shared.timer_ops.is_empty()
&& self.wheel.borrow().armed() == 0
&& self.precise.borrow().armed() == 0
{
return;
}
self.fire_armed_timers();
}
fn fire_armed_timers(&self) {
if !self.shared.timer_ops.is_empty() {
let mut wheel = self.wheel.borrow_mut();
self.shared
.timer_ops
.take_all(|op| op.apply(self, &mut wheel));
}
if self.precise.borrow().armed() != 0 {
self.fire_precise();
}
if self.wheel.borrow().armed() == 0 {
return;
}
let now = self.wheel_now();
// The buffer is taken out, so the wheel and the buffer are let go
// while the wakers run (a wake may arm a timer again).
let mut fired = self.fired.take();
self.wheel.borrow_mut().advance(now, &mut fired);
for waker in fired.drain(..) {
waker.wake();
}
self.fired.replace(fired);
}
/// Fires the precise timers that came due, as `fire_timers` does the
/// wheel's: out of line, as most shards never arm one.
#[inline(never)]
fn fire_precise(&self) {
let now = self.precise_now();
let mut fired = self.fired.take();
self.precise.borrow_mut().advance(now, &mut fired);
for waker in fired.drain(..) {
waker.wake();
}
self.fired.replace(fired);
}
}