1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
//! Private bookkeeping behind the `ledger` feature.
//!
//! With the feature off, [`Track`] and its guards are zero-sized and every
//! method is an empty inline function, so the hot path carries no cost.
#[cfg(not(feature = "ledger"))]
pub(crate) use off::*;
#[cfg(feature = "ledger")]
pub(crate) use on::*;
#[cfg(not(feature = "ledger"))]
mod off {
pub(crate) struct Track;
pub(crate) struct Guard;
impl Track {
pub(crate) const NONE: Track = Track;
#[inline(always)]
pub(crate) fn root() -> Self {
Track
}
#[inline(always)]
pub(crate) fn child(&self) -> Self {
Track
}
#[inline(always)]
pub(crate) fn alias(&self) -> Self {
Track
}
#[inline(always)]
pub(crate) fn task(&self) -> Guard {
Guard
}
#[inline(always)]
pub(crate) fn release(&self) -> Guard {
Guard
}
#[inline(always)]
#[allow(dead_code)] // only called by the parallel scheduler paths
pub(crate) fn note_install(&self) {}
}
}
#[cfg(feature = "ledger")]
mod on {
use crate::ledger::{register, Counts};
use std::sync::atomic::Ordering;
use std::sync::Arc;
/// Ledger handle of a token: its root's counters and whether the thread
/// holding the token is itself counted as a live task of that root.
pub(crate) struct Track {
counts: Option<Arc<Counts>>,
counted: bool,
}
pub(crate) struct Guard<'a> {
counts: Option<&'a Counts>,
// +1 on drop (a released slot being taken back) or -1 on drop (a task ending).
end_is_release: bool,
}
impl Track {
pub(crate) const NONE: Track = Track {
counts: None,
counted: false,
};
pub(crate) fn root() -> Self {
Track {
counts: Some(register()),
counted: false,
}
}
/// Handle given to a job of `fork`: the job runs inside a counted task.
pub(crate) fn child(&self) -> Self {
Track {
counts: self.counts.clone(),
counted: true,
}
}
/// Handle of a `reborrow`: identical accounting to the parent.
pub(crate) fn alias(&self) -> Self {
Track {
counts: self.counts.clone(),
counted: self.counted,
}
}
pub(crate) fn root_id(&self) -> Option<u64> {
self.counts.as_ref().map(|c| c.id)
}
/// One task of this root starts running; it ends when the guard drops.
pub(crate) fn task(&self) -> Guard<'_> {
if let Some(c) = &self.counts {
c.enter();
}
Guard {
counts: self.counts.as_deref(),
end_is_release: false,
}
}
/// The calling task stops counting while its slots are handed to
/// children; it counts again when the guard drops.
pub(crate) fn release(&self) -> Guard<'_> {
let counts = if self.counted {
self.counts.as_deref()
} else {
None
};
if let Some(c) = counts {
c.live.fetch_sub(1, Ordering::SeqCst);
}
Guard {
counts,
end_is_release: true,
}
}
#[allow(dead_code)] // only called by the parallel scheduler paths
pub(crate) fn note_install(&self) {
if let Some(c) = &self.counts {
c.installs.fetch_add(1, Ordering::SeqCst);
}
}
}
impl Drop for Guard<'_> {
fn drop(&mut self) {
if let Some(c) = self.counts {
if self.end_is_release {
c.enter();
} else {
c.live.fetch_sub(1, Ordering::SeqCst);
}
}
}
}
}