1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
//! Changing how many instances an app runs.
//!
//! Scaling up registers and starts new ids; scaling down picks the highest
//! ones and stops them. Either way the answer waits until every instance it
//! touched is terminal or online, so the reply reflects what actually
//! happened rather than what was asked for.
use super::*;
impl<R: ProcessRunner> Actor<R> {
/// Sets `name`'s instance count to `count`.
///
/// Scaling up, [`instance_slots`] hands out the lowest free slots, exactly
/// as a `Start` does. Scaling down, the highest-numbered slots are
/// deregistered first, the same thing `Delete` does: a `Stop` would leave
/// them holding their slots. The reply does not wait for the departures,
/// which report themselves on the bus as `process.delete`.
///
/// Refused while departures are in flight, since a departing instance stays
/// registered and [`SheepSlot::pending_delete`] until its exit lands. That
/// reaches the operator as [`SupervisorError::InvalidScale`].
pub(super) fn handle_scale(
&mut self,
name: &str,
count: u32,
reply: oneshot::Sender<Result<Scaled, SupervisorError>>,
) {
let mut slots: Vec<(u32, u32)> = self
.sheep
.iter()
.filter(|(_, slot)| slot.entry.spec.config().name == name)
.map(|(id, slot)| (slot.entry.instance, *id))
.collect();
if slots.is_empty() {
let _ = reply.send(Err(SupervisorError::NotFound));
return;
}
slots.sort_unstable();
if count == 0 {
let _ = reply.send(Err(SupervisorError::InvalidScale(format!(
"an app runs at least one instance; use `shep delete {name}` to remove it"
))));
return;
}
if self
.sheep
.get(&slots[0].1)
.is_some_and(|slot| slot.entry.dog.is_some())
{
let _ = reply.send(Err(SupervisorError::InvalidScale(format!(
"{name} is a dog, and a dog runs one process"
))));
return;
}
if self.reloads.contains_key(name) {
let _ = reply.send(Err(SupervisorError::ReloadInFlight(name.to_string())));
return;
}
// Counted rather than merely detected: the number tells the operator
// how much of the flock is still moving.
let leaving = slots
.iter()
.filter(|(_, id)| self.sheep.get(id).is_some_and(|slot| slot.pending_delete))
.count();
if leaving > 0 {
let _ = reply.send(Err(SupervisorError::InvalidScale(format!(
"{name} has {leaving} instance(s) still shutting down from an \
earlier command; wait for them to leave `shep flock` and scale \
again"
))));
return;
}
// Re-normalized rather than mutated in place: holding a `ResolvedApp`
// proves it passed `normalize`, and editing the field behind that door
// would be the one place in the tree holding one that had not.
let rescaled = match try_with_count(
&self
.sheep
.get(&slots[0].1)
.expect("handle_scale: id read off this map a moment ago")
.entry
.spec,
count,
) {
Ok(app) => app,
Err(err) => {
let _ = reply.send(Err(SupervisorError::InvalidScale(err.to_string())));
return;
}
};
let current = u32::try_from(slots.len()).unwrap_or(u32::MAX);
// The spawn/remove pass runs first and the config write-back second:
// writing `rescaled` onto every slot up front and then failing a spawn
// leaves every survivor claiming `instances = 4` in a flock of three.
let mut failure = None;
// The one slot `spawn_fresh` registers on a failed attempt. Kept out of
// `survivors`, since it is not a running instance, but the config
// write-back below still has to reach it.
let mut orphaned_by_failed_spawn = None;
let survivors: Vec<u32> = match count.cmp(¤t) {
Ordering::Equal => slots.iter().map(|(_, id)| *id).collect(),
Ordering::Greater => {
if let Err(err) = self.refuse_disabled([name]) {
let _ = reply.send(Err(err));
return;
}
// Inside this arm, since it is the only one that spawns:
// resolving an identity for `Equal` or `Less` meant a lookup
// that could refuse a call using no credentials at all.
// `CannotStart`: nothing has been spawned or removed yet.
let credentials = match self.credentials_for_spawn(slots[0].1) {
Ok(credentials) => credentials,
Err(err) => {
let _ =
reply.send(Err(SupervisorError::CannotStart(format!("{name}: {err}"))));
return;
}
};
let existing: Vec<u32> = slots.iter().map(|(instance, _)| *instance).collect();
let mut ids: Vec<u32> = slots.iter().map(|(_, id)| *id).collect();
// One read for the whole batch: every instance this scale-up
// adds resolves against the same store.
let secrets = self.secret_view(&rescaled);
for instance in instance_slots(&existing, count - current) {
let attempted_id = self.next_id;
match self.spawn_fresh(
&rescaled,
instance,
credentials,
&secrets,
None,
&BTreeSet::new(),
) {
Ok(info) => ids.push(info.id),
Err(message) => {
// Partial, and said so: the instances already
// spawned are serving real traffic, and unwinding
// them would turn one failed spawn into an outage.
orphaned_by_failed_spawn = Some(attempted_id);
failure = Some(message);
break;
}
}
}
ids
}
Ordering::Less => {
let cut = usize::try_from(count).unwrap_or(usize::MAX);
let (keep, remove) = slots.split_at(cut);
let removed: Vec<u32> = remove.iter().map(|(_, id)| *id).collect();
self.begin_manual_ids(
removed,
ManualKind::Delete,
CommandOrigin::Operator,
// The removals' own terminal snapshots go nowhere: this
// reply is the survivors.
ReplyKind::Ids(oneshot::channel().0),
);
keep.iter().map(|(_, id)| *id).collect()
}
};
// The count actually achieved: `count` on every path but a partial
// scale-up. Re-normalized rather than assigned, for the same reason
// `rescaled` was: a `ResolvedApp` is a proof token.
let achieved = u32::try_from(survivors.len()).unwrap_or(u32::MAX);
let stored = match try_with_count(&rescaled, achieved) {
Ok(app) => app,
Err(err) => {
let _ = reply.send(Err(SupervisorError::InvalidScale(err.to_string())));
return;
}
};
// The parked config travels onto the slots this call created, which
// `spawn_fresh` registers with `pending: None`; the loop below also
// writes `stored` onto the `Errored` slot a failed spawn left, which a
// later `handle_scale` counts. `with_count` keeps the count agreeing.
let owed = self.sheep.get(&slots[0].1).and_then(|slot| {
let parked = slot.entry.pending.clone()?;
let parked = with_count(&parked, achieved).unwrap_or(parked);
Some((parked, slot.entry.pending_reidentifies))
});
for id in survivors.iter().chain(orphaned_by_failed_spawn.iter()) {
if let Some(slot) = self.sheep.get_mut(id) {
// `out_file`/`err_file` need no refresh here: `stored` only
// moves `instances`, and no token an accepted log path may
// carry reads that. `normalize` refuses a `{{secret:...}}`
// in either field (`SecretInLogPath`), which leaves
// `{{instance}}` and `{{name}}`; a survivor's own `instance`
// is untouched by a scale and its name cannot move. Note it
// is `normalize` that narrows this and not `render`, which
// resolves secret references too.
slot.entry.spec = stored.clone();
match &mut slot.entry.pending {
// A slot already owed a config keeps it, with the count
// brought forward: a parked count of 2 against a spec of 4
// is drift that never clears.
Some(parked) => {
if let Some(recounted) = with_count(parked, achieved) {
*parked = recounted;
}
}
None => {
if let Some((parked, reidentifies)) = &owed {
slot.entry.pending = Some(parked.clone());
slot.entry.pending_reidentifies = *reidentifies;
}
}
}
}
}
let mut instances: Vec<ProcessInfo> = survivors
.iter()
.filter_map(|id| {
self.sheep
.get(id)
.map(|slot| to_info(&slot.entry, &self.smits))
})
.collect();
sort_flock(&mut instances);
// `Ok` even when `failure` is set: the caller records `app`
// unconditionally and turns `shortfall` into the operator's error,
// where an `Err` would leave the muster roll on the pre-scale count.
let _ = reply.send(Ok(Scaled {
instances,
app: stored,
requested: count,
shortfall: failure,
}));
}
}