runnerd-core 0.1.2

Config, domain logic and API types shared by runnerd and its clients
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
//! What a slot's runner is doing as far as this daemon knows: why it is there, whether GitHub's listing shows it busy,
//! the job a capture or the history named for it, that job as admission tells jobs apart, and a forced pause's
//! deadline for it. It owns that value and must not call Docker, GitHub or the store, nor log: the engine reads
//! those and tells it what each said.

use std::time::{Duration, SystemTime};

use tokio::time::Instant;

use crate::admission::Key;

use super::Purpose;

/// How a slot's runner came to be there, and what that says of its job.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Origin {
    /// This daemon started it, for `purpose`, and behind the barrier of job `behind` if it passed one (ADR 0131).
    Started {
        purpose: Purpose,
        behind: Option<u64>,
    },
    /// A restart found its container running and adopted it.
    Adopted(Adoption),
}

/// What the runners listings showed of an adopted runner since its adoption, which says whether its job started
/// under this daemon.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Adoption {
    /// No listing showed it waiting with no job named: it may have taken its job while no daemon ran, and the
    /// capture that names it after the restart does not say when.
    Unknown,
    /// A listing showed it waiting with no job named: the job it takes next starts under this daemon. A capture names
    /// a job before a listing that lags it shows the runner busy, so a listing that shows it idle once its job is
    /// named says nothing.
    Waited,
}

/// What GitHub's runners listings showed of a runner.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Listed {
    /// Idle: the last listing showed it waiting for a job.
    Idle,
    /// Adopted, and no listing answered for it yet. It counts as busy, but no listing showed it so: the first that
    /// does may still show the job it took queued. Only `adopted` makes it.
    Unlisted,
    /// Busy, as the listing before the last already showed it.
    Busy,
    /// Busy since the listing or the answer that `Mark` says: the queued jobs listings of its repository after it may
    /// still show the job it took, as GitHub lists one queued for a few seconds after. Only a listing of those that
    /// began after the mark, and at least `poll_interval` after it, ends it (`queued_listed`); a later runners listing
    /// that shows it busy again leaves the mark as it was.
    NewlyBusy(Mark),
}

/// A listing of GitHub's, as the claims on listed jobs count it: its number, counted from 1 since the engine's start,
/// and when it began on Tokio's clock. A claim is marked with the listing begun last when it was made: a runner first
/// seen busy, with the runners listing that showed it, or the one before GitHub said so; a runner gone, with the one
/// before it went.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) struct Mark {
    pub(super) listing: u64,
    pub(super) at: Instant,
}

impl Mark {
    /// Whether a queued jobs listing `self` ends a claim marked with `claim`: it is a later listing, begun at least
    /// `floor` after the claim's, so GitHub's lag in taking a job off the queued listing has as long as when every
    /// listing came one `poll_interval` after the last, whatever the listings' pace (ADR 0128).
    pub(super) fn ends(self, claim: Mark, floor: Duration) -> bool {
        self.listing > claim.listing && self.at >= claim.at + floor
    }
}

/// A job named for a runner, with when this daemon's capture found it.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct TakenJob {
    id: u64,
    /// When this daemon's capture found it; `None` for one only the history named, after a restart.
    found_at: Option<SystemTime>,
}

/// A slot's runner's engagement: whether it holds a job, which, and how it came to be in its slot.
///
/// All four combinations of listed busy and a job named are meant: the runners listing lags a capture that names a
/// job, and a capture lags a listing that shows the runner busy. The key is beside the job, not in it: admission keys
/// a job by the history's record, which a capture that names the job again does not change.
///
/// A restart rebuilds it in `adopted`: adopted and not yet listed, counted busy until a listing says otherwise, with
/// the job its history row names, a forced pause's deadline as its row kept it, and no key until `find_keys` reads its
/// record. Whether a listing showed it newly busy or waiting, when its job was found and the barrier it started behind
/// are lost: every barrier after a restart matures after it, so a backfill would not count an adopted runner anyway. A
/// reload does not touch it, so a newly busy runner keeps its mark through one.
#[derive(Debug, Clone)]
pub(super) struct Engagement {
    origin: Origin,
    /// What GitHub's last runners listing showed of it, or what GitHub said since.
    listing: Listed,
    /// The job its capture or its history row named.
    job: Option<TakenJob>,
    /// That job as admission tells jobs apart, once the history had its record.
    key: Option<Key>,
    /// When a forced pause that cancelled its run removes it if it still runs.
    cancel_by: Option<Instant>,
}

impl Engagement {
    /// A runner this daemon starts for `purpose`, behind the barrier of job `behind` if its plan passed one: idle,
    /// with no job.
    pub(super) fn started(purpose: Purpose, behind: Option<u64>) -> Self {
        Self {
            origin: Origin::Started { purpose, behind },
            listing: Listed::Idle,
            job: None,
            key: None,
            cancel_by: None,
        }
    }

    /// A runner a restart adopts, running the job its history row names, if any: counted busy until a listing says
    /// otherwise. `cancel_by` is the deadline a forced pause before the restart gave it, as its row kept it.
    pub(super) fn adopted(job: Option<u64>, cancel_by: Option<Instant>) -> Self {
        Self {
            origin: Origin::Adopted(Adoption::Unknown),
            listing: Listed::Unlisted,
            job: job.map(|id| TakenJob { id, found_at: None }),
            key: None,
            cancel_by,
        }
    }

    /// A listing of its repository's queued jobs begun at `listing` answered: it is no longer newly busy if that
    /// listing ends its mark (`Mark::ends`).
    pub(super) fn queued_listed(&mut self, listing: Mark, floor: Duration) {
        if let Listed::NewlyBusy(mark) = self.listing
            && listing.ends(mark, floor)
        {
            self.listing = Listed::Busy;
        }
    }

    /// A runners listing that `mark` says showed it busy or idle. The first listing that shows an adopted runner busy
    /// shows it newly busy, though adoption counted it busy already. One that shows a newly busy runner busy again
    /// keeps its mark. One that shows an adopted runner waiting with no job named says its next job starts under this
    /// daemon.
    pub(super) fn listed(&mut self, busy: bool, mark: Mark) {
        self.listing = match (busy, self.listing) {
            (false, _) => Listed::Idle,
            (true, Listed::Idle | Listed::Unlisted) => Listed::NewlyBusy(mark),
            (true, listed @ (Listed::Busy | Listed::NewlyBusy(_))) => listed,
        };
        if let Origin::Adopted(adoption) = &mut self.origin
            && !busy
            && self.job.is_none()
        {
            *adoption = Adoption::Waited;
        }
    }

    /// GitHub said it took a job, as a removal of it refused, when `mark` says: newly busy unless a listing showed it
    /// busy already, and a newly busy one keeps its mark. One no listing answered for yet stays so, for the first that
    /// shows it busy to mark it newly busy.
    pub(super) fn kept_busy(&mut self, mark: Mark) {
        self.listing = match self.listing {
            Listed::Idle => Listed::NewlyBusy(mark),
            listed @ (Listed::Busy | Listed::NewlyBusy(_) | Listed::Unlisted) => listed,
        };
    }

    /// Its capture found `job` at `at`. Its key stays: the history's record of the job says what it is.
    pub(super) fn found(&mut self, job: u64, at: SystemTime) {
        self.job = Some(TakenJob {
            id: job,
            found_at: Some(at),
        });
    }

    /// The job its worker log names, once it stopped, for a runner no capture named a job for.
    pub(super) fn named_by_worker_log(&mut self, job: u64) {
        if !self.runs(job) {
            self.job = Some(TakenJob {
                id: job,
                found_at: None,
            });
        }
    }

    /// The history has a record of its job, which admission tells apart by `key`.
    pub(super) fn keyed(&mut self, key: Key) {
        self.key = Some(key);
    }

    /// A forced pause cancelled its run: it is removed if it still runs at `deadline`.
    pub(super) fn cancelled(&mut self, deadline: Instant) {
        self.cancel_by = Some(deadline);
    }

    /// Whether GitHub's last runners listing showed it busy, or GitHub said so since; an adopted runner no listing
    /// answered for yet counts as busy. Alone it says what the listing says, which lags a capture that named a job:
    /// `engaged` is whether it holds one.
    pub(super) fn listed_busy(&self) -> bool {
        self.listing != Listed::Idle
    }

    /// The job its capture or its history row named; a listing that shows it idle does not unname it.
    pub(super) fn named_job(&self) -> Option<u64> {
        self.job.map(|job| job.id)
    }

    /// Whether `job` is the one named for it.
    pub(super) fn runs(&self, job: u64) -> bool {
        self.named_job() == Some(job)
    }

    /// Whether it holds a job as far as this daemon knows: the listing shows it busy, or a capture named its job
    /// first.
    pub(super) fn engaged(&self) -> bool {
        self.listed_busy() || self.job.is_some()
    }

    /// Whether it may be running one of its pool's listed jobs that no capture named yet: the listings of its
    /// repository until `queued_listed` ends it can still show that job queued.
    pub(super) fn claims_a_listed_job(&self) -> bool {
        matches!(self.listing, Listed::NewlyBusy(_)) && self.job.is_none()
    }

    /// Whether a detaching stop removes it: no job named and either listed idle or adopted, as adoption counts a
    /// runner busy until a listing says otherwise. One started here that a listing shows busy is left to its job.
    pub(super) fn goes_when_detaching(&self) -> bool {
        self.job.is_none() && (!self.listed_busy() || !self.was_started_here())
    }

    /// Whether this daemon started it: a restart adopted it otherwise.
    pub(super) fn was_started_here(&self) -> bool {
        self.purpose().is_some()
    }

    /// Why this daemon started it; `None` for one a restart adopted.
    pub(super) fn purpose(&self) -> Option<Purpose> {
        match self.origin {
            Origin::Started { purpose, .. } => Some(purpose),
            Origin::Adopted(_) => None,
        }
    }

    /// When this daemon's capture found its job, if that job started under this daemon: `None` for an adopted runner
    /// no listing showed waiting with no job named, whose job may have started before the restart.
    pub(super) fn job_found_here(&self) -> Option<SystemTime> {
        let started_here = match self.origin {
            Origin::Started { .. } | Origin::Adopted(Adoption::Waited) => true,
            Origin::Adopted(Adoption::Unknown) => false,
        };
        self.job
            .and_then(|job| job.found_at)
            .filter(|_| started_here)
    }

    /// The owner of the barrier it started behind; `None` for one started otherwise or adopted by a restart.
    pub(super) fn behind(&self) -> Option<u64> {
        match self.origin {
            Origin::Started { behind, .. } => behind,
            Origin::Adopted(_) => None,
        }
    }

    /// Its job as admission tells jobs apart, once the history had its record.
    pub(super) fn key(&self) -> Option<&Key> {
        self.key.as_ref()
    }

    /// Whether a forced pause cancelled its run and it still runs past the deadline that gave it to exit.
    pub(super) fn cancel_overdue(&self, now: Instant) -> bool {
        self.cancel_by.is_some_and(|deadline| deadline <= now)
    }
}

#[cfg(test)]
impl Engagement {
    /// When its capture found its job; `None` before.
    pub(super) fn job_found_at(&self) -> Option<SystemTime> {
        self.job.and_then(|job| job.found_at)
    }

    /// As if its capture had found its job at `at`.
    pub(super) fn found_job_at(&mut self, at: SystemTime) {
        if let Some(job) = &mut self.job {
            job.found_at = Some(at);
        }
    }

    /// As if its capture had found its job `by` earlier.
    pub(super) fn age_job_start(&mut self, by: Duration) {
        if let Some(TakenJob {
            found_at: Some(at), ..
        }) = &mut self.job
        {
            *at -= by;
        }
    }

    /// As a listing shows it busy, with no job named.
    pub(super) fn set_listed_busy(&mut self) {
        self.count_busy();
    }

    /// As if it had taken `job`, which a listing shows; when a capture found a job stays.
    pub(super) fn has_taken(&mut self, job: u64) {
        self.job = Some(TakenJob {
            id: job,
            found_at: self.job.and_then(|job| job.found_at),
        });
        self.count_busy();
    }

    /// Busy, newly or not as it was: an idle one becomes busy, not newly.
    fn count_busy(&mut self) {
        if self.listing == Listed::Idle {
            self.listing = Listed::Busy;
        }
    }

    /// As if a listing showed it idle before any job was named, nor its key known.
    pub(super) fn unname(&mut self) {
        self.job = None;
        self.key = None;
        self.listing = Listed::Idle;
    }

    pub(super) fn set_key(&mut self, key: Key) {
        self.key = Some(key);
    }

    /// As if it was first seen busy `by` earlier.
    pub(super) fn age_mark(&mut self, by: Duration) {
        if let Listed::NewlyBusy(mark) = &mut self.listing
            && let Some(at) = mark.at.checked_sub(by)
        {
            mark.at = at;
        }
    }

    /// Whether a forced pause cancelled its run.
    pub(super) fn is_cancelling(&self) -> bool {
        self.cancel_by.is_some()
    }

    /// As if its forced pause's grace ran out.
    pub(super) fn expire_cancel(&mut self, now: Instant) {
        if self.cancel_by.is_some() {
            self.cancel_by = Some(now);
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    const POLL: Duration = Duration::from_secs(15);

    /// Listing `listing`, begun `secs` after `start`.
    fn mark(start: Instant, listing: u64, secs: u64) -> Mark {
        Mark {
            listing,
            at: start + Duration::from_secs(secs),
        }
    }

    /// A runner this daemon started, which listing 1 shows busy before any capture names its job.
    fn newly_busy(start: Instant) -> Engagement {
        let mut engagement = Engagement::started(Purpose::Job, None);
        engagement.listed(true, mark(start, 1, 0));
        assert!(engagement.claims_a_listed_job(), "listing 1 marks it");
        engagement
    }

    #[test]
    fn runners_listings_that_show_it_busy_again_leave_its_mark_as_it_was() {
        // Its repository is listed every 5 s while active, its runners every third probe.
        let start = Instant::now();
        let mut engagement = newly_busy(start);
        for (listing, secs) in [(2, 5), (3, 10)] {
            engagement.listed(true, mark(start, listing, secs));
            engagement.queued_listed(mark(start, listing, secs), POLL);
            assert!(
                engagement.claims_a_listed_job(),
                "listing {listing}, {secs} s after the mark, may still show its job"
            );
        }
        engagement.listed(true, mark(start, 4, 15));
        engagement.queued_listed(mark(start, 4, 15), POLL);
        assert!(
            !engagement.claims_a_listed_job(),
            "the first listing a poll interval after the first mark ends the claim"
        );
        assert!(engagement.listed_busy(), "it stays busy");
    }

    #[test]
    fn only_a_later_listing_ends_the_claim_however_late_it_answers() {
        let start = Instant::now();
        let mut engagement = newly_busy(start);
        // Listing 1's queued jobs answer long after it began, which is what counts.
        engagement.queued_listed(mark(start, 1, 60), POLL);
        assert!(
            engagement.claims_a_listed_job(),
            "the listing that marked it may still show its job"
        );
    }

    #[test]
    fn github_saying_a_newly_busy_runner_is_busy_keeps_its_mark() {
        let start = Instant::now();
        let mut engagement = newly_busy(start);
        engagement.kept_busy(mark(start, 3, 10));
        engagement.queued_listed(mark(start, 4, 15), POLL);
        assert!(
            !engagement.claims_a_listed_job(),
            "the claim ends a poll interval after the first mark, not the refused removal"
        );
    }
}