runnerd-core 0.1.5

Config, domain logic and API types shared by runnerd and its clients
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
//! Schema 16: a history at 15 brought up to it, or read as it by a read-only look, and waits that leave out the
//! time no runnerd ran.

use rusqlite::Connection;

use super::{keep_since, queued, runner, user_version};
use crate::api::{Bucket, HostSample, StatsQuery};
use crate::config::Config;
use crate::github::QueuedJob;
use crate::store::{SCHEMA, Store, VERSION};
use crate::testdir::TestDir;

/// The schema at version 15, as runnerd 0.0.19 to 0.0.22 created it.
const SCHEMA_15: &str = include_str!("schema-15.sql");

const SECOND: u64 = 1000;
const HOUR: u64 = 3600 * SECOND;

/// A history at version 15 at `path`, holding every job since 1970, with one job that ran: queued at 1000 s,
/// started 30 s later, finished at 1100 s, peaking at 1G.
fn history_15(path: &std::path::Path) {
    let db = Connection::open(path).unwrap();
    db.execute_batch(SCHEMA_15).unwrap();
    db.execute_batch(
        "UPDATE history_kept SET jobs_since = 0;
         INSERT INTO runners (name, repository, profile, slot, container, started_at, removed_at, memory_cap)
         VALUES ('r', 'o/r', 'app', 1, 'c', 1000000, 1100000, 4294967296);
         INSERT INTO jobs (id, repository, run_id, run_attempt, name, labels, state, conclusion, runner, queued_at,
         started_at, finished_at, memory_peak)
         VALUES (1, 'o/r', 10, 1, 'build', '[]', 'completed', 'success', 'r', 1000000, 1030000, 1100000, 1073741824);",
    )
    .unwrap();
    db.pragma_update(None, "user_version", 15).unwrap();
}

#[tokio::test]
async fn a_history_at_15_reads_read_only_as_it_is_and_opens_at_16_with_its_rows_and_their_waits() {
    let dir = TestDir::new("store-15");
    let path = dir.0.join("runnerd.db");
    history_15(&path);
    // `runnerd doctor` and `simulate` read it before the daemon restarts, and leave it at 15.
    let history = super::super::sizing_at(&path, 0).unwrap().unwrap();
    let wait = history.jobs.first().and_then(crate::sizing::Ran::wait);
    assert_eq!(
        wait.map(|wait| wait.counted(history.kept_since)),
        Some(30 * SECOND)
    );
    assert_eq!(
        history.jobs.first().map(|job| job.slot_waited),
        Some(0),
        "a row from before schema 18 counts as no wait for a slot"
    );
    assert_eq!(user_version(&path), 15);
    let store = Store::open(&path).unwrap();
    assert_eq!(user_version(&path), VERSION);
    let job = store.job(1).await.unwrap().unwrap().summary;
    assert_eq!((job.timeline.unseen, job.waited(0)), (0, Some(30)));
    let activity = store.activity(1000, 1100).await.unwrap();
    assert_eq!(activity.wait_p95(), Some(30), "an old row keeps its wait");
}

/// Each table's columns and each index, as SQLite describes them.
fn described(db: &Connection) -> Vec<String> {
    let mut tables: Vec<String> = db
        .prepare("SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name")
        .unwrap()
        .query_map([], |row| row.get(0))
        .unwrap()
        .collect::<rusqlite::Result<_>>()
        .unwrap();
    tables.retain(|table| table != "sqlite_sequence");
    let mut described = Vec::new();
    for table in tables {
        let columns = db
            .prepare(
                "SELECT cid, name, type, \"notnull\", dflt_value, pk FROM pragma_table_info(?1) ORDER BY cid",
            )
            .unwrap()
            .query_map([&table], |row| {
                Ok(format!(
                    "{table}: {} {} {} not null {} default {:?} pk {}",
                    row.get::<_, i64>(0)?,
                    row.get::<_, String>(1)?,
                    row.get::<_, String>(2)?,
                    row.get::<_, bool>(3)?,
                    row.get::<_, Option<String>>(4)?,
                    row.get::<_, i64>(5)?,
                ))
            })
            .unwrap()
            .collect::<rusqlite::Result<Vec<_>>>()
            .unwrap();
        described.extend(columns);
    }
    let indexes = db
        .prepare("SELECT tbl_name, name, sql FROM sqlite_master WHERE type = 'index' ORDER BY name")
        .unwrap()
        .query_map([], |row| {
            Ok(format!(
                "index {}.{}: {:?}",
                row.get::<_, String>(0)?,
                row.get::<_, String>(1)?,
                row.get::<_, Option<String>>(2)?,
            ))
        })
        .unwrap()
        .collect::<rusqlite::Result<Vec<_>>>()
        .unwrap();
    described.extend(indexes);
    described
}

#[test]
fn a_history_brought_up_from_15_has_the_schema_a_new_one_gets() {
    let dir = TestDir::new("store-drift");
    let old = dir.0.join("old.db");
    history_15(&old);
    drop(Store::open(&old).unwrap());
    let new = Connection::open_in_memory().unwrap();
    new.execute_batch(SCHEMA).unwrap();
    // A schema.sql change without its migration, or the other way round, differs here.
    assert_eq!(described(&Connection::open(&old).unwrap()), described(&new));
}

/// `at` (ms) as GitHub writes a time.
fn rfc3339(at: u64) -> String {
    Connection::open_in_memory()
        .unwrap()
        .query_row(
            "SELECT strftime('%Y-%m-%dT%H:%M:%SZ', ?1, 'unixepoch')",
            [at / SECOND],
            |row| row.get(0),
        )
        .unwrap()
}

const CONFIG: &str = r#"
[host]
slice = "runnerd.slice"
isolation = "sysbox"

[caches]
mirror = { enabled = false, memory = "512M", cpus = 1, ttl = "168h" }
pnpm = { max_size = "5G" }
maven = { max_size = "5G" }

[profiles.app]
image_stage = "node"
cpus = 4
memory = "4G"
slots = 2

[[repositories]]
name = "o/r"
profile = "app"
labels = ["self-hosted"]
"#;

#[tokio::test]
async fn waits_leave_out_the_time_no_runnerd_ran_wherever_they_are_counted() {
    let dir = TestDir::new("store-downtime");
    let path = dir.0.join("runnerd.db");
    // From a whole second at the history's start, so every wait is whole seconds.
    let start = crate::time::now_ms() / SECOND * SECOND + SECOND;
    let queue = |store: &Store, id: u64, at: u64| {
        store.queued(
            "o/r",
            &QueuedJob {
                created_at: at,
                ..queued(id)
            },
            crate::time::now_ms(),
        );
    };
    // The first daemon sees jobs 1 and 5, and stops 20 s after it started.
    let first = Store::open_at(&path, start).unwrap();
    keep_since(&path, start);
    queue(&first, 1, start + 10 * SECOND);
    queue(&first, 5, start + 5 * SECOND);
    first.host_sample(HostSample {
        at: (start + 20 * SECOND) / SECOND,
        cpu: 0,
        memory: 0,
    });
    first.flush().await;
    drop(first);
    // Eight hours later the next one sees job 2, queued before the history's start, and job 3, queued while none
    // ran; job 4 it sees at once.
    let restart = start + 8 * HOUR;
    let second = Store::open_at(&path, restart).unwrap();
    queue(&second, 2, start - HOUR);
    queue(&second, 3, start + HOUR);
    queue(&second, 4, restart + 60 * SECOND);
    let started = restart + 100 * SECOND;
    for job in 1..=5 {
        second.claimed(job, 10, "r", started);
    }
    // A runner that outlived the first daemon took job 5 two hours into the downtime, as GitHub then says.
    second.details(crate::github::JobDetails {
        id: 5,
        run_id: 10,
        run_attempt: 1,
        name: "job 5".to_owned(),
        workflow_name: None,
        head_branch: None,
        html_url: None,
        labels: Vec::new(),
        created_at: None,
        status: "in_progress".to_owned(),
        conclusion: None,
        started_at: Some(rfc3339(start + 2 * HOUR)),
        completed_at: None,
        runner_name: Some("r".to_owned()),
        steps: Vec::new(),
    });
    for job in 1..=5 {
        second.lost(job, started + 60 * SECOND);
    }
    second.flush().await;
    // Job 1 waited 10 s before the first daemon stopped and 100 s after the second started; job 2 the first
    // daemon's 20 s and the 100 s; job 3 and job 4 the time from the second's start and from their queueing; job 5
    // the 15 s before the first daemon stopped.
    let waits = [110, 120, 100, 40, 15];
    let now = (started + HOUR) / SECOND;
    let activity = second.activity(0, now).await.unwrap();
    let mut counted: Vec<u64> = activity.started.iter().map(|job| job.waited).collect();
    counted.sort_unstable();
    let mut expected = waits.to_vec();
    expected.sort_unstable();
    assert_eq!(counted, expected, "the dashboard's waits");
    let config = Config::parse(CONFIG).unwrap();
    let history = second.insights(0).await.unwrap();
    let insights = crate::insights::insights(&history, &config, 14, now * SECOND);
    let week = &insights.waits.first().unwrap().now.all;
    assert_eq!(
        (week.jobs, week.p95),
        (5, activity.wait_p95()),
        "insights' p95 is the dashboard's"
    );
    let stats = second
        .stats(
            StatsQuery {
                repository: None,
                since: Some(0),
                until: None,
                bucket: Bucket::Day,
            },
            now,
        )
        .await
        .unwrap();
    let queued: u64 = stats.rows.iter().map(|row| row.queue_seconds).sum();
    assert_eq!(queued, waits.iter().sum::<u64>(), "the stats' summed waits");
    // Its own page keeps its whole wait, and says how much of it passed while none ran.
    let job = second.job(3).await.unwrap().unwrap().summary;
    assert_eq!(job.waited(now), Some(7 * 3600 + 100));
    assert_eq!(
        job.unseen_note(now).as_deref(),
        Some("7h 00m of it while runnerd was not running")
    );
    let seen = second.job(4).await.unwrap().unwrap().summary;
    assert_eq!((seen.waited(now), seen.unseen_note(now)), (Some(40), None));
}

#[tokio::test]
async fn an_open_whose_start_then_fails_does_not_hand_out_the_same_downtime_again() {
    let dir = TestDir::new("store-reopen");
    let path = dir.0.join("runnerd.db");
    let start = crate::time::now_ms() / SECOND * SECOND + SECOND;
    let first = Store::open_at(&path, start).unwrap();
    first.queued(
        "o/r",
        &QueuedJob {
            created_at: start + 10 * SECOND,
            ..queued(1)
        },
        crate::time::now_ms(),
    );
    first.flush().await;
    drop(first);
    // Eight hours later a start fails after the history opened, writing nothing; systemd retries 3 s later, and
    // that one runs.
    let restart = start + 8 * HOUR;
    drop(Store::open_at(&path, restart).unwrap());
    let retried = Store::open_at(&path, restart + 3 * SECOND).unwrap();
    // An open at the same time as the last adds nothing at all.
    drop(retried);
    let store = Store::open_at(&path, restart + 3 * SECOND).unwrap();
    let job = store.job(1).await.unwrap().unwrap().summary;
    assert_eq!(
        job.timeline.unseen,
        (8 * HOUR - 10 * SECOND + 3 * SECOND) / SECOND
    );
}

#[tokio::test]
async fn an_open_waits_for_a_writer_that_still_holds_the_history() {
    let dir = TestDir::new("store-busy");
    let path = dir.0.join("runnerd.db");
    drop(Store::open(&path).unwrap());
    // As the writer of a store just dropped, finishing its last writes.
    let writer = Connection::open(&path).unwrap();
    writer
        .execute_batch(
            "BEGIN IMMEDIATE; INSERT INTO host_samples (at, cpu, memory) VALUES (1000, 0, 0)",
        )
        .unwrap();
    let release = std::thread::spawn(move || {
        std::thread::sleep(std::time::Duration::from_millis(300));
        writer.execute_batch("COMMIT").unwrap();
    });
    let opened = Store::open(&path);
    release.join().unwrap();
    assert!(opened.is_ok(), "the open waits: {:?}", opened.err());
}

// Catches a row from before schema 26 read as a wait of 0 for every cause, a cause added to such a row, two writes of
// one cause not adding up, and a cause a later runnerd stored failing the read (ADR 0154).
#[tokio::test]
async fn a_jobs_causes_add_up_and_a_row_from_before_schema_26_counts_none() {
    use crate::api::WaitCause;
    let dir = TestDir::new("store-26");
    let path = dir.0.join("runnerd.db");
    history_15(&path);
    let store = Store::open(&path).unwrap();
    for id in [2, 3] {
        store.queued("o/r", &queued(id), 1_000_000);
    }
    let wait = |job, cause, ms| crate::store::CauseWait { job, cause, ms };
    store.causes_waited(vec![
        wait(1, WaitCause::Host, 40 * SECOND),
        wait(2, WaitCause::Host, 40 * SECOND),
    ]);
    store.causes_waited(vec![
        wait(2, WaitCause::Host, 5 * SECOND),
        wait(2, WaitCause::NoPool, SECOND),
    ]);
    store.flush().await;
    Connection::open(&path)
        .unwrap()
        .execute_batch(
            "UPDATE jobs SET state = 'completed', conclusion = 'success', started_at = 1100000, \
             finished_at = 1200000 WHERE id IN (2, 3);
             UPDATE jobs SET wait_causes = json_set(wait_causes, '$.later', 7000) WHERE id = 3;",
        )
        .unwrap();
    let history = store.insights(0).await.unwrap();
    let causes: Vec<_> = history
        .runs
        .iter()
        .map(|run| (run.id, run.causes.clone()))
        .collect();
    assert_eq!(
        causes,
        [
            (1, None),
            (
                2,
                Some([(WaitCause::Host, 45 * SECOND), (WaitCause::NoPool, SECOND)].into())
            ),
            (3, Some([(WaitCause::Other, 7 * SECOND)].into())),
        ]
    );
}

// Catches the downtime of a restart while a listed job still queues counted as having come before its first listing,
// which takes it off the wrong part of the wait (ADR 0154).
#[tokio::test]
async fn only_the_downtime_a_jobs_first_listing_found_came_before_that_listing() {
    let dir = TestDir::new("store-unlisted");
    let path = dir.0.join("runnerd.db");
    let start = crate::time::now_ms() / SECOND * SECOND + SECOND;
    let queue = |store: &Store, id: u64, at: u64| {
        store.queued(
            "o/r",
            &QueuedJob {
                created_at: at,
                ..queued(id)
            },
            crate::time::now_ms(),
        );
    };
    let first = Store::open_at(&path, start).unwrap();
    keep_since(&path, start);
    queue(&first, 1, start + 5 * SECOND);
    first.host_sample(HostSample {
        at: (start + 20 * SECOND) / SECOND,
        cpu: 0,
        memory: 0,
    });
    first.flush().await;
    drop(first);
    // An hour without runnerd: job 1, listed before it, still queues; job 2 was queued during it.
    let second = Store::open_at(&path, start + HOUR).unwrap();
    queue(&second, 2, start + 30 * SECOND);
    second.flush().await;
    let unseen: Vec<(u64, u64)> = Connection::open(&path)
        .unwrap()
        .prepare("SELECT unseen, unseen_unlisted FROM jobs ORDER BY id")
        .unwrap()
        .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
        .unwrap()
        .collect::<rusqlite::Result<_>>()
        .unwrap();
    let down = HOUR - 20 * SECOND;
    assert_eq!(
        unseen,
        [(down, 0), (HOUR - 30 * SECOND, HOUR - 30 * SECOND)],
        "job 1's downtime came after its listing; job 2's before"
    );
}

// Catches the downtime of a job a capture recorded first being reported as GitHub's listing lag: its row starts with
// the capture's time as both its queueing and its start, so the restart gave it no downtime, and GitHub's real times,
// when they came, only took downtime off (ADR 0154). A runner that outlived the first daemon took job 7 and job 8
// while none ran; job 8's queued listing, taken before the capture, comes after it.
#[tokio::test]
async fn a_job_a_capture_recorded_first_leaves_out_the_downtime_its_wait_went_through() {
    let dir = TestDir::new("store-capture-first");
    let path = dir.0.join("runnerd.db");
    let start = crate::time::now_ms() / SECOND * SECOND + SECOND;
    let first = Store::open_at(&path, start).unwrap();
    keep_since(&path, start);
    first.host_sample(HostSample {
        at: (start + 20 * SECOND) / SECOND,
        cpu: 0,
        memory: 0,
    });
    first.flush().await;
    drop(first);
    let restart = start + HOUR;
    let second = Store::open_at(&path, restart).unwrap();
    second.runner_started(runner("r", restart));
    let captured = restart + 5 * SECOND;
    second.claimed(7, 10, "r", captured);
    second.claimed(8, 10, "r", captured);
    second.queued(
        "o/r",
        &QueuedJob {
            created_at: start + 60 * SECOND,
            ..queued(8)
        },
        captured - SECOND,
    );
    for job in [7, 8] {
        let details = crate::github::JobDetails {
            id: job,
            run_id: 10,
            run_attempt: 1,
            name: "build".to_owned(),
            workflow_name: None,
            head_branch: None,
            html_url: None,
            labels: Vec::new(),
            created_at: Some(rfc3339(start + 60 * SECOND)),
            status: "in_progress".to_owned(),
            conclusion: None,
            started_at: Some(rfc3339(start + 180 * SECOND)),
            completed_at: None,
            runner_name: Some("r".to_owned()),
            steps: Vec::new(),
        };
        // GitHub is read again: the downtime is not given twice.
        second.details(details.clone());
        second.details(details);
    }
    second.flush().await;
    let unseen: Vec<(u64, u64)> = Connection::open(&path)
        .unwrap()
        .prepare("SELECT unseen, unseen_unlisted FROM jobs ORDER BY id")
        .unwrap()
        .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
        .unwrap()
        .collect::<rusqlite::Result<_>>()
        .unwrap();
    let waited = 120 * SECOND;
    assert_eq!(
        unseen,
        [(waited, waited), (waited, waited)],
        "both waited out the downtime whole, before any listing"
    );
}