runnerd-core 0.1.4

Config, domain logic and API types shared by runnerd and its clients
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
//! The job history: a SQLite database of jobs, their runners, steps, samples and logs, the events clients resume
//! from, cache sizes, the paused repositories and where the notifier got to. Owns the schema, the one writer and
//! the event broadcast. Writes are messages to a writer thread, so the engine never waits on the disk; reads and
//! pruning run on Tokio's blocking pool, pruning on its own connection, a batch at a time. It decides nothing about
//! scheduling and stores no secret: no token, no JIT config, no runner `_diag` log.

use std::collections::{BTreeMap, BTreeSet};
use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, TryLockError};
use std::time::Duration;

use rusqlite::{Connection, OpenFlags};
use tokio::sync::{OwnedSemaphorePermit, Semaphore, broadcast, mpsc, oneshot};
use tracing::warn;

use crate::admission::trace::Trace;
use crate::api::{self, Event, EventEnvelope};
use crate::config::History;
use crate::github::{JobDetails, NamedRun, QueuedJob};
use crate::observe::disk::{Window, Written};
use crate::observe::metrics::{CpuHeld, Events};
use crate::scheduler::warm;
use crate::sizing;
use crate::sync::lock;

mod lens;
mod outline;
mod prune;
mod read;
mod run;
mod write;

pub use run::RunJobs;

/// The schema a new database is created with, at [`VERSION`].
const SCHEMA: &str = include_str!("store/schema.sql");

/// The oldest schema version a database is brought up from: 0.0.19's (ADR 0073).
const OLDEST: u32 = 15;

/// The steps from [`OLDEST`] on, in order, each with the version it brings a database to. A change to the schema
/// adds one here and makes the same change to `schema.sql`, which a new database gets instead.
const MIGRATIONS: &[(u32, &str)] = &[
    (
        16,
        "ALTER TABLE jobs ADD COLUMN unseen INTEGER NOT NULL DEFAULT 0;",
    ),
    (
        17,
        "ALTER TABLE jobs ADD COLUMN cpu_periods INTEGER;\n\
         ALTER TABLE jobs ADD COLUMN cpu_throttled INTEGER;\n\
         ALTER TABLE jobs ADD COLUMN cpu_stalled INTEGER;\n\
         ALTER TABLE samples ADD COLUMN cpu_periods INTEGER;\n\
         ALTER TABLE samples ADD COLUMN cpu_throttled INTEGER;\n\
         ALTER TABLE samples ADD COLUMN cpu_stalled INTEGER;",
    ),
    (
        18,
        "ALTER TABLE jobs ADD COLUMN slot_waited INTEGER NOT NULL DEFAULT 0;",
    ),
    (
        19,
        "ALTER TABLE jobs ADD COLUMN seen_at INTEGER;\n\
         ALTER TABLE runners ADD COLUMN purpose TEXT;\n\
         ALTER TABLE runners ADD COLUMN stopped_for TEXT;",
    ),
    (
        20,
        "ALTER TABLE jobs ADD COLUMN runner_fault INTEGER NOT NULL DEFAULT 0;",
    ),
    (21, "ALTER TABLE runners ADD COLUMN cancel_by INTEGER;"),
    (
        22,
        "CREATE TABLE log_outlines (job INTEGER PRIMARY KEY NOT NULL, from_line INTEGER NOT NULL, \
         errors INTEGER NOT NULL, warnings INTEGER NOT NULL, groups_kept INTEGER NOT NULL, \
         marks_kept INTEGER NOT NULL, sections_kept INTEGER NOT NULL, dropped INTEGER NOT NULL, \
         hidden INTEGER NOT NULL);\n\
         CREATE TABLE log_groups (job INTEGER NOT NULL, start INTEGER NOT NULL, end_line INTEGER, \
         depth INTEGER NOT NULL, title TEXT NOT NULL, PRIMARY KEY (job, start)) WITHOUT ROWID;\n\
         CREATE INDEX log_groups_open ON log_groups (job, start) WHERE end_line IS NULL;\n\
         CREATE TABLE log_marks (job INTEGER NOT NULL, line INTEGER NOT NULL, kind TEXT NOT NULL, \
         PRIMARY KEY (job, line)) WITHOUT ROWID;",
    ),
    (
        23,
        "CREATE TABLE sessions (id INTEGER PRIMARY KEY, started_at INTEGER NOT NULL, sampled_at INTEGER);\n\
         ALTER TABLE records ADD COLUMN gone_at INTEGER;",
    ),
    (
        24,
        "ALTER TABLE jobs ADD COLUMN disk_written INTEGER;\n\
         ALTER TABLE jobs ADD COLUMN disk_write_ios INTEGER;\n\
         ALTER TABLE jobs ADD COLUMN disk_dirty INTEGER;\n\
         ALTER TABLE jobs ADD COLUMN disk_measure TEXT;\n\
         ALTER TABLE runners ADD COLUMN cgroup_gone_at INTEGER;\n\
         ALTER TABLE samples ADD COLUMN written INTEGER;\n\
         ALTER TABLE samples ADD COLUMN write_ios INTEGER;\n\
         ALTER TABLE samples ADD COLUMN dirty INTEGER;\n\
         CREATE TABLE disk_windows (start INTEGER PRIMARY KEY NOT NULL, end_at INTEGER NOT NULL, \
         device TEXT NOT NULL, store TEXT, drive INTEGER, slice INTEGER, daemon INTEGER, commits INTEGER) WITHOUT ROWID;\n\
         CREATE INDEX jobs_by_runner ON jobs (runner);",
    ),
];

/// The schema version this build reads and writes, the database's `user_version`: its last migration's.
const VERSION: u32 = match MIGRATIONS.last() {
    Some((version, _)) => *version,
    None => OLDEST,
};

/// Bytes of log lines sent and not yet stored, at most: a job's capture waits beyond it, so a noisy job or a slow
/// disk holds back that job's log, not the daemon's memory.
pub(crate) const LOG_BACKLOG: u64 = 8 << 20;

/// Host samples the writer holds before it stores them in one transaction: a minute's worth. One transaction per
/// sample wrote about 300 MB a day to an idle daemon's WAL. A crash loses the samples held.
const HOST_SAMPLE_BATCH: usize = 30;

/// How many events a slow client may fall behind before its stream ends; it reconnects with `Last-Event-ID`.
const EVENT_BUFFER: usize = 4096;

/// Why the database cannot be used.
#[derive(Debug, thiserror::Error)]
pub enum StoreError {
    /// SQLite failed.
    #[error("{0}")]
    Sqlite(#[from] rusqlite::Error),
    /// A newer runnerd wrote the database.
    #[error(
        "the job history database is version {found}, newer than the {supported} this runnerd reads; \
         run a newer runnerd or move the database aside"
    )]
    Newer {
        /// The database's version.
        found: u32,
        /// The newest version this build knows.
        supported: u32,
    },
    /// A runnerd before 0.0.19 wrote the database.
    #[error(
        "the job history database is version {found}, from a runnerd before 0.0.19; this runnerd reads versions \
         from {oldest} on: move the database aside"
    )]
    Older {
        /// The database's version.
        found: u32,
        /// The oldest version this build reads.
        oldest: u32,
    },
    /// The directory or file cannot be created.
    #[error("{0}")]
    Io(#[from] std::io::Error),
    /// A blocking read did not finish.
    #[error("a database read stopped: {0}")]
    Join(#[from] tokio::task::JoinError),
    /// The database's `-wal` file holds changes not yet in it, and this user cannot open the database the way that
    /// reads them.
    #[error(
        "{} holds changes not yet in the job history, which this user cannot read; run runnerd doctor as the user \
         the daemon runs as",
        .0.display()
    )]
    Unread(PathBuf),
}

impl StoreError {
    /// Whether a read stopped because the runtime shuts down, as when the daemon exits, rather than because the
    /// history failed: the runtime cancels the blocking reads it has not run.
    pub fn is_shutdown(&self) -> bool {
        matches!(self, StoreError::Join(join) if join.is_cancelled())
    }
}

/// A runner container as the history records it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RunnerRecord {
    /// The runner's name, also its container's.
    pub name: String,
    /// `owner/name`.
    pub repository: String,
    /// The profile it runs in.
    pub profile: String,
    /// 1-based.
    pub slot: usize,
    /// The Docker container ID.
    pub container: String,
    /// When the container started, in milliseconds since the epoch.
    pub started_at: u64,
    /// The container's memory cap in bytes.
    pub memory_cap: Option<u64>,
    /// The container's CPU limit in millicores.
    pub cpu_cap: Option<u64>,
    /// How it was admitted; `None` for one a restart adopted without a record of it.
    pub admission: Option<api::Admission>,
    /// Why it started: `job`, `warm` or `burst`; `None` for one a restart adopted.
    pub purpose: Option<&'static str>,
}

/// One sample of one runner's container.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SampleRecord {
    /// The runner.
    pub runner: String,
    /// When, in milliseconds since the epoch.
    pub at: u64,
    /// Millicores over the interval before it.
    pub cpu: u64,
    /// CPU time so far, in microseconds.
    pub cpu_usage: u64,
    /// Bytes without page cache (`observe::metrics::Usage::memory`).
    pub memory: u64,
    /// Bytes received since the container started.
    pub network_received: u64,
    /// Bytes sent since the container started.
    pub network_sent: u64,
    /// The container's `memory.events` so far.
    pub events: Events,
    /// What held the container's CPU back so far.
    pub cpu_held: CpuHeld,
    /// What it wrote to the disks under Docker's root so far; `None` where it was not read.
    pub written: Option<Written>,
    /// Page cache it had written and the kernel not yet to the disk; `None` where it was not read.
    pub dirty: Option<u64>,
}

/// The last measurement of the cache volumes.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Measured {
    /// When it was taken (seconds); none before the first measurement.
    pub at: Option<u64>,
    /// Each volume's size in bytes, by name.
    pub bytes: BTreeMap<String, u64>,
}

/// The profiles and repositories as the stored events last announced them: what clients resuming after those
/// events have.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Announced {
    /// The last `profile.changed` of each profile not removed since, by name.
    pub profiles: Vec<api::ProfileStatus>,
    /// The last `repository.changed` of each repository not removed since, by name.
    pub repositories: Vec<api::RepositoryStatus>,
}

/// A job the history has not closed, as the engine needs it after a restart.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OpenJob {
    /// GitHub's job ID.
    pub id: u64,
    /// `owner/name`.
    pub repository: String,
    /// `Queued` or `Running`.
    pub state: api::JobState,
    /// Its runner, once one took it.
    pub runner: Option<String>,
    /// When its runner was removed, in milliseconds since the epoch.
    pub runner_removed_at: Option<u64>,
    /// The workflow run it belongs to.
    pub run_id: u64,
    /// 1 for the run's first attempt.
    pub run_attempt: u32,
    /// The workflow's name, when known.
    pub workflow: Option<String>,
    /// When GitHub queued it, in milliseconds since the epoch.
    pub queued_at: u64,
    /// The labels it asked for, which tell its repository's pool.
    pub labels: Vec<String>,
}

/// The job a slot ran last, as the history has it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SlotJob {
    /// The profile the slot is in.
    pub profile: String,
    /// 1-based.
    pub slot: usize,
    /// The job, as admission tells jobs apart.
    pub key: crate::admission::Key,
}

/// Where a job's log capture continues: the runner's page file and the byte offset in it.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct LogPosition {
    /// The page number, from 1.
    pub page: u64,
    /// Bytes of that page already stored as lines.
    pub offset: u64,
}

/// How far the mirror's log was counted: the time of the last response counted, in nanoseconds since the epoch as
/// Docker logs it, and how many responses logged at exactly that time were counted, so the next read counts the
/// ones it was too early to see.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct MirrorPosition {
    /// Nanoseconds since the epoch.
    pub at: u64,
    /// Responses logged at `at` already counted.
    pub seen: u64,
}

/// What the writer thread is asked to do, in order.
#[derive(Debug)]
enum Write {
    Queued {
        repository: String,
        job: QueuedJob,
        seen_at: u64,
    },
    RunnerStarted(RunnerRecord),
    RunnerAdmission {
        runner: String,
        admission: api::Admission,
    },
    RunnerOomKilled(String),
    RunnerStoppedFor {
        runner: String,
        reason: &'static str,
    },
    RunnerCancelBy {
        runner: String,
        at: u64,
    },
    HostWaited {
        jobs: Vec<u64>,
        ms: u64,
    },
    SlotWaited {
        jobs: Vec<u64>,
        ms: u64,
    },
    BuildCache {
        runner: String,
        counts: api::BuildCacheCounts,
    },
    RunnerRemoved {
        runner: String,
        at: u64,
    },
    RunnersGone {
        running: Vec<String>,
        at: u64,
    },
    Claimed {
        job: u64,
        run_id: u64,
        runner: String,
        at: u64,
    },
    Details {
        details: JobDetails,
    },
    WorkflowNamed(NamedRun),
    Elsewhere {
        details: JobDetails,
    },
    Lost {
        job: u64,
        at: u64,
    },
    Unanswered {
        job: u64,
        at: u64,
    },
    Log {
        job: u64,
        lines: Vec<String>,
        /// Where the capture resumes once these lines are stored; `None` for all but the last part of a batch.
        position: Option<LogPosition>,
        max_bytes: u64,
        /// The lines' share of `LOG_BACKLOG`, given back once they are stored.
        _backlog: OwnedSemaphorePermit,
    },
    Samples(Vec<SampleRecord>),
    CgroupsGone {
        runners: Vec<String>,
        at: u64,
    },
    DiskWindow(Window),
    HostSample(api::HostSample),
    MirrorBytes {
        bytes: Vec<(String, u64)>,
        until: MirrorPosition,
        /// Whether the count committed.
        done: oneshot::Sender<bool>,
    },
    CacheSizes {
        at: u64,
        sizes: BTreeMap<String, u64>,
    },
    /// Boxed: an event with a job's summary is many times the other writes' size.
    Event(Box<Event>),
    RecordMade(api::Record),
    RecordSkipped {
        job: u64,
        why: String,
    },
    RecordRemoved {
        job: u64,
        at: u64,
        by: api::RecordRemoval,
    },
    RecordsListed {
        jobs: Vec<u64>,
        at: u64,
    },
    Paused {
        repositories: Vec<String>,
        /// Whether it committed.
        done: oneshot::Sender<bool>,
    },
    Visibility {
        repository: String,
        visibility: api::Visibility,
        at: u64,
    },
    SizingIntent {
        change: api::SizingChange,
        /// Its row, once it committed.
        done: oneshot::Sender<Option<i64>>,
    },
    SizingResolved {
        id: i64,
        resolution: Resolution,
        /// Whether it committed.
        done: oneshot::Sender<bool>,
    },
    Notify {
        change: NotifyChange,
        /// Whether it committed.
        done: oneshot::Sender<bool>,
    },
    RunnerNamed {
        secs: u64,
        /// Whether it committed.
        done: oneshot::Sender<bool>,
    },
    Flush(oneshot::Sender<()>),
}

/// What the notifier records.
#[derive(Debug)]
enum NotifyChange {
    /// Sets a target's keys and removes others.
    Record {
        target: String,
        set: Vec<(String, u64)>,
        remove: Vec<String>,
    },
    /// Forgets every target but these.
    Keep(Vec<String>),
}

/// What became of a pending auto sizing change.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Resolution {
    /// The config file has it: announced as `sizing.changed`.
    Done,
    /// The file was not written.
    Failed,
    /// Left pending, and the config no longer says whether it was written: it keeps its interval and stays listed,
    /// with what it replaced.
    Uncertain,
}

/// Auto sizing as the history keeps it: when each profile last changed, and changes not yet known to be written.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct AutoSizing {
    /// The latest pending or done change of each profile, in seconds, however old.
    pub last: BTreeMap<String, u64>,
    /// Changes recorded before their file was written and not resolved since, by row.
    pub pending: Vec<(i64, api::SizingChange)>,
    /// Each profile's `cpus` as the daemon ran them, from its `profile.changed` events, oldest first: those `[history]
    /// events` keeps, and each profile's latest however old.
    pub cpus: Vec<sizing::CpusSet>,
    /// Each profile's `memory` as the daemon ran it, from the same events.
    pub memory: Vec<sizing::MemorySet>,
    /// Each profile's slots as the daemon ran them, from the same events: one more for each slot a lowering left a
    /// runner in until it ends.
    pub slots: Vec<sizing::SlotsSet>,
}

/// The job history. Cloning shares it.
#[derive(Debug, Clone)]
pub struct Store {
    writes: mpsc::UnboundedSender<Write>,
    reader: Arc<Mutex<Connection>>,
    /// Taken with `try_lock`, so one prune runs at a time; one that panicked midway left rows the next one deletes.
    pruner: Arc<Mutex<Connection>>,
    /// Set by `close`: a running prune stops after its batch, and none starts.
    closed: Arc<AtomicBool>,
    /// Host samples sent and not yet stored, oldest first; reads include them. The writer removes them once stored.
    // Locked with `sync::lock`: every critical section is a push, a copy or a drain.
    held: Arc<Mutex<Vec<api::HostSample>>>,
    /// The last host sample sent, for admission to read without a query.
    latest_host: Arc<Mutex<Option<api::HostSample>>>,
    /// The last reading of the host's memory, which is not stored.
    latest_host_memory: Arc<Mutex<Option<api::HostMemory>>>,
    /// The jobs the last plan had waiting for the host's memory, and when (ms); insights read it.
    host_waiting: Arc<Mutex<(BTreeSet<u64>, u64)>>,
    /// Each running runner's last sample, for admission to read without a query.
    latest_samples: Arc<Mutex<BTreeMap<String, SampleRecord>>>,
    /// The runners whose container's cgroup the sampler saw go, until they are removed or it reads them again.
    gone: Arc<Mutex<BTreeSet<String>>>,
    events: broadcast::Sender<Arc<EventEnvelope>>,
    backlog: Arc<Semaphore>,
}

impl Store {
    /// Opens the database at `path`, creating it and its directory (mode 0700) if missing, with the schema when it
    /// is new, and brings it to the schema this build reads. Refuses a database from before 0.0.19 or from a newer
    /// runnerd. The time since the history's last sign of life is taken as time no runnerd ran: the jobs queued in
    /// it leave it out of their waits.
    pub fn open(path: &Path) -> Result<Store, StoreError> {
        Store::open_at(path, crate::time::now_ms())
    }

    /// [`Store::open`] at `now` (ms).
    fn open_at(path: &Path, now: u64) -> Result<Store, StoreError> {
        if let Some(dir) = path.parent().filter(|dir| !dir.as_os_str().is_empty()) {
            std::fs::DirBuilder::new()
                .recursive(true)
                .mode(0o700)
                .create(dir)?;
        }
        // Owner-only, as SQLite then makes its `-wal` and `-shm` files: logs may say more than the owner wants shared.
        std::fs::OpenOptions::new()
            .create(true)
            .append(true)
            .mode(0o600)
            .open(path)?;
        let mut db = Connection::open(path)?;
        configure(&db)?;
        prepare(&mut db)?;
        let downtime = Downtime::before(&mut db, now)?;
        db.execute("INSERT INTO sessions (started_at) VALUES (?1)", [now])?;
        let session = db.last_insert_rowid();
        let reader = Connection::open_with_flags(
            path,
            OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
        )?;
        configure(&reader)?;
        let pruner = Connection::open(path)?;
        configure(&pruner)?;
        let (events, _) = broadcast::channel(EVENT_BUFFER);
        let (writes, receiver) = mpsc::unbounded_channel();
        let thread_events = events.clone();
        let held = Arc::new(Mutex::new(Vec::new()));
        let thread_held = held.clone();
        std::thread::Builder::new()
            .name("runnerd-store".to_owned())
            .spawn(move || {
                write::run(
                    write::Started {
                        db,
                        downtime,
                        session,
                    },
                    receiver,
                    &thread_events,
                    &thread_held,
                );
            })?;
        Ok(Store {
            writes,
            reader: Arc::new(Mutex::new(reader)),
            pruner: Arc::new(Mutex::new(pruner)),
            closed: Arc::new(AtomicBool::new(false)),
            held,
            latest_host: Arc::new(Mutex::new(None)),
            latest_host_memory: Arc::new(Mutex::new(None)),
            host_waiting: Arc::new(Mutex::new((BTreeSet::new(), 0))),
            latest_samples: Arc::new(Mutex::new(BTreeMap::new())),
            gone: Arc::new(Mutex::new(BTreeSet::new())),
            events,
            backlog: Arc::new(Semaphore::new(
                usize::try_from(LOG_BACKLOG).unwrap_or(usize::MAX),
            )),
        })
    }

    fn send(&self, write: Write) {
        // Fails only once the writer thread is gone, after a SQLite error it logged.
        let _ = self.writes.send(write);
    }

    /// Records a queued job the repository's runners may run, which a listing answered at `seen_at` (ms) showed;
    /// announces it the first time, and keeps the first listing's time. A job a runner claimed first keeps its state
    /// and gets `seen_at` if no listing showed it before and this one answered before it started.
    pub fn queued(&self, repository: &str, job: &QueuedJob, seen_at: u64) {
        self.send(Write::Queued {
            repository: repository.to_owned(),
            job: job.clone(),
            seen_at,
        });
    }

    /// Records a runner container that started, or one an earlier daemon started and this one adopted.
    pub fn runner_started(&self, runner: RunnerRecord) {
        self.send(Write::RunnerStarted(runner));
    }

    /// Records that a runner's container and registration were removed at `at` (ms).
    pub fn runner_removed(&self, runner: &str, at: u64) {
        lock(&self.latest_samples).remove(runner);
        lock(&self.gone).remove(runner);
        self.send(Write::RunnerRemoved {
            runner: runner.to_owned(),
            at,
        });
    }

    /// Records every runner the history holds open, except the `running` ones, removed at `at` (ms): what recovery
    /// found gone, so it stops counting as holding a slot.
    pub fn runners_gone(&self, running: Vec<String>, at: u64) {
        lock(&self.latest_samples).retain(|runner, _| running.contains(runner));
        lock(&self.gone).retain(|runner| running.contains(runner));
        self.send(Write::RunnersGone { running, at });
    }

    /// Records that a runner took a job at `at` (ms).
    pub fn claimed(&self, job: u64, run_id: u64, runner: &str, at: u64) {
        self.send(Write::Claimed {
            job,
            run_id,
            runner: runner.to_owned(),
            at,
        });
    }

    /// Records a running job's progress and steps as GitHub reports them; announces changed steps.
    pub fn details(&self, details: JobDetails) {
        self.send(Write::Details { details });
    }

    /// Names the workflow of a run's jobs, which were recorded under GitHub's name for the run because the workflow's
    /// could not be read then (ADR 0140).
    pub fn workflow_named(&self, named: NamedRun) {
        self.send(Write::WorkflowNamed(named));
    }

    /// Closes a queued job that left the queue without a runner of this daemon: another machine's runner took it, or
    /// none did and it has no start.
    pub fn elsewhere(&self, details: JobDetails) {
        self.send(Write::Elsewhere { details });
    }

    /// Closes a queued job that left the queue and that GitHub no longer answers for, at `at`: none of this daemon's
    /// runners took it, so it reads as run elsewhere, with no conclusion.
    pub fn unanswered(&self, job: u64, at: u64) {
        self.send(Write::Unanswered { job, at });
    }

    /// Closes a job whose container is gone and that GitHub did not report completed in time.
    pub fn lost(&self, job: u64, at: u64) {
        self.send(Write::Lost { job, at });
    }

    /// Appends log lines to a job, up to `max_bytes` per job, and records where the capture continues. Waits while
    /// `LOG_BACKLOG` bytes of lines are waiting to be stored; a larger batch goes in parts, each waiting its turn.
    pub async fn log(&self, job: u64, lines: Vec<String>, position: LogPosition, max_bytes: u64) {
        let size = |line: &String| line.len() as u64 + 1;
        let mut lines = lines.into_iter().map(|mut line| {
            // No line takes the whole backlog; the capture already cuts lines far shorter.
            if size(&line) > LOG_BACKLOG {
                let mut end = usize::try_from(LOG_BACKLOG / 2).unwrap_or(usize::MAX);
                while !line.is_char_boundary(end) {
                    end -= 1;
                }
                line.truncate(end);
            }
            line
        });
        let mut next = lines.next();
        loop {
            let (mut part, mut bytes) = (Vec::new(), 0);
            while let Some(line) = next.take_if(|line| bytes + size(line) <= LOG_BACKLOG) {
                bytes += size(&line);
                part.push(line);
                next = lines.next();
            }
            let last = next.is_none();
            let share = u32::try_from(bytes.max(1)).unwrap_or(u32::MAX);
            // The semaphore is never closed.
            let Ok(backlog) = self.backlog.clone().acquire_many_owned(share).await else {
                return;
            };
            self.send(Write::Log {
                job,
                lines: part,
                position: last.then_some(position),
                max_bytes,
                _backlog: backlog,
            });
            if last {
                return;
            }
        }
    }

    /// Records samples of runner containers.
    pub fn samples(&self, samples: Vec<SampleRecord>) {
        let mut latest = lock(&self.latest_samples);
        for sample in &samples {
            latest.insert(sample.runner.clone(), sample.clone());
        }
        drop(latest);
        self.send(Write::Samples(samples));
    }

    /// Records a sample of the slice and sends it to every subscriber at once; it is stored with the next ones, a
    /// minute's worth in one transaction, or at `flush`.
    pub fn host_sample(&self, sample: api::HostSample) {
        *lock(&self.latest_host) = Some(sample.clone());
        lock(&self.held).push(sample.clone());
        self.send(Write::HostSample(sample));
    }

    /// A runner's last sample, stored or not; `None` before its first sample or once it is removed.
    pub fn latest_sample(&self, runner: &str) -> Option<SampleRecord> {
        lock(&self.latest_samples).get(runner).cloned()
    }

    /// Notes that the cgroups of these runners' containers are gone (`metrics::Sampled::gone`), for admission to read
    /// until they are removed. Kept in memory only: a restart finds such a container stopped.
    pub fn containers_gone(&self, runners: Vec<String>) {
        lock(&self.gone).extend(runners);
    }

    /// Records that these runners' containers' cgroups were gone at `at` (ms), one sample after they were last read
    /// (`metrics::Sampled::ended`): what they wrote is measured to their end (ADR 0143). A job already closed gets
    /// its figure again.
    pub fn containers_ended(&self, runners: Vec<String>, at: u64) {
        if !runners.is_empty() {
            self.send(Write::CgroupsGone { runners, at });
        }
    }

    /// Records a window of what the disks under Docker's root wrote.
    pub fn disk_window(&self, window: Window) {
        self.send(Write::DiskWindow(window));
    }

    /// Notes that these runners' containers' cgroups were read (`metrics::Sampled::read`): one marked gone came back,
    /// as a `docker restart` brings it, and holds memory again.
    pub fn containers_read(&self, runners: &[String]) {
        let mut gone = lock(&self.gone);
        for runner in runners {
            gone.remove(runner);
        }
    }

    /// Whether a runner's container's cgroup is gone, though its slot may hold it a while yet.
    pub fn container_gone(&self, runner: &str) -> bool {
        lock(&self.gone).contains(runner)
    }

    /// Records what a runner reserves now, when measured admission changed it after the start.
    pub fn runner_admission(&self, runner: &str, admission: api::Admission) {
        self.send(Write::RunnerAdmission {
            runner: runner.to_owned(),
            admission,
        });
    }

    /// Records that Docker saw the kernel kill a process in a runner's container for memory, which its samples may
    /// have missed: one in the last interval before the container stopped.
    pub fn runner_oom_killed(&self, runner: &str) {
        self.send(Write::RunnerOomKilled(runner.to_owned()));
    }

    /// Records when a forced pause that cancelled a runner's run removes it if it still runs, in ms since the epoch,
    /// so a restart keeps that deadline.
    pub fn runner_cancel_by(&self, runner: &str, at: u64) {
        self.send(Write::RunnerCancelBy {
            runner: runner.to_owned(),
            at,
        });
    }

    /// Records why an idle runner was removed, as `scheduler::StopReason::as_str` names it.
    pub fn runner_stopped_for(&self, runner: &str, reason: &'static str) {
        self.send(Write::RunnerStoppedFor {
            runner: runner.to_owned(),
            reason,
        });
    }

    /// Adds what the shared build cache served a runner and took from it, and records whether its job could write.
    pub fn build_cache(&self, runner: &str, counts: api::BuildCacheCounts) {
        self.send(Write::BuildCache {
            runner: runner.to_owned(),
            counts,
        });
    }

    /// Keeps the latest reading of the host's memory for admission and the status; it is not stored.
    pub fn host_memory(&self, memory: api::HostMemory) {
        *lock(&self.latest_host_memory) = Some(memory);
    }

    /// The latest reading of the host's memory; `None` before the first.
    pub fn latest_host_memory(&self) -> Option<api::HostMemory> {
        lock(&self.latest_host_memory).clone()
    }

    /// Keeps which `jobs` the plan at `at` (ms) had waiting for memory the host lacks (ADR 0103), for insights to read
    /// with the reading; it is not stored. A plan that finds none replaces the last one's.
    pub fn host_waiting(&self, jobs: BTreeSet<u64>, at: u64) {
        *lock(&self.host_waiting) = (jobs, at);
    }

    /// Adds `ms` to how long each of `jobs` waited because the host's memory could not take its runner, which
    /// sizing does not read as a wait for a slot.
    pub fn host_waited(&self, jobs: Vec<u64>, ms: u64) {
        self.send(Write::HostWaited { jobs, ms });
    }

    /// Adds `ms` to how long each of `jobs` waited with no slot of its profile free while the budgets had room for
    /// its runner: what auto sizing reads as a wait for a slot (ADR 0090).
    pub fn slot_waited(&self, jobs: Vec<u64>, ms: u64) {
        self.send(Write::SlotWaited { jobs, ms });
    }

    /// The last host sample sent, stored or not; `None` before the first.
    pub fn latest_host_sample(&self) -> Option<api::HostSample> {
        lock(&self.latest_host).clone()
    }

    /// Adds bytes the mirror served to runners, counted from its log up to `until`, and says whether that was
    /// stored: if not, the caller counts the same lines again.
    pub async fn mirror_bytes(&self, bytes: Vec<(String, u64)>, until: MirrorPosition) -> bool {
        let (done, committed) = oneshot::channel();
        self.send(Write::MirrorBytes { bytes, until, done });
        committed.await.unwrap_or(false)
    }

    /// Records volume sizes in bytes, by name, measured at `at` (ms).
    pub fn cache_sizes(&self, at: u64, sizes: BTreeMap<String, u64>) {
        self.send(Write::CacheSizes { at, sizes });
    }

    /// Stores a durable event, or sends a live one, to every subscriber.
    pub fn event(&self, event: Event) {
        self.send(Write::Event(Box::new(event)));
    }

    /// Records that a failed job's runner was kept as `record`, and announces it.
    pub fn record_made(&self, record: api::Record) {
        self.send(Write::RecordMade(record));
    }

    /// Records why a failed job's runner was not kept.
    pub fn record_skipped(&self, job: u64, why: String) {
        self.send(Write::RecordSkipped { job, why });
    }

    /// Records that a job's record was removed at `at` (ms), and why, and announces it.
    pub fn record_removed(&self, job: u64, at: u64, by: api::RecordRemoval) {
        self.send(Write::RecordRemoved { job, at, by });
    }

    /// Records that Docker listed the records of `jobs`, and no others, at `at` (ms): a record the history notes as
    /// kept whose job is not among them was removed outside runnerd, and is noted gone and announced as
    /// `record.gone`. Only an inventory Docker answered is sent; one it did not changes nothing.
    pub fn records_listed(&self, jobs: Vec<u64>, at: u64) {
        self.send(Write::RecordsListed { jobs, at });
    }

    /// Records the paused repositories, all of them, in place of the ones recorded before. The write is sent at
    /// once, in order with every other; the future says whether it committed.
    pub fn paused(&self, repositories: Vec<String>) -> impl Future<Output = bool> + use<> {
        let (done, committed) = oneshot::channel();
        self.send(Write::Paused { repositories, done });
        async move { committed.await.unwrap_or(false) }
    }

    /// Records that this machine named a runner for `secs`, the second its name ends in, before GitHub or Docker
    /// knows the name: a restart names the next runner later even when nothing else recorded this one. Says whether
    /// it committed.
    pub async fn runner_named(&self, secs: u64) -> bool {
        let (done, committed) = oneshot::channel();
        self.send(Write::RunnerNamed { secs, done });
        // Waited for on a blocking thread, as a read is: the runtime then knows the start waits for the disk, and a
        // paused test clock does not run on meanwhile.
        tokio::task::spawn_blocking(move || committed.blocking_recv().unwrap_or(false))
            .await
            .unwrap_or(false)
    }

    /// Records a repository's visibility as GitHub gave it at `at` (ms), in place of the one recorded before.
    pub fn visibility(&self, repository: String, visibility: api::Visibility, at: u64) {
        self.send(Write::Visibility {
            repository,
            visibility,
            at,
        });
    }

    /// Records a change auto sizing is about to write, as pending, before the file is, synced to disk. The future
    /// has its row once it committed, `None` if it did not: then nothing may be written.
    pub fn sizing_intent(
        &self,
        change: api::SizingChange,
    ) -> impl Future<Output = Option<i64>> + use<> {
        let (done, committed) = oneshot::channel();
        self.send(Write::SizingIntent { change, done });
        async move { committed.await.ok().flatten() }
    }

    /// Records what became of a pending change, synced to disk; a done one is announced as `sizing.changed` in the
    /// same transaction. The future says whether it committed.
    pub fn sizing_resolved(
        &self,
        id: i64,
        resolution: Resolution,
    ) -> impl Future<Output = bool> + use<> {
        let (done, committed) = oneshot::channel();
        self.send(Write::SizingResolved {
            id,
            resolution,
            done,
        });
        async move { committed.await.unwrap_or(false) }
    }

    /// Records what the notifier delivered to a target: sets `set`'s keys and removes `remove`'s, in one
    /// transaction. The future says whether it committed.
    pub fn notify_record(
        &self,
        target: &str,
        set: Vec<(String, u64)>,
        remove: Vec<String>,
    ) -> impl Future<Output = bool> + use<> {
        let (done, committed) = oneshot::channel();
        self.send(Write::Notify {
            change: NotifyChange::Record {
                target: target.to_owned(),
                set,
                remove,
            },
            done,
        });
        async move { committed.await.unwrap_or(false) }
    }

    /// Forgets what the notifier kept for every target but `targets`. The future says whether it committed.
    pub fn notify_keep(&self, targets: Vec<String>) -> impl Future<Output = bool> + use<> {
        let (done, committed) = oneshot::channel();
        self.send(Write::Notify {
            change: NotifyChange::Keep(targets),
            done,
        });
        async move { committed.await.unwrap_or(false) }
    }

    /// Deletes what is older than `history` keeps, at `now` (ms), off the writer, in batches the writer's
    /// transactions fit between. A prune while the last one runs does nothing. The handle finishes with it; dropping
    /// it lets it run.
    pub fn prune(&self, now: u64, history: &History) -> tokio::task::JoinHandle<()> {
        let pruner = self.pruner.clone();
        let closed = self.closed.clone();
        let history = history.clone();
        tokio::task::spawn_blocking(move || {
            let db = match pruner.try_lock() {
                Ok(db) => db,
                Err(TryLockError::Poisoned(poisoned)) => poisoned.into_inner(),
                Err(TryLockError::WouldBlock) => return,
            };
            let db = prune::Db {
                connection: &db,
                stop: &closed,
            };
            if !db.stop.load(Ordering::Relaxed) {
                prune::run(&db, now, &history);
            }
        })
    }

    /// Stops a running prune after its batch, and any later one before it starts, so a shutdown does not wait for
    /// one. What a stopped prune left, the next daemon's deletes. Writes go on as before.
    pub fn close(&self) {
        self.closed.store(true, Ordering::Relaxed);
    }

    /// Waits until every write sent before is done.
    pub async fn flush(&self) {
        let (done, wait) = oneshot::channel();
        self.send(Write::Flush(done));
        let _ = wait.await;
    }

    /// Every event from now on, durable or live. Subscribe before reading what was stored, then skip what the read
    /// returned, and nothing is missed or sent twice.
    pub fn subscribe(&self) -> broadcast::Receiver<Arc<EventEnvelope>> {
        self.events.subscribe()
    }

    async fn read<T: Send + 'static>(
        &self,
        query: impl FnOnce(&Connection) -> rusqlite::Result<T> + Send + 'static,
    ) -> Result<T, StoreError> {
        let reader = self.reader.clone();
        // A panic during a read leaves the connection usable: reads change nothing.
        Ok(tokio::task::spawn_blocking(move || query(&lock(&reader))).await??)
    }

    /// One page of jobs as `query` asks, which [`api::JobQuery::check`] passed (ADR 0135).
    pub async fn jobs(&self, query: api::JobQuery) -> Result<api::JobList, StoreError> {
        self.read(move |db| read::jobs(db, &query)).await
    }

    /// What each slot ran over `window` (ADR 0135), with the host samples not yet stored counted as watched time.
    pub async fn timeline(&self, window: api::Span) -> Result<api::SlotTimeline, StoreError> {
        let held: Vec<u64> = lock(&self.held).iter().map(|sample| sample.at).collect();
        self.read(move |db| lens::timeline(db, window, &held, api::SAMPLE_INTERVAL))
            .await
    }

    /// Job `id` beside the job `with` names, or its baseline (ADR 0135); `None` without either job.
    pub async fn compare(
        &self,
        id: u64,
        with: Option<u64>,
    ) -> Result<Option<api::JobComparison>, StoreError> {
        self.read(move |db| lens::compare(db, id, with)).await
    }

    /// A job with its steps.
    pub async fn job(&self, id: u64) -> Result<Option<api::Job>, StoreError> {
        self.read(move |db| read::job(db, id)).await
    }

    /// Up to `limit` other jobs of `repository`'s run `run` than `except`, newest attempt first (ADR 0134).
    pub async fn run_jobs(
        &self,
        repository: &str,
        run: u64,
        except: u64,
        limit: usize,
    ) -> Result<RunJobs, StoreError> {
        let repository = repository.to_owned();
        self.read(move |db| run::run_jobs(db, &repository, run, except, limit))
            .await
    }

    /// A job's samples, oldest first; `None` if there is no such job.
    pub async fn job_samples(
        &self,
        id: u64,
        interval: Duration,
    ) -> Result<Option<api::SampleSeries>, StoreError> {
        self.read(move |db| read::job_samples(db, id, interval))
            .await
    }

    /// Up to `limit` log lines of a job from line `from`, with their numbers.
    pub async fn log_lines(
        &self,
        job: u64,
        from: u64,
        limit: u64,
    ) -> Result<Vec<(u64, String)>, StoreError> {
        self.read(move |db| read::log_lines(db, job, from, limit))
            .await
    }

    /// A job's log as structure (ADR 0133); `None` without the job.
    pub async fn log_outline(&self, job: u64) -> Result<Option<api::LogOutline>, StoreError> {
        self.read(move |db| outline::outline(db, job)).await
    }

    /// Up to `limit` styled lines of a job's log from line `from`, fewer once they hold 4 MiB of text; each shows at most
    /// [`api::LINE_BYTES_MAX`] bytes of its line.
    pub async fn styled_lines(
        &self,
        job: u64,
        from: u64,
        limit: u64,
    ) -> Result<Vec<api::StyledLine>, StoreError> {
        self.read(move |db| outline::styled_lines(db, job, from, limit))
            .await
    }

    /// One page of a search of a job's log, each batch of lines its own read, so the search never holds the reader
    /// for a whole log.
    pub async fn search_log(
        &self,
        job: u64,
        mut search: api::Search,
    ) -> Result<api::LogSearch, StoreError> {
        // Each batch is styled and marked where it is read, off the async workers.
        while let Some(want) = search.wants() {
            search = self
                .read(move |db| {
                    search.read(read::lines_from(db, job, want)?);
                    Ok(search)
                })
                .await?;
        }
        Ok(search.finish())
    }

    /// Up to `limit` durable events after `after`, oldest first.
    pub async fn events_after(
        &self,
        after: u64,
        limit: u64,
    ) -> Result<Vec<EventEnvelope>, StoreError> {
        self.read(move |db| read::events_after(db, after, limit))
            .await
    }

    /// The changes auto sizing wrote since `since` (ms), oldest first.
    pub async fn sizing_changes(&self, since: u64) -> Result<Vec<api::SizingChange>, StoreError> {
        self.read(move |db| read::sizing_changes(db, since)).await
    }

    /// When each profile last changed, and the changes still pending.
    pub async fn auto_sizing(&self) -> Result<AutoSizing, StoreError> {
        self.read(read::auto_sizing).await
    }

    /// The profiles and repositories as the stored events last announced them, so a restarted daemon announces
    /// only what changed since, including what its config no longer has. Blocks on the database: for startup.
    pub fn announced(&self) -> Result<Announced, StoreError> {
        Ok(read::announced(&lock(&self.reader))?)
    }

    /// The repositories recorded paused, for the engine to start with. Blocks on the database: for startup.
    pub fn paused_repositories(&self) -> Result<Vec<String>, StoreError> {
        Ok(read::paused(&lock(&self.reader))?)
    }

    /// The slice's samples since `since` (seconds), oldest first, the ones not yet stored included.
    pub async fn host_samples(&self, since: u64) -> Result<Vec<api::HostSample>, StoreError> {
        // Copied before the read: the writer removes a sample only once it is stored, so each one is in the copy,
        // the read, or both.
        let held = lock(&self.held).clone();
        let mut samples = self.read(move |db| read::host_samples(db, since)).await?;
        let stored = samples.last().map_or(0, |sample| sample.at);
        samples.extend(
            held.into_iter()
                .filter(|sample| sample.at >= since && sample.at > stored),
        );
        Ok(samples)
    }

    /// The jobs runnerd's runners started and finished from `since` until `now` (seconds).
    pub async fn activity(&self, since: u64, now: u64) -> Result<api::Activity, StoreError> {
        self.read(move |db| read::activity(db, since, now)).await
    }

    /// Finished jobs per repository and bucket.
    pub async fn stats(&self, query: api::StatsQuery, now: u64) -> Result<api::Stats, StoreError> {
        self.read(move |db| read::stats(db, &query, now)).await
    }

    /// What sizing reads: the finished jobs queued at or after `since` (ms) and the runners since.
    pub async fn sizing(&self, since: u64) -> Result<sizing::History, StoreError> {
        self.read(move |db| read::sizing(db, since)).await
    }

    /// What pricing at GitHub's rates reads: the completed and lost jobs dated at or after `since` (ms), each
    /// repository's visibility and since when the history holds every final job.
    pub async fn pricing(&self, since: u64) -> Result<api::PricingHistory, StoreError> {
        self.read(move |db| read::pricing(db, since)).await
    }

    /// The jobs completed or lost that were queued at or after `since` (ms), as insights read them, from when the
    /// history holds every job, and the queued jobs the last plan had waiting for the host's memory.
    pub async fn insights(&self, since: u64) -> Result<crate::insights::History, StoreError> {
        let memory = self.latest_host_memory();
        let (waiting, seen) = lock(&self.host_waiting).clone();
        self.read(move |db| {
            let mut history = read::insights(db, since)?;
            if let Some(memory) = memory.filter(|_| !waiting.is_empty()) {
                let jobs = read::held_jobs(db, &waiting)?;
                history.host =
                    (!jobs.is_empty()).then_some(crate::insights::HostHold { memory, seen, jobs });
            }
            Ok(history)
        })
        .await
    }

    /// The newest durable event's ID; 0 before the first.
    pub async fn last_event(&self) -> Result<u64, StoreError> {
        self.read(read::last_event).await
    }

    /// What the notifier kept for each target, by target, then key.
    pub async fn notify_state(
        &self,
    ) -> Result<BTreeMap<String, BTreeMap<String, u64>>, StoreError> {
        self.read(read::notify_state).await
    }

    /// The jobs queued at or after `since` (ms), whatever became of them, as warm runners learn from them.
    pub async fn warm(&self, since: u64) -> Result<Vec<warm::Seen>, StoreError> {
        self.read(move |db| read::warm(db, since)).await
    }

    /// The last measurement of every volume, with its time (seconds), by name.
    pub async fn cache_sizes_now(&self) -> Result<Measured, StoreError> {
        self.read(read::latest_cache_sizes).await
    }

    /// What the build cache served and took from the runners started since `since` (ms), summed.
    pub async fn build_cache_since(&self, since: u64) -> Result<api::BuildCacheCounts, StoreError> {
        self.read(move |db| read::build_cache_since(db, since))
            .await
    }

    /// Every volume measurement since `since` (seconds).
    pub async fn cache_history(&self, since: u64) -> Result<Vec<api::CacheSize>, StoreError> {
        self.read(move |db| read::cache_history(db, since)).await
    }

    /// The jobs not yet closed, for the engine after a restart.
    pub async fn open_jobs(&self) -> Result<Vec<OpenJob>, StoreError> {
        self.read(read::open_jobs).await
    }

    /// The newest named job each slot ran, for the engine after a restart.
    pub async fn slot_jobs(&self) -> Result<Vec<SlotJob>, StoreError> {
        self.read(read::slot_jobs).await
    }

    /// The latest second a runner was named for (`runner_named`), or any runner name starting with `prefix` carries,
    /// of a runner or of a job's runner: the engine names runners later after a restart, so no name it used before
    /// is used twice.
    pub async fn last_runner_second(&self, prefix: String) -> Result<Option<u64>, StoreError> {
        self.read(move |db| read::last_runner_second(db, &prefix))
            .await
    }

    /// The job a runner took, if the history knows it.
    pub async fn runner_job(&self, runner: &str) -> Result<Option<u64>, StoreError> {
        let runner = runner.to_owned();
        self.read(move |db| read::runner_job(db, &runner)).await
    }

    /// How a runner was admitted, with what it reserves since, if the history knows it.
    pub async fn runner_admitted(
        &self,
        runner: &str,
    ) -> Result<Option<api::Admission>, StoreError> {
        let runner = runner.to_owned();
        self.read(move |db| read::runner_admission(db, &runner))
            .await
    }

    /// When a forced pause that cancelled a runner's run removes it if it still runs, in ms since the epoch, if one
    /// did.
    pub async fn runner_cancel_deadline(&self, runner: &str) -> Result<Option<u64>, StoreError> {
        let runner = runner.to_owned();
        self.read(move |db| read::runner_cancel_deadline(db, &runner))
            .await
    }

    /// The memory cap a runner started with, as its record kept it, for when Docker cannot tell.
    pub async fn runner_memory_cap(&self, runner: &str) -> Result<Option<u64>, StoreError> {
        let runner = runner.to_owned();
        self.read(move |db| read::runner_memory_cap(db, &runner))
            .await
    }

    /// Where a job's log capture continues.
    pub async fn log_position(&self, job: u64) -> Result<LogPosition, StoreError> {
        self.read(move |db| read::log_position(db, job)).await
    }

    /// How far the mirror's log was counted.
    pub async fn mirror_log_until(&self) -> Result<MirrorPosition, StoreError> {
        self.read(read::mirror_log_until).await
    }
}

/// What sizing reads from the database at `path`, opened read-only and left as it is, as `runnerd doctor` reads it
/// while no daemon runs; `None` when there is no database yet. In a directory it may write, SQLite keeps the
/// `-wal` and `-shm` files a read of a WAL database needs next to it; in one it may not, it reads the file as it is,
/// or refuses when the `-wal` holds changes the file does not have yet.
pub fn sizing_at(path: &Path, since: u64) -> Result<Option<sizing::History>, StoreError> {
    read_only(path, |db| sizing_of(db, since))
}

/// How the jobs of the history at `path` that started from `since` (ms) ran, read as [`sizing_at`] reads, for a replay
/// that runs them so (`admission::simulate`): their samples, when the daemon could name each, what the host's programs
/// held. `overhead` is what the slice holds beside its runners, in bytes; `None` measures it from the history's busy
/// moments, as `read::trace` says. `named_by` is the longest a known job waits to be read: the poll interval.
pub fn trace_at(
    path: &Path,
    since: u64,
    named_by: u64,
    overhead: Option<u64>,
) -> Result<Option<Trace>, StoreError> {
    read_only(path, |db| {
        if version(db)? == 0 {
            return Ok(Trace::default());
        }
        Ok(read::trace(db, since, named_by, overhead)?)
    })
}

/// When the last job of the history at `path` was queued, in ms, read as [`sizing_at`] reads; `None` with no job yet.
/// A replay's window counts back from it, so a backup replays its own last hours.
pub fn last_queued_at(path: &Path) -> Result<Option<u64>, StoreError> {
    let last = read_only(path, |db| {
        if version(db)? == 0 {
            return Ok(None);
        }
        Ok(db.query_row("SELECT MAX(queued_at) FROM jobs", [], |row| row.get(0))?)
    })?;
    Ok(last.flatten())
}

/// What `read` finds in the database at `path`, opened read-only and left as it is; `None` when there is none.
fn read_only<T>(
    path: &Path,
    read: impl Fn(&Connection) -> Result<T, StoreError>,
) -> Result<Option<T>, StoreError> {
    if !path.exists() {
        return Ok(None);
    }
    let read = |path: &str, flags| -> Result<T, StoreError> {
        let db = Connection::open_with_flags(path, flags)?;
        db.busy_timeout(Duration::from_secs(5))?;
        read(&db)
    };
    let name = path.to_string_lossy();
    match read(&name, OpenFlags::SQLITE_OPEN_READ_ONLY) {
        Err(StoreError::Sqlite(error))
            if matches!(
                error.sqlite_error_code(),
                Some(rusqlite::ErrorCode::ReadOnly | rusqlite::ErrorCode::CannotOpen)
            ) =>
        {
            // No `-shm` can be made: read the file alone, unless the `-wal` holds what it lacks.
            let mut wal = path.as_os_str().to_owned();
            wal.push("-wal");
            let wal = PathBuf::from(wal);
            if std::fs::metadata(&wal).is_ok_and(|wal| wal.len() > 0) {
                return Err(StoreError::Unread(wal));
            }
            let uri = format!(
                "file:{}?mode=ro&immutable=1",
                name.replace('%', "%25")
                    .replace('?', "%3f")
                    .replace('#', "%23")
            );
            read(
                &uri,
                OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_URI,
            )
        }
        history => history,
    }
    .map(Some)
}

/// What sizing reads from the database, without creating its schema; an older one reads as [`VERSION`].
fn sizing_of(db: &Connection, since: u64) -> Result<sizing::History, StoreError> {
    match version(db)? {
        0 => return Ok(sizing::History::default()),
        VERSION => {}
        _ => read_as_current(db)?,
    }
    let mut history = read::sizing(db, since)?;
    let auto = read::auto_sizing(db)?;
    history.memory_set = auto.memory;
    history.slots_set = auto.slots;
    history.changes = read::sizing_changes(db, since.saturating_sub(sizing::memory::HISTORY))?;
    Ok(history)
}

/// The database's schema version: 0 for a new one, else from [`OLDEST`] to [`VERSION`]; any other is refused.
fn version(db: &Connection) -> Result<u32, StoreError> {
    let found: u32 = db.pragma_query_value(None, "user_version", |row| row.get(0))?;
    match found {
        0 | OLDEST..=VERSION => Ok(found),
        found if found > VERSION => Err(StoreError::Newer {
            found,
            supported: VERSION,
        }),
        found => Err(StoreError::Older {
            found,
            oldest: OLDEST,
        }),
    }
}

/// Lets a read-only look at a database from before [`VERSION`], which it cannot migrate, read it as this build
/// does: a table it lacks becomes an empty temporary one, and a table that lacks columns a temporary view of it
/// that adds them, each with its default. Temporary objects come first in name lookups.
fn read_as_current(db: &Connection) -> Result<(), StoreError> {
    let current = Connection::open_in_memory()?;
    current.execute_batch(SCHEMA)?;
    let tables: Vec<(String, String)> = current
        .prepare(
            "SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
        )?
        .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
        .collect::<rusqlite::Result<_>>()?;
    let columns =
        |db: &Connection, table: &str| -> rusqlite::Result<Vec<(String, Option<String>)>> {
            db.prepare("SELECT name, dflt_value FROM pragma_table_info(?1, 'main')")?
                .query_map([table], |row| Ok((row.get(0)?, row.get(1)?)))?
                .collect()
        };
    for (table, sql) in tables {
        let found: Vec<String> = columns(db, &table)?
            .into_iter()
            .map(|(name, _)| name)
            .collect();
        if found.is_empty() {
            db.execute_batch(&sql.replacen("CREATE TABLE", "CREATE TEMP TABLE", 1))?;
            continue;
        }
        let added: Vec<String> = columns(&current, &table)?
            .into_iter()
            .filter(|(name, _)| !found.contains(name))
            .map(|(name, default)| format!("{} AS {name}", default.as_deref().unwrap_or("NULL")))
            .collect();
        if !added.is_empty() {
            db.execute_batch(&format!(
                "CREATE TEMP VIEW {table} AS SELECT *, {} FROM main.{table}",
                added.join(", ")
            ))?;
        }
    }
    Ok(())
}

/// WAL, so the API reads while the writer writes; a busy timeout instead of an immediate error.
fn configure(db: &Connection) -> rusqlite::Result<()> {
    db.pragma_update(None, "journal_mode", "wal")?;
    db.pragma_update(None, "synchronous", "normal")?;
    db.pragma_update(None, "foreign_keys", "on")?;
    db.busy_timeout(Duration::from_secs(5))
}

/// Creates the schema in a new database, in one transaction with its version, or runs the migrations an older one
/// lacks, each in its own transaction with the version it brings it to.
fn prepare(db: &mut Connection) -> Result<(), StoreError> {
    let found = version(db)?;
    if found == 0 {
        let transaction = db.transaction()?;
        transaction.execute_batch(SCHEMA)?;
        transaction.pragma_update(None, "user_version", VERSION)?;
        transaction.commit()?;
        return Ok(());
    }
    for (version, migration) in MIGRATIONS.iter().filter(|(version, _)| *version > found) {
        let transaction = db.transaction()?;
        transaction.execute_batch(migration)?;
        transaction.pragma_update(None, "user_version", version)?;
        transaction.commit()?;
    }
    Ok(())
}

/// The time before this daemon when no runnerd ran, in ms: from the history's last sign of life, its newest host
/// sample, event or open, to this daemon's start. Host samples are stored a minute's worth at a time, so a crash
/// may count up to a minute more. Without a sign of life, as in a new history, there is none: the history's start
/// clips the waits of jobs queued before it (`history_kept`).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct Downtime {
    from: u64,
    until: u64,
}

impl Downtime {
    /// The downtime before a daemon that starts at `now`; the jobs still queued in the history leave it out of
    /// their waits, as the ones first seen after do when they are recorded. The open is a sign of life itself, kept
    /// with them: a start that fails after it, and is retried, does not hand out the same downtime again.
    fn before(db: &mut Connection, now: u64) -> Result<Option<Downtime>, StoreError> {
        // Immediate: it reads, then writes. A deferred transaction that has read cannot wait for the write lock, and
        // fails at once while another connection, such as a dropped store's writer finishing, holds it.
        let transaction = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
        let last: Option<u64> = transaction.query_row(
            "SELECT max(at) FROM (SELECT max(at) AS at FROM host_samples UNION ALL SELECT max(at) FROM events \
             UNION ALL SELECT value FROM meta WHERE key = 'opened_at')",
            [],
            |row| row.get(0),
        )?;
        let downtime = last
            .filter(|last| *last < now)
            .map(|from| Downtime { from, until: now });
        if let Some(downtime) = downtime {
            let queued: Vec<(u64, u64)> = transaction
                .prepare("SELECT id, queued_at FROM jobs WHERE state = 'queued'")?
                .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
                .collect::<rusqlite::Result<_>>()?;
            for (job, queued_at) in queued {
                transaction.execute(
                    "UPDATE jobs SET unseen = unseen + ?2 WHERE id = ?1",
                    rusqlite::params![job, downtime.of(queued_at)],
                )?;
            }
        }
        transaction.execute(
            "INSERT OR REPLACE INTO meta (key, value) VALUES ('opened_at', ?1)",
            [now],
        )?;
        transaction.commit()?;
        Ok(downtime)
    }

    /// How much of it a job queued at `queued_at` (ms) waited through.
    fn of(self, queued_at: u64) -> u64 {
        self.until.saturating_sub(queued_at.max(self.from))
    }

    /// How much of what [`Downtime::of`] gave a job queued at `queued_at` came after it started at `started_at`
    /// (ms), as for a job a runner that outlived the last daemon took while none ran.
    fn after(self, queued_at: u64, started_at: u64) -> u64 {
        self.of(queued_at)
            .saturating_sub(started_at.saturating_sub(queued_at.max(self.from)))
    }
}

/// Logs a failed write; the writer goes on with the next one.
fn logged<T>(what: &str, result: rusqlite::Result<T>) -> Option<T> {
    result
        .map_err(|error| warn!("job history: cannot {what}: {error}"))
        .ok()
}

#[cfg(test)]
mod tests;